diff --git a/Config/config.json b/Config/config.json index 4850723..b5bda8b 100644 --- a/Config/config.json +++ b/Config/config.json @@ -1,9 +1,12 @@ { - "serverLocation": "D:\\Spt Stuff\\server", - "threads": 20, + "serverLocation": "E:\\spt\\Server", + "threads": 10, "threadPoolingTimeoutMs": 1000, "jsonSerializer": "DotNet", "manualGarbageCollectionCalls": false, + "dumpProcessorConfig": { + "spawnContainerChanceIncludeAfterDate": "2023-08-10 00:00:01" + }, "dataStorageConfig": { "dataStorageType": "Memory", "fileDataStorageTempLocation": "D:\\Spt Stuff\\Lootgenerator\\Dumps\\cache" @@ -21,13 +24,14 @@ "cleanupTempFolderAfterProcess": true }, "intakeReaderConfig": { + "maxDumpsPerMap": 1500, "readerType": "Json", "ignoredDumpLocations": [ "Hideout" ] }, "dumpFilesLocation": [ - "D:\\Spt Stuff\\Lootgenerator\\dumps\\input" + "E:\\spt\\dumps\\input" ], "thresholdDate": "2023-01-08", "acceptedFileExtensions": [ @@ -43,6 +47,6 @@ "spawnPointToleranceForForced": 99.5 }, "writerConfig": { - "outputLocation": "D:\\Spt Stuff\\Lootgenerator\\dumps\\output" + "outputLocation": "E:\\spt\\dumps\\output" } } \ No newline at end of file diff --git a/Model/Config/Config.cs b/Model/Config/Config.cs index e888406..6249a27 100644 --- a/Model/Config/Config.cs +++ b/Model/Config/Config.cs @@ -41,6 +41,10 @@ public class Config [JsonProperty("processorConfig")] [JsonPropertyName("processorConfig")] public ProcessorConfig ProcessorConfig { get; set; } + + [JsonProperty("dumpProcessorConfig")] + [JsonPropertyName("dumpProcessorConfig")] + public DumpProcessorConfig DumpProcessorConfig { get; set; } [JsonProperty("writerConfig")] [JsonPropertyName("writerConfig")] diff --git a/Model/Config/DumpProcessorConfig.cs b/Model/Config/DumpProcessorConfig.cs new file mode 100644 index 0000000..b7f7630 --- /dev/null +++ b/Model/Config/DumpProcessorConfig.cs @@ -0,0 +1,14 @@ +using System.Text.Json.Serialization; +using LootDumpProcessor.Serializers.Json.Converters; +using Newtonsoft.Json; + +namespace LootDumpProcessor.Model.Config; + +public class DumpProcessorConfig +{ + [JsonProperty("spawnContainerChanceIncludeAfterDate")] + [JsonPropertyName("spawnContainerChanceIncludeAfterDate")] + [Newtonsoft.Json.JsonConverter(typeof(NewtonsoftDateTimeConverter))] + [System.Text.Json.Serialization.JsonConverter(typeof(NetDateTimeConverter))] + public DateTime SpawnContainerChanceIncludeAfterDate { get; set; } +} \ No newline at end of file diff --git a/Model/Config/IntakeReaderConfig.cs b/Model/Config/IntakeReaderConfig.cs index b3e820f..9522bed 100644 --- a/Model/Config/IntakeReaderConfig.cs +++ b/Model/Config/IntakeReaderConfig.cs @@ -10,6 +10,11 @@ public class IntakeReaderConfig [JsonPropertyName("readerType")] public IntakeReaderTypes IntakeReaderType { get; set; } = IntakeReaderTypes.Json; + [JsonProperty("maxDumpsPerMap")] + [JsonPropertyName("maxDumpsPerMap")] + public int MaxDumpsPerMap { get; set; } = 1500; + + [JsonProperty("ignoredDumpLocations")] [JsonPropertyName("ignoredDumpLocations")] public List IgnoredDumpLocations { get; set; } = new List(); diff --git a/Model/Processing/PreProcessedLooseLoot.cs b/Model/Processing/PreProcessedLooseLoot.cs index ecf6f33..73ccc9a 100644 --- a/Model/Processing/PreProcessedLooseLoot.cs +++ b/Model/Processing/PreProcessedLooseLoot.cs @@ -1,5 +1,5 @@ -using LootDumpProcessor.Storage; -using Newtonsoft.Json; +using LootDumpProcessor.Serializers.Json.Converters; +using LootDumpProcessor.Storage; namespace LootDumpProcessor.Model.Processing; @@ -7,7 +7,8 @@ public class PreProcessedLooseLoot : IKeyable { public Dictionary Counts { get; set; } - [JsonConverter(typeof(NewtonsoftJsonKeyConverter))] + [Newtonsoft.Json.JsonConverter(typeof(NewtonsoftJsonKeyConverter))] + [System.Text.Json.Serialization.JsonConverter(typeof(NetJsonKeyConverter))] public IKey ItemProperties { get; set; } public int MapSpawnpointCount { get; set; } diff --git a/Model/Tarkov/Handbook.cs b/Model/Tarkov/Handbook.cs index 5d4c8d9..c78f848 100644 --- a/Model/Tarkov/Handbook.cs +++ b/Model/Tarkov/Handbook.cs @@ -20,4 +20,17 @@ public class HandbookRoot { public List Categories { get; set; } public List Items { get; set; } +} + +public class StaticContainerRoot +{ + public decimal probability { get; set; } + public StaticContainerTemplate template { get; set; } +} + +public class StaticContainerTemplate +{ + public string Id { get; set; } + public decimal SpawnChance { get; set; } + public bool IsAlwaysSpawn { get; set; } } \ No newline at end of file diff --git a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs index b75f7ba..b9802dd 100644 --- a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs +++ b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs @@ -39,29 +39,24 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor // dictionary of maps, that has a dictionary of template and hit count var mapStaticContainersAggregated = new Dictionary>(); var mapStaticContainersAggregatedLock = new object(); - + Runners.Clear(); // BSG changed the map data so static containers are now dynamic, so we need to scan all dumps for the static containers. + LoggerFactory.GetInstance().Log("Queuing dumps for static data processing", LogLevel.Info); foreach (var dumped in dumps) { Runners.Add( Task.Factory.StartNew(() => { + LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", LogLevel.Info); var data = _jsonSerializer.Deserialize(File.ReadAllText(dumped.BasicInfo.FileName)); - // the if statement below will keep track of how many dumps we have for each map - lock (mapDumpCounterLock) - { - if (mapDumpCounter.ContainsKey(data.Data.Name)) - mapDumpCounter[data.Data.Name] += 1; - else - mapDumpCounter.Add(data.Data.Name, 1); - } // the if statement below takes care of processing "forced" or real static data for each map, we only need // to do this once per map, so we dont care about doing it again lock (staticContainersLock) { if (!staticContainers.ContainsKey(data.Data.Name)) { + LoggerFactory.GetInstance().Log($"Doing first time process for map {data.Data.Name} of real static data", LogLevel.Info); var mapStaticLoot = StaticLootProcessor.CreateRealStaticContainers(data); staticContainers[mapStaticLoot.Item1] = mapStaticLoot.Item2; } @@ -78,14 +73,30 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor } } - foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(data)) + // Only process the dump file if the date is higher (after) the configuration date + if (FileDateParser.TryParseFileDate(dumped.BasicInfo.FileName, out var fileDate) && + fileDate.HasValue && + fileDate.Value > LootDumpProcessorContext.GetConfig().DumpProcessorConfig + .SpawnContainerChanceIncludeAfterDate) { - lock (mapStaticContainersAggregatedLock) + // the if statement below will keep track of how many dumps we have for each map + lock (mapDumpCounterLock) { - if (mapAggregatedData.ContainsKey(dynamicStaticContainer)) - mapAggregatedData[dynamicStaticContainer] += 1; + if (mapDumpCounter.ContainsKey(data.Data.Name)) + mapDumpCounter[data.Data.Name] += 1; else - mapAggregatedData.Add(dynamicStaticContainer, 1); + mapDumpCounter.Add(data.Data.Name, 1); + } + + foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(data)) + { + lock (mapStaticContainersAggregatedLock) + { + if (mapAggregatedData.ContainsKey(dynamicStaticContainer)) + mapAggregatedData[dynamicStaticContainer] += 1; + else + mapAggregatedData.Add(dynamicStaticContainer, 1); + } } } @@ -95,6 +106,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor } Task.WaitAll(Runners.ToArray()); + LoggerFactory.GetInstance().Log("All static data processing threads finished", LogLevel.Info); // Aggregate and calculate the probability of a static container mapStaticContainersAggregated.ToDictionary( kv => kv.Key, @@ -102,7 +114,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor td => new StaticDataPoint { Template = td.Key, - Probability = Math.Round((double)((decimal)td.Value / (decimal)mapDumpCounter[kv.Key]), 2) + Probability = GetStaticProbability(kv.Key, td, mapDumpCounter) } ).ToList() ).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); @@ -110,32 +122,42 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor // Static containers output.Add(OutputFileType.StaticContainer, staticContainers); + LoggerFactory.GetInstance().Log("Processing ammo distribution", LogLevel.Info); // Ammo distribution output.Add( OutputFileType.StaticAmmo, StaticLootProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts) ); + LoggerFactory.GetInstance().Log("Processing static loot distribution", LogLevel.Info); // Static loot distribution output.Add( OutputFileType.StaticLoot, StaticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts) ); + LoggerFactory.GetInstance().Log("Processing loose loot distribution", LogLevel.Info); // Loose loot distribution var looseLootDistribution = LooseLootProcessor.CreateLooseLootDistribution( dumpProcessData.MapCounts, dumpProcessData.LooseLootCounts ); + LoggerFactory.GetInstance().Log("Collecting loose loot distribution information", LogLevel.Info); var loot = dumpProcessData.MapCounts .Select(mapCount => mapCount.Key) .ToDictionary(mi => mi, mi => looseLootDistribution[mi]); output.Add(OutputFileType.LooseLoot, loot); + LoggerFactory.GetInstance().Log("Dump processing fully completed!", LogLevel.Info); return output; } + private static double GetStaticProbability(string mapName, KeyValuePair td, Dictionary mapDumpCounter) + { + return Math.Round((double)((decimal)td.Value / (decimal)mapDumpCounter[mapName]), 2); + } + private DumpProcessData GetDumpProcessData(List dumps) { var dumpProcessData = new DumpProcessData(); diff --git a/Process/Processor/StaticLootProcessor.cs b/Process/Processor/StaticLootProcessor.cs index f882790..7f8dc79 100644 --- a/Process/Processor/StaticLootProcessor.cs +++ b/Process/Processor/StaticLootProcessor.cs @@ -60,9 +60,17 @@ public class StaticLootProcessor public static List