From 7002b60e02b622ec3809867d4fff67e742650eee Mon Sep 17 00:00:00 2001 From: Dev Date: Mon, 22 Apr 2024 19:08:40 +0100 Subject: [PATCH 01/15] Pass Map list into CreateStaticLootDistribution --- .../MultithreadSteppedDumpProcessor.cs | 2 +- Process/Processor/StaticLootProcessor.cs | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs index 1dc2265..4dbbeba 100644 --- a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs +++ b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs @@ -146,7 +146,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor // Static loot distribution output.Add( OutputFileType.StaticLoot, - StaticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts) + StaticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts, staticContainers) ); if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) diff --git a/Process/Processor/StaticLootProcessor.cs b/Process/Processor/StaticLootProcessor.cs index 30ac69c..300a03c 100644 --- a/Process/Processor/StaticLootProcessor.cs +++ b/Process/Processor/StaticLootProcessor.cs @@ -113,17 +113,24 @@ public static class StaticLootProcessor } public static Dictionary CreateStaticLootDistribution( - List container_counts - ) + List container_counts, + Dictionary staticContainers) { + // Iterate over each map we have containers for + foreach (var map in staticContainers) + { + var mapName = map.Key; + var mapContainers = map.Value; + } + var static_loot_distribution = new Dictionary(); - var types = Enumerable.Distinct((from ci in container_counts + var uniqueContainerTypeIds = Enumerable.Distinct((from ci in container_counts select ci.Type).ToList()); - foreach (var typei in types) + foreach (var typeId in uniqueContainerTypeIds) { var container_counts_selected = (from ci in container_counts - where ci.Type == typei + where ci.Type == typeId select ci).ToList(); var itemscounts = new List(); foreach (var ci in container_counts_selected) From 8a541316d42c20ace705a03e5033e8052d96564e Mon Sep 17 00:00:00 2001 From: Dev Date: Mon, 22 Apr 2024 19:09:20 +0100 Subject: [PATCH 02/15] Cleanup of var names --- .../MultithreadSteppedDumpProcessor.cs | 17 ++++++++--------- Process/Processor/StaticLootProcessor.cs | 6 +++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs index 4dbbeba..16303ea 100644 --- a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs +++ b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs @@ -52,11 +52,10 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor Runners.Add( Task.Factory.StartNew(() => { - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug)) LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", LogLevel.Debug); - var data = _jsonSerializer.Deserialize(File.ReadAllText(dumped.BasicInfo.FileName)); - var mapName = data.Data.Name; + var dataDump = _jsonSerializer.Deserialize(File.ReadAllText(dumped.BasicInfo.FileName)); + var mapName = dataDump.Data.Name; // the if statement below takes care of processing "forced" or real static data for each map, only need // to do this once per map, we dont care about doing it again @@ -66,7 +65,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor { if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) LoggerFactory.GetInstance().Log($"Doing first time process for map {mapName} of real static data", LogLevel.Info); - var mapStaticContainers = StaticLootProcessor.CreateStaticWeaponsAndStaticForcedContainers(data); + var mapStaticContainers = StaticLootProcessor.CreateStaticWeaponsAndStaticForcedContainers(dataDump); // .Item1 = map name staticContainers[mapStaticContainers.Item1] = mapStaticContainers.Item2; } @@ -93,18 +92,18 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor IncrementMapCounterDictionaryValue(mapDumpCounter, mapName); } - var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(data.Data.Id.ToLower(), out string[]? ignoreListForMap); - foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(data)) + var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(dataDump.Data.Id.ToLower(), out string[]? ignoreListForMap); + foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(dataDump)) { lock (mapStaticContainersAggregatedLock) { - // Skip adding containers to aggredated data if container id is in ignore list if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id)) { + // Skip adding containers to aggregated data if container id is in ignore list continue; } - // Increment count by 1 + // Increment times container seen in dump by 1 if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1)) mapAggregatedDataDict[dynamicStaticContainer] += 1; } @@ -126,7 +125,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor td => new StaticDataPoint { Template = td.Key, - Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) + Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) // kv.Key = map name } ).ToList() ).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); diff --git a/Process/Processor/StaticLootProcessor.cs b/Process/Processor/StaticLootProcessor.cs index 300a03c..9b96d55 100644 --- a/Process/Processor/StaticLootProcessor.cs +++ b/Process/Processor/StaticLootProcessor.cs @@ -140,8 +140,8 @@ public static class StaticLootProcessor select cii).ToList().Count); } - static_loot_distribution[typei] = new StaticItemDistribution(); - static_loot_distribution[typei].ItemCountDistribution = itemscounts.GroupBy(i => i) + static_loot_distribution[typeId] = new StaticItemDistribution(); + static_loot_distribution[typeId].ItemCountDistribution = itemscounts.GroupBy(i => i) .Select(g => new ItemCountDistribution { Count = g.Key, @@ -161,7 +161,7 @@ public static class StaticLootProcessor } } - static_loot_distribution[typei].ItemDistribution = itemsHitCounts.Select(v => new StaticDistribution + static_loot_distribution[typeId].ItemDistribution = itemsHitCounts.Select(v => new StaticDistribution { Tpl = v.Key, RelativeProbability = v.Value From 892c7c548e843d501d4c1aa6a5720df0e63fcb2e Mon Sep 17 00:00:00 2001 From: Dev Date: Mon, 22 Apr 2024 20:59:48 +0100 Subject: [PATCH 03/15] Add new sandbox map --- Config/map_directory_mapping.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Config/map_directory_mapping.yaml b/Config/map_directory_mapping.yaml index 72154b2..a340500 100644 --- a/Config/map_directory_mapping.yaml +++ b/Config/map_directory_mapping.yaml @@ -29,4 +29,7 @@ Streets of Tarkov: - tarkovstreets Sandbox: name: - - Sandbox \ No newline at end of file + - Sandbox +SandboxHigh: + name: + - sandbox_high \ No newline at end of file From 8834775186bcb2bfb95188e4ed4978412e9714e2 Mon Sep 17 00:00:00 2001 From: Dev Date: Mon, 22 Apr 2024 21:01:02 +0100 Subject: [PATCH 04/15] First attempt at splitting static loot per map, static ammo is disabled for now --- Model/Processing/DumpProcessData.cs | 2 +- .../MultithreadSteppedDumpProcessor.cs | 109 +++++++------ Process/Processor/StaticLootProcessor.cs | 147 ++++++++++++------ Process/Writer/FileWriter.cs | 19 ++- 4 files changed, 180 insertions(+), 97 deletions(-) diff --git a/Model/Processing/DumpProcessData.cs b/Model/Processing/DumpProcessData.cs index b091d18..b627f05 100644 --- a/Model/Processing/DumpProcessData.cs +++ b/Model/Processing/DumpProcessData.cs @@ -5,6 +5,6 @@ namespace LootDumpProcessor.Model.Processing; public class DumpProcessData { public Dictionary LooseLootCounts { get; set; } = new(); - public List ContainerCounts { get; set; } = new(); + public Dictionary> ContainerCounts { get; set; } = new(); public Dictionary MapCounts { get; set; } = new(); } \ No newline at end of file diff --git a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs index 16303ea..4eba38f 100644 --- a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs +++ b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs @@ -67,6 +67,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor LoggerFactory.GetInstance().Log($"Doing first time process for map {mapName} of real static data", LogLevel.Info); var mapStaticContainers = StaticLootProcessor.CreateStaticWeaponsAndStaticForcedContainers(dataDump); // .Item1 = map name + // .Item2 = force/weapon static arrays staticContainers[mapStaticContainers.Item1] = mapStaticContainers.Item2; } } @@ -84,28 +85,32 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor } // Only process the dump file if the date is higher (after) the configuration date - if (DumpWasMadeAfterConfigThresholdDate(dumped)) + if (!DumpWasMadeAfterConfigThresholdDate(dumped)) { - // Keep track of how many dumps we have for each map - lock (mapDumpCounterLock) - { - IncrementMapCounterDictionaryValue(mapDumpCounter, mapName); - } + return; + } - var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(dataDump.Data.Id.ToLower(), out string[]? ignoreListForMap); - foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(dataDump)) + // Keep track of how many dumps we have for each map + lock (mapDumpCounterLock) + { + IncrementMapCounterDictionaryValue(mapDumpCounter, mapName); + } + + var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(dataDump.Data.Id.ToLower(), out string[]? ignoreListForMap); + foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(dataDump)) + { + lock (mapStaticContainersAggregatedLock) { - lock (mapStaticContainersAggregatedLock) + if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id)) { - if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id)) - { - // Skip adding containers to aggregated data if container id is in ignore list - continue; - } + // Skip adding containers to aggregated data if container id is in ignore list + continue; + } - // Increment times container seen in dump by 1 - if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1)) - mapAggregatedDataDict[dynamicStaticContainer] += 1; + // Increment times container seen in dump by 1 + if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1)) + { + mapAggregatedDataDict[dynamicStaticContainer] += 1; } } } @@ -128,17 +133,17 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) // kv.Key = map name } ).ToList() - ).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); + ).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); // Hydrate staticContainers.StaticContainers // Static containers output.Add(OutputFileType.StaticContainer, staticContainers); if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) LoggerFactory.GetInstance().Log("Processing ammo distribution", LogLevel.Info); // Ammo distribution - output.Add( - OutputFileType.StaticAmmo, - StaticLootProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts) - ); + //output.Add( + // OutputFileType.StaticAmmo, + // StaticLootProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts) + //); if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) LoggerFactory.GetInstance().Log("Processing static loot distribution", LogLevel.Info); @@ -198,23 +203,23 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor .ToList() .ForEach(tuple => { - var mapi = tuple.Key; - var g = tuple.ToList(); + var mapName = tuple.Key; + var partialFileMetaData = tuple.ToList(); if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) LoggerFactory.GetInstance().Log( - $"Processing map {mapi}, total dump data to process: {g.Count}", + $"Processing map {mapName}, total dump data to process: {partialFileMetaData.Count}", LogLevel.Info ); - dumpProcessData.MapCounts[mapi] = g.Count; + dumpProcessData.MapCounts[mapName] = partialFileMetaData.Count; var lockObjectContainerCounts = new object(); var lockObjectCounts = new object(); - var counts = new LooseLootCounts(); + var looseLootCounts = new LooseLootCounts(); var lockObjectDictionaryCounts = new object(); var dictionaryCounts = new FlatKeyableDictionary(); - counts.Counts = dictionaryCounts.GetKey(); + looseLootCounts.Counts = dictionaryCounts.GetKey(); /* var dictionaryItemCounts = new FlatKeyableDictionary>(); @@ -225,22 +230,21 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor var dictionaryItemProperties = new FlatKeyableDictionary>(); var actualDictionaryItemProperties = new FlatKeyableDictionary(); - counts.ItemProperties = actualDictionaryItemProperties.GetKey(); + looseLootCounts.ItemProperties = actualDictionaryItemProperties.GetKey(); - dumpProcessData.LooseLootCounts.Add(mapi, counts.GetKey()); + dumpProcessData.LooseLootCounts.Add(mapName, looseLootCounts.GetKey()); // add the items to the queue - foreach (var gi in g) + foreach (var partialData in partialFileMetaData) { - _partialDataToProcess.Add(gi); + _partialDataToProcess.Add(partialData); } // Call GC before running threads - g = null; + partialFileMetaData = null; tuple = null; GCHandler.Collect(); // The data storage factory has a lock, we dont want the locks to occur when multithreading - for (int i = 0; i < LootDumpProcessorContext.GetConfig().Threads; i++) { Runners.Add( @@ -253,26 +257,35 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor try { var dumpData = _dataStorage.GetItem(partialData.ParsedDumpKey); + + // Static containers lock (lockObjectContainerCounts) { - dumpProcessData.ContainerCounts.AddRange(dumpData.Containers); + if (!dumpProcessData.ContainerCounts.ContainsKey(mapName)) + { + dumpProcessData.ContainerCounts.Add(mapName, dumpData.Containers); + } + else + { + dumpProcessData.ContainerCounts[mapName].AddRange(dumpData.Containers); + } } - // loose loot into ids on files + // Loose loot into ids on files var loadedDictionary = _dataStorage .GetItem>>( dumpData.LooseLoot.ItemProperties ); - foreach (var (k, v) in loadedDictionary) + foreach (var (uniqueKey, containerTemplate) in loadedDictionary) { - var count = dumpData.LooseLoot.Counts[k]; + var count = dumpData.LooseLoot.Counts[uniqueKey]; lock (lockObjectDictionaryCounts) { - if (dictionaryCounts.ContainsKey(k)) - dictionaryCounts[k] += count; + if (dictionaryCounts.ContainsKey(uniqueKey)) + dictionaryCounts[uniqueKey] += count; else - dictionaryCounts[k] = count; + dictionaryCounts[uniqueKey] = count; } /* @@ -287,20 +300,20 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor lock (lockObjectDictionaryItemProperties) { - if (!dictionaryItemProperties.TryGetValue(k, out var values)) + if (!dictionaryItemProperties.TryGetValue(uniqueKey, out var values)) { values = new FlatKeyableList