From 8834775186bcb2bfb95188e4ed4978412e9714e2 Mon Sep 17 00:00:00 2001
From: Dev <dev@dev.sp-tarkov.com>
Date: Mon, 22 Apr 2024 21:01:02 +0100
Subject: [PATCH] First attempt at splitting static loot per map, static ammo
 is disabled for now

---
 Model/Processing/DumpProcessData.cs           |   2 +-
 .../MultithreadSteppedDumpProcessor.cs        | 109 +++++++------
 Process/Processor/StaticLootProcessor.cs      | 147 ++++++++++++------
 Process/Writer/FileWriter.cs                  |  19 ++-
 4 files changed, 180 insertions(+), 97 deletions(-)

diff --git a/Model/Processing/DumpProcessData.cs b/Model/Processing/DumpProcessData.cs
index b091d18..b627f05 100644
--- a/Model/Processing/DumpProcessData.cs
+++ b/Model/Processing/DumpProcessData.cs
@@ -5,6 +5,6 @@ namespace LootDumpProcessor.Model.Processing;
 public class DumpProcessData
 {
     public Dictionary<string, IKey> LooseLootCounts { get; set; } = new();
-    public List<PreProcessedStaticLoot> ContainerCounts { get; set; } = new();
+    public Dictionary<string, List<PreProcessedStaticLoot>> ContainerCounts { get; set; } = new();
     public Dictionary<string, int> MapCounts { get; set; } = new();
 }
\ No newline at end of file
diff --git a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
index 16303ea..4eba38f 100644
--- a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
+++ b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
@@ -67,6 +67,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
                                 LoggerFactory.GetInstance().Log($"Doing first time process for map {mapName} of real static data", LogLevel.Info);
                             var mapStaticContainers = StaticLootProcessor.CreateStaticWeaponsAndStaticForcedContainers(dataDump);
                             // .Item1 = map name
+                            // .Item2 = force/weapon static arrays
                             staticContainers[mapStaticContainers.Item1] = mapStaticContainers.Item2;
                         }
                     }
@@ -84,28 +85,32 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
                     }
 
                     // Only process the dump file if the date is higher (after) the configuration date
-                    if (DumpWasMadeAfterConfigThresholdDate(dumped))
+                    if (!DumpWasMadeAfterConfigThresholdDate(dumped))
                     {
-                        // Keep track of how many dumps we have for each map
-                        lock (mapDumpCounterLock)
-                        {
-                            IncrementMapCounterDictionaryValue(mapDumpCounter, mapName);
-                        }
+                        return;
+                    }
 
-                        var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(dataDump.Data.Id.ToLower(), out string[]? ignoreListForMap);
-                        foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(dataDump))
+                    // Keep track of how many dumps we have for each map
+                    lock (mapDumpCounterLock)
+                    {
+                        IncrementMapCounterDictionaryValue(mapDumpCounter, mapName);
+                    }
+
+                    var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(dataDump.Data.Id.ToLower(), out string[]? ignoreListForMap);
+                    foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(dataDump))
+                    {
+                        lock (mapStaticContainersAggregatedLock)
                         {
-                            lock (mapStaticContainersAggregatedLock)
+                            if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id))
                             {
-                                if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id))
-                                {
-                                    // Skip adding containers to aggregated data if container id is in ignore list
-                                    continue;
-                                }
+                                // Skip adding containers to aggregated data if container id is in ignore list
+                                continue;
+                            }
 
-                                // Increment times container seen in dump by 1
-                                if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1))
-                                    mapAggregatedDataDict[dynamicStaticContainer] += 1;
+                            // Increment times container seen in dump by 1
+                            if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1))
+                            {
+                                mapAggregatedDataDict[dynamicStaticContainer] += 1;
                             }
                         }
                     }
@@ -128,17 +133,17 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
                     Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) // kv.Key = map name
                 }
             ).ToList()
-        ).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value);
+        ).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); // Hydrate staticContainers.StaticContainers
 
         // Static containers
         output.Add(OutputFileType.StaticContainer, staticContainers);
         if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Processing ammo distribution", LogLevel.Info);
         // Ammo distribution
-        output.Add(
-            OutputFileType.StaticAmmo,
-            StaticLootProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts)
-        );
+        //output.Add(
+        //    OutputFileType.StaticAmmo,
+        //    StaticLootProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts)
+        //);
 
         if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Processing static loot distribution", LogLevel.Info);
@@ -198,23 +203,23 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
             .ToList()
             .ForEach(tuple =>
             {
-                var mapi = tuple.Key;
-                var g = tuple.ToList();
+                var mapName = tuple.Key;
+                var partialFileMetaData = tuple.ToList();
                 if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
                     LoggerFactory.GetInstance().Log(
-                        $"Processing map {mapi}, total dump data to process: {g.Count}",
+                        $"Processing map {mapName}, total dump data to process: {partialFileMetaData.Count}",
                         LogLevel.Info
                     );
-                dumpProcessData.MapCounts[mapi] = g.Count;
+                dumpProcessData.MapCounts[mapName] = partialFileMetaData.Count;
 
                 var lockObjectContainerCounts = new object();
 
                 var lockObjectCounts = new object();
-                var counts = new LooseLootCounts();
+                var looseLootCounts = new LooseLootCounts();
 
                 var lockObjectDictionaryCounts = new object();
                 var dictionaryCounts = new FlatKeyableDictionary<string, int>();
-                counts.Counts = dictionaryCounts.GetKey();
+                looseLootCounts.Counts = dictionaryCounts.GetKey();
 
                 /*
                 var dictionaryItemCounts = new FlatKeyableDictionary<string, List<string>>();
@@ -225,22 +230,21 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
                 var dictionaryItemProperties = new FlatKeyableDictionary<string, FlatKeyableList<Template>>();
 
                 var actualDictionaryItemProperties = new FlatKeyableDictionary<string, IKey>();
-                counts.ItemProperties = actualDictionaryItemProperties.GetKey();
+                looseLootCounts.ItemProperties = actualDictionaryItemProperties.GetKey();
 
-                dumpProcessData.LooseLootCounts.Add(mapi, counts.GetKey());
+                dumpProcessData.LooseLootCounts.Add(mapName, looseLootCounts.GetKey());
                 // add the items to the queue
-                foreach (var gi in g)
+                foreach (var partialData in partialFileMetaData)
                 {
-                    _partialDataToProcess.Add(gi);
+                    _partialDataToProcess.Add(partialData);
                 }
 
                 // Call GC before running threads
-                g = null;
+                partialFileMetaData = null;
                 tuple = null;
                 GCHandler.Collect();
 
                 // The data storage factory has a lock, we dont want the locks to occur when multithreading
-
                 for (int i = 0; i < LootDumpProcessorContext.GetConfig().Threads; i++)
                 {
                     Runners.Add(
@@ -253,26 +257,35 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
                                     try
                                     {
                                         var dumpData = _dataStorage.GetItem<ParsedDump>(partialData.ParsedDumpKey);
+
+                                        // Static containers
                                         lock (lockObjectContainerCounts)
                                         {
-                                            dumpProcessData.ContainerCounts.AddRange(dumpData.Containers);
+                                            if (!dumpProcessData.ContainerCounts.ContainsKey(mapName))
+                                            {
+                                                dumpProcessData.ContainerCounts.Add(mapName, dumpData.Containers);
+                                            }
+                                            else
+                                            {
+                                                dumpProcessData.ContainerCounts[mapName].AddRange(dumpData.Containers);
+                                            }
                                         }
 
-                                        // loose loot into ids on files
+                                        // Loose loot into ids on files
                                         var loadedDictionary =
                                             _dataStorage
                                                 .GetItem<SubdivisionedKeyableDictionary<string, List<Template>>>(
                                                     dumpData.LooseLoot.ItemProperties
                                                 );
-                                        foreach (var (k, v) in loadedDictionary)
+                                        foreach (var (uniqueKey, containerTemplate) in loadedDictionary)
                                         {
-                                            var count = dumpData.LooseLoot.Counts[k];
+                                            var count = dumpData.LooseLoot.Counts[uniqueKey];
                                             lock (lockObjectDictionaryCounts)
                                             {
-                                                if (dictionaryCounts.ContainsKey(k))
-                                                    dictionaryCounts[k] += count;
+                                                if (dictionaryCounts.ContainsKey(uniqueKey))
+                                                    dictionaryCounts[uniqueKey] += count;
                                                 else
-                                                    dictionaryCounts[k] = count;
+                                                    dictionaryCounts[uniqueKey] = count;
                                             }
 
                                             /*
@@ -287,20 +300,20 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
 
                                             lock (lockObjectDictionaryItemProperties)
                                             {
-                                                if (!dictionaryItemProperties.TryGetValue(k, out var values))
+                                                if (!dictionaryItemProperties.TryGetValue(uniqueKey, out var values))
                                                 {
                                                     values = new FlatKeyableList<Template>();
-                                                    dictionaryItemProperties.Add(k, values);
-                                                    actualDictionaryItemProperties.Add(k, values.GetKey());
+                                                    dictionaryItemProperties.Add(uniqueKey, values);
+                                                    actualDictionaryItemProperties.Add(uniqueKey, values.GetKey());
                                                 }
 
-                                                values.AddRange(v);
+                                                values.AddRange(containerTemplate);
                                             }
                                         }
 
                                         lock (lockObjectCounts)
                                         {
-                                            counts.MapSpawnpointCount.Add(dumpData.LooseLoot.MapSpawnpointCount);
+                                            looseLootCounts.MapSpawnpointCount.Add(dumpData.LooseLoot.MapSpawnpointCount);
                                         }
                                     }
                                     catch (Exception e)
@@ -345,8 +358,8 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
                 _dataStorage.Store(actualDictionaryItemProperties);
                 actualDictionaryItemProperties = null;
                 GCHandler.Collect();
-                _dataStorage.Store(counts);
-                counts = null;
+                _dataStorage.Store(looseLootCounts);
+                looseLootCounts = null;
                 GCHandler.Collect();
             });
         return dumpProcessData;
diff --git a/Process/Processor/StaticLootProcessor.cs b/Process/Processor/StaticLootProcessor.cs
index 9b96d55..3e211d1 100644
--- a/Process/Processor/StaticLootProcessor.cs
+++ b/Process/Processor/StaticLootProcessor.cs
@@ -112,62 +112,119 @@ public static class StaticLootProcessor
         return ammo_distribution;
     }
 
-    public static Dictionary<string, StaticItemDistribution> CreateStaticLootDistribution(
-        List<PreProcessedStaticLoot> container_counts,
+    /// <summary>
+    /// Dict key = map,
+    /// value = sub dit:
+    ///     key = container Ids
+    ///     value = items + counts
+    /// </summary>
+    public static Dictionary<string, Dictionary<string, StaticItemDistribution>> CreateStaticLootDistribution(
+        Dictionary<string, List<PreProcessedStaticLoot>> container_counts,
         Dictionary<string, MapStaticLoot> staticContainers)
     {
+        var allMapsStaticLootDisto = new Dictionary< string, Dictionary<string, StaticItemDistribution>>();
         // Iterate over each map we have containers for
-        foreach (var map in staticContainers)
+        foreach (var mapContainersKvp in container_counts)
         {
-            var mapName = map.Key;
-            var mapContainers = map.Value;
+            var mapName = mapContainersKvp.Key;
+            var containers = mapContainersKvp.Value;
+
+            var static_loot_distribution = new Dictionary<string, StaticItemDistribution>();
+            var uniqueContainerTypeIds = Enumerable.Distinct((from ci in containers
+                                                              select ci.Type).ToList());
+
+            foreach (var typeId in uniqueContainerTypeIds)
+            {
+                var container_counts_selected = (from ci in containers
+                                                 where ci.Type == typeId
+                                                 select ci).ToList();
+
+                // Get array of all times a count of items was found in container
+                List<int> itemCountsInContainer = GetCountOfItemsInContainer(container_counts_selected);
+
+                // Create structure to hold item count + weight that it will be picked
+                // Group same counts together
+                static_loot_distribution[typeId] = new StaticItemDistribution();
+                static_loot_distribution[typeId].ItemCountDistribution = itemCountsInContainer.GroupBy(i => i)
+                    .Select(g => new ItemCountDistribution
+                    {
+                        Count = g.Key,
+                        RelativeProbability = g.Count()
+                    }).ToList();
+
+                static_loot_distribution[typeId].ItemDistribution = CreateItemDistribution(container_counts_selected);
+            }
+            // Key = containers tpl, value = items + count weights
+            allMapsStaticLootDisto.TryAdd(mapName, static_loot_distribution);
+
         }
 
-        var static_loot_distribution = new Dictionary<string, StaticItemDistribution>();
-        var uniqueContainerTypeIds = Enumerable.Distinct((from ci in container_counts
-            select ci.Type).ToList());
+        return allMapsStaticLootDisto;
 
-        foreach (var typeId in uniqueContainerTypeIds)
+        //var static_loot_distribution = new Dictionary<string, StaticItemDistribution>();
+        //var uniqueContainerTypeIds = Enumerable.Distinct((from ci in container_counts
+        //    select ci.Type).ToList());
+
+        //foreach (var typeId in uniqueContainerTypeIds)
+        //{
+        //    var container_counts_selected = (from ci in container_counts
+        //                                     where ci.Type == typeId
+        //                                     select ci).ToList();
+
+        //    // Get array of all times a count of items was found in container
+        //    List<int> itemCountsInContainer = GetCountOfItemsInContainer(container_counts_selected);
+
+        //    // Create structure to hold item count + weight that it will be picked
+        //    // Group same counts together
+        //    static_loot_distribution[typeId] = new StaticItemDistribution();
+        //    static_loot_distribution[typeId].ItemCountDistribution = itemCountsInContainer.GroupBy(i => i)
+        //        .Select(g => new ItemCountDistribution
+        //        {
+        //            Count = g.Key,
+        //            RelativeProbability = g.Count()
+        //        }).ToList();
+
+        //    static_loot_distribution[typeId].ItemDistribution = CreateItemDistribution(container_counts_selected);
+        //}
+        //// Key = containers tpl, value = items + count weights
+        //return static_loot_distribution;
+    }
+
+    private static List<StaticDistribution> CreateItemDistribution(List<PreProcessedStaticLoot> container_counts_selected)
+    {
+        // TODO: Change for different algo that splits items per parent once parentid = containerid, then compose
+        // TODO: key and finally create distribution based on composed Id instead
+        var itemsHitCounts = new Dictionary<string, int>();
+        foreach (var ci in container_counts_selected)
         {
-            var container_counts_selected = (from ci in container_counts
-                where ci.Type == typeId
-                select ci).ToList();
-            var itemscounts = new List<int>();
-            foreach (var ci in container_counts_selected)
+            foreach (var cii in ci.Items.Where(cii => cii.ParentId == ci.ContainerId))
             {
-                itemscounts.Add((from cii in ci.Items
-                    where cii.ParentId == ci.ContainerId
-                    select cii).ToList().Count);
+                if (itemsHitCounts.ContainsKey(cii.Tpl))
+                    itemsHitCounts[cii.Tpl] += 1;
+                else
+                    itemsHitCounts[cii.Tpl] = 1;
             }
-
-            static_loot_distribution[typeId] = new StaticItemDistribution();
-            static_loot_distribution[typeId].ItemCountDistribution = itemscounts.GroupBy(i => i)
-                .Select(g => new ItemCountDistribution
-                {
-                    Count = g.Key,
-                    RelativeProbability = g.Count()
-                }).ToList();
-            // TODO: Change for different algo that splits items per parent once parentid = containerid, then compose
-            // TODO: key and finally create distribution based on composed Id instead
-            var itemsHitCounts = new Dictionary<string, int>();
-            foreach (var ci in container_counts_selected)
-            {
-                foreach (var cii in ci.Items.Where(cii => cii.ParentId == ci.ContainerId))
-                {
-                    if (itemsHitCounts.ContainsKey(cii.Tpl))
-                        itemsHitCounts[cii.Tpl] += 1;
-                    else
-                        itemsHitCounts[cii.Tpl] = 1;
-                }
-            }
-
-            static_loot_distribution[typeId].ItemDistribution = itemsHitCounts.Select(v => new StaticDistribution
-            {
-                Tpl = v.Key,
-                RelativeProbability = v.Value
-            }).ToList();
         }
 
-        return static_loot_distribution;
+        // WIll create array of objects that have a tpl + relative probability weight value
+        return itemsHitCounts.Select(v => new StaticDistribution
+        {
+            Tpl = v.Key,
+            RelativeProbability = v.Value
+        }).ToList();
+    }
+
+    private static List<int> GetCountOfItemsInContainer(List<PreProcessedStaticLoot> container_counts_selected)
+    {
+        var itemCountsInContainer = new List<int>();
+        foreach (var containerWithItems in container_counts_selected)
+        {
+            // Only count item if its parent is the container, only root items are counted (not mod/attachment items)
+            itemCountsInContainer.Add((from cii in containerWithItems.Items
+                                       where cii.ParentId == containerWithItems.ContainerId
+                                       select cii).ToList().Count);
+        }
+
+        return itemCountsInContainer;
     }
 }
\ No newline at end of file
diff --git a/Process/Writer/FileWriter.cs b/Process/Writer/FileWriter.cs
index 6f6e08d..9b3adb8 100644
--- a/Process/Writer/FileWriter.cs
+++ b/Process/Writer/FileWriter.cs
@@ -1,7 +1,9 @@
 using LootDumpProcessor.Model.Output;
 using LootDumpProcessor.Model.Output.LooseLoot;
 using LootDumpProcessor.Model.Output.StaticContainer;
+using LootDumpProcessor.Model.Processing;
 using LootDumpProcessor.Serializers.Json;
+using System.Collections.Generic;
 
 namespace LootDumpProcessor.Process.Writer;
 
@@ -59,16 +61,27 @@ public class FileWriter : IWriter
                 var staticContainer = (Dictionary<string, MapStaticLoot>)data;
                 File.WriteAllText($@"{_outputPath}\loot\staticContainers.json",
                     _jsonSerializer.Serialize(staticContainer));
+
                 break;
             case OutputFileType.StaticLoot:
-                var staticLoot = (Dictionary<string, StaticItemDistribution>)data;
-                File.WriteAllText($@"{_outputPath}\loot\staticLoot.json",
-                    _jsonSerializer.Serialize(staticLoot));
+                var staticLootData = (Dictionary<string, Dictionary<string, StaticItemDistribution>>)data;
+                foreach (var (key, value) in staticLootData)
+                {
+                    foreach (var s in LootDumpProcessorContext.GetDirectoryMappings()[key].Name)
+                    {
+                        if (!Directory.Exists($@"{_outputPath}\locations\{s}"))
+                            Directory.CreateDirectory($@"{_outputPath}\locations\{s}");
+                        File.WriteAllText($@"{_outputPath}\locations\{s}\staticLoot.json",
+                            _jsonSerializer.Serialize(value));
+                    }
+                }
+
                 break;
             case OutputFileType.StaticAmmo:
                 var staticAmmo = (Dictionary<string, List<AmmoDistribution>>)data;
                 File.WriteAllText($@"{_outputPath}\loot\staticAmmo.json",
                     _jsonSerializer.Serialize(staticAmmo));
+
                 break;
             default:
                 throw new ArgumentOutOfRangeException(nameof(type), type, null);