From 1a4003595a32760a671ef2830a91f616e6d65f1b Mon Sep 17 00:00:00 2001
From: bluextx <bluextchannel@gmail.com>
Date: Sat, 11 Jan 2025 07:41:15 +0300
Subject: [PATCH] Refactored loose loot processor to process single map

---
 .../MultithreadSteppedDumpProcessor.cs        |  18 +-
 .../LooseLootProcessor/ILooseLootProcessor.cs |   7 +-
 .../LooseLootProcessor/LooseLootProcessor.cs  | 278 +++++++++---------
 3 files changed, 151 insertions(+), 152 deletions(-)

diff --git a/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs b/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
index 277591c..663971a 100644
--- a/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
+++ b/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
@@ -3,6 +3,7 @@ using LootDumpProcessor.Logger;
 using LootDumpProcessor.Model;
 using LootDumpProcessor.Model.Input;
 using LootDumpProcessor.Model.Output;
+using LootDumpProcessor.Model.Output.LooseLoot;
 using LootDumpProcessor.Model.Output.StaticContainer;
 using LootDumpProcessor.Model.Processing;
 using LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
@@ -222,17 +223,22 @@ public class MultithreadSteppedDumpProcessor(
 
         if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Processing loose loot distribution", LogLevel.Info);
+        
         // Loose loot distribution
-        var looseLootDistribution = _looseLootProcessor.CreateLooseLootDistribution(
-            dumpProcessData.MapCounts,
-            dumpProcessData.LooseLootCounts
-        );
-
+        var looseLoot = new ConcurrentDictionary<string, LooseLootRoot>();
+        Parallel.ForEach(dumpProcessData.MapCounts.Keys, mapId =>
+        {
+            var mapCount = dumpProcessData.MapCounts[mapId];
+            var looseLootCount = dumpProcessData.LooseLootCounts[mapId];
+            var looseLootDistribution =
+                _looseLootProcessor.CreateLooseLootDistribution(mapId, mapCount, looseLootCount);
+            looseLoot[mapId] = looseLootDistribution;
+        });
         if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Collecting loose loot distribution information", LogLevel.Info);
         var loot = dumpProcessData.MapCounts
             .Select(mapCount => mapCount.Key)
-            .ToDictionary(mi => mi, mi => looseLootDistribution[mi]);
+            .ToDictionary(mi => mi, mi => looseLoot[mi]);
 
         output.Add(OutputFileType.LooseLoot, loot);
         if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
diff --git a/source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/ILooseLootProcessor.cs b/source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/ILooseLootProcessor.cs
index edb54ee..5a6e637 100644
--- a/source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/ILooseLootProcessor.cs
+++ b/source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/ILooseLootProcessor.cs
@@ -9,9 +9,10 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
     {
         PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseLoot);
 
-        Dictionary<string, LooseLootRoot> CreateLooseLootDistribution(
-            Dictionary<string, int> mapCounts,
-            Dictionary<string, IKey> looseLootCounts
+        LooseLootRoot CreateLooseLootDistribution(
+            string mapId,
+            int mapCount,
+            IKey looseLootCountKey
         );
     }
 }
\ No newline at end of file
diff --git a/source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/LooseLootProcessor.cs b/source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/LooseLootProcessor.cs
index e60e755..e15eda0 100644
--- a/source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/LooseLootProcessor.cs
+++ b/source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/LooseLootProcessor.cs
@@ -36,8 +36,8 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
                 if (!uniqueLootIds.ContainsKey(sanitizedId))
                 {
                     uniqueLootIds[sanitizedId] = template.Id;
-                    preProcessedLoot.Counts[sanitizedId] = preProcessedLoot.Counts.ContainsKey(sanitizedId)
-                        ? preProcessedLoot.Counts[sanitizedId] + 1
+                    preProcessedLoot.Counts[sanitizedId] = preProcessedLoot.Counts.TryGetValue(sanitizedId, out var count)
+                        ? count + 1
                         : 1;
                 }
 
@@ -54,182 +54,174 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
             return preProcessedLoot;
         }
 
-        public Dictionary<string, LooseLootRoot> CreateLooseLootDistribution(
-            Dictionary<string, int> mapCounts,
-            Dictionary<string, IKey> looseLootCounts
+        public LooseLootRoot CreateLooseLootDistribution(
+            string mapId,
+            int mapCount,
+            IKey looseLootCountKey
         )
         {
             var config = LootDumpProcessorContext.GetConfig();
             var spawnPointTolerance = config.ProcessorConfig.SpawnPointToleranceForForced / 100;
-            var looseLootDistribution = new Dictionary<string, LooseLootRoot>();
+            var looseLootDistribution = new LooseLootRoot();
 
-            foreach (var map in mapCounts)
+            var probabilities = new Dictionary<string, double>();
+            var looseLootCountsItem = _dataStorage.GetItem<LooseLootCounts>(looseLootCountKey);
+
+            var counts = _dataStorage.GetItem<FlatKeyableDictionary<string, int>>(looseLootCountsItem.Counts);
+            foreach (var (itemId, count) in counts)
             {
-                var mapName = map.Key;
-                var mapCount = map.Value;
+                probabilities[itemId] = (double)count / mapCount;
+            }
 
-                var probabilities = new Dictionary<string, double>();
-                var looseLootCountsItem = _dataStorage.GetItem<LooseLootCounts>(looseLootCounts[mapName]);
+            var spawnPointCount = looseLootCountsItem.MapSpawnpointCount.Select(Convert.ToDouble);
+            var initialMean = np.mean(np.array(spawnPointCount)).ToArray<double>().First();
+            var tolerancePercentage = config.ProcessorConfig.LooseLootCountTolerancePercentage / 100;
+            var highThreshold = initialMean * (1 + tolerancePercentage);
 
-                var counts = _dataStorage.GetItem<FlatKeyableDictionary<string, int>>(looseLootCountsItem.Counts);
-                foreach (var (itemId, count) in counts)
+            looseLootCountsItem.MapSpawnpointCount = looseLootCountsItem.MapSpawnpointCount
+                .Where(count => count <= highThreshold)
+                .ToList();
+
+            looseLootDistribution.SpawnPointCount = new SpawnPointCount
+            {
+                Mean = np.mean(np.array(looseLootCountsItem.MapSpawnpointCount)),
+                Std = np.std(np.array(looseLootCountsItem.MapSpawnpointCount))
+            };
+            looseLootDistribution.SpawnPointsForced = new List<SpawnPointsForced>();
+            looseLootDistribution.SpawnPoints = new List<SpawnPoint>();
+
+            var itemProperties = _dataStorage.GetItem<FlatKeyableDictionary<string, IKey>>(looseLootCountsItem.ItemProperties);
+            foreach (var (spawnPoint, itemListKey) in itemProperties)
+            {
+                var itemCounts = new Dictionary<ComposedKey, int>();
+                var savedTemplates = _dataStorage.GetItem<FlatKeyableList<Template>>(itemListKey);
+
+                foreach (var template in savedTemplates)
                 {
-                    probabilities[itemId] = (double)count / mapCount;
+                    var composedKey = new ComposedKey(template);
+                    if (!itemCounts.TryAdd(composedKey, 1))
+                        itemCounts[composedKey]++;
                 }
 
-                var spawnPointCount = looseLootCountsItem.MapSpawnpointCount.Select(Convert.ToDouble);
-                var initialMean = np.mean(np.array(spawnPointCount)).ToArray<double>().First();
-                var tolerancePercentage = config.ProcessorConfig.LooseLootCountTolerancePercentage / 100;
-                var highThreshold = initialMean * (1 + tolerancePercentage);
-
-                looseLootCountsItem.MapSpawnpointCount = looseLootCountsItem.MapSpawnpointCount
-                    .Where(count => count <= highThreshold)
+                var groupedTemplates = savedTemplates
+                    .Select(t => (t.GetSaneId(), t))
+                    .GroupBy(g => g.Item1)
                     .ToList();
 
-                looseLootDistribution[mapName] = new LooseLootRoot
+                if (groupedTemplates.Count > 1)
                 {
-                    SpawnPointCount = new SpawnPointCount
-                    {
-                        Mean = np.mean(np.array(looseLootCountsItem.MapSpawnpointCount)),
-                        Std = np.std(np.array(looseLootCountsItem.MapSpawnpointCount))
-                    },
-                    SpawnPointsForced = new List<SpawnPointsForced>(),
-                    SpawnPoints = new List<SpawnPoint>()
-                };
+                    throw new Exception("Multiple sanitized IDs found for a single spawn point.");
+                }
 
-                var itemProperties = _dataStorage.GetItem<FlatKeyableDictionary<string, IKey>>(looseLootCountsItem.ItemProperties);
-                foreach (var (spawnPoint, itemListKey) in itemProperties)
+                var spawnPoints = groupedTemplates.First().Select(g => g.t).ToList();
+                var locationId = spawnPoints[0].GetLocationId();
+                var templateCopy = ProcessorUtil.Copy(spawnPoints[0]);
+                var itemDistributions = itemCounts.Select(kv => new ItemDistribution
                 {
-                    var itemCounts = new Dictionary<ComposedKey, int>();
-                    var savedTemplates = _dataStorage.GetItem<FlatKeyableList<Template>>(itemListKey);
+                    ComposedKey = kv.Key,
+                    RelativeProbability = kv.Value
+                }).ToList();
 
-                    foreach (var template in savedTemplates)
+                if (itemDistributions.Count == 1 &&
+                    (LootDumpProcessorContext.GetTarkovItems().IsQuestItem(itemDistributions[0].ComposedKey?.FirstItem?.Tpl) ||
+                     LootDumpProcessorContext.GetForcedLooseItems()[mapId].Contains(itemDistributions[0].ComposedKey?.FirstItem?.Tpl)))
+                {
+                    var forcedSpawnPoint = new SpawnPointsForced
                     {
-                        var composedKey = new ComposedKey(template);
-                        if (!itemCounts.TryAdd(composedKey, 1))
-                            itemCounts[composedKey]++;
-                    }
-
-                    var groupedTemplates = savedTemplates
-                        .Select(t => (t.GetSaneId(), t))
-                        .GroupBy(g => g.Item1)
-                        .ToList();
-
-                    if (groupedTemplates.Count > 1)
+                        LocationId = locationId,
+                        Probability = probabilities[spawnPoint],
+                        Template = templateCopy
+                    };
+                    looseLootDistribution.SpawnPointsForced.Add(forcedSpawnPoint);
+                }
+                else if (probabilities[spawnPoint] > spawnPointTolerance)
+                {
+                    var forcedSpawnPoint = new SpawnPointsForced
                     {
-                        throw new Exception("Multiple sanitized IDs found for a single spawn point.");
-                    }
-
-                    var spawnPoints = groupedTemplates.First().Select(g => g.t).ToList();
-                    var locationId = spawnPoints[0].GetLocationId();
-                    var templateCopy = ProcessorUtil.Copy(spawnPoints[0]);
-                    var itemDistributions = itemCounts.Select(kv => new ItemDistribution
+                        LocationId = locationId,
+                        Probability = probabilities[spawnPoint],
+                        Template = templateCopy
+                    };
+                    looseLootDistribution.SpawnPointsForced.Add(forcedSpawnPoint);
+                    _logger.LogWarning(
+                        "Item: {ItemId} has > {Tolerance}% spawn chance in spawn point: {LocationId} but isn't in forced loot, adding to forced",
+                        templateCopy.Id,
+                        config.ProcessorConfig.SpawnPointToleranceForForced,
+                        forcedSpawnPoint.LocationId
+                    );
+                }
+                else
+                {
+                    var spawnPointEntry = new SpawnPoint
                     {
-                        ComposedKey = kv.Key,
-                        RelativeProbability = kv.Value
-                    }).ToList();
+                        LocationId = locationId,
+                        Probability = probabilities[spawnPoint],
+                        Template = templateCopy,
+                        ItemDistribution = itemDistributions
+                    };
 
-                    if (itemDistributions.Count == 1 &&
-                        (LootDumpProcessorContext.GetTarkovItems().IsQuestItem(itemDistributions[0].ComposedKey?.FirstItem?.Tpl) ||
-                         LootDumpProcessorContext.GetForcedLooseItems()[mapName].Contains(itemDistributions[0].ComposedKey?.FirstItem?.Tpl)))
+                    templateCopy.Items = new List<Item>();
+
+                    var groupedByKey = spawnPoints
+                        .GroupBy(t => new ComposedKey(t))
+                        .ToDictionary(g => g.Key, g => g.ToList());
+
+                    foreach (var distribution in itemDistributions)
                     {
-                        var forcedSpawnPoint = new SpawnPointsForced
+                        if (groupedByKey.TryGetValue(distribution.ComposedKey, out var items))
                         {
-                            LocationId = locationId,
-                            Probability = probabilities[spawnPoint],
-                            Template = templateCopy
-                        };
-                        looseLootDistribution[mapName].SpawnPointsForced.Add(forcedSpawnPoint);
-                    }
-                    else if (probabilities[spawnPoint] > spawnPointTolerance)
-                    {
-                        var forcedSpawnPoint = new SpawnPointsForced
-                        {
-                            LocationId = locationId,
-                            Probability = probabilities[spawnPoint],
-                            Template = templateCopy
-                        };
-                        looseLootDistribution[mapName].SpawnPointsForced.Add(forcedSpawnPoint);
-                        _logger.LogWarning(
-                            "Item: {ItemId} has > {Tolerance}% spawn chance in spawn point: {LocationId} but isn't in forced loot, adding to forced",
-                            templateCopy.Id,
-                            config.ProcessorConfig.SpawnPointToleranceForForced,
-                            forcedSpawnPoint.LocationId
-                        );
-                    }
-                    else
-                    {
-                        var spawnPointEntry = new SpawnPoint
-                        {
-                            LocationId = locationId,
-                            Probability = probabilities[spawnPoint],
-                            Template = templateCopy,
-                            ItemDistribution = itemDistributions
-                        };
-
-                        templateCopy.Items = new List<Item>();
-
-                        var groupedByKey = spawnPoints
-                            .GroupBy(t => new ComposedKey(t))
-                            .ToDictionary(g => g.Key, g => g.ToList());
-
-                        foreach (var distribution in itemDistributions)
-                        {
-                            if (groupedByKey.TryGetValue(distribution.ComposedKey, out var items))
+                            var itemList = items.First().Items;
+                            var originalItem = itemList.Find(i => string.IsNullOrEmpty(i.ParentId));
+                            var originalId = originalItem.Id;
+                            originalItem.Id = distribution.ComposedKey.Key;
+                            foreach (var childItem in itemList.Where(i => i.ParentId == originalId))
                             {
-                                var itemList = items.First().Items;
-                                var originalItem = itemList.Find(i => string.IsNullOrEmpty(i.ParentId));
-                                var originalId = originalItem.Id;
-                                originalItem.Id = distribution.ComposedKey.Key;
-                                foreach (var childItem in itemList.Where(i => i.ParentId == originalId))
-                                {
-                                    childItem.ParentId = originalItem.Id;
-                                }
+                                childItem.ParentId = originalItem.Id;
+                            }
 
-                                templateCopy.Items.AddRange(itemList);
-                            }
-                            else
-                            {
-                                _logger.LogError(
-                                    "Item template {TemplateId} on loose loot distribution for spawn point {SpawnPointId} not found in spawn points.",
-                                    distribution.ComposedKey?.FirstItem?.Tpl,
-                                    templateCopy.Id
-                                );
-                            }
+                            templateCopy.Items.AddRange(itemList);
+                        }
+                        else
+                        {
+                            _logger.LogError(
+                                "Item template {TemplateId} on loose loot distribution for spawn point {SpawnPointId} not found in spawn points.",
+                                distribution.ComposedKey?.FirstItem?.Tpl,
+                                templateCopy.Id
+                            );
                         }
-
-                        looseLootDistribution[mapName].SpawnPoints.Add(spawnPointEntry);
                     }
+
+                    looseLootDistribution.SpawnPoints.Add(spawnPointEntry);
                 }
+            }
 
-                looseLootDistribution[mapName].SpawnPoints = looseLootDistribution[mapName].SpawnPoints
-                    .OrderBy(x => x.Template.Id)
-                    .ToList();
+            looseLootDistribution.SpawnPoints = looseLootDistribution.SpawnPoints
+                .OrderBy(x => x.Template.Id)
+                .ToList();
 
-                var configuredForcedTemplates = new HashSet<string>(LootDumpProcessorContext.GetForcedLooseItems()[mapName].Select(item => item));
-                var foundForcedTemplates = new HashSet<string>(looseLootDistribution[mapName].SpawnPointsForced.Select(fp => fp.Template.Items[0].Tpl));
+            var configuredForcedTemplates = new HashSet<string>(LootDumpProcessorContext.GetForcedLooseItems()[mapId].Select(item => item));
+            var foundForcedTemplates = new HashSet<string>(looseLootDistribution.SpawnPointsForced.Select(fp => fp.Template.Items[0].Tpl));
 
-                foreach (var expectedTpl in configuredForcedTemplates)
+            foreach (var expectedTpl in configuredForcedTemplates)
+            {
+                if (!foundForcedTemplates.Contains(expectedTpl))
                 {
-                    if (!foundForcedTemplates.Contains(expectedTpl))
-                    {
-                        _logger.LogError(
-                            "Expected item: {ItemTpl} defined in forced_loose.yaml config not found in forced loot",
-                            expectedTpl
-                        );
-                    }
+                    _logger.LogError(
+                        "Expected item: {ItemTpl} defined in forced_loose.yaml config not found in forced loot",
+                        expectedTpl
+                    );
                 }
+            }
 
-                foreach (var foundTpl in foundForcedTemplates)
+            foreach (var foundTpl in foundForcedTemplates)
+            {
+                if (!configuredForcedTemplates.Contains(foundTpl))
                 {
-                    if (!configuredForcedTemplates.Contains(foundTpl))
-                    {
-                        _logger.LogWarning(
-                            "Map: {MapName} Item: {ItemTpl} not defined in forced_loose.yaml config but was flagged as forced by code",
-                            mapName,
-                            foundTpl
-                        );
-                    }
+                    _logger.LogWarning(
+                        "Map: {MapName} Item: {ItemTpl} not defined in forced_loose.yaml config but was flagged as forced by code",
+                        mapId,
+                        foundTpl
+                    );
                 }
             }