Fix static loot output (#6)

* Improved thread safety and async processing in dump processor components

* Removed unused _processedDumps field and simplified variable scope in QueuePipeline

source/LootDumpProcessor/Model/Template.cs

@@ -5,38 +5,25 @@ using LootDumpProcessor.Utils;
 
 namespace LootDumpProcessor.Model;
 
-public class Template : IKeyable, ICloneable
+public class Template(
+    string internalId, string? id, bool isContainer, bool? useGravity, bool? randomRotation,
+    Vector3? position, Vector3? rotation, bool? isGroupPosition, List<GroupPosition>? groupPositions,
+    bool? isAlwaysSpawn, string? root, List<Item> items
+)
+    : IKeyable, ICloneable
 {
-    [JsonIgnore] public string InternalId { get; }
+    [JsonIgnore] public string InternalId { get; } = internalId;
-    [JsonPropertyName("Id")] public string Id { get; set; }
+    [JsonPropertyName("Id")] public string? Id { get; set; } = id;
-    [JsonPropertyName("IsContainer")] public bool IsContainer { get; set; }
+    [JsonPropertyName("IsContainer")] public bool IsContainer { get; set; } = isContainer;
-    public bool UseGravity { get; set; }
+    public bool? UseGravity { get; set; } = useGravity;
-    public bool RandomRotation { get; set; }
+    public bool? RandomRotation { get; set; } = randomRotation;
-    [JsonPropertyName("Position")] public Vector3 Position { get; set; }
+    [JsonPropertyName("Position")] public Vector3? Position { get; set; } = position;
-    [JsonPropertyName("Rotation")] public Vector3 Rotation { get; set; }
+    [JsonPropertyName("Rotation")] public Vector3? Rotation { get; set; } = rotation;
-    [JsonPropertyName("IsGroupPosition")] public bool IsGroupPosition { get; set; }
+    [JsonPropertyName("IsGroupPosition")] public bool? IsGroupPosition { get; set; } = isGroupPosition;
-    [JsonPropertyName("GroupPositions")] public List<GroupPosition> GroupPositions { get; set; }
+    [JsonPropertyName("GroupPositions")] public List<GroupPosition>? GroupPositions { get; set; } = groupPositions;
-    [JsonPropertyName("IsAlwaysSpawn")] public bool IsAlwaysSpawn { get; set; }
+    [JsonPropertyName("IsAlwaysSpawn")] public bool? IsAlwaysSpawn { get; set; } = isAlwaysSpawn;
-    [JsonPropertyName("Root")] public string Root { get; set; }
+    [JsonPropertyName("Root")] public string? Root { get; set; } = root;
-    [JsonPropertyName("Items")] public List<Item> Items { get; set; }
+    [JsonPropertyName("Items")] public List<Item> Items { get; set; } = items;
-
-    public Template(string internalId, string id, bool isContainer, bool useGravity, bool randomRotation,
-        Vector3 position, Vector3 rotation, bool isGroupPosition, List<GroupPosition> groupPositions,
-        bool isAlwaysSpawn, string root, List<Item> items)
-    {
-        this.InternalId = internalId;
-        Id = id;
-        IsContainer = isContainer;
-        UseGravity = useGravity;
-        RandomRotation = randomRotation;
-        Position = position;
-        Rotation = rotation;
-        IsGroupPosition = isGroupPosition;
-        GroupPositions = groupPositions;
-        IsAlwaysSpawn = isAlwaysSpawn;
-        Root = root;
-        Items = items;
-    }
 
     private bool Equals(Template other) => Id == other.Id;
 

source/LootDumpProcessor/Process/Processor/DumpProcessor/IDumpProcessor.cs

@@ -4,5 +4,5 @@ namespace LootDumpProcessor.Process.Processor.DumpProcessor;
 
 public interface IDumpProcessor
 {
-    Dictionary<OutputFileType, object> ProcessDumps(List<PartialData> dumps);
+    Task<Dictionary<OutputFileType, object>> ProcessDumps(List<PartialData> dumps);
 }

source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs

@@ -47,7 +47,7 @@ public class MultithreadSteppedDumpProcessor(
 
     private static readonly IDataStorage _dataStorage = DataStorageFactory.GetInstance();
 
-    public Dictionary<OutputFileType, object> ProcessDumps(List<PartialData> dumps)
+    public async Task<Dictionary<OutputFileType, object>> ProcessDumps(List<PartialData> dumps)
     {
         _logger.LogInformation("Starting final dump processing");
         var output = new Dictionary<OutputFileType, object>();
@@ -65,8 +65,8 @@ public class MultithreadSteppedDumpProcessor(
         {
             MaxDegreeOfParallelism = Environment.ProcessorCount
         };
-        Parallel.ForEachAsync(dumps, parallelOptions,
+        await Parallel.ForEachAsync(dumps, parallelOptions,
-            async (partialData, cancellationToken) =>
+            async (partialData, _) =>
                 await Process(partialData, staticContainers, mapStaticContainersAggregated, mapDumpCounter));
 
         _logger.LogInformation("All static data processing threads finished");
@@ -202,7 +202,7 @@ public class MultithreadSteppedDumpProcessor(
         fileDate.Value > LootDumpProcessorContext.GetConfig().DumpProcessorConfig
             .SpawnContainerChanceIncludeAfterDate;
 
-    private static void IncrementMapCounterDictionaryValue(IDictionary<string, int> mapDumpCounter, string mapName)
+    private static void IncrementMapCounterDictionaryValue(ConcurrentDictionary<string, int> mapDumpCounter, string mapName)
     {
         if (!mapDumpCounter.TryAdd(mapName, 1)) mapDumpCounter[mapName] += 1;
     }
@@ -305,13 +305,12 @@ public class MultithreadSteppedDumpProcessor(
                     );
             foreach (var (uniqueKey, containerTemplate) in loadedDictionary)
             {
-                var count = dumpData.LooseLoot.Counts[uniqueKey];
+                var isValueFound = dumpData.LooseLoot.Counts.TryGetValue(uniqueKey, out var count);
+                if (!isValueFound) _logger.LogError("Value for {UniqueKey} not found", uniqueKey);
                 lock (lockObjectDictionaryCounts)
                 {
-                    if (dictionaryCounts.ContainsKey(uniqueKey))
+                    if (!dictionaryCounts.TryAdd(uniqueKey, count))
                         dictionaryCounts[uniqueKey] += count;
-                    else
-                        dictionaryCounts[uniqueKey] = count;
                 }
 
                 lock (lockObjectDictionaryItemProperties)
@@ -319,8 +318,8 @@ public class MultithreadSteppedDumpProcessor(
                     if (!dictionaryItemProperties.TryGetValue(uniqueKey, out var values))
                     {
                         values = new FlatKeyableList<Template>(_keyGenerator.Generate());
-                        dictionaryItemProperties.Add(uniqueKey, values);
+                        dictionaryItemProperties.TryAdd(uniqueKey, values);
-                        actualDictionaryItemProperties.Add(uniqueKey, values.GetKey());
+                        actualDictionaryItemProperties.TryAdd(uniqueKey, values.GetKey());
                     }
 
                     values.AddRange(containerTemplate);

source/LootDumpProcessor/Process/QueuePipeline.cs

@@ -52,7 +52,7 @@ public class QueuePipeline(
         try
         {
             await FixFilesFromDumps();
-            foreach (var mapName in _mapNames) ProcessFilesFromDumpsPerMap(collector, mapName);
+            foreach (var mapName in _mapNames) await ProcessFilesFromDumpsPerMap(collector, mapName);
         }
         finally
         {
@@ -126,7 +126,7 @@ public class QueuePipeline(
         return queuedFilesToProcess;
     }
 
-    private void ProcessFilesFromDumpsPerMap(ICollector collector, string mapName)
+    private async Task ProcessFilesFromDumpsPerMap(ICollector collector, string mapName)
     {
         // Gather all files, sort them by date descending and then add them into the processing queue
         GatherFiles().FindAll(f => f.ToLower().Contains($"{mapName}--")).OrderByDescending(f =>
@@ -158,7 +158,9 @@ public class QueuePipeline(
         // Single writer instance to collect results
         var writer = WriterFactory.GetInstance();
         // Single collector instance to collect results
-        writer.WriteAll(_dumpProcessor.ProcessDumps(collector.Retrieve()));
+        var partialData = collector.Retrieve();
+        var processedDumps = await _dumpProcessor.ProcessDumps(partialData);
+        writer.WriteAll(processedDumps);
 
         // clear collector and datastorage as we process per map now
         collector.Clear();

source/LootDumpProcessor/Serializers/Json/JsonSerializerSettings.cs

@@ -15,7 +15,6 @@ public static class JsonSerializerSettings
             new NetJsonKeyConverter(),
             new JsonStringEnumConverter(),
             new NetDateTimeConverter()
-        },
+        }
-        WriteIndented = true
     };
 }

source/LootDumpProcessor/Storage/Collections/FlatKeyableDictionary.cs

@@ -1,8 +1,9 @@
+using System.Collections.Concurrent;
 using System.Text.Json.Serialization;
 
 namespace LootDumpProcessor.Storage.Collections;
 
-public class FlatKeyableDictionary<K, V> : Dictionary<K, V>, IKeyable
+public class FlatKeyableDictionary<K, V> : ConcurrentDictionary<K, V>, IKeyable
 {
     [JsonPropertyName("__id__")] private string Id { get; set; }
 

source/LootDumpProcessor/Utils/ProcessorUtil.cs

@@ -7,11 +7,12 @@ namespace LootDumpProcessor.Utils;
 public static class ProcessorUtil
 {
     public static string GetSaneId(this Template x) =>
-        $"({x.Position.X}, {x.Position.Y}, {x.Position.Z}, {Math.Round(x.Rotation.X, 3)}," +
+        $"({x.Position.GetValueOrDefault().X}, {x.Position.GetValueOrDefault().Y}, {x.Position.GetValueOrDefault().Z}, {Math.Round(x.Rotation.GetValueOrDefault().X, 3)}," +
-        $" {Math.Round(x.Rotation.Y, 3)}, {Math.Round(x.Rotation.Z, 3)}," +
+        $" {Math.Round(x.Rotation.GetValueOrDefault().Y, 3)}, {Math.Round(x.Rotation.GetValueOrDefault().Z, 3)}," +
         $" {x.UseGravity}, {x.IsGroupPosition})";
 
-    public static string GetLocationId(this Template x) => $"({x.Position.X}, {x.Position.Y}, {x.Position.Z})";
+    public static string GetLocationId(this Template x) =>
+        $"({x.Position.GetValueOrDefault().X}, {x.Position.GetValueOrDefault().Y}, {x.Position.GetValueOrDefault().Z})";
 
     public static T? Copy<T>(T? obj) where T : ICloneable
     {