Improved further memory usage and added a subsystem for the collector to dump into a file data while it keeps the pre-processor going

Config/config.json

@@ -47,6 +47,11 @@
     "spawnPointToleranceForForced": 99.5,
     "looseLootCountTolerancePercentage": 75
   },
+  "collectorConfig": {
+    "collectorType": "Dump",
+    "maxEntitiesBeforeDumping": 1000,
+      "dumpLocation": "C:\\Users\\USER\\SPT\\tmp"
+  },
   "writerConfig": {
     "outputLocation": "E:\\spt\\dumps\\output"
   },

Model/ComposedKey.cs

@@ -1,5 +1,6 @@
 using System.Text.Json.Serialization;
 using LootDumpProcessor.Model.Processing;
+using LootDumpProcessor.Utils;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model;
@@ -26,7 +27,7 @@ public class ComposedKey
             .Cast<string>()
             .Select(i => (double)i.GetHashCode())
             .Sum()
-            .ToString() ?? Guid.NewGuid().ToString();
+            .ToString() ?? KeyGenerator.GetNextKey();
         FirstItem = items?[0];
     }
 

Model/Config/CollectorConfig.cs

@@ -0,0 +1,20 @@
+using System.Text.Json.Serialization;
+using LootDumpProcessor.Process.Collector;
+using Newtonsoft.Json;
+
+namespace LootDumpProcessor.Model.Config;
+
+public class CollectorConfig
+{
+    [JsonProperty("collectorType")]
+    [JsonPropertyName("collectorType")]
+    public CollectorType CollectorType { get; set; }
+    
+    [JsonProperty("maxEntitiesBeforeDumping")]
+    [JsonPropertyName("maxEntitiesBeforeDumping")]
+    public int MaxEntitiesBeforeDumping { get; set; }
+    
+    [JsonProperty("dumpLocation")]
+    [JsonPropertyName("dumpLocation")]
+    public string DumpLocation { get; set; }
+}

Model/Config/Config.cs

@@ -49,6 +49,10 @@ public class Config
     [JsonProperty("writerConfig")]
     [JsonPropertyName("writerConfig")]
     public WriterConfig WriterConfig { get; set; }
+    
+    [JsonProperty("collectorConfig")]
+    [JsonPropertyName("collectorConfig")]
+    public CollectorConfig CollectorConfig { get; set; }
 
     [JsonProperty("containerIgnoreList")]
     [JsonPropertyName("containerIgnoreList")]

Model/Processing/LooseLootCounts.cs

@@ -1,5 +1,6 @@
 using System.Text.Json.Serialization;
 using LootDumpProcessor.Storage;
+using LootDumpProcessor.Utils;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Processing;
@@ -8,7 +9,7 @@ public class LooseLootCounts : IKeyable
 {
     [JsonProperty("__id__")]
     [JsonPropertyName("__id__")]
-    public string __ID { get; set; } = Guid.NewGuid().ToString();
+    public string __ID { get; set; } = KeyGenerator.GetNextKey();
 
     public IKey Counts { get; set; }
 

Model/Template.cs

@@ -9,7 +9,7 @@ namespace LootDumpProcessor.Model
     {
         [Newtonsoft.Json.JsonIgnore]
         [System.Text.Json.Serialization.JsonIgnore]
-        public string __ID { get; } = Guid.NewGuid().ToString();
+        public string __ID { get; } = KeyGenerator.GetNextKey();
 
         [JsonProperty("Id", NullValueHandling = NullValueHandling.Ignore)]
         [JsonPropertyName("Id")]

Process/Collector/CollectorFactory.cs

@@ -2,9 +2,16 @@
 
 public static class CollectorFactory
 {
+    private static ICollector? _collector;
     public static ICollector GetInstance()
     {
-        // TODO: implement real factory
+        if (_collector == null)
-        return new HashSetCollector();
+            _collector = LootDumpProcessorContext.GetConfig().CollectorConfig.CollectorType switch
+            {
+                CollectorType.Memory => new HashSetCollector(),
+                CollectorType.Dump => new DumpCollector(),
+                _ => throw new ArgumentOutOfRangeException()
+            };
+        return _collector;
     }
 }

Process/Collector/CollectorType.cs

@@ -0,0 +1,7 @@
+namespace LootDumpProcessor.Process.Collector;
+
+public enum CollectorType
+{
+    Memory,
+    Dump
+}

Process/Collector/DumpCollector.cs

@@ -0,0 +1,45 @@
+using LootDumpProcessor.Model.Processing;
+using LootDumpProcessor.Serializers.Json;
+
+namespace LootDumpProcessor.Process.Collector;
+
+public class DumpCollector : ICollector
+{
+    private static readonly string DumpLocation = $"{LootDumpProcessorContext.GetConfig().CollectorConfig.DumpLocation}/collector/";
+    private readonly List<PartialData> processedDumps = new(LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping + 50);
+    private readonly object lockObject = new();
+    
+    public void Setup()
+    {
+        if (Directory.Exists(DumpLocation))
+        {
+            Directory.Delete(DumpLocation, true);
+        }
+        
+        Directory.CreateDirectory(DumpLocation);
+    }
+
+    public void Hold(PartialData parsedDump)
+    {
+        lock (lockObject)
+        {
+            processedDumps.Add(parsedDump);
+            if (processedDumps.Count > LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping)
+            {
+                var fileName = $"collector-{DateTime.Now.ToString("yyyyMMddHHmmssfffff")}.json";
+                File.WriteAllText($"{DumpLocation}{fileName}", JsonSerializerFactory.GetInstance().Serialize(processedDumps));
+                processedDumps.Clear();
+            }
+        }
+    }
+
+    public List<PartialData> Retrieve()
+    {
+        foreach (var file in Directory.GetFiles(DumpLocation))
+        {
+            processedDumps.AddRange(JsonSerializerFactory.GetInstance().Deserialize<List<PartialData>>(File.ReadAllText(file)));
+        }
+
+        return processedDumps;
+    }
+}

Process/Collector/HashSetCollector.cs

@@ -4,11 +4,9 @@ namespace LootDumpProcessor.Process.Collector;
 
 public class HashSetCollector : ICollector
 {
-    private readonly HashSet<PartialData> processedDumps = new();
+    private readonly HashSet<PartialData> processedDumps = new(100_000);
-
     private readonly object lockObject = new();
 
-
     public void Setup()
     {
     }

Process/Processor/FileProcessor/FileProcessorFactory.cs

@@ -2,7 +2,7 @@
 
 public static class FileProcessorFactory
 {
-    private static IFileProcessor _fileProcessor;
+    private static IFileProcessor? _fileProcessor;
     public static IFileProcessor GetInstance()
     {
         // TODO: implement actual factory someday

Storage/Collections/FlatKeyableDictionary.cs

@@ -1,4 +1,5 @@
 using System.Text.Json.Serialization;
+using LootDumpProcessor.Utils;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Storage.Collections;
@@ -7,10 +8,10 @@ public class FlatKeyableDictionary<K, V> : Dictionary<K, V>, IKeyable
 {
     [JsonProperty("__id__")]
     [JsonPropertyName("__id__")]
-    public string __ID { get; set; } = Guid.NewGuid().ToString();
+    public string __ID { get; set; } = KeyGenerator.GetNextKey();
 
     public IKey GetKey()
     {
-        return new FlatUniqueKey(new[] { __ID });
+        return new FlatUniqueKey([__ID]);
     }
 }

Storage/Collections/FlatKeyableList.cs

@@ -1,11 +1,13 @@
-namespace LootDumpProcessor.Storage.Collections;
+using LootDumpProcessor.Utils;
+
+namespace LootDumpProcessor.Storage.Collections;
 
 public class FlatKeyableList<T> : List<T>, IKeyable
 {
-    public string __ID { get; } = Guid.NewGuid().ToString();
+    public string __ID { get; } = KeyGenerator.GetNextKey();
 
     public IKey GetKey()
     {
-        return new FlatUniqueKey(new[] { __ID });
+        return new FlatUniqueKey([__ID]);
     }
 }

Storage/Collections/SubdivisionedKeyableDictionary.cs

@@ -1,4 +1,5 @@
 using System.Text.Json.Serialization;
+using LootDumpProcessor.Utils;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Storage.Collections;
@@ -7,7 +8,7 @@ public class SubdivisionedKeyableDictionary<K, V> : Dictionary<K, V>, IKeyable
 {
     [JsonProperty("__id__")]
     [JsonPropertyName("__id__")]
-    public string __ID { get; set; } = Guid.NewGuid().ToString();
+    public string __ID { get; set; } = KeyGenerator.GetNextKey();
 
     [JsonProperty("extras")]
     [JsonPropertyName("extras")]
@@ -38,10 +39,8 @@ public class SubdivisionedKeyableDictionary<K, V> : Dictionary<K, V>, IKeyable
             subdivisions.Add(__ID);
             return new SubdivisionedUniqueKey(subdivisions.ToArray());
         }
-        else
+        
-        {
+        return new SubdivisionedUniqueKey(["dictionaries", __ID]);
-            return new SubdivisionedUniqueKey(new[] { "dictionaries", __ID });
-        }
     }
 
     public void AddExtraSubdivisions(string[] extras)

Utils/KeyGenerator.cs

@@ -0,0 +1,15 @@
+namespace LootDumpProcessor.Utils;
+
+public class KeyGenerator
+{
+    private static long currentKey = 0L;
+    private static object lockObject = new();
+
+    public static string GetNextKey()
+    {
+        lock (lockObject)
+        {
+            return $"{++currentKey}";
+        }
+    }
+}