Reverted back BOM encoding of UTF-8 files

Config/config.json

@@ -1,4 +1,4 @@
-{
+{
   "serverLocation": "E:\\spt\\Server",
   "threads": 10,
   "threadPoolingTimeoutMs": 1000,
@@ -47,6 +47,11 @@
     "spawnPointToleranceForForced": 99.5,
     "looseLootCountTolerancePercentage": 75
   },
+  "collectorConfig": {
+    "collectorType": "Dump",
+    "maxEntitiesBeforeDumping": 1000,
+      "dumpLocation": "C:\\Users\\USER\\SPT\\tmp"
+  },
   "writerConfig": {
     "outputLocation": "E:\\spt\\dumps\\output"
   },

GCHandler.cs

@@ -1,6 +1,6 @@
-namespace LootDumpProcessor;
+namespace LootDumpProcessor;
 
-public class GCHandler
+public static class GCHandler
 {
     public static void Collect()
     {

Logger/ILogger.cs

@@ -1,8 +1,9 @@
-namespace LootDumpProcessor.Logger;
+namespace LootDumpProcessor.Logger;
 
 public interface ILogger
 {
     void Setup();
     void Log(string message, LogLevel level);
+    bool CanBeLogged(LogLevel level);
     void Stop();
 }

Logger/LogLevel.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Logger;
+namespace LootDumpProcessor.Logger;
 
 public enum LogLevel
 {

Logger/LoggerFactory.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Logger;
+namespace LootDumpProcessor.Logger;
 
 public static class LoggerFactory
 {

Logger/QueueLogger.cs

@@ -1,15 +1,15 @@
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 
 namespace LootDumpProcessor.Logger;
 
 public class QueueLogger : ILogger
 {
-    private BlockingCollection<LoggedMessage> queuedMessages = new BlockingCollection<LoggedMessage>();
+    private readonly BlockingCollection<LoggedMessage> queuedMessages = new();
     private Task? loggerThread;
     private bool isRunning;
     private int logLevel;
-    private static readonly int _logTerminationTimeoutMs = 1000;
-    private static readonly int _logTerminationRetryCount = 3;
+    private const int LogTerminationTimeoutMs = 1000;
+    private const int LogTerminationRetryCount = 3;
 
     public void Setup()
     {
@@ -56,19 +56,14 @@ public class QueueLogger : ILogger
 
     private int GetLogLevel(LogLevel level)
     {
-        switch (level)
+        return level switch
         {
-            case LogLevel.Error:
-                return 1;
-            case LogLevel.Warning:
-                return 2;
-            case LogLevel.Info:
-                return 3;
-            case LogLevel.Debug:
-                return 4;
-            default:
-                throw new ArgumentOutOfRangeException();
-        }
+            LogLevel.Error => 1,
+            LogLevel.Warning => 2,
+            LogLevel.Info => 3,
+            LogLevel.Debug => 4,
+            _ => throw new ArgumentOutOfRangeException()
+        };
     }
 
     public void Log(string message, LogLevel level)
@@ -77,6 +72,11 @@ public class QueueLogger : ILogger
             queuedMessages.Add(new LoggedMessage { Message = message, LogLevel = level });
     }
 
+    public bool CanBeLogged(LogLevel level)
+    {
+        return GetLogLevel(level) <= logLevel;
+    }
+
     // Wait for graceful termination of the logging thread
     public void Stop()
     {
@@ -84,24 +84,24 @@ public class QueueLogger : ILogger
         if (loggerThread != null)
         {
             Console.ResetColor();
-            int retryCount = 0;
+            var retryCount = 0;
             while (!loggerThread.IsCompleted)
             {
-                if (retryCount == _logTerminationRetryCount)
+                if (retryCount == LogTerminationRetryCount)
                 {
                     Console.WriteLine(
                         $"Logger thread did not terminate by itself after {retryCount} retries. Some log messages may be lost.");
                     break;
                 }
 
-                Console.WriteLine($"Waiting {_logTerminationTimeoutMs}ms for logger termination");
-                Thread.Sleep(_logTerminationTimeoutMs);
+                Console.WriteLine($"Waiting {LogTerminationTimeoutMs}ms for logger termination");
+                Thread.Sleep(LogTerminationTimeoutMs);
                 retryCount++;
             }
         }
     }
 
-    class LoggedMessage
+    private class LoggedMessage
     {
         public string Message { get; init; }
         public LogLevel LogLevel { get; init; }

LootDumpProcessor.sln

@@ -1,4 +1,4 @@
-
+
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.4.33213.308

LootDumpProcessorContext.cs

@@ -1,4 +1,4 @@
-using LootDumpProcessor.Model.Config;
+using LootDumpProcessor.Model.Config;
 using LootDumpProcessor.Model.Output.StaticContainer;
 using LootDumpProcessor.Process;
 using LootDumpProcessor.Serializers.Json;
@@ -6,22 +6,22 @@ using LootDumpProcessor.Serializers.Yaml;
 
 namespace LootDumpProcessor;
 
-public class LootDumpProcessorContext
+public static class LootDumpProcessorContext
 {
     private static Config? _config;
-    private static readonly object _configLock = new object();
+    private static readonly object _configLock = new();
     private static ForcedStatic? _forcedStatic;
-    private static readonly object _forcedStaticLock = new object();
+    private static readonly object _forcedStaticLock = new();
     private static Dictionary<string, MapDirectoryMapping>? _mapDirectoryMappings;
-    private static readonly object _mapDirectoryMappingsLock = new object();
+    private static readonly object _mapDirectoryMappingsLock = new();
     private static HashSet<string>? _staticWeaponIds;
-    private static readonly object _staticWeaponIdsLock = new object();
+    private static readonly object _staticWeaponIdsLock = new();
     private static Dictionary<string, List<StaticForced>>? _forcedItems;
-    private static readonly object _forcedItemsLock = new object();
+    private static readonly object _forcedItemsLock = new();
     private static Dictionary<string, HashSet<string>>? _forcedLoose;
-    private static readonly object _forcedLooseLock = new object();
+    private static readonly object _forcedLooseLock = new();
     private static TarkovItems? _tarkovItems;
-    private static readonly object _tarkovItemsLock = new object();
+    private static readonly object _tarkovItemsLock = new();
 
     public static Config GetConfig()
     {

Model/ComposedKey.cs

@@ -1,5 +1,6 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using LootDumpProcessor.Model.Processing;
+using LootDumpProcessor.Utils;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model;
@@ -26,7 +27,7 @@ public class ComposedKey
             .Cast<string>()
             .Select(i => (double)i.GetHashCode())
             .Sum()
-            .ToString() ?? Guid.NewGuid().ToString();
+            .ToString() ?? KeyGenerator.GetNextKey();
         FirstItem = items?[0];
     }
 
@@ -34,7 +35,7 @@ public class ComposedKey
     {
         if (obj is not ComposedKey key)
             return false;
-        return this.Key == key.Key;
+        return Key == key.Key;
     }
 
     public override int GetHashCode() => Key.GetHashCode();

Model/Config/CollectorConfig.cs

@@ -0,0 +1,20 @@
+using System.Text.Json.Serialization;
+using LootDumpProcessor.Process.Collector;
+using Newtonsoft.Json;
+
+namespace LootDumpProcessor.Model.Config;
+
+public class CollectorConfig
+{
+    [JsonProperty("collectorType")]
+    [JsonPropertyName("collectorType")]
+    public CollectorType CollectorType { get; set; }
+    
+    [JsonProperty("maxEntitiesBeforeDumping")]
+    [JsonPropertyName("maxEntitiesBeforeDumping")]
+    public int MaxEntitiesBeforeDumping { get; set; }
+    
+    [JsonProperty("dumpLocation")]
+    [JsonPropertyName("dumpLocation")]
+    public string DumpLocation { get; set; }
+}

Model/Config/Config.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using LootDumpProcessor.Serializers.Json;
 using Newtonsoft.Json;
 
@@ -50,6 +50,10 @@ public class Config
     [JsonPropertyName("writerConfig")]
     public WriterConfig WriterConfig { get; set; }
     
+    [JsonProperty("collectorConfig")]
+    [JsonPropertyName("collectorConfig")]
+    public CollectorConfig CollectorConfig { get; set; }
+
     [JsonProperty("containerIgnoreList")]
     [JsonPropertyName("containerIgnoreList")]
     public Dictionary<string, string[]> ContainerIgnoreList { get; set; }

Model/Config/DataStorageConfig.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using LootDumpProcessor.Storage;
 using Newtonsoft.Json;
 

Model/Config/ForcedStatic.cs

@@ -1,4 +1,4 @@
-using LootDumpProcessor.Model.Output.StaticContainer;
+using LootDumpProcessor.Model.Output.StaticContainer;
 using YamlDotNet.Serialization;
 
 namespace LootDumpProcessor.Model.Config;

Model/Config/IntakeReaderConfig.cs

@@ -1,5 +1,6 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using LootDumpProcessor.Process.Reader;
+using LootDumpProcessor.Process.Reader.Intake;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Config;
@@ -17,5 +18,5 @@ public class IntakeReaderConfig
     
     [JsonProperty("ignoredDumpLocations")]
     [JsonPropertyName("ignoredDumpLocations")]
-    public List<string> IgnoredDumpLocations { get; set; } = new List<string>();
+    public List<string> IgnoredDumpLocations { get; set; } = new();
 }

Model/Config/LoggerConfig.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using LootDumpProcessor.Logger;
 using Newtonsoft.Json;
 

Model/Config/MapDirectoryMapping.cs

@@ -1,4 +1,4 @@
-using YamlDotNet.Serialization;
+using YamlDotNet.Serialization;
 
 namespace LootDumpProcessor.Model.Config;
 

Model/Config/PreProcessorConfig.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using LootDumpProcessor.Process.Reader.PreProcess;
 using Newtonsoft.Json;
 

Model/Config/ProcessorConfig.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Config;

Model/Config/ReaderConfig.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using LootDumpProcessor.Process.Reader.Filters;
 using Newtonsoft.Json;
 
@@ -24,7 +24,7 @@ public class ReaderConfig
 
     [JsonProperty("acceptedFileExtensions")]
     [JsonPropertyName("acceptedFileExtensions")]
-    public List<string> AcceptedFileExtensions { get; set; } = new List<string>();
+    public List<string> AcceptedFileExtensions { get; set; } = new();
 
     [JsonProperty("processSubFolders")]
     [JsonPropertyName("processSubFolders")]

Model/Config/WriterConfig.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Config;

Model/FireMode.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model
@@ -13,7 +13,7 @@ namespace LootDumpProcessor.Model
         {
             return new FireMode
             {
-                FireModeType = this.FireModeType
+                FireModeType = FireModeType
             };
         }
     }

Model/Foldable.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model
@@ -13,7 +13,7 @@ namespace LootDumpProcessor.Model
         {
             return new Foldable
             {
-                Folded = this.Folded
+                Folded = Folded
             };
         }
     }

Model/GroupPosition.cs

@@ -1,5 +1,5 @@
-using System.Text.Json.Serialization;
-using LootDumpProcessor.Process.Processor;
+using System.Text.Json.Serialization;
+using LootDumpProcessor.Utils;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model
@@ -26,10 +26,10 @@ namespace LootDumpProcessor.Model
         {
             return new GroupPosition
             {
-                Name = this.Name,
-                Weight = this.Weight,
-                Position = ProcessorUtil.Copy(this.Position),
-                Rotation = ProcessorUtil.Copy(this.Rotation)
+                Name = Name,
+                Weight = Weight,
+                Position = ProcessorUtil.Copy(Position),
+                Rotation = ProcessorUtil.Copy(Rotation)
             };
         }
     }

Model/Input/AirdropParameter.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Input/Banner.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Input/BotLocationModifier.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Input/ColliderParams.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Input/Data.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Input/Exit.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Input/ExitZones.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Input/Preview.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Input/Props.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Input/RootData.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Input/SpawnPointParam.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Input/Wave.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Input

Model/Item.cs

@@ -1,5 +1,5 @@
-using System.Text.Json.Serialization;
-using LootDumpProcessor.Process.Processor;
+using System.Text.Json.Serialization;
+using LootDumpProcessor.Utils;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model
@@ -34,24 +34,24 @@ namespace LootDumpProcessor.Model
         {
             if (obj is not Item parsed)
                 return false;
-            return parsed.Tpl == this.Tpl && parsed.ParentId == this.ParentId;
+            return parsed.Tpl == Tpl && parsed.ParentId == ParentId;
         }
 
         public override int GetHashCode()
         {
-            return (this.Tpl?.GetHashCode() + this.ParentId?.GetHashCode()) ?? base.GetHashCode();
+            return (Tpl?.GetHashCode() + ParentId?.GetHashCode()) ?? base.GetHashCode();
         }
 
         public object Clone()
         {
             return new Item
             {
-                Id = this.Id,
-                Tpl = this.Tpl,
-                ParentId = this.ParentId,
-                SlotId = this.SlotId,
-                Location = this.Location,
-                Upd = ProcessorUtil.Copy(this.Upd)
+                Id = Id,
+                Tpl = Tpl,
+                ParentId = ParentId,
+                SlotId = SlotId,
+                Location = Location,
+                Upd = ProcessorUtil.Copy(Upd)
             };
         }
     }

Model/Output/AbstractDistribution.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output;

Model/Output/AmmoDistribution.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output;

Model/Output/ItemCountDistribution.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output

Model/Output/ItemDistribution.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output

Model/Output/LooseLoot/LooseLootRoot.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output.LooseLoot

Model/Output/LooseLoot/SpawnPoint.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output.LooseLoot

Model/Output/LooseLoot/SpawnPointCount.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output.LooseLoot

Model/Output/LooseLoot/SpawnPointsForced.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output.LooseLoot

Model/Output/StaticContainer/MapStaticLoot.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output.StaticContainer

Model/Output/StaticContainer/StaticContainerRoot.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output.StaticContainer

Model/Output/StaticContainer/StaticDataPoint.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output.StaticContainer;

Model/Output/StaticContainer/StaticForced.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 using YamlDotNet.Serialization;
 

Model/Output/StaticDistribution.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output;

Model/Output/StaticItemDistribution.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Output

Model/Processing/BaseClasses.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Model.Processing;
+namespace LootDumpProcessor.Model.Processing;
 
 public static class BaseClasses
 {

Model/Processing/BasicInfo.cs

@@ -1,4 +1,4 @@
-using LootDumpProcessor.Model.Input;
+using LootDumpProcessor.Model.Input;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Processing;

Model/Processing/CaliberTemplateCount.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Model.Processing;
+namespace LootDumpProcessor.Model.Processing;
 
 public class CaliberTemplateCount
 {

Model/Processing/DumpProcessData.cs

@@ -1,4 +1,4 @@
-using LootDumpProcessor.Storage;
+using LootDumpProcessor.Storage;
 
 namespace LootDumpProcessor.Model.Processing;
 

Model/Processing/LooseLootCounts.cs

@@ -1,5 +1,6 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using LootDumpProcessor.Storage;
+using LootDumpProcessor.Utils;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Processing;
@@ -8,7 +9,7 @@ public class LooseLootCounts : IKeyable
 {
     [JsonProperty("__id__")]
     [JsonPropertyName("__id__")]
-    public string __ID { get; set; } = Guid.NewGuid().ToString();
+    public string __ID { get; set; } = KeyGenerator.GetNextKey();
 
     public IKey Counts { get; set; }
 

Model/Processing/ParsedDump.cs

@@ -1,11 +1,11 @@
-using System.Text.RegularExpressions;
+using System.Text.RegularExpressions;
 using LootDumpProcessor.Storage;
 
 namespace LootDumpProcessor.Model.Processing;
 
 public class ParsedDump : IKeyable
 {
-    private static readonly Regex _hashRegex = new Regex("([^a-zA-Z0-9])");
+    private static readonly Regex _hashRegex = new("([^a-zA-Z0-9])");
     public BasicInfo BasicInfo { get; set; }
     public PreProcessedLooseLoot LooseLoot { get; set; }
     public List<PreProcessedStaticLoot> Containers { get; set; }
@@ -13,13 +13,13 @@ public class ParsedDump : IKeyable
     public override bool Equals(object? obj)
     {
         if (obj is ParsedDump dump)
-            return dump.BasicInfo.Equals(this.BasicInfo);
+            return dump.BasicInfo.Equals(BasicInfo);
         return false;
     }
 
     public override int GetHashCode()
     {
-        return this.BasicInfo.GetHashCode();
+        return BasicInfo.GetHashCode();
     }
 
     public IKey GetKey()

Model/Processing/PartialData.cs

@@ -1,4 +1,4 @@
-using LootDumpProcessor.Storage;
+using LootDumpProcessor.Storage;
 
 namespace LootDumpProcessor.Model.Processing;
 
@@ -10,12 +10,12 @@ public class PartialData
     public override bool Equals(object? obj)
     {
         if (obj is ParsedDump dump)
-            return dump.BasicInfo.Equals(this.BasicInfo);
+            return dump.BasicInfo.Equals(BasicInfo);
         return false;
     }
 
     public override int GetHashCode()
     {
-        return this.BasicInfo.GetHashCode();
+        return BasicInfo.GetHashCode();
     }
 }

Model/Processing/PreProcessedLooseLoot.cs

@@ -1,4 +1,4 @@
-using LootDumpProcessor.Serializers.Json.Converters;
+using LootDumpProcessor.Serializers.Json.Converters;
 using LootDumpProcessor.Storage;
 
 namespace LootDumpProcessor.Model.Processing;

Model/Processing/PreProcessedStaticLoot.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Model.Processing;
+namespace LootDumpProcessor.Model.Processing;
 
 public class PreProcessedStaticLoot
 {

Model/Repairable.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model
@@ -17,8 +17,8 @@ namespace LootDumpProcessor.Model
         {
             return new Repairable
             {
-                Durability = this.Durability,
-                MaxDurability = this.MaxDurability
+                Durability = Durability,
+                MaxDurability = MaxDurability
             };
         }
     }

Model/Tarkov/Handbook.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Model.Tarkov;
+namespace LootDumpProcessor.Model.Tarkov;
 
 public class Category
 {

Model/Tarkov/TemplateItems.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model.Tarkov;

Model/Template.cs

@@ -1,6 +1,6 @@
-using System.Text.Json.Serialization;
-using LootDumpProcessor.Process.Processor;
+using System.Text.Json.Serialization;
 using LootDumpProcessor.Storage;
+using LootDumpProcessor.Utils;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model
@@ -9,7 +9,7 @@ namespace LootDumpProcessor.Model
     {
         [Newtonsoft.Json.JsonIgnore]
         [System.Text.Json.Serialization.JsonIgnore]
-        public string __ID { get; } = Guid.NewGuid().ToString();
+        public string __ID { get; } = KeyGenerator.GetNextKey();
 
         [JsonProperty("Id", NullValueHandling = NullValueHandling.Ignore)]
         [JsonPropertyName("Id")]
@@ -82,17 +82,17 @@ namespace LootDumpProcessor.Model
         {
             return new Template
             {
-                Id = this.Id,
-                IsContainer = this.IsContainer,
-                UseGravity = this.UseGravity,
-                RandomRotation = this.RandomRotation,
-                Position = ProcessorUtil.Copy(this.Position),
-                Rotation = ProcessorUtil.Copy(this.Rotation),
-                IsGroupPosition = this.IsGroupPosition,
-                GroupPositions = ProcessorUtil.Copy(this.GroupPositions),
-                IsAlwaysSpawn = this.IsAlwaysSpawn,
-                Root = this.Root,
-                Items = ProcessorUtil.Copy(this.Items)
+                Id = Id,
+                IsContainer = IsContainer,
+                UseGravity = UseGravity,
+                RandomRotation = RandomRotation,
+                Position = ProcessorUtil.Copy(Position),
+                Rotation = ProcessorUtil.Copy(Rotation),
+                IsGroupPosition = IsGroupPosition,
+                GroupPositions = ProcessorUtil.Copy(GroupPositions),
+                IsAlwaysSpawn = IsAlwaysSpawn,
+                Root = Root,
+                Items = ProcessorUtil.Copy(Items)
             };
         }
     }

Model/Upd.cs

@@ -1,5 +1,6 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using LootDumpProcessor.Process.Processor;
+using LootDumpProcessor.Utils;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model
@@ -26,10 +27,10 @@ namespace LootDumpProcessor.Model
         {
             return new Upd
             {
-                StackObjectsCount = this.StackObjectsCount,
-                FireMode = ProcessorUtil.Copy(this.FireMode),
-                Foldable = ProcessorUtil.Copy(this.Foldable),
-                Repairable = ProcessorUtil.Copy(this.Repairable)
+                StackObjectsCount = StackObjectsCount,
+                FireMode = ProcessorUtil.Copy(FireMode),
+                Foldable = ProcessorUtil.Copy(Foldable),
+                Repairable = ProcessorUtil.Copy(Repairable)
             };
         }
     }

Model/Vector3.cs

@@ -1,4 +1,4 @@
-using System.Text.Json.Serialization;
+using System.Text.Json.Serialization;
 using Newtonsoft.Json;
 
 namespace LootDumpProcessor.Model
@@ -21,9 +21,9 @@ namespace LootDumpProcessor.Model
         {
             return new Vector3
             {
-                X = this.X,
-                Y = this.Y,
-                Z = this.Z
+                X = X,
+                Y = Y,
+                Z = Z
             };
         }
     }

Process/Collector/CollectorFactory.cs

@@ -1,10 +1,17 @@
-namespace LootDumpProcessor.Process.Collector;
+namespace LootDumpProcessor.Process.Collector;
 
 public static class CollectorFactory
 {
+    private static ICollector? _collector;
     public static ICollector GetInstance()
     {
-        // TODO: implement real factory
-        return new HashSetCollector();
+        if (_collector == null)
+            _collector = LootDumpProcessorContext.GetConfig().CollectorConfig.CollectorType switch
+            {
+                CollectorType.Memory => new HashSetCollector(),
+                CollectorType.Dump => new DumpCollector(),
+                _ => throw new ArgumentOutOfRangeException()
+            };
+        return _collector;
     }
 }

Process/Collector/CollectorType.cs

@@ -0,0 +1,7 @@
+namespace LootDumpProcessor.Process.Collector;
+
+public enum CollectorType
+{
+    Memory,
+    Dump
+}

Process/Collector/DumpCollector.cs

@@ -0,0 +1,45 @@
+using LootDumpProcessor.Model.Processing;
+using LootDumpProcessor.Serializers.Json;
+
+namespace LootDumpProcessor.Process.Collector;
+
+public class DumpCollector : ICollector
+{
+    private static readonly string DumpLocation = $"{LootDumpProcessorContext.GetConfig().CollectorConfig.DumpLocation}/collector/";
+    private readonly List<PartialData> processedDumps = new(LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping + 50);
+    private readonly object lockObject = new();
+    
+    public void Setup()
+    {
+        if (Directory.Exists(DumpLocation))
+        {
+            Directory.Delete(DumpLocation, true);
+        }
+        
+        Directory.CreateDirectory(DumpLocation);
+    }
+
+    public void Hold(PartialData parsedDump)
+    {
+        lock (lockObject)
+        {
+            processedDumps.Add(parsedDump);
+            if (processedDumps.Count > LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping)
+            {
+                var fileName = $"collector-{DateTime.Now.ToString("yyyyMMddHHmmssfffff")}.json";
+                File.WriteAllText($"{DumpLocation}{fileName}", JsonSerializerFactory.GetInstance().Serialize(processedDumps));
+                processedDumps.Clear();
+            }
+        }
+    }
+
+    public List<PartialData> Retrieve()
+    {
+        foreach (var file in Directory.GetFiles(DumpLocation))
+        {
+            processedDumps.AddRange(JsonSerializerFactory.GetInstance().Deserialize<List<PartialData>>(File.ReadAllText(file)));
+        }
+
+        return processedDumps;
+    }
+}

Process/Collector/HashSetCollector.cs

@@ -1,13 +1,11 @@
-using LootDumpProcessor.Model.Processing;
+using LootDumpProcessor.Model.Processing;
 
 namespace LootDumpProcessor.Process.Collector;
 
 public class HashSetCollector : ICollector
 {
-    private HashSet<PartialData> processedDumps = new HashSet<PartialData>();
-
-    private object lockObject = new object();
-
+    private readonly HashSet<PartialData> processedDumps = new(100_000);
+    private readonly object lockObject = new();
 
     public void Setup()
     {

Process/Collector/ICollector.cs

@@ -1,4 +1,3 @@
-using LootDumpProcessor.Model.Config;
 using LootDumpProcessor.Model.Processing;
 
 namespace LootDumpProcessor.Process.Collector;

Process/FilesGatherer.cs

@@ -1,6 +0,0 @@
-namespace LootDumpProcessor.Process;
-
-public class FilesGatherer
-{
-    
-}

Process/IPipeline.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Process;
+namespace LootDumpProcessor.Process;
 
 public interface IPipeline
 {

Process/OutputFileType.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Process;
+namespace LootDumpProcessor.Process;
 
 public enum OutputFileType
 {

Process/PipelineFactory.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Process;
+namespace LootDumpProcessor.Process;
 
 public static class PipelineFactory
 {

Process/Processor/DumpProcessor/DumpProcessorFactory.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Process.Processor.DumpProcessor;
+namespace LootDumpProcessor.Process.Processor.DumpProcessor;
 
 public static class DumpProcessorFactory
 {

Process/Processor/DumpProcessor/IDumpProcessor.cs

@@ -1,4 +1,4 @@
-using LootDumpProcessor.Model.Processing;
+using LootDumpProcessor.Model.Processing;
 
 namespace LootDumpProcessor.Process.Processor.DumpProcessor;
 

Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs

@@ -1,4 +1,4 @@
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 using LootDumpProcessor.Logger;
 using LootDumpProcessor.Model;
 using LootDumpProcessor.Model.Input;
@@ -7,6 +7,7 @@ using LootDumpProcessor.Model.Processing;
 using LootDumpProcessor.Serializers.Json;
 using LootDumpProcessor.Storage;
 using LootDumpProcessor.Storage.Collections;
+using LootDumpProcessor.Utils;
 
 namespace LootDumpProcessor.Process.Processor.DumpProcessor;
 
@@ -25,10 +26,12 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
 
     public Dictionary<OutputFileType, object> ProcessDumps(List<PartialData> dumps)
     {
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Starting final dump processing", LogLevel.Info);
         var output = new Dictionary<OutputFileType, object>();
 
         var dumpProcessData = GetDumpProcessData(dumps);
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Heavy processing done!", LogLevel.Info);
 
         var staticContainers = new Dictionary<string, MapStaticLoot>();
@@ -42,13 +45,15 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
 
         Runners.Clear();
         // BSG changed the map data so static containers are now dynamic, so we need to scan all dumps for the static containers.
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Queuing dumps for static data processing", LogLevel.Info);
         foreach (var dumped in dumps)
         {
             Runners.Add(
                 Task.Factory.StartNew(() =>
                 {
-                    LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", LogLevel.Info);
+                    if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
+                        LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", LogLevel.Debug);
                     var data = _jsonSerializer.Deserialize<RootData>(File.ReadAllText(dumped.BasicInfo.FileName));
                     // the if statement below takes care of processing "forced" or real static data for each map, we only need
                     // to do this once per map, so we dont care about doing it again
@@ -56,6 +61,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
                     {
                         if (!staticContainers.ContainsKey(data.Data.Name))
                         {
+                            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
                                 LoggerFactory.GetInstance().Log($"Doing first time process for map {data.Data.Name} of real static data", LogLevel.Info);
                             var mapStaticLoot = StaticLootProcessor.CreateRealStaticContainers(data);
                             staticContainers[mapStaticLoot.Item1] = mapStaticLoot.Item2;
@@ -113,6 +119,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
         }
 
         Task.WaitAll(Runners.ToArray());
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("All static data processing threads finished", LogLevel.Info);
         // Aggregate and calculate the probability of a static container
         mapStaticContainersAggregated.ToDictionary(
@@ -128,7 +135,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
 
         // Static containers
         output.Add(OutputFileType.StaticContainer, staticContainers);
-
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Processing ammo distribution", LogLevel.Info);
         // Ammo distribution
         output.Add(
@@ -136,6 +143,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
             StaticLootProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts)
         );
 
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Processing static loot distribution", LogLevel.Info);
         // Static loot distribution
         output.Add(
@@ -143,6 +151,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
             StaticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts)
         );
 
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Processing loose loot distribution", LogLevel.Info);
         // Loose loot distribution
         var looseLootDistribution = LooseLootProcessor.CreateLooseLootDistribution(
@@ -150,12 +159,14 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
             dumpProcessData.LooseLootCounts
         );
 
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Collecting loose loot distribution information", LogLevel.Info);
         var loot = dumpProcessData.MapCounts
             .Select(mapCount => mapCount.Key)
             .ToDictionary(mi => mi, mi => looseLootDistribution[mi]);
 
         output.Add(OutputFileType.LooseLoot, loot);
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log("Dump processing fully completed!", LogLevel.Info);
         return output;
     }
@@ -175,6 +186,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
             {
                 var mapi = tuple.Key;
                 var g = tuple.ToList();
+                if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
                     LoggerFactory.GetInstance().Log(
                         $"Processing map {mapi}, total dump data to process: {g.Count}",
                         LogLevel.Info
@@ -274,14 +286,12 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
 
                                         lock (lockObjectCounts)
                                         {
-                                            counts.MapSpawnpointCount.AddRange(new List<int>
-                                            {
-                                                dumpData.LooseLoot.MapSpawnpointCount
-                                            });
+                                            counts.MapSpawnpointCount.Add(dumpData.LooseLoot.MapSpawnpointCount);
                                         }
                                     }
                                     catch (Exception e)
                                     {
+                                        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
                                             LoggerFactory.GetInstance().Log(
                                                 $"ERROR OCCURRED:{e.Message}\n{e.StackTrace}",
                                                 LogLevel.Error
@@ -296,6 +306,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
                 // Wait until all runners are done processing
                 while (!Runners.All(r => r.IsCompleted))
                 {
+                    if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
                         LoggerFactory.GetInstance().Log(
                             $"One or more file processors are still processing files. Waiting {LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs}ms before checking again",
                             LogLevel.Info

Process/Processor/FileProcessor/FileProcessor.cs

@@ -1,16 +1,16 @@
-using LootDumpProcessor.Logger;
+using LootDumpProcessor.Logger;
 using LootDumpProcessor.Model;
 using LootDumpProcessor.Model.Processing;
-using LootDumpProcessor.Process.Processor;
 using LootDumpProcessor.Storage;
 
-namespace LootDumpProcessor.Process.Impl;
+namespace LootDumpProcessor.Process.Processor.FileProcessor;
 
 public class FileProcessor : IFileProcessor
 {
     public PartialData Process(BasicInfo parsedData)
     {
-        LoggerFactory.GetInstance().Log($"Processing file {parsedData.FileName}...", LogLevel.Info);
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
+            LoggerFactory.GetInstance().Log($"Processing file {parsedData.FileName}...", LogLevel.Debug);
         List<Template> looseLoot = new List<Template>();
         List<Template> staticLoot = new List<Template>();
 
@@ -29,7 +29,7 @@ public class FileProcessor : IFileProcessor
             BasicInfo = parsedData
         };
 
-        PartialData data = new PartialData
+        var data = new PartialData
         {
             BasicInfo = parsedData,
             ParsedDumpKey = (AbstractKey)dumpData.GetKey()
@@ -37,16 +37,17 @@ public class FileProcessor : IFileProcessor
 
         if (!DataStorageFactory.GetInstance().Exists(dumpData.GetKey()))
         {
+            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
                 LoggerFactory.GetInstance().Log(
                     $"Cached not found for {string.Join("/", dumpData.GetKey().GetLookupIndex())} processing.",
-                LogLevel.Info
+                    LogLevel.Debug
                 );
             dumpData.Containers = StaticLootProcessor.PreProcessStaticLoot(staticLoot);
             dumpData.LooseLoot = LooseLootProcessor.PreProcessLooseLoot(looseLoot);
             DataStorageFactory.GetInstance().Store(dumpData);
         }
-
-        LoggerFactory.GetInstance().Log($"File {parsedData.FileName} finished processing!", LogLevel.Info);
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
+            LoggerFactory.GetInstance().Log($"File {parsedData.FileName} finished processing!", LogLevel.Debug);
         return data;
     }
 }

Process/Processor/FileProcessor/FileProcessorFactory.cs

@@ -1,12 +1,13 @@
-using LootDumpProcessor.Process.Impl;
-
-namespace LootDumpProcessor.Process.Processor;
+namespace LootDumpProcessor.Process.Processor.FileProcessor;
 
 public static class FileProcessorFactory
 {
+    private static IFileProcessor? _fileProcessor;
     public static IFileProcessor GetInstance()
     {
-        // implement actual factory someday
-        return new FileProcessor();
+        // TODO: implement actual factory someday
+        if (_fileProcessor == null)
+            _fileProcessor = new FileProcessor();
+        return _fileProcessor;
     }
 }

Process/Processor/FileProcessor/IFileProcessor.cs

@@ -1,6 +1,6 @@
-using LootDumpProcessor.Model.Processing;
+using LootDumpProcessor.Model.Processing;
 
-namespace LootDumpProcessor.Process;
+namespace LootDumpProcessor.Process.Processor.FileProcessor;
 
 public interface IFileProcessor
 {

Process/Processor/LooseLootProcessor.cs

@@ -1,15 +1,16 @@
-using LootDumpProcessor.Logger;
+using LootDumpProcessor.Logger;
 using LootDumpProcessor.Model;
 using LootDumpProcessor.Model.Output;
 using LootDumpProcessor.Model.Output.LooseLoot;
 using LootDumpProcessor.Model.Processing;
 using LootDumpProcessor.Storage;
 using LootDumpProcessor.Storage.Collections;
+using LootDumpProcessor.Utils;
 using NumSharp;
 
 namespace LootDumpProcessor.Process.Processor;
 
-public class LooseLootProcessor
+public static class LooseLootProcessor
 {
     public static PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseloot)
     {
@@ -163,6 +164,7 @@ public class LooseLootProcessor
                         Template = template
                     };
                     looseLootDistribution[mapName].SpawnPointsForced.Add(spawnPointToAdd);
+                    if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
                         LoggerFactory.GetInstance().Log(
                             $"Item: {template.Id} has > {LootDumpProcessorContext.GetConfig().ProcessorConfig.SpawnPointToleranceForForced}% spawn chance in spawn point: {spawnPointToAdd.LocationId} but isn't in forced loot, adding to forced",
                             LogLevel.Warning
@@ -204,6 +206,7 @@ public class LooseLootProcessor
                         }
                         else
                         {
+                            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
                                 LoggerFactory.GetInstance().Log(
                                     $"Item template {distribution.ComposedKey?.FirstItem?.Tpl} was on loose loot distribution for spawn point {template.Id} but the spawn points didnt contain a template matching it.",
                                     LogLevel.Error
@@ -248,6 +251,7 @@ public class LooseLootProcessor
             {
                 if (!forcedTplsFound.Contains(itemTpl))
                 {
+                    if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
                         LoggerFactory.GetInstance().Log(
                             $"Expected item: {itemTpl} defined in forced_loose.yaml config not found in forced loot",
                             LogLevel.Error
@@ -260,6 +264,7 @@ public class LooseLootProcessor
             {
                 if (!forcedTplsInConfig.Contains(itemTpl))
                 {
+                    if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
                         LoggerFactory.GetInstance().Log(
                             $"Map: {mapName} Item: {itemTpl} not defined in forced_loose.yaml config but was flagged as forced by code",
                             LogLevel.Warning

Process/Processor/StaticLootProcessor.cs

@@ -1,12 +1,13 @@
-using LootDumpProcessor.Model;
+using LootDumpProcessor.Model;
 using LootDumpProcessor.Model.Input;
 using LootDumpProcessor.Model.Output;
 using LootDumpProcessor.Model.Output.StaticContainer;
 using LootDumpProcessor.Model.Processing;
+using LootDumpProcessor.Utils;
 
 namespace LootDumpProcessor.Process.Processor;
 
-public class StaticLootProcessor
+public static class StaticLootProcessor
 {
     public static List<PreProcessedStaticLoot> PreProcessStaticLoot(List<Template> staticloot)
     {
@@ -30,7 +31,6 @@ public class StaticLootProcessor
 
     public static Tuple<string, MapStaticLoot> CreateRealStaticContainers(RootData rawMapDump)
     {
-        List<StaticForced> forcedStaticItems;
         var mapName = rawMapDump.Data.Name;
         var staticLootPositions = (from li in rawMapDump.Data.Loot
             where li.IsContainer ?? false
@@ -46,7 +46,7 @@ public class StaticLootProcessor
             }
         }
 
-        forcedStaticItems = LootDumpProcessorContext.GetForcedItems().ContainsKey(mapName)
+        var forcedStaticItems = LootDumpProcessorContext.GetForcedItems().ContainsKey(mapName)
             ? LootDumpProcessorContext.GetForcedItems()[mapName]
             : new List<StaticForced>();
 
@@ -145,9 +145,7 @@ public class StaticLootProcessor
             var itemsHitCounts = new Dictionary<string, int>();
             foreach (var ci in container_counts_selected)
             {
-                foreach (var cii in ci.Items)
-                {
-                    if (cii.ParentId == ci.ContainerId)
+                foreach (var cii in ci.Items.Where(cii => cii.ParentId == ci.ContainerId))
                 {
                     if (itemsHitCounts.ContainsKey(cii.Tpl))
                         itemsHitCounts[cii.Tpl] += 1;
@@ -155,7 +153,6 @@ public class StaticLootProcessor
                         itemsHitCounts[cii.Tpl] = 1;
                 }
             }
-            }
 
             static_loot_distribution[typei].ItemDistribution = itemsHitCounts.Select(v => new StaticDistribution
             {

Process/QueuePipeline.cs

@@ -1,12 +1,15 @@
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 using LootDumpProcessor.Logger;
 using LootDumpProcessor.Process.Collector;
 using LootDumpProcessor.Process.Processor;
 using LootDumpProcessor.Process.Processor.DumpProcessor;
+using LootDumpProcessor.Process.Processor.FileProcessor;
 using LootDumpProcessor.Process.Reader;
 using LootDumpProcessor.Process.Reader.Filters;
+using LootDumpProcessor.Process.Reader.Intake;
 using LootDumpProcessor.Process.Reader.PreProcess;
 using LootDumpProcessor.Process.Writer;
+using LootDumpProcessor.Utils;
 
 namespace LootDumpProcessor.Process;
 
@@ -38,7 +41,8 @@ public class QueuePipeline : IPipeline
         // We add 2 more threads to the total count to account for subprocesses and others
         int threads = LootDumpProcessorContext.GetConfig().Threads;
         ThreadPool.SetMaxThreads(threads + 2, threads + 2);
-
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
+            LoggerFactory.GetInstance().Log("Gathering files to begin processing", LogLevel.Info);
         try
         {
             // Gather all files, sort them by date descending and then add them into the processing queue
@@ -49,9 +53,14 @@ public class QueuePipeline : IPipeline
                 }
             ).ToList().ForEach(f => _filesToProcess.Add(f));
             
+            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
+                LoggerFactory.GetInstance().Log("Files sorted and ready to begin pre-processing", LogLevel.Info);
+            
             // We startup all the threads and collect them into a runners list
             for (int i = 0; i < threads; i++)
             {
+                if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
+                    LoggerFactory.GetInstance().Log("Creating pre-processing threads", LogLevel.Info);
                 Runners.Add(
                     Task.Factory.StartNew(
                         () =>
@@ -67,6 +76,7 @@ public class QueuePipeline : IPipeline
                                 }
                                 catch (Exception e)
                                 {
+                                    if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
                                         LoggerFactory.GetInstance().Log(
                                             $"Error occurred while processing file {file}\n{e.Message}\n{e.StackTrace}",
                                             LogLevel.Error);
@@ -80,12 +90,14 @@ public class QueuePipeline : IPipeline
             // Wait until all runners are done processing
             while (!Runners.All(r => r.IsCompleted))
             {
+                if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
                     LoggerFactory.GetInstance().Log(
                         $"One or more file processors are still processing files. Waiting {LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs}ms before checking again",
                         LogLevel.Info);
                 Thread.Sleep(TimeSpan.FromMilliseconds(LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs));
             }
-
+            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
+                LoggerFactory.GetInstance().Log("Pre-processing finished", LogLevel.Info);
             // Single writer instance to collect results
             var writer = WriterFactory.GetInstance();
             // Single collector instance to collect results

Process/Reader/Filters/FileFilterFactory.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Process.Reader.Filters;
+namespace LootDumpProcessor.Process.Reader.Filters;
 
 public static class FileFilterFactory
 {

Process/Reader/Filters/FileFilterTypes.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Process.Reader.Filters;
+namespace LootDumpProcessor.Process.Reader.Filters;
 
 public enum FileFilterTypes
 {

Process/Reader/Filters/IFileFilter.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Process.Reader.Filters;
+namespace LootDumpProcessor.Process.Reader.Filters;
 
 public interface IFileFilter
 {

Process/Reader/Filters/JsonDumpFileFilter.cs

@@ -1,4 +1,4 @@
-using System.Globalization;
+using System.Globalization;
 using System.Text.RegularExpressions;
 using LootDumpProcessor.Logger;
 
@@ -6,22 +6,23 @@ namespace LootDumpProcessor.Process.Reader.Filters;
 
 public class JsonDumpFileFilter : IFileFilter
 {
-    private static Regex FileNameDateRegex = new("([0-9]{4}(-[0-9]{2}){2}_((-){0,1}[0-9]{2}){3})");
-    private static DateTime parsedThresholdDate;
+    private static readonly Regex _fileNameDateRegex = new("([0-9]{4}(-[0-9]{2}){2}_((-){0,1}[0-9]{2}){3})");
+    private static readonly DateTime _parsedThresholdDate;
 
     static JsonDumpFileFilter()
     {
         // Calculate parsed date from config threshold
         if (string.IsNullOrEmpty(LootDumpProcessorContext.GetConfig().ReaderConfig.ThresholdDate))
         {
+            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
                 LoggerFactory.GetInstance()
                     .Log($"ThresholdDate is null or empty in configs, defaulting to current day minus 30 days",
                         LogLevel.Warning);
-            parsedThresholdDate = (DateTime.Now - TimeSpan.FromDays(30));
+            _parsedThresholdDate = (DateTime.Now - TimeSpan.FromDays(30));
         }
         else
         {
-            parsedThresholdDate = DateTime.ParseExact(
+            _parsedThresholdDate = DateTime.ParseExact(
                 LootDumpProcessorContext.GetConfig().ReaderConfig.ThresholdDate,
                 "yyyy-MM-dd",
                 CultureInfo.InvariantCulture
@@ -33,8 +34,8 @@ public class JsonDumpFileFilter : IFileFilter
 
     public bool Accept(string filename)
     {
-        var unparsedDate = FileNameDateRegex.Match(filename).Groups[1].Value;
+        var unparsedDate = _fileNameDateRegex.Match(filename).Groups[1].Value;
         var date = DateTime.ParseExact(unparsedDate, "yyyy-MM-dd_HH-mm-ss", CultureInfo.InvariantCulture);
-        return date > parsedThresholdDate;
+        return date > _parsedThresholdDate;
     }
 }

Process/Reader/Intake/IIntakeReader.cs

@@ -1,6 +1,6 @@
-using LootDumpProcessor.Model.Processing;
+using LootDumpProcessor.Model.Processing;
 
-namespace LootDumpProcessor.Process;
+namespace LootDumpProcessor.Process.Reader.Intake;
 
 public interface IIntakeReader
 {

Process/Reader/Intake/IntakeReaderFactory.cs

@@ -1,12 +1,18 @@
-using LootDumpProcessor.Process.Impl;
-
-namespace LootDumpProcessor.Process.Reader;
+namespace LootDumpProcessor.Process.Reader.Intake;
 
 public static class IntakeReaderFactory
 {
+    private static readonly Dictionary<IntakeReaderTypes, IIntakeReader> Instances = new();
+    private static readonly object DictionaryLock = new();
     public static IIntakeReader GetInstance()
     {
-        return LootDumpProcessorContext.GetConfig().ReaderConfig.IntakeReaderConfig.IntakeReaderType switch
+        var type = LootDumpProcessorContext.GetConfig().ReaderConfig.IntakeReaderConfig?.IntakeReaderType ??
+                   IntakeReaderTypes.Json;
+        lock (DictionaryLock)
+        {
+            if (!Instances.TryGetValue(type, out var intakeReader))
+            {
+                intakeReader = type switch
                 {
                     IntakeReaderTypes.Json => new JsonFileIntakeReader(),
                     _ => throw new ArgumentOutOfRangeException(
@@ -14,5 +20,9 @@ public static class IntakeReaderFactory
                         "Value was not defined on IntakeReaderConfig"
                     )
                 };
+                Instances.Add(type, intakeReader);
+            }
+            return intakeReader;
+        }
     }
 }

Process/Reader/Intake/IntakeReaderTypes.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Process.Reader;
+namespace LootDumpProcessor.Process.Reader.Intake;
 
 public enum IntakeReaderTypes
 {

Process/Reader/Intake/JsonFileIntakeReader.cs

@@ -1,11 +1,11 @@
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 using LootDumpProcessor.Logger;
 using LootDumpProcessor.Model.Input;
 using LootDumpProcessor.Model.Processing;
-using LootDumpProcessor.Process.Processor;
 using LootDumpProcessor.Serializers.Json;
+using LootDumpProcessor.Utils;
 
-namespace LootDumpProcessor.Process.Impl;
+namespace LootDumpProcessor.Process.Reader.Intake;
 
 public class JsonFileIntakeReader : IIntakeReader
 {
@@ -21,19 +21,21 @@ public class JsonFileIntakeReader : IIntakeReader
         var fileData = File.ReadAllText(file);
         // If the file format changes it may screw up this date parser
         if (!FileDateParser.TryParseFileDate(file, out var date))
+        {
+            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
                 LoggerFactory.GetInstance().Log($"Couldnt parse date from file: {file}", LogLevel.Error);
+        }
 
         var fi = _jsonSerializer.Deserialize<RootData>(fileData);
         if (fi.Data?.Name != null && (!_ignoredLocations?.Contains(fi.Data.Name) ?? true))
         {
-            int counter;
-            if (!_totalMapDumpsCounter.TryGetValue(fi.Data.Name, out counter))
+            if (!_totalMapDumpsCounter.TryGetValue(fi.Data.Name, out var counter))
             {
                 counter = 0;
                 _totalMapDumpsCounter[fi.Data.Name] = counter;
             }
 
-            if (counter < LootDumpProcessorContext.GetConfig().ReaderConfig.IntakeReaderConfig.MaxDumpsPerMap)
+            if (counter < (LootDumpProcessorContext.GetConfig().ReaderConfig.IntakeReaderConfig?.MaxDumpsPerMap ?? 1500))
             {
                 basicInfo = new BasicInfo
                 {
@@ -44,15 +46,18 @@ public class JsonFileIntakeReader : IIntakeReader
                     FileName = file
                 };
                 _totalMapDumpsCounter[fi.Data.Name] += 1;
-                LoggerFactory.GetInstance().Log($"File {file} fully read, returning data", LogLevel.Info);
+                if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
+                    LoggerFactory.GetInstance().Log($"File {file} fully read, returning data", LogLevel.Debug);
                 return true;
             }
-            LoggerFactory.GetInstance().Log($"Ignoring file {file} as the file cap for map {fi.Data.Name} has been reached", LogLevel.Info);
+            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
+                LoggerFactory.GetInstance().Log($"Ignoring file {file} as the file cap for map {fi.Data.Name} has been reached", LogLevel.Debug);
         }
 
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
             LoggerFactory.GetInstance().Log(
                 $"File {file} was not eligible for dump data, it did not contain a location name or it was on ignored locations config",
-            LogLevel.Info
+                LogLevel.Warning
             );
         basicInfo = null;
         return false;

Process/Reader/PreProcess/AbstractPreProcessReader.cs

@@ -1,10 +1,10 @@
-using LootDumpProcessor.Logger;
+using LootDumpProcessor.Logger;
 
 namespace LootDumpProcessor.Process.Reader.PreProcess;
 
 public abstract class AbstractPreProcessReader : IPreProcessReader
 {
-    protected string _tempFolder;
+    protected readonly string _tempFolder;
 
     public AbstractPreProcessReader()
     {
@@ -12,6 +12,7 @@ public abstract class AbstractPreProcessReader : IPreProcessReader
         if (string.IsNullOrEmpty(tempFolder))
         {
             tempFolder = GetBaseDirectory();
+            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
                 LoggerFactory.GetInstance()
                     .Log(
                         $"No temp folder was assigned preProcessorTempFolder in PreProcessorConfig, defaulting to {tempFolder}",
@@ -35,7 +36,7 @@ public abstract class AbstractPreProcessReader : IPreProcessReader
 
     protected string GetBaseDirectory()
     {
-        return $"{Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)}\\SPT\\tmp\\PreProcessor";
+        return $@"{Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)}\SPT\tmp\PreProcessor";
     }
 
     public void Dispose()

Process/Reader/PreProcess/IPreProcessReader.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Process.Reader.PreProcess;
+namespace LootDumpProcessor.Process.Reader.PreProcess;
 
 public interface IPreProcessReader
 {

Process/Reader/PreProcess/PreProcessReaderFactory.cs

@@ -1,9 +1,8 @@
-namespace LootDumpProcessor.Process.Reader.PreProcess;
+namespace LootDumpProcessor.Process.Reader.PreProcess;
 
 public static class PreProcessReaderFactory
 {
     private static readonly Dictionary<PreProcessReaderTypes, IPreProcessReader> _proProcessReaders = new();
-    private static object lockObject = new object();
 
     public static IPreProcessReader GetInstance(PreProcessReaderTypes type)
     {

Process/Reader/PreProcess/PreProcessReaderTypes.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Process.Reader.PreProcess;
+namespace LootDumpProcessor.Process.Reader.PreProcess;
 
 public enum PreProcessReaderTypes
 {

Process/Reader/PreProcess/SevenZipPreProcessReader.cs

@@ -1,4 +1,4 @@
-using LootDumpProcessor.Logger;
+using LootDumpProcessor.Logger;
 using SevenZip;
 using SevenZip.Sdk.Compression.Lzma;
 
@@ -15,21 +15,25 @@ public class SevenZipPreProcessReader : AbstractPreProcessReader
 
     public override bool TryPreProcess(string file, out List<string> files, out List<string> directories)
     {
-        Decoder decoder = new Decoder();
-
         var fileRaw = Path.GetFileNameWithoutExtension(file);
         // SevenZip library doesnt like forward slashes for some reason
         var outPath = $"{_tempFolder}\\{fileRaw}".Replace("/", "\\");
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log(
                 $"Unzipping {file} into temp path {outPath}, this may take a while...",
                 LogLevel.Info);
         var extractor = new SevenZipExtractor(file);
-        extractor.Extracting += (sender, args) =>
+        // Only log process on debug mode
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
+        {
+            extractor.Extracting += (_, args) =>
             {
                 if (args.PercentDone % 10 == 0)
-                LoggerFactory.GetInstance().Log($"Unzip progress: {args.PercentDone}%", LogLevel.Info);
+                    LoggerFactory.GetInstance().Log($"Unzip progress: {args.PercentDone}%", LogLevel.Debug);
             };
+        }
         extractor.ExtractArchive(outPath);
+        if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
             LoggerFactory.GetInstance().Log($"Finished unzipping {file} into temp path {outPath}", LogLevel.Info);
 
         files = Directory.GetFiles(outPath).ToList();

Process/TarkovItems.cs

@@ -1,24 +1,22 @@
-using LootDumpProcessor.Logger;
+using LootDumpProcessor.Logger;
 using LootDumpProcessor.Model.Tarkov;
 using LootDumpProcessor.Serializers.Json;
 
 namespace LootDumpProcessor.Process;
 
-public class TarkovItems
+public class TarkovItems(string items)
 {
     private static readonly IJsonSerializer _jsonSerializer = JsonSerializerFactory.GetInstance();
 
-    private Dictionary<string, TemplateFileItem> _items;
-    
-    public TarkovItems(string items)
-    {
-        _items = _jsonSerializer.Deserialize<Dictionary<string, TemplateFileItem>>(File.ReadAllText(items));
-    }
+    private readonly Dictionary<string, TemplateFileItem>? _items = _jsonSerializer.Deserialize<Dictionary<string, TemplateFileItem>>(File.ReadAllText(items));
 
     public virtual bool IsBaseClass(string tpl, string baseclass_id)
     {
+        if (_items == null)
+            throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
         if (!_items.TryGetValue(tpl, out var item_template))
         {
+            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
                 LoggerFactory.GetInstance().Log($"[IsBaseClass] Item template '{tpl}' with base class id '{baseclass_id}' was not found on the server items!", LogLevel.Error);
             return false;
         }
@@ -31,8 +29,11 @@ public class TarkovItems
 
     public virtual bool IsQuestItem(string tpl)
     {
+        if (_items == null)
+            throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
         if (!_items.TryGetValue(tpl, out var item_template))
         {
+            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
                 LoggerFactory.GetInstance().Log($"[IsQuestItem] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
             return false;
         }
@@ -41,8 +42,11 @@ public class TarkovItems
 
     public virtual string? MaxDurability(string tpl)
     {
+        if (_items == null)
+            throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
         if (!_items.TryGetValue(tpl, out var item_template))
         {
+            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
                 LoggerFactory.GetInstance().Log($"[MaxDurability] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
             return null;
         }
@@ -51,8 +55,11 @@ public class TarkovItems
 
     public virtual string? AmmoCaliber(string tpl)
     {
+        if (_items == null)
+            throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
         if (!_items.TryGetValue(tpl, out var item_template))
         {
+            if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
                 LoggerFactory.GetInstance().Log($"[AmmoCaliber] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
             return null;
         }

Process/Writer/FileWriter.cs

@@ -1,14 +1,14 @@
-using LootDumpProcessor.Model.Output;
+using LootDumpProcessor.Model.Output;
 using LootDumpProcessor.Model.Output.LooseLoot;
 using LootDumpProcessor.Model.Output.StaticContainer;
 using LootDumpProcessor.Serializers.Json;
 
-namespace LootDumpProcessor.Process.Impl;
+namespace LootDumpProcessor.Process.Writer;
 
 public class FileWriter : IWriter
 {
     private static readonly IJsonSerializer _jsonSerializer = JsonSerializerFactory.GetInstance();
-    private static string _outputPath;
+    private static readonly string _outputPath;
 
     static FileWriter()
     {
@@ -47,9 +47,9 @@ public class FileWriter : IWriter
                 {
                     foreach (var s in LootDumpProcessorContext.GetDirectoryMappings()[key].Name)
                     {
-                        if (!Directory.Exists($"{_outputPath}\\locations\\{s}"))
-                            Directory.CreateDirectory($"{_outputPath}\\locations\\{s}");
-                        File.WriteAllText($"{_outputPath}\\locations\\{s}\\looseLoot.json",
+                        if (!Directory.Exists($@"{_outputPath}\locations\{s}"))
+                            Directory.CreateDirectory($@"{_outputPath}\locations\{s}");
+                        File.WriteAllText($@"{_outputPath}\locations\{s}\looseLoot.json",
                             _jsonSerializer.Serialize(value));
                     }
                 }
@@ -57,17 +57,17 @@ public class FileWriter : IWriter
                 break;
             case OutputFileType.StaticContainer:
                 var staticContainer = (Dictionary<string, MapStaticLoot>)data;
-                File.WriteAllText($"{_outputPath}\\loot\\staticContainers.json",
+                File.WriteAllText($@"{_outputPath}\loot\staticContainers.json",
                     _jsonSerializer.Serialize(staticContainer));
                 break;
             case OutputFileType.StaticLoot:
                 var staticLoot = (Dictionary<string, StaticItemDistribution>)data;
-                File.WriteAllText($"{_outputPath}\\loot\\staticLoot.json",
+                File.WriteAllText($@"{_outputPath}\loot\staticLoot.json",
                     _jsonSerializer.Serialize(staticLoot));
                 break;
             case OutputFileType.StaticAmmo:
                 var staticAmmo = (Dictionary<string, List<AmmoDistribution>>)data;
-                File.WriteAllText($"{_outputPath}\\loot\\staticAmmo.json",
+                File.WriteAllText($@"{_outputPath}\loot\staticAmmo.json",
                     _jsonSerializer.Serialize(staticAmmo));
                 break;
             default:

Process/Writer/IWriter.cs

@@ -1,4 +1,4 @@
-namespace LootDumpProcessor.Process;
+namespace LootDumpProcessor.Process.Writer;
 
 public interface IWriter
 {

Process/Writer/WriterFactory.cs

@@ -1,5 +1,3 @@
-using LootDumpProcessor.Process.Impl;
-
 namespace LootDumpProcessor.Process.Writer;
 
 public static class WriterFactory