From 9be5d6e342a6f7af4407c7b6becc4d52e1c3181c Mon Sep 17 00:00:00 2001 From: bluextx Date: Sat, 11 Jan 2025 06:54:59 +0300 Subject: [PATCH] Refactored processors to use dependency injection and improved immutability --- .../LootDumpProcessor.csproj | 2 + .../Model/Output/StaticItemDistribution.cs | 4 +- .../Model/Processing/ParsedDump.cs | 2 +- .../Process/PipelineFactory.cs | 10 - .../DumpProcessor/DumpProcessorFactory.cs | 12 - .../MultithreadSteppedDumpProcessor.cs | 82 +++-- .../Processor/FileProcessor/FileProcessor.cs | 8 +- .../FileProcessor/FileProcessorFactory.cs | 13 - .../Process/Processor/StaticLootProcessor.cs | 281 ------------------ .../v2/AmmoProcessor/AmmoProcessor.cs | 56 ++++ .../v2/AmmoProcessor/IAmmoProcessor.cs | 10 + .../IStaticContainersProcessor.cs | 11 + .../StaticContainerProcessor.cs | 79 +++++ .../IStaticLootProcessor.cs | 15 + .../StaticLootProcessor.cs | 117 ++++++++ .../Process/QueuePipeline.cs | 24 +- .../Process/Writer/FileWriter.cs | 4 +- source/LootDumpProcessor/Program.cs | 28 +- .../Serializers/Json/NetJsonSerializer.cs | 3 +- 19 files changed, 398 insertions(+), 363 deletions(-) delete mode 100644 source/LootDumpProcessor/Process/PipelineFactory.cs delete mode 100644 source/LootDumpProcessor/Process/Processor/DumpProcessor/DumpProcessorFactory.cs delete mode 100644 source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessorFactory.cs delete mode 100644 source/LootDumpProcessor/Process/Processor/StaticLootProcessor.cs create mode 100644 source/LootDumpProcessor/Process/Processor/v2/AmmoProcessor/AmmoProcessor.cs create mode 100644 source/LootDumpProcessor/Process/Processor/v2/AmmoProcessor/IAmmoProcessor.cs create mode 100644 source/LootDumpProcessor/Process/Processor/v2/StaticContainersProcessor/IStaticContainersProcessor.cs create mode 100644 source/LootDumpProcessor/Process/Processor/v2/StaticContainersProcessor/StaticContainerProcessor.cs create mode 100644 source/LootDumpProcessor/Process/Processor/v2/StaticLootProcessor/IStaticLootProcessor.cs create mode 100644 source/LootDumpProcessor/Process/Processor/v2/StaticLootProcessor/StaticLootProcessor.cs diff --git a/source/LootDumpProcessor/LootDumpProcessor.csproj b/source/LootDumpProcessor/LootDumpProcessor.csproj index 13a5a2c..c4bbc05 100644 --- a/source/LootDumpProcessor/LootDumpProcessor.csproj +++ b/source/LootDumpProcessor/LootDumpProcessor.csproj @@ -12,6 +12,8 @@ + + diff --git a/source/LootDumpProcessor/Model/Output/StaticItemDistribution.cs b/source/LootDumpProcessor/Model/Output/StaticItemDistribution.cs index b074afc..ac281a1 100644 --- a/source/LootDumpProcessor/Model/Output/StaticItemDistribution.cs +++ b/source/LootDumpProcessor/Model/Output/StaticItemDistribution.cs @@ -7,10 +7,10 @@ namespace LootDumpProcessor.Model.Output { [JsonProperty("itemcountDistribution", NullValueHandling = NullValueHandling.Ignore)] [JsonPropertyName("itemcountDistribution")] - public List? ItemCountDistribution { get; set; } + public IReadOnlyList? ItemCountDistribution { get; set; } [JsonProperty("itemDistribution", NullValueHandling = NullValueHandling.Ignore)] [JsonPropertyName("itemDistribution")] - public List? ItemDistribution { get; set; } + public IReadOnlyList? ItemDistribution { get; set; } } } \ No newline at end of file diff --git a/source/LootDumpProcessor/Model/Processing/ParsedDump.cs b/source/LootDumpProcessor/Model/Processing/ParsedDump.cs index 7c62a3b..a78b96e 100644 --- a/source/LootDumpProcessor/Model/Processing/ParsedDump.cs +++ b/source/LootDumpProcessor/Model/Processing/ParsedDump.cs @@ -8,7 +8,7 @@ public class ParsedDump : IKeyable private static readonly Regex _hashRegex = new("([^a-zA-Z0-9])"); public BasicInfo BasicInfo { get; set; } public PreProcessedLooseLoot LooseLoot { get; set; } - public List Containers { get; set; } + public IReadOnlyList Containers { get; set; } public override bool Equals(object? obj) { diff --git a/source/LootDumpProcessor/Process/PipelineFactory.cs b/source/LootDumpProcessor/Process/PipelineFactory.cs deleted file mode 100644 index 08867a2..0000000 --- a/source/LootDumpProcessor/Process/PipelineFactory.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace LootDumpProcessor.Process; - -public static class PipelineFactory -{ - public static IPipeline GetInstance() - { - // implement actual factory at some point - return new QueuePipeline(); - } -} \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/Processor/DumpProcessor/DumpProcessorFactory.cs b/source/LootDumpProcessor/Process/Processor/DumpProcessor/DumpProcessorFactory.cs deleted file mode 100644 index eaf3ee8..0000000 --- a/source/LootDumpProcessor/Process/Processor/DumpProcessor/DumpProcessorFactory.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace LootDumpProcessor.Process.Processor.DumpProcessor; - -public static class DumpProcessorFactory -{ - - public static IDumpProcessor GetInstance() - { - // Implement real factory - return new MultithreadSteppedDumpProcessor(); - } - -} \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs b/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs index d89e148..f665f8d 100644 --- a/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs +++ b/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs @@ -4,6 +4,9 @@ using LootDumpProcessor.Model; using LootDumpProcessor.Model.Input; using LootDumpProcessor.Model.Output.StaticContainer; using LootDumpProcessor.Model.Processing; +using LootDumpProcessor.Process.Processor.v2.AmmoProcessor; +using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor; +using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor; using LootDumpProcessor.Serializers.Json; using LootDumpProcessor.Storage; using LootDumpProcessor.Storage.Collections; @@ -11,8 +14,20 @@ using LootDumpProcessor.Utils; namespace LootDumpProcessor.Process.Processor.DumpProcessor; -public class MultithreadSteppedDumpProcessor : IDumpProcessor +public class MultithreadSteppedDumpProcessor( + IStaticLootProcessor staticLootProcessor, IStaticContainersProcessor staticContainersProcessor, + IAmmoProcessor ammoProcessor +) : IDumpProcessor { + private readonly IStaticLootProcessor _staticLootProcessor = + staticLootProcessor ?? throw new ArgumentNullException(nameof(staticLootProcessor)); + + private readonly IStaticContainersProcessor _staticContainersProcessor = + staticContainersProcessor ?? throw new ArgumentNullException(nameof(staticContainersProcessor)); + + private readonly IAmmoProcessor _ammoProcessor = + ammoProcessor ?? throw new ArgumentNullException(nameof(ammoProcessor)); + private static IJsonSerializer _jsonSerializer = JsonSerializerFactory.GetInstance(); private static readonly List Runners = new(); @@ -51,14 +66,17 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor Task.Factory.StartNew(() => { if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug)) - LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", LogLevel.Debug); + LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", + LogLevel.Debug); var dataDump = _jsonSerializer.Deserialize(File.ReadAllText(dumped.BasicInfo.FileName)); if (dataDump == null) { if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error)) - LoggerFactory.GetInstance().Log($"Failed to deserialize data from file {dumped.BasicInfo.FileName}", LogLevel.Error); + LoggerFactory.GetInstance() + .Log($"Failed to deserialize data from file {dumped.BasicInfo.FileName}", + LogLevel.Error); return; // Skip processing this dump } @@ -70,7 +88,9 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor if (!staticContainers.TryGetValue(mapId, out var mapStaticLoot)) { if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log($"Doing first time process for map {mapId} of real static data", LogLevel.Info); + LoggerFactory.GetInstance() + .Log($"Doing first time process for map {mapId} of real static data", + LogLevel.Info); staticContainers[mapId] = new MapStaticLoot { @@ -82,10 +102,13 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor { // .Item1 = map name // .Item2 = force/weapon static arrays - var mapStaticContainers = StaticLootProcessor.CreateStaticWeaponsAndStaticForcedContainers(dataDump); + var mapStaticContainers = + _staticContainersProcessor.CreateStaticWeaponsAndForcedContainers(dataDump); - var newStaticWeapons = mapStaticContainers.Item2.StaticWeapons.Where(x => !mapStaticLoot.StaticWeapons.Exists(y => y.Id == x.Id)); - var newStaticForced = mapStaticContainers.Item2.StaticForced.Where(x => !mapStaticLoot.StaticForced.Exists(y => y.ContainerId == x.ContainerId)); + var newStaticWeapons = mapStaticContainers.Item2.StaticWeapons.Where(x => + !mapStaticLoot.StaticWeapons.Exists(y => y.Id == x.Id)); + var newStaticForced = mapStaticContainers.Item2.StaticForced.Where(x => + !mapStaticLoot.StaticForced.Exists(y => y.ContainerId == x.ContainerId)); mapStaticLoot.StaticWeapons.AddRange(newStaticWeapons); mapStaticLoot.StaticForced.AddRange(newStaticForced); @@ -116,8 +139,10 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor IncrementMapCounterDictionaryValue(mapDumpCounter, mapId); } - var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(mapId, out string[]? ignoreListForMap); - foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(dataDump)) + var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList + .TryGetValue(mapId, out string[]? ignoreListForMap); + foreach (var dynamicStaticContainer in _staticContainersProcessor.CreateDynamicStaticContainers( + dataDump)) { lock (mapStaticContainersAggregatedLock) { @@ -145,15 +170,17 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor LoggerFactory.GetInstance().Log("All static data processing threads finished", LogLevel.Info); // Aggregate and calculate the probability of a static container mapStaticContainersAggregated.ToDictionary( - kv => kv.Key, - kv => kv.Value.Select( - td => new StaticDataPoint - { - Template = td.Key, - Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) // kv.Key = map name - } + kv => kv.Key, + kv => kv.Value.Select( + td => new StaticDataPoint + { + Template = td.Key, + Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) // kv.Key = map name + } + ).ToList() ).ToList() - ).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); // Hydrate staticContainers.StaticContainers + .ForEach(kv => + staticContainers[kv.Key].StaticContainers = kv.Value); // Hydrate staticContainers.StaticContainers // Static containers output.Add(OutputFileType.StaticContainer, staticContainers); @@ -162,7 +189,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor // Ammo distribution output.Add( OutputFileType.StaticAmmo, - StaticLootProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts) + _ammoProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts) ); if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) @@ -170,7 +197,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor // Static loot distribution output.Add( OutputFileType.StaticLoot, - StaticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts, staticContainers) + _staticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts, staticContainers) ); if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) @@ -196,9 +223,9 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor private static bool DumpWasMadeAfterConfigThresholdDate(PartialData dataDump) { return FileDateParser.TryParseFileDate(dataDump.BasicInfo.FileName, out var fileDate) && - fileDate.HasValue && - fileDate.Value > LootDumpProcessorContext.GetConfig().DumpProcessorConfig - .SpawnContainerChanceIncludeAfterDate; + fileDate.HasValue && + fileDate.Value > LootDumpProcessorContext.GetConfig().DumpProcessorConfig + .SpawnContainerChanceIncludeAfterDate; } private static void IncrementMapCounterDictionaryValue(Dictionary mapDumpCounter, string mapName) @@ -210,7 +237,8 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor } } - private static double GetStaticContainerProbability(string mapName, KeyValuePair td, Dictionary mapDumpCounter) + private static double GetStaticContainerProbability(string mapName, KeyValuePair td, + Dictionary mapDumpCounter) { return Math.Round((double)((decimal)td.Value / (decimal)mapDumpCounter[mapName]), 2); } @@ -255,7 +283,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor dumpProcessData.LooseLootCounts.Add(mapName, looseLootCounts.GetKey()); BlockingCollection _partialDataToProcess = new(); - + // add the items to the queue foreach (var partialData in partialFileMetaData) { @@ -286,7 +314,8 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor { if (!dumpProcessData.ContainerCounts.ContainsKey(mapName)) { - dumpProcessData.ContainerCounts.Add(mapName, dumpData.Containers); + dumpProcessData.ContainerCounts.Add(mapName, + dumpData.Containers.ToList()); } else { @@ -336,7 +365,8 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor lock (lockObjectCounts) { - looseLootCounts.MapSpawnpointCount.Add(dumpData.LooseLoot.MapSpawnpointCount); + looseLootCounts.MapSpawnpointCount.Add( + dumpData.LooseLoot.MapSpawnpointCount); } } catch (Exception e) diff --git a/source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessor.cs b/source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessor.cs index 9b3d387..3d9af88 100644 --- a/source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessor.cs +++ b/source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessor.cs @@ -1,12 +1,15 @@ using LootDumpProcessor.Logger; using LootDumpProcessor.Model; using LootDumpProcessor.Model.Processing; +using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor; using LootDumpProcessor.Storage; namespace LootDumpProcessor.Process.Processor.FileProcessor; -public class FileProcessor : IFileProcessor +public class FileProcessor(IStaticLootProcessor staticLootProcessor) : IFileProcessor { + private readonly IStaticLootProcessor _staticLootProcessor = staticLootProcessor ?? throw new ArgumentNullException(nameof(staticLootProcessor)); + public PartialData Process(BasicInfo parsedData) { if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug)) @@ -43,10 +46,11 @@ public class FileProcessor : IFileProcessor $"Cached not found for {string.Join("/", dumpData.GetKey().GetLookupIndex())} processing.", LogLevel.Debug ); - dumpData.Containers = StaticLootProcessor.PreProcessStaticLoot(staticLoot); + dumpData.Containers = _staticLootProcessor.PreProcessStaticLoot(staticLoot); dumpData.LooseLoot = LooseLootProcessor.PreProcessLooseLoot(looseLoot); DataStorageFactory.GetInstance().Store(dumpData); } + if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug)) LoggerFactory.GetInstance().Log($"File {parsedData.FileName} finished processing!", LogLevel.Debug); return data; diff --git a/source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessorFactory.cs b/source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessorFactory.cs deleted file mode 100644 index a31961e..0000000 --- a/source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessorFactory.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace LootDumpProcessor.Process.Processor.FileProcessor; - -public static class FileProcessorFactory -{ - private static IFileProcessor? _fileProcessor; - public static IFileProcessor GetInstance() - { - // TODO: implement actual factory someday - if (_fileProcessor == null) - _fileProcessor = new FileProcessor(); - return _fileProcessor; - } -} \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/Processor/StaticLootProcessor.cs b/source/LootDumpProcessor/Process/Processor/StaticLootProcessor.cs deleted file mode 100644 index 2dfe1e7..0000000 --- a/source/LootDumpProcessor/Process/Processor/StaticLootProcessor.cs +++ /dev/null @@ -1,281 +0,0 @@ -using LootDumpProcessor.Model; -using LootDumpProcessor.Model.Input; -using LootDumpProcessor.Model.Output; -using LootDumpProcessor.Model.Output.StaticContainer; -using LootDumpProcessor.Model.Processing; -using LootDumpProcessor.Utils; -using System.Collections.Generic; - -namespace LootDumpProcessor.Process.Processor; - -public static class StaticLootProcessor -{ - public static List PreProcessStaticLoot(List