From 54243f0932a90b9782ea6d2c3333f65549d06065 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 30 Jan 2024 20:22:32 +0000 Subject: [PATCH] Improved further memory usage and added a subsystem for the collector to dump into a file data while it keeps the pre-processor going --- Config/config.json | 5 +++ Model/ComposedKey.cs | 3 +- Model/Config/CollectorConfig.cs | 20 +++++++++ Model/Config/Config.cs | 4 ++ Model/Processing/LooseLootCounts.cs | 3 +- Model/Template.cs | 2 +- Process/Collector/CollectorFactory.cs | 11 ++++- Process/Collector/CollectorType.cs | 7 +++ Process/Collector/DumpCollector.cs | 45 +++++++++++++++++++ Process/Collector/HashSetCollector.cs | 4 +- .../FileProcessor/FileProcessorFactory.cs | 2 +- Storage/Collections/FlatKeyableDictionary.cs | 5 ++- Storage/Collections/FlatKeyableList.cs | 8 ++-- .../SubdivisionedKeyableDictionary.cs | 9 ++-- Utils/KeyGenerator.cs | 15 +++++++ 15 files changed, 124 insertions(+), 19 deletions(-) create mode 100644 Model/Config/CollectorConfig.cs create mode 100644 Process/Collector/CollectorType.cs create mode 100644 Process/Collector/DumpCollector.cs create mode 100644 Utils/KeyGenerator.cs diff --git a/Config/config.json b/Config/config.json index 7a4ce4f..1b1c211 100644 --- a/Config/config.json +++ b/Config/config.json @@ -47,6 +47,11 @@ "spawnPointToleranceForForced": 99.5, "looseLootCountTolerancePercentage": 75 }, + "collectorConfig": { + "collectorType": "Dump", + "maxEntitiesBeforeDumping": 1000, + "dumpLocation": "C:\\Users\\USER\\SPT\\tmp" + }, "writerConfig": { "outputLocation": "E:\\spt\\dumps\\output" }, diff --git a/Model/ComposedKey.cs b/Model/ComposedKey.cs index 432b40a..da574b1 100644 --- a/Model/ComposedKey.cs +++ b/Model/ComposedKey.cs @@ -1,5 +1,6 @@ using System.Text.Json.Serialization; using LootDumpProcessor.Model.Processing; +using LootDumpProcessor.Utils; using Newtonsoft.Json; namespace LootDumpProcessor.Model; @@ -26,7 +27,7 @@ public class ComposedKey .Cast() .Select(i => (double)i.GetHashCode()) .Sum() - .ToString() ?? Guid.NewGuid().ToString(); + .ToString() ?? KeyGenerator.GetNextKey(); FirstItem = items?[0]; } diff --git a/Model/Config/CollectorConfig.cs b/Model/Config/CollectorConfig.cs new file mode 100644 index 0000000..0d51aa0 --- /dev/null +++ b/Model/Config/CollectorConfig.cs @@ -0,0 +1,20 @@ +using System.Text.Json.Serialization; +using LootDumpProcessor.Process.Collector; +using Newtonsoft.Json; + +namespace LootDumpProcessor.Model.Config; + +public class CollectorConfig +{ + [JsonProperty("collectorType")] + [JsonPropertyName("collectorType")] + public CollectorType CollectorType { get; set; } + + [JsonProperty("maxEntitiesBeforeDumping")] + [JsonPropertyName("maxEntitiesBeforeDumping")] + public int MaxEntitiesBeforeDumping { get; set; } + + [JsonProperty("dumpLocation")] + [JsonPropertyName("dumpLocation")] + public string DumpLocation { get; set; } +} \ No newline at end of file diff --git a/Model/Config/Config.cs b/Model/Config/Config.cs index ef765fe..a38e459 100644 --- a/Model/Config/Config.cs +++ b/Model/Config/Config.cs @@ -49,6 +49,10 @@ public class Config [JsonProperty("writerConfig")] [JsonPropertyName("writerConfig")] public WriterConfig WriterConfig { get; set; } + + [JsonProperty("collectorConfig")] + [JsonPropertyName("collectorConfig")] + public CollectorConfig CollectorConfig { get; set; } [JsonProperty("containerIgnoreList")] [JsonPropertyName("containerIgnoreList")] diff --git a/Model/Processing/LooseLootCounts.cs b/Model/Processing/LooseLootCounts.cs index 99f604d..4cbe0af 100644 --- a/Model/Processing/LooseLootCounts.cs +++ b/Model/Processing/LooseLootCounts.cs @@ -1,5 +1,6 @@ using System.Text.Json.Serialization; using LootDumpProcessor.Storage; +using LootDumpProcessor.Utils; using Newtonsoft.Json; namespace LootDumpProcessor.Model.Processing; @@ -8,7 +9,7 @@ public class LooseLootCounts : IKeyable { [JsonProperty("__id__")] [JsonPropertyName("__id__")] - public string __ID { get; set; } = Guid.NewGuid().ToString(); + public string __ID { get; set; } = KeyGenerator.GetNextKey(); public IKey Counts { get; set; } diff --git a/Model/Template.cs b/Model/Template.cs index d3d55ba..0c62565 100644 --- a/Model/Template.cs +++ b/Model/Template.cs @@ -9,7 +9,7 @@ namespace LootDumpProcessor.Model { [Newtonsoft.Json.JsonIgnore] [System.Text.Json.Serialization.JsonIgnore] - public string __ID { get; } = Guid.NewGuid().ToString(); + public string __ID { get; } = KeyGenerator.GetNextKey(); [JsonProperty("Id", NullValueHandling = NullValueHandling.Ignore)] [JsonPropertyName("Id")] diff --git a/Process/Collector/CollectorFactory.cs b/Process/Collector/CollectorFactory.cs index 2aa20ae..8c2b1d9 100644 --- a/Process/Collector/CollectorFactory.cs +++ b/Process/Collector/CollectorFactory.cs @@ -2,9 +2,16 @@ public static class CollectorFactory { + private static ICollector? _collector; public static ICollector GetInstance() { - // TODO: implement real factory - return new HashSetCollector(); + if (_collector == null) + _collector = LootDumpProcessorContext.GetConfig().CollectorConfig.CollectorType switch + { + CollectorType.Memory => new HashSetCollector(), + CollectorType.Dump => new DumpCollector(), + _ => throw new ArgumentOutOfRangeException() + }; + return _collector; } } \ No newline at end of file diff --git a/Process/Collector/CollectorType.cs b/Process/Collector/CollectorType.cs new file mode 100644 index 0000000..bbbe6b0 --- /dev/null +++ b/Process/Collector/CollectorType.cs @@ -0,0 +1,7 @@ +namespace LootDumpProcessor.Process.Collector; + +public enum CollectorType +{ + Memory, + Dump +} \ No newline at end of file diff --git a/Process/Collector/DumpCollector.cs b/Process/Collector/DumpCollector.cs new file mode 100644 index 0000000..ab9a355 --- /dev/null +++ b/Process/Collector/DumpCollector.cs @@ -0,0 +1,45 @@ +using LootDumpProcessor.Model.Processing; +using LootDumpProcessor.Serializers.Json; + +namespace LootDumpProcessor.Process.Collector; + +public class DumpCollector : ICollector +{ + private static readonly string DumpLocation = $"{LootDumpProcessorContext.GetConfig().CollectorConfig.DumpLocation}/collector/"; + private readonly List processedDumps = new(LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping + 50); + private readonly object lockObject = new(); + + public void Setup() + { + if (Directory.Exists(DumpLocation)) + { + Directory.Delete(DumpLocation, true); + } + + Directory.CreateDirectory(DumpLocation); + } + + public void Hold(PartialData parsedDump) + { + lock (lockObject) + { + processedDumps.Add(parsedDump); + if (processedDumps.Count > LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping) + { + var fileName = $"collector-{DateTime.Now.ToString("yyyyMMddHHmmssfffff")}.json"; + File.WriteAllText($"{DumpLocation}{fileName}", JsonSerializerFactory.GetInstance().Serialize(processedDumps)); + processedDumps.Clear(); + } + } + } + + public List Retrieve() + { + foreach (var file in Directory.GetFiles(DumpLocation)) + { + processedDumps.AddRange(JsonSerializerFactory.GetInstance().Deserialize>(File.ReadAllText(file))); + } + + return processedDumps; + } +} \ No newline at end of file diff --git a/Process/Collector/HashSetCollector.cs b/Process/Collector/HashSetCollector.cs index 1739dc4..989ecf6 100644 --- a/Process/Collector/HashSetCollector.cs +++ b/Process/Collector/HashSetCollector.cs @@ -4,11 +4,9 @@ namespace LootDumpProcessor.Process.Collector; public class HashSetCollector : ICollector { - private readonly HashSet processedDumps = new(); - + private readonly HashSet processedDumps = new(100_000); private readonly object lockObject = new(); - public void Setup() { } diff --git a/Process/Processor/FileProcessor/FileProcessorFactory.cs b/Process/Processor/FileProcessor/FileProcessorFactory.cs index 1230e19..be961ed 100644 --- a/Process/Processor/FileProcessor/FileProcessorFactory.cs +++ b/Process/Processor/FileProcessor/FileProcessorFactory.cs @@ -2,7 +2,7 @@ public static class FileProcessorFactory { - private static IFileProcessor _fileProcessor; + private static IFileProcessor? _fileProcessor; public static IFileProcessor GetInstance() { // TODO: implement actual factory someday diff --git a/Storage/Collections/FlatKeyableDictionary.cs b/Storage/Collections/FlatKeyableDictionary.cs index 0561583..fbd1c07 100644 --- a/Storage/Collections/FlatKeyableDictionary.cs +++ b/Storage/Collections/FlatKeyableDictionary.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using LootDumpProcessor.Utils; using Newtonsoft.Json; namespace LootDumpProcessor.Storage.Collections; @@ -7,10 +8,10 @@ public class FlatKeyableDictionary : Dictionary, IKeyable { [JsonProperty("__id__")] [JsonPropertyName("__id__")] - public string __ID { get; set; } = Guid.NewGuid().ToString(); + public string __ID { get; set; } = KeyGenerator.GetNextKey(); public IKey GetKey() { - return new FlatUniqueKey(new[] { __ID }); + return new FlatUniqueKey([__ID]); } } \ No newline at end of file diff --git a/Storage/Collections/FlatKeyableList.cs b/Storage/Collections/FlatKeyableList.cs index 7942504..6ff56e2 100644 --- a/Storage/Collections/FlatKeyableList.cs +++ b/Storage/Collections/FlatKeyableList.cs @@ -1,11 +1,13 @@ -namespace LootDumpProcessor.Storage.Collections; +using LootDumpProcessor.Utils; + +namespace LootDumpProcessor.Storage.Collections; public class FlatKeyableList : List, IKeyable { - public string __ID { get; } = Guid.NewGuid().ToString(); + public string __ID { get; } = KeyGenerator.GetNextKey(); public IKey GetKey() { - return new FlatUniqueKey(new[] { __ID }); + return new FlatUniqueKey([__ID]); } } \ No newline at end of file diff --git a/Storage/Collections/SubdivisionedKeyableDictionary.cs b/Storage/Collections/SubdivisionedKeyableDictionary.cs index 35efc07..c5560dc 100644 --- a/Storage/Collections/SubdivisionedKeyableDictionary.cs +++ b/Storage/Collections/SubdivisionedKeyableDictionary.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using LootDumpProcessor.Utils; using Newtonsoft.Json; namespace LootDumpProcessor.Storage.Collections; @@ -7,7 +8,7 @@ public class SubdivisionedKeyableDictionary : Dictionary, IKeyable { [JsonProperty("__id__")] [JsonPropertyName("__id__")] - public string __ID { get; set; } = Guid.NewGuid().ToString(); + public string __ID { get; set; } = KeyGenerator.GetNextKey(); [JsonProperty("extras")] [JsonPropertyName("extras")] @@ -38,10 +39,8 @@ public class SubdivisionedKeyableDictionary : Dictionary, IKeyable subdivisions.Add(__ID); return new SubdivisionedUniqueKey(subdivisions.ToArray()); } - else - { - return new SubdivisionedUniqueKey(new[] { "dictionaries", __ID }); - } + + return new SubdivisionedUniqueKey(["dictionaries", __ID]); } public void AddExtraSubdivisions(string[] extras) diff --git a/Utils/KeyGenerator.cs b/Utils/KeyGenerator.cs new file mode 100644 index 0000000..e6af455 --- /dev/null +++ b/Utils/KeyGenerator.cs @@ -0,0 +1,15 @@ +namespace LootDumpProcessor.Utils; + +public class KeyGenerator +{ + private static long currentKey = 0L; + private static object lockObject = new(); + + public static string GetNextKey() + { + lock (lockObject) + { + return $"{++currentKey}"; + } + } +} \ No newline at end of file