diff --git a/source/LootDumpProcessor/Logger/ILogger.cs b/source/LootDumpProcessor/Logger/ILogger.cs deleted file mode 100644 index e19a74b..0000000 --- a/source/LootDumpProcessor/Logger/ILogger.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace LootDumpProcessor.Logger; - -public interface ILogger -{ - void Setup(); - void Log(string message, LogLevel level); - bool CanBeLogged(LogLevel level); - void Stop(); -} \ No newline at end of file diff --git a/source/LootDumpProcessor/Logger/LogLevel.cs b/source/LootDumpProcessor/Logger/LogLevel.cs deleted file mode 100644 index cb28877..0000000 --- a/source/LootDumpProcessor/Logger/LogLevel.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace LootDumpProcessor.Logger; - -public enum LogLevel -{ - Error, - Warning, - Info, - Debug -} \ No newline at end of file diff --git a/source/LootDumpProcessor/Logger/LoggerFactory.cs b/source/LootDumpProcessor/Logger/LoggerFactory.cs deleted file mode 100644 index 71b9650..0000000 --- a/source/LootDumpProcessor/Logger/LoggerFactory.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace LootDumpProcessor.Logger; - -public static class LoggerFactory -{ - private static ILogger? _logger; - - public static ILogger GetInstance() - { - if (_logger == null) - _logger = new QueueLogger(); - // TODO: implement factory - return _logger; - } -} \ No newline at end of file diff --git a/source/LootDumpProcessor/Logger/QueueLogger.cs b/source/LootDumpProcessor/Logger/QueueLogger.cs deleted file mode 100644 index 62fad18..0000000 --- a/source/LootDumpProcessor/Logger/QueueLogger.cs +++ /dev/null @@ -1,109 +0,0 @@ -using System.Collections.Concurrent; - -namespace LootDumpProcessor.Logger; - -public class QueueLogger : ILogger -{ - private readonly BlockingCollection queuedMessages = new(); - private Task? loggerThread; - private bool isRunning; - private int logLevel; - private const int LogTerminationTimeoutMs = 1000; - private const int LogTerminationRetryCount = 3; - - public void Setup() - { - SetLogLevel(); - isRunning = true; - loggerThread = Task.Factory.StartNew(() => - { - while (isRunning) - { - while (queuedMessages.TryTake(out var value)) - { - Console.ResetColor(); - switch (value.LogLevel) - { - case LogLevel.Error: - Console.BackgroundColor = ConsoleColor.Red; - Console.ForegroundColor = ConsoleColor.Black; - break; - case LogLevel.Warning: - Console.BackgroundColor = ConsoleColor.Yellow; - Console.ForegroundColor = ConsoleColor.Black; - break; - case LogLevel.Debug: - Console.ForegroundColor = ConsoleColor.DarkCyan; - break; - case LogLevel.Info: - default: - break; - } - - Console.WriteLine(value.Message); - } - - Thread.Sleep( - TimeSpan.FromMilliseconds(LootDumpProcessorContext.GetConfig().LoggerConfig.QueueLoggerPoolingTimeoutMs)); - } - }, TaskCreationOptions.LongRunning); - } - - private void SetLogLevel() - { - logLevel = GetLogLevel(LootDumpProcessorContext.GetConfig().LoggerConfig.LogLevel); - } - - private int GetLogLevel(LogLevel level) - { - return level switch - { - LogLevel.Error => 1, - LogLevel.Warning => 2, - LogLevel.Info => 3, - LogLevel.Debug => 4, - _ => throw new ArgumentOutOfRangeException() - }; - } - - public void Log(string message, LogLevel level) - { - if (GetLogLevel(level) <= logLevel) - queuedMessages.Add(new LoggedMessage { Message = message, LogLevel = level }); - } - - public bool CanBeLogged(LogLevel level) - { - return GetLogLevel(level) <= logLevel; - } - - // Wait for graceful termination of the logging thread - public void Stop() - { - isRunning = false; - if (loggerThread != null) - { - Console.ResetColor(); - var retryCount = 0; - while (!loggerThread.IsCompleted) - { - if (retryCount == LogTerminationRetryCount) - { - Console.WriteLine( - $"Logger thread did not terminate by itself after {retryCount} retries. Some log messages may be lost."); - break; - } - - Console.WriteLine($"Waiting {LogTerminationTimeoutMs}ms for logger termination"); - Thread.Sleep(LogTerminationTimeoutMs); - retryCount++; - } - } - } - - private class LoggedMessage - { - public string Message { get; init; } - public LogLevel LogLevel { get; init; } - } -} \ No newline at end of file diff --git a/source/LootDumpProcessor/LootDumpProcessorContext.cs b/source/LootDumpProcessor/LootDumpProcessorContext.cs index 1f2d404..147f39b 100644 --- a/source/LootDumpProcessor/LootDumpProcessorContext.cs +++ b/source/LootDumpProcessor/LootDumpProcessorContext.cs @@ -20,7 +20,7 @@ public static class LootDumpProcessorContext private static readonly object _forcedItemsLock = new(); private static Dictionary>? _forcedLoose; private static readonly object _forcedLooseLock = new(); - private static TarkovItems? _tarkovItems; + private static TarkovItemsProvider? _tarkovItems; private static readonly object _tarkovItemsLock = new(); public static Config GetConfig() @@ -54,25 +54,6 @@ public static class LootDumpProcessorContext return _forcedStatic; } - /// - /// Not Used - /// - /// - public static Dictionary GetDirectoryMappings() - { - lock (_mapDirectoryMappingsLock) - { - if (_mapDirectoryMappings == null) - { - _mapDirectoryMappings = YamlSerializerFactory.GetInstance() - .Deserialize>( - File.ReadAllText("./Config/map_directory_mapping.yaml")); - } - } - - return _mapDirectoryMappings; - } - public static HashSet GetStaticWeaponIds() { lock (_staticWeaponIdsLock) @@ -112,19 +93,4 @@ public static class LootDumpProcessorContext return _forcedLoose; } - - public static TarkovItems GetTarkovItems() - { - lock (_tarkovItemsLock) - { - if (_tarkovItems == null) - { - _tarkovItems = new TarkovItems( - $"{GetConfig().ServerLocation}/project/assets/database/templates/items.json" - ); - } - } - - return _tarkovItems; - } } \ No newline at end of file diff --git a/source/LootDumpProcessor/Model/ComposedKey.cs b/source/LootDumpProcessor/Model/ComposedKey.cs index cbe1d5b..eb4246b 100644 --- a/source/LootDumpProcessor/Model/ComposedKey.cs +++ b/source/LootDumpProcessor/Model/ComposedKey.cs @@ -1,35 +1,15 @@ using System.Text.Json.Serialization; -using LootDumpProcessor.Model.Processing; -using LootDumpProcessor.Utils; using Newtonsoft.Json; namespace LootDumpProcessor.Model; -public class ComposedKey +public class ComposedKey(string key, Item? firstItem) { - [JsonProperty("key")] - [JsonPropertyName("key")] - public string Key { get; init; } + [JsonProperty("key")] [JsonPropertyName("key")] public string Key { get; init; } = key; [Newtonsoft.Json.JsonIgnore] [System.Text.Json.Serialization.JsonIgnore] - public Item? FirstItem { get; } - - public ComposedKey(Template template) : this(template.Items) - { - } - - public ComposedKey(List? items) - { - Key = items?.Select(i => i.Tpl) - .Where(i => !string.IsNullOrEmpty(i) && - !LootDumpProcessorContext.GetTarkovItems().IsBaseClass(i, BaseClasses.Ammo)) - .Cast() - .Select(i => (double)i.GetHashCode()) - .Sum() - .ToString() ?? KeyGenerator.GetNextKey(); - FirstItem = items?[0]; - } + public Item? FirstItem { get; } = firstItem; public override bool Equals(object? obj) { diff --git a/source/LootDumpProcessor/Model/Config/Config.cs b/source/LootDumpProcessor/Model/Config/Config.cs index 1fb1345..bfd145f 100644 --- a/source/LootDumpProcessor/Model/Config/Config.cs +++ b/source/LootDumpProcessor/Model/Config/Config.cs @@ -8,7 +8,7 @@ public class Config { [JsonProperty("serverLocation")] [JsonPropertyName("serverLocation")] - public string? ServerLocation { get; set; } + public string ServerLocation { get; set; } = string.Empty; [JsonProperty("threads")] [JsonPropertyName("threads")] diff --git a/source/LootDumpProcessor/Model/Config/IntakeReaderConfig.cs b/source/LootDumpProcessor/Model/Config/IntakeReaderConfig.cs index f8fb7b0..5414fb4 100644 --- a/source/LootDumpProcessor/Model/Config/IntakeReaderConfig.cs +++ b/source/LootDumpProcessor/Model/Config/IntakeReaderConfig.cs @@ -6,10 +6,6 @@ namespace LootDumpProcessor.Model.Config; public class IntakeReaderConfig { - [JsonProperty("readerType")] - [JsonPropertyName("readerType")] - public IntakeReaderTypes IntakeReaderType { get; set; } = IntakeReaderTypes.Json; - [JsonProperty("maxDumpsPerMap")] [JsonPropertyName("maxDumpsPerMap")] public int MaxDumpsPerMap { get; set; } = 1500; diff --git a/source/LootDumpProcessor/Model/Config/LoggerConfig.cs b/source/LootDumpProcessor/Model/Config/LoggerConfig.cs index 01e1035..2344399 100644 --- a/source/LootDumpProcessor/Model/Config/LoggerConfig.cs +++ b/source/LootDumpProcessor/Model/Config/LoggerConfig.cs @@ -1,5 +1,5 @@ using System.Text.Json.Serialization; -using LootDumpProcessor.Logger; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; namespace LootDumpProcessor.Model.Config; @@ -8,7 +8,7 @@ public class LoggerConfig { [JsonProperty("logLevel")] [JsonPropertyName("logLevel")] - public LogLevel LogLevel { get; set; } = LogLevel.Info; + public LogLevel LogLevel { get; set; } [JsonProperty("queueLoggerPoolingTimeoutMs")] [JsonPropertyName("queueLoggerPoolingTimeoutMs")] diff --git a/source/LootDumpProcessor/Model/Config/PreProcessorConfig.cs b/source/LootDumpProcessor/Model/Config/PreProcessorConfig.cs index 1fd2bf3..d09c0e9 100644 --- a/source/LootDumpProcessor/Model/Config/PreProcessorConfig.cs +++ b/source/LootDumpProcessor/Model/Config/PreProcessorConfig.cs @@ -6,10 +6,6 @@ namespace LootDumpProcessor.Model.Config; public class PreProcessorConfig { - [JsonProperty("preProcessors")] - [JsonPropertyName("preProcessors")] - public List? PreProcessors { get; set; } - [JsonProperty("preProcessorTempFolder")] [JsonPropertyName("preProcessorTempFolder")] public string? PreProcessorTempFolder { get; set; } diff --git a/source/LootDumpProcessor/Model/Config/ReaderConfig.cs b/source/LootDumpProcessor/Model/Config/ReaderConfig.cs index 87f0a9e..f66d219 100644 --- a/source/LootDumpProcessor/Model/Config/ReaderConfig.cs +++ b/source/LootDumpProcessor/Model/Config/ReaderConfig.cs @@ -22,15 +22,7 @@ public class ReaderConfig [JsonPropertyName("thresholdDate")] public string? ThresholdDate { get; set; } - [JsonProperty("acceptedFileExtensions")] - [JsonPropertyName("acceptedFileExtensions")] - public List AcceptedFileExtensions { get; set; } = new(); - [JsonProperty("processSubFolders")] [JsonPropertyName("processSubFolders")] public bool ProcessSubFolders { get; set; } - - [JsonProperty("fileFilters")] - [JsonPropertyName("fileFilters")] - public List? FileFilters { get; set; } } \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/ComposedKeyGenerator.cs b/source/LootDumpProcessor/Process/ComposedKeyGenerator.cs new file mode 100644 index 0000000..968d672 --- /dev/null +++ b/source/LootDumpProcessor/Process/ComposedKeyGenerator.cs @@ -0,0 +1,24 @@ +using LootDumpProcessor.Model; +using LootDumpProcessor.Model.Processing; +using LootDumpProcessor.Utils; + +namespace LootDumpProcessor.Process; + +public class ComposedKeyGenerator(ITarkovItemsProvider tarkovItemsProvider) : IComposedKeyGenerator +{ + private readonly ITarkovItemsProvider _tarkovItemsProvider = + tarkovItemsProvider ?? throw new ArgumentNullException(nameof(tarkovItemsProvider)); + + public ComposedKey Generate(IEnumerable items) + { + var key = items?.Select(i => i.Tpl) + .Where(i => !string.IsNullOrEmpty(i) && + !_tarkovItemsProvider.IsBaseClass(i, BaseClasses.Ammo)) + .Cast() + .Select(i => (double)i.GetHashCode()) + .Sum() + .ToString() ?? KeyGenerator.GetNextKey(); + var firstItem = items?.FirstOrDefault(); + return new ComposedKey(key, firstItem); + } +} \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/IComposedKeyGenerator.cs b/source/LootDumpProcessor/Process/IComposedKeyGenerator.cs new file mode 100644 index 0000000..ccde32a --- /dev/null +++ b/source/LootDumpProcessor/Process/IComposedKeyGenerator.cs @@ -0,0 +1,8 @@ +using LootDumpProcessor.Model; + +namespace LootDumpProcessor.Process; + +public interface IComposedKeyGenerator +{ + ComposedKey Generate(IEnumerable items); +} \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/IPipeline.cs b/source/LootDumpProcessor/Process/IPipeline.cs index 70b796c..ec83152 100644 --- a/source/LootDumpProcessor/Process/IPipeline.cs +++ b/source/LootDumpProcessor/Process/IPipeline.cs @@ -2,5 +2,5 @@ namespace LootDumpProcessor.Process; public interface IPipeline { - Task DoProcess(); + Task Execute(); } \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/ITarkovItemsProvider.cs b/source/LootDumpProcessor/Process/ITarkovItemsProvider.cs new file mode 100644 index 0000000..ca0c9fc --- /dev/null +++ b/source/LootDumpProcessor/Process/ITarkovItemsProvider.cs @@ -0,0 +1,9 @@ +namespace LootDumpProcessor.Process; + +public interface ITarkovItemsProvider +{ + bool IsBaseClass(string tpl, string baseclassId); + bool IsQuestItem(string tpl); + string? MaxDurability(string tpl); + string? AmmoCaliber(string tpl); +} \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs b/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs index 05bd92c..ce0262d 100644 --- a/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs +++ b/source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs @@ -1,11 +1,13 @@ using System.Collections.Concurrent; -using LootDumpProcessor.Logger; +using LootDumpProcessor; using LootDumpProcessor.Model; using LootDumpProcessor.Model.Input; using LootDumpProcessor.Model.Output; using LootDumpProcessor.Model.Output.LooseLoot; using LootDumpProcessor.Model.Output.StaticContainer; using LootDumpProcessor.Model.Processing; +using LootDumpProcessor.Process; +using LootDumpProcessor.Process.Processor.DumpProcessor; using LootDumpProcessor.Process.Processor.v2.AmmoProcessor; using LootDumpProcessor.Process.Processor.v2.LooseLootProcessor; using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor; @@ -14,52 +16,39 @@ using LootDumpProcessor.Serializers.Json; using LootDumpProcessor.Storage; using LootDumpProcessor.Storage.Collections; using LootDumpProcessor.Utils; - -namespace LootDumpProcessor.Process.Processor.DumpProcessor; +using Microsoft.Extensions.Logging; public class MultithreadSteppedDumpProcessor( - IStaticLootProcessor staticLootProcessor, IStaticContainersProcessor staticContainersProcessor, - IAmmoProcessor ammoProcessor, ILooseLootProcessor looseLootProcessor -) : IDumpProcessor + IStaticLootProcessor staticLootProcessor, + IStaticContainersProcessor staticContainersProcessor, + IAmmoProcessor ammoProcessor, + ILooseLootProcessor looseLootProcessor, + ILogger logger +) + : IDumpProcessor { - private readonly IStaticLootProcessor _staticLootProcessor = - staticLootProcessor ?? throw new ArgumentNullException(nameof(staticLootProcessor)); - - private readonly IStaticContainersProcessor _staticContainersProcessor = - staticContainersProcessor ?? throw new ArgumentNullException(nameof(staticContainersProcessor)); - - private readonly IAmmoProcessor _ammoProcessor = - ammoProcessor ?? throw new ArgumentNullException(nameof(ammoProcessor)); - - private readonly ILooseLootProcessor _looseLootProcessor = - looseLootProcessor ?? throw new ArgumentNullException(nameof(looseLootProcessor)); + private readonly IStaticLootProcessor _staticLootProcessor = staticLootProcessor ?? throw new ArgumentNullException(nameof(staticLootProcessor)); + private readonly IStaticContainersProcessor _staticContainersProcessor = staticContainersProcessor ?? throw new ArgumentNullException(nameof(staticContainersProcessor)); + private readonly IAmmoProcessor _ammoProcessor = ammoProcessor ?? throw new ArgumentNullException(nameof(ammoProcessor)); + private readonly ILooseLootProcessor _looseLootProcessor = looseLootProcessor ?? throw new ArgumentNullException(nameof(looseLootProcessor)); + private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger)); private static readonly IJsonSerializer _jsonSerializer = JsonSerializerFactory.GetInstance(); - - // if we need to, this variable can be moved to use the factory, but since the factory - // needs a locking mechanism to prevent dictionary access exceptions, its better to keep - // a reference to use here private static readonly IDataStorage _dataStorage = DataStorageFactory.GetInstance(); public Dictionary ProcessDumps(List dumps) { - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log("Starting final dump processing", LogLevel.Info); + _logger.LogInformation("Starting final dump processing"); var output = new Dictionary(); var dumpProcessData = GetDumpProcessData(dumps); - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log("Heavy processing done!", LogLevel.Info); + _logger.LogInformation("Heavy processing done!"); var staticContainers = new ConcurrentDictionary(); - // We need to count how many dumps we have for each map var mapDumpCounter = new ConcurrentDictionary(); - // dictionary of maps, that has a dictionary of template and hit count var mapStaticContainersAggregated = new ConcurrentDictionary>(); - // BSG changed the map data so static containers are now dynamic, so we need to scan all dumps for the static containers. - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log("Queuing dumps for static data processing", LogLevel.Info); + _logger.LogInformation("Queuing dumps for static data processing"); var parallelOptions = new ParallelOptions { @@ -69,26 +58,23 @@ public class MultithreadSteppedDumpProcessor( async (partialData, cancellationToken) => await Process(partialData, staticContainers, mapStaticContainersAggregated, mapDumpCounter)); - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log("All static data processing threads finished", LogLevel.Info); - // Aggregate and calculate the probability of a static container + _logger.LogInformation("All static data processing threads finished"); + mapStaticContainersAggregated.ToDictionary( kv => kv.Key, kv => kv.Value.Select( td => new StaticDataPoint { Template = td.Key, - Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) // kv.Key = map name + Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) } ).ToList() ).ToList() .ForEach(kv => - staticContainers[kv.Key].StaticContainers = kv.Value); // Hydrate staticContainers.StaticContainers + staticContainers[kv.Key].StaticContainers = kv.Value); - // Static containers output.Add(OutputFileType.StaticContainer, staticContainers); - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log("Processing ammo distribution", LogLevel.Info); + _logger.LogInformation("Processing ammo distribution"); var staticAmmo = new ConcurrentDictionary>>(); Parallel.ForEach(dumpProcessData.ContainerCounts.Keys, parallelOptions: parallelOptions, mapId => @@ -97,14 +83,9 @@ public class MultithreadSteppedDumpProcessor( var ammoDistribution = _ammoProcessor.CreateAmmoDistribution(mapId, preProcessedStaticLoots); staticAmmo[mapId] = ammoDistribution; }); - // Ammo distribution - output.Add( - OutputFileType.StaticAmmo, - staticAmmo - ); + output.Add(OutputFileType.StaticAmmo, staticAmmo); - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log("Processing static loot distribution", LogLevel.Info); + _logger.LogInformation("Processing static loot distribution"); var staticLoot = new ConcurrentDictionary>(); Parallel.ForEach(dumpProcessData.ContainerCounts.Keys, parallelOptions: parallelOptions, mapId => @@ -114,16 +95,10 @@ public class MultithreadSteppedDumpProcessor( _staticLootProcessor.CreateStaticLootDistribution(mapId, preProcessedStaticLoots); staticLoot[mapId] = staticLootDistribution; }); - // Static loot distribution - output.Add( - OutputFileType.StaticLoot, - staticLoot - ); + output.Add(OutputFileType.StaticLoot, staticLoot); - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log("Processing loose loot distribution", LogLevel.Info); + _logger.LogInformation("Processing loose loot distribution"); - // Loose loot distribution var looseLoot = new ConcurrentDictionary(); Parallel.ForEach(dumpProcessData.MapCounts.Keys, parallelOptions: parallelOptions, mapId => { @@ -133,15 +108,13 @@ public class MultithreadSteppedDumpProcessor( _looseLootProcessor.CreateLooseLootDistribution(mapId, mapCount, looseLootCount); looseLoot[mapId] = looseLootDistribution; }); - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log("Collecting loose loot distribution information", LogLevel.Info); + _logger.LogInformation("Collecting loose loot distribution information"); var loot = dumpProcessData.MapCounts .Select(mapCount => mapCount.Key) .ToDictionary(mi => mi, mi => looseLoot[mi]); output.Add(OutputFileType.LooseLoot, loot); - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log("Dump processing fully completed!", LogLevel.Info); + _logger.LogInformation("Dump processing fully completed!"); return output; } @@ -150,32 +123,22 @@ public class MultithreadSteppedDumpProcessor( ConcurrentDictionary> mapStaticContainersAggregated, ConcurrentDictionary mapDumpCounter) { - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug)) - LoggerFactory.GetInstance().Log($"Processing static data for file {partialData.BasicInfo.FileName}", - LogLevel.Debug); + _logger.LogDebug("Processing static data for file {FileName}", partialData.BasicInfo.FileName); var fileContent = await File.ReadAllTextAsync(partialData.BasicInfo.FileName); var dataDump = _jsonSerializer.Deserialize(fileContent); if (dataDump == null) { - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error)) - LoggerFactory.GetInstance() - .Log($"Failed to deserialize data from file {partialData.BasicInfo.FileName}", - LogLevel.Error); + _logger.LogError("Failed to deserialize data from file {FileName}", partialData.BasicInfo.FileName); return; } var mapId = dataDump.Data.LocationLoot.Id.ToLower(); - // Because we may use multiple version dumps for map data, merge the static loot between dumps - if (!staticContainers.TryGetValue(mapId, out var mapStaticLoot)) { - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance() - .Log($"Doing first time process for map {mapId} of real static data", - LogLevel.Info); + _logger.LogInformation("Doing first time process for map {MapId} of real static data", mapId); staticContainers[mapId] = new MapStaticLoot { @@ -197,29 +160,19 @@ public class MultithreadSteppedDumpProcessor( mapStaticLoot.StaticForced.AddRange(newStaticForced); } - - // Takes care of finding how many "dynamic static containers" we have on the map - ConcurrentDictionary mapAggregatedDataDict; - - // Init dict if map key doesnt exist - if (!mapStaticContainersAggregated.TryGetValue(mapId, out mapAggregatedDataDict)) + if (!mapStaticContainersAggregated.TryGetValue(mapId, out ConcurrentDictionary mapAggregatedDataDict)) { mapAggregatedDataDict = new ConcurrentDictionary(); mapStaticContainersAggregated.TryAdd(mapId, mapAggregatedDataDict); } - - // Only process the dump file if the date is higher (after) the configuration date if (!DumpWasMadeAfterConfigThresholdDate(partialData)) { return; } - // Keep track of how many dumps we have for each map - IncrementMapCounterDictionaryValue(mapDumpCounter, mapId); - var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList .TryGetValue(mapId, out var ignoreListForMap); foreach (var dynamicStaticContainer in _staticContainersProcessor.CreateDynamicStaticContainers( @@ -227,11 +180,9 @@ public class MultithreadSteppedDumpProcessor( { if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id)) { - // Skip adding containers to aggregated data if container id is in ignore list continue; } - // Increment times container seen in dump by 1 if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1)) { mapAggregatedDataDict[dynamicStaticContainer] += 1; @@ -253,7 +204,6 @@ public class MultithreadSteppedDumpProcessor( { if (!mapDumpCounter.TryAdd(mapName, 1)) { - // Dict has map, increment count by 1 mapDumpCounter[mapName] += 1; } } @@ -264,7 +214,7 @@ public class MultithreadSteppedDumpProcessor( return Math.Round((double)((decimal)td.Value / (decimal)mapDumpCounter[mapName]), 2); } - private static DumpProcessData GetDumpProcessData(List dumps) + private DumpProcessData GetDumpProcessData(List dumps) { var dumpProcessData = new DumpProcessData(); @@ -274,15 +224,10 @@ public class MultithreadSteppedDumpProcessor( { var mapName = tuple.Key; var partialFileMetaData = tuple.ToList(); - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log( - $"Processing map {mapName}, total dump data to process: {partialFileMetaData.Count}", - LogLevel.Info - ); + _logger.LogInformation("Processing map {MapName}, total dump data to process: {Count}", mapName, partialFileMetaData.Count); dumpProcessData.MapCounts[mapName] = partialFileMetaData.Count; var lockObjectContainerCounts = new object(); - var lockObjectCounts = new object(); var looseLootCounts = new LooseLootCounts(); @@ -290,11 +235,6 @@ public class MultithreadSteppedDumpProcessor( var dictionaryCounts = new FlatKeyableDictionary(); looseLootCounts.Counts = dictionaryCounts.GetKey(); - /* - var dictionaryItemCounts = new FlatKeyableDictionary>(); - counts.Items = dictionaryItemCounts.GetKey(); - */ - var lockObjectDictionaryItemProperties = new object(); var dictionaryItemProperties = new FlatKeyableDictionary>(); @@ -305,13 +245,11 @@ public class MultithreadSteppedDumpProcessor( BlockingCollection _partialDataToProcess = new(); - // add the items to the queue foreach (var partialData in partialFileMetaData) { _partialDataToProcess.Add(partialData); } - // Call GC before running threads partialFileMetaData = null; tuple = null; GCHandler.Collect(); @@ -337,11 +275,7 @@ public class MultithreadSteppedDumpProcessor( _dataStorage.Store(dictionaryCounts); dictionaryCounts = null; GCHandler.Collect(); - /* - DataStorageFactory.GetInstance().Store(dictionaryItemCounts); - dictionaryItemCounts = null; - GC.Collect(); - */ + _dataStorage.Store(actualDictionaryItemProperties); actualDictionaryItemProperties = null; GCHandler.Collect(); @@ -352,7 +286,7 @@ public class MultithreadSteppedDumpProcessor( return dumpProcessData; } - private static void ProcessPartialData(PartialData partialDataToProcess, + private void ProcessPartialData(PartialData partialDataToProcess, object lockObjectContainerCounts, DumpProcessData dumpProcessData, string mapName, object lockObjectDictionaryCounts, FlatKeyableDictionary? dictionaryCounts, object lockObjectDictionaryItemProperties, @@ -364,7 +298,6 @@ public class MultithreadSteppedDumpProcessor( { var dumpData = _dataStorage.GetItem(partialDataToProcess.ParsedDumpKey); - // Static containers lock (lockObjectContainerCounts) { if (!dumpProcessData.ContainerCounts.ContainsKey(mapName)) @@ -378,7 +311,6 @@ public class MultithreadSteppedDumpProcessor( } } - // Loose loot into ids on files var loadedDictionary = _dataStorage .GetItem>>( @@ -416,11 +348,7 @@ public class MultithreadSteppedDumpProcessor( } catch (Exception e) { - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error)) - LoggerFactory.GetInstance().Log( - $"ERROR OCCURRED:{e.Message}\n{e.StackTrace}", - LogLevel.Error - ); + _logger.LogError("ERROR OCCURRED: {Message}\n{StackTrace}", e.Message, e.StackTrace); } } } \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessor.cs b/source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessor.cs index 76bfb3f..7952c6c 100644 --- a/source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessor.cs +++ b/source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessor.cs @@ -1,35 +1,41 @@ -using LootDumpProcessor.Logger; using LootDumpProcessor.Model; using LootDumpProcessor.Model.Processing; +using LootDumpProcessor.Process.Processor.FileProcessor; using LootDumpProcessor.Process.Processor.v2.LooseLootProcessor; using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor; using LootDumpProcessor.Storage; +using Microsoft.Extensions.Logging; -namespace LootDumpProcessor.Process.Processor.FileProcessor; - -public class FileProcessor(IStaticLootProcessor staticLootProcessor, ILooseLootProcessor looseLootProcessor) - : IFileProcessor +public class FileProcessor : IFileProcessor { - private readonly IStaticLootProcessor _staticLootProcessor = - staticLootProcessor ?? throw new ArgumentNullException(nameof(staticLootProcessor)); + private readonly IStaticLootProcessor _staticLootProcessor; + private readonly ILooseLootProcessor _looseLootProcessor; + private readonly ILogger _logger; - private readonly ILooseLootProcessor _looseLootProcessor = - looseLootProcessor ?? throw new ArgumentNullException(nameof(looseLootProcessor)); + public FileProcessor( + IStaticLootProcessor staticLootProcessor, + ILooseLootProcessor looseLootProcessor, + ILogger logger) + { + _staticLootProcessor = staticLootProcessor + ?? throw new ArgumentNullException(nameof(staticLootProcessor)); + _looseLootProcessor = looseLootProcessor + ?? throw new ArgumentNullException(nameof(looseLootProcessor)); + _logger = logger + ?? throw new ArgumentNullException(nameof(logger)); + } public PartialData Process(BasicInfo parsedData) { - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug)) - LoggerFactory.GetInstance().Log($"Processing file {parsedData.FileName}...", LogLevel.Debug); + _logger.LogDebug("Processing file {FileName}...", parsedData.FileName); - List