spt/loot-dump-processor
spt/loot-dump-processor
0
0
Fork 0
mirror of https://github.com/sp-tarkov/loot-dump-processor.git synced 2025-02-13 02:10:46 -05:00
Code

Refactored dependency injection and logging infrastructure

Browse Source 
The changes include:
- Replaced custom logging with Microsoft.Extensions.Logging
- Added TarkovItemsProvider and ComposedKeyGenerator services
- Simplified configuration models by removing redundant options
- Improved dependency injection in processors and readers
- Removed unused factory methods and simplified service registration
This commit is contained in:
bluextx 2025-01-11 09:12:21 +03:00
parent 8c0d46585a
commit 047372b6dc
32 changed files with 404 additions and 703 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
source/LootDumpProcessor/Logger/ILogger.cs
View File

@ -1,9 +0,0 @@
namespace LootDumpProcessor.Logger;
public interface ILogger
{
void Setup();
void Log(string message, LogLevel level);
bool CanBeLogged(LogLevel level);
void Stop();
}

9
source/LootDumpProcessor/Logger/LogLevel.cs
View File

@ -1,9 +0,0 @@
namespace LootDumpProcessor.Logger;
public enum LogLevel
{
Error,
Warning,
Info,
Debug
}

14
source/LootDumpProcessor/Logger/LoggerFactory.cs
View File

@ -1,14 +0,0 @@
namespace LootDumpProcessor.Logger;
public static class LoggerFactory
{
private static ILogger? _logger;
public static ILogger GetInstance()
{
if (_logger == null)
_logger = new QueueLogger();
// TODO: implement factory
return _logger;
}
}

109
source/LootDumpProcessor/Logger/QueueLogger.cs
View File

@ -1,109 +0,0 @@
using System.Collections.Concurrent;
namespace LootDumpProcessor.Logger;
public class QueueLogger : ILogger
{
private readonly BlockingCollection<LoggedMessage> queuedMessages = new();
private Task? loggerThread;
private bool isRunning;
private int logLevel;
private const int LogTerminationTimeoutMs = 1000;
private const int LogTerminationRetryCount = 3;
public void Setup()
{
SetLogLevel();
isRunning = true;
loggerThread = Task.Factory.StartNew(() =>
{
while (isRunning)
{
while (queuedMessages.TryTake(out var value))
{
Console.ResetColor();
switch (value.LogLevel)
{
case LogLevel.Error:
Console.BackgroundColor = ConsoleColor.Red;
Console.ForegroundColor = ConsoleColor.Black;
break;
case LogLevel.Warning:
Console.BackgroundColor = ConsoleColor.Yellow;
Console.ForegroundColor = ConsoleColor.Black;
break;
case LogLevel.Debug:
Console.ForegroundColor = ConsoleColor.DarkCyan;
break;
case LogLevel.Info:
default:
break;
}
Console.WriteLine(value.Message);
}
Thread.Sleep(
TimeSpan.FromMilliseconds(LootDumpProcessorContext.GetConfig().LoggerConfig.QueueLoggerPoolingTimeoutMs));
}
}, TaskCreationOptions.LongRunning);
}
private void SetLogLevel()
{
logLevel = GetLogLevel(LootDumpProcessorContext.GetConfig().LoggerConfig.LogLevel);
}
private int GetLogLevel(LogLevel level)
{
return level switch
{
LogLevel.Error => 1,
LogLevel.Warning => 2,
LogLevel.Info => 3,
LogLevel.Debug => 4,
_ => throw new ArgumentOutOfRangeException()
};
}
public void Log(string message, LogLevel level)
{
if (GetLogLevel(level) <= logLevel)
queuedMessages.Add(new LoggedMessage { Message = message, LogLevel = level });
}
public bool CanBeLogged(LogLevel level)
{
return GetLogLevel(level) <= logLevel;
}
// Wait for graceful termination of the logging thread
public void Stop()
{
isRunning = false;
if (loggerThread != null)
{
Console.ResetColor();
var retryCount = 0;
while (!loggerThread.IsCompleted)
{
if (retryCount == LogTerminationRetryCount)
{
Console.WriteLine(
$"Logger thread did not terminate by itself after {retryCount} retries. Some log messages may be lost.");
break;
}
Console.WriteLine($"Waiting {LogTerminationTimeoutMs}ms for logger termination");
Thread.Sleep(LogTerminationTimeoutMs);
retryCount++;
}
}
}
private class LoggedMessage
{
public string Message { get; init; }
public LogLevel LogLevel { get; init; }
}
}

36
source/LootDumpProcessor/LootDumpProcessorContext.cs
View File

@ -20,7 +20,7 @@ public static class LootDumpProcessorContext
private static readonly object _forcedItemsLock = new();
private static Dictionary<string, HashSet<string>>? _forcedLoose;
private static readonly object _forcedLooseLock = new();
private static TarkovItems? _tarkovItems;
private static TarkovItemsProvider? _tarkovItems;
private static readonly object _tarkovItemsLock = new();
public static Config GetConfig()
@ -54,25 +54,6 @@ public static class LootDumpProcessorContext
return _forcedStatic;
}
/// <summary>
/// Not Used
/// </summary>
/// <returns></returns>
public static Dictionary<string, MapDirectoryMapping> GetDirectoryMappings()
{
lock (_mapDirectoryMappingsLock)
{
if (_mapDirectoryMappings == null)
{
_mapDirectoryMappings = YamlSerializerFactory.GetInstance()
.Deserialize<Dictionary<string, MapDirectoryMapping>>(
File.ReadAllText("./Config/map_directory_mapping.yaml"));
}
}
return _mapDirectoryMappings;
}
public static HashSet<string> GetStaticWeaponIds()
{
lock (_staticWeaponIdsLock)
@ -112,19 +93,4 @@ public static class LootDumpProcessorContext
return _forcedLoose;
}
public static TarkovItems GetTarkovItems()
{
lock (_tarkovItemsLock)
{
if (_tarkovItems == null)
{
_tarkovItems = new TarkovItems(
$"{GetConfig().ServerLocation}/project/assets/database/templates/items.json"
);
}
}
return _tarkovItems;
}
}

26
source/LootDumpProcessor/Model/ComposedKey.cs
View File

@ -1,35 +1,15 @@
using System.Text.Json.Serialization;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Utils;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model;
public class ComposedKey
public class ComposedKey(string key, Item? firstItem)
{
[JsonProperty("key")]
[JsonPropertyName("key")]
public string Key { get; init; }
[JsonProperty("key")] [JsonPropertyName("key")] public string Key { get; init; } = key;
[Newtonsoft.Json.JsonIgnore]
[System.Text.Json.Serialization.JsonIgnore]
public Item? FirstItem { get; }
public ComposedKey(Template template) : this(template.Items)
{
}
public ComposedKey(List<Item>? items)
{
Key = items?.Select(i => i.Tpl)
.Where(i => !string.IsNullOrEmpty(i) &&
!LootDumpProcessorContext.GetTarkovItems().IsBaseClass(i, BaseClasses.Ammo))
.Cast<string>()
.Select(i => (double)i.GetHashCode())
.Sum()
.ToString() ?? KeyGenerator.GetNextKey();
FirstItem = items?[0];
}
public Item? FirstItem { get; } = firstItem;
public override bool Equals(object? obj)
{

2
source/LootDumpProcessor/Model/Config/Config.cs
View File

@ -8,7 +8,7 @@ public class Config
{
[JsonProperty("serverLocation")]
[JsonPropertyName("serverLocation")]
public string? ServerLocation { get; set; }
public string ServerLocation { get; set; } = string.Empty;
[JsonProperty("threads")]
[JsonPropertyName("threads")]

4
source/LootDumpProcessor/Model/Config/IntakeReaderConfig.cs
View File

@ -6,10 +6,6 @@ namespace LootDumpProcessor.Model.Config;
public class IntakeReaderConfig
{
[JsonProperty("readerType")]
[JsonPropertyName("readerType")]
public IntakeReaderTypes IntakeReaderType { get; set; } = IntakeReaderTypes.Json;
[JsonProperty("maxDumpsPerMap")]
[JsonPropertyName("maxDumpsPerMap")]
public int MaxDumpsPerMap { get; set; } = 1500;

4
source/LootDumpProcessor/Model/Config/LoggerConfig.cs
View File

@ -1,5 +1,5 @@
using System.Text.Json.Serialization;
using LootDumpProcessor.Logger;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Config;
@ -8,7 +8,7 @@ public class LoggerConfig
{
[JsonProperty("logLevel")]
[JsonPropertyName("logLevel")]
public LogLevel LogLevel { get; set; } = LogLevel.Info;
public LogLevel LogLevel { get; set; }
[JsonProperty("queueLoggerPoolingTimeoutMs")]
[JsonPropertyName("queueLoggerPoolingTimeoutMs")]

4
source/LootDumpProcessor/Model/Config/PreProcessorConfig.cs
View File

@ -6,10 +6,6 @@ namespace LootDumpProcessor.Model.Config;
public class PreProcessorConfig
{
[JsonProperty("preProcessors")]
[JsonPropertyName("preProcessors")]
public List<PreProcessReaderTypes>? PreProcessors { get; set; }
[JsonProperty("preProcessorTempFolder")]
[JsonPropertyName("preProcessorTempFolder")]
public string? PreProcessorTempFolder { get; set; }

8
source/LootDumpProcessor/Model/Config/ReaderConfig.cs
View File

@ -22,15 +22,7 @@ public class ReaderConfig
[JsonPropertyName("thresholdDate")]
public string? ThresholdDate { get; set; }
[JsonProperty("acceptedFileExtensions")]
[JsonPropertyName("acceptedFileExtensions")]
public List<string> AcceptedFileExtensions { get; set; } = new();
[JsonProperty("processSubFolders")]
[JsonPropertyName("processSubFolders")]
public bool ProcessSubFolders { get; set; }
[JsonProperty("fileFilters")]
[JsonPropertyName("fileFilters")]
public List<FileFilterTypes>? FileFilters { get; set; }
}

24
source/LootDumpProcessor/Process/ComposedKeyGenerator.cs Normal file
View File

@ -0,0 +1,24 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Utils;
namespace LootDumpProcessor.Process;
public class ComposedKeyGenerator(ITarkovItemsProvider tarkovItemsProvider) : IComposedKeyGenerator
{
private readonly ITarkovItemsProvider _tarkovItemsProvider =
tarkovItemsProvider ?? throw new ArgumentNullException(nameof(tarkovItemsProvider));
public ComposedKey Generate(IEnumerable<Item> items)
{
var key = items?.Select(i => i.Tpl)
.Where(i => !string.IsNullOrEmpty(i) &&
!_tarkovItemsProvider.IsBaseClass(i, BaseClasses.Ammo))
.Cast<string>()
.Select(i => (double)i.GetHashCode())
.Sum()
.ToString() ?? KeyGenerator.GetNextKey();
var firstItem = items?.FirstOrDefault();
return new ComposedKey(key, firstItem);
}
}

8
source/LootDumpProcessor/Process/IComposedKeyGenerator.cs Normal file
View File

@ -0,0 +1,8 @@
using LootDumpProcessor.Model;
namespace LootDumpProcessor.Process;
public interface IComposedKeyGenerator
{
ComposedKey Generate(IEnumerable<Item> items);
}

2
source/LootDumpProcessor/Process/IPipeline.cs
View File

@ -2,5 +2,5 @@ namespace LootDumpProcessor.Process;
public interface IPipeline
{
Task DoProcess();
Task Execute();
}

9
source/LootDumpProcessor/Process/ITarkovItemsProvider.cs Normal file
View File

@ -0,0 +1,9 @@
namespace LootDumpProcessor.Process;
public interface ITarkovItemsProvider
{
bool IsBaseClass(string tpl, string baseclassId);
bool IsQuestItem(string tpl);
string? MaxDurability(string tpl);
string? AmmoCaliber(string tpl);
}

150
source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
View File

@ -1,11 +1,13 @@
using System.Collections.Concurrent;
using LootDumpProcessor.Logger;
using LootDumpProcessor;
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process;
using LootDumpProcessor.Process.Processor.DumpProcessor;
using LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
using LootDumpProcessor.Process.Processor.v2.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
@ -14,52 +16,39 @@ using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage.Collections;
using LootDumpProcessor.Utils;
namespace LootDumpProcessor.Process.Processor.DumpProcessor;
using Microsoft.Extensions.Logging;
public class MultithreadSteppedDumpProcessor(
IStaticLootProcessor staticLootProcessor, IStaticContainersProcessor staticContainersProcessor,
IAmmoProcessor ammoProcessor, ILooseLootProcessor looseLootProcessor
) : IDumpProcessor
IStaticLootProcessor staticLootProcessor,
IStaticContainersProcessor staticContainersProcessor,
IAmmoProcessor ammoProcessor,
ILooseLootProcessor looseLootProcessor,
ILogger<MultithreadSteppedDumpProcessor> logger
)
: IDumpProcessor
{
private readonly IStaticLootProcessor _staticLootProcessor =
staticLootProcessor ?? throw new ArgumentNullException(nameof(staticLootProcessor));
private readonly IStaticContainersProcessor _staticContainersProcessor =
staticContainersProcessor ?? throw new ArgumentNullException(nameof(staticContainersProcessor));
private readonly IAmmoProcessor _ammoProcessor =
ammoProcessor ?? throw new ArgumentNullException(nameof(ammoProcessor));
private readonly ILooseLootProcessor _looseLootProcessor =
looseLootProcessor ?? throw new ArgumentNullException(nameof(looseLootProcessor));
private readonly IStaticLootProcessor _staticLootProcessor = staticLootProcessor ?? throw new ArgumentNullException(nameof(staticLootProcessor));
private readonly IStaticContainersProcessor _staticContainersProcessor = staticContainersProcessor ?? throw new ArgumentNullException(nameof(staticContainersProcessor));
private readonly IAmmoProcessor _ammoProcessor = ammoProcessor ?? throw new ArgumentNullException(nameof(ammoProcessor));
private readonly ILooseLootProcessor _looseLootProcessor = looseLootProcessor ?? throw new ArgumentNullException(nameof(looseLootProcessor));
private readonly ILogger<MultithreadSteppedDumpProcessor> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
private static readonly IJsonSerializer _jsonSerializer = JsonSerializerFactory.GetInstance();
// if we need to, this variable can be moved to use the factory, but since the factory
// needs a locking mechanism to prevent dictionary access exceptions, its better to keep
// a reference to use here
private static readonly IDataStorage _dataStorage = DataStorageFactory.GetInstance();
public Dictionary<OutputFileType, object> ProcessDumps(List<PartialData> dumps)
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Starting final dump processing", LogLevel.Info);
_logger.LogInformation("Starting final dump processing");
var output = new Dictionary<OutputFileType, object>();
var dumpProcessData = GetDumpProcessData(dumps);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Heavy processing done!", LogLevel.Info);
_logger.LogInformation("Heavy processing done!");
var staticContainers = new ConcurrentDictionary<string, MapStaticLoot>();
// We need to count how many dumps we have for each map
var mapDumpCounter = new ConcurrentDictionary<string, int>();
// dictionary of maps, that has a dictionary of template and hit count
var mapStaticContainersAggregated = new ConcurrentDictionary<string, ConcurrentDictionary<Template, int>>();
// BSG changed the map data so static containers are now dynamic, so we need to scan all dumps for the static containers.
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Queuing dumps for static data processing", LogLevel.Info);
_logger.LogInformation("Queuing dumps for static data processing");
var parallelOptions = new ParallelOptions
{
@ -69,26 +58,23 @@ public class MultithreadSteppedDumpProcessor(
async (partialData, cancellationToken) =>
await Process(partialData, staticContainers, mapStaticContainersAggregated, mapDumpCounter));
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("All static data processing threads finished", LogLevel.Info);
// Aggregate and calculate the probability of a static container
_logger.LogInformation("All static data processing threads finished");
mapStaticContainersAggregated.ToDictionary(
kv => kv.Key,
kv => kv.Value.Select(
td => new StaticDataPoint
{
Template = td.Key,
Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) // kv.Key = map name
Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter)
}
).ToList()
).ToList()
.ForEach(kv =>
staticContainers[kv.Key].StaticContainers = kv.Value); // Hydrate staticContainers.StaticContainers
staticContainers[kv.Key].StaticContainers = kv.Value);
// Static containers
output.Add(OutputFileType.StaticContainer, staticContainers);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Processing ammo distribution", LogLevel.Info);
_logger.LogInformation("Processing ammo distribution");
var staticAmmo = new ConcurrentDictionary<string, IReadOnlyDictionary<string, List<AmmoDistribution>>>();
Parallel.ForEach(dumpProcessData.ContainerCounts.Keys, parallelOptions: parallelOptions, mapId =>
@ -97,14 +83,9 @@ public class MultithreadSteppedDumpProcessor(
var ammoDistribution = _ammoProcessor.CreateAmmoDistribution(mapId, preProcessedStaticLoots);
staticAmmo[mapId] = ammoDistribution;
});
// Ammo distribution
output.Add(
OutputFileType.StaticAmmo,
staticAmmo
);
output.Add(OutputFileType.StaticAmmo, staticAmmo);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Processing static loot distribution", LogLevel.Info);
_logger.LogInformation("Processing static loot distribution");
var staticLoot = new ConcurrentDictionary<string, IReadOnlyDictionary<string, StaticItemDistribution>>();
Parallel.ForEach(dumpProcessData.ContainerCounts.Keys, parallelOptions: parallelOptions, mapId =>
@ -114,16 +95,10 @@ public class MultithreadSteppedDumpProcessor(
_staticLootProcessor.CreateStaticLootDistribution(mapId, preProcessedStaticLoots);
staticLoot[mapId] = staticLootDistribution;
});
// Static loot distribution
output.Add(
OutputFileType.StaticLoot,
staticLoot
);
output.Add(OutputFileType.StaticLoot, staticLoot);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Processing loose loot distribution", LogLevel.Info);
_logger.LogInformation("Processing loose loot distribution");
// Loose loot distribution
var looseLoot = new ConcurrentDictionary<string, LooseLootRoot>();
Parallel.ForEach(dumpProcessData.MapCounts.Keys, parallelOptions: parallelOptions, mapId =>
{
@ -133,15 +108,13 @@ public class MultithreadSteppedDumpProcessor(
_looseLootProcessor.CreateLooseLootDistribution(mapId, mapCount, looseLootCount);
looseLoot[mapId] = looseLootDistribution;
});
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Collecting loose loot distribution information", LogLevel.Info);
_logger.LogInformation("Collecting loose loot distribution information");
var loot = dumpProcessData.MapCounts
.Select(mapCount => mapCount.Key)
.ToDictionary(mi => mi, mi => looseLoot[mi]);
output.Add(OutputFileType.LooseLoot, loot);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Dump processing fully completed!", LogLevel.Info);
_logger.LogInformation("Dump processing fully completed!");
return output;
}
@ -150,32 +123,22 @@ public class MultithreadSteppedDumpProcessor(
ConcurrentDictionary<string, ConcurrentDictionary<Template, int>> mapStaticContainersAggregated,
ConcurrentDictionary<string, int> mapDumpCounter)
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"Processing static data for file {partialData.BasicInfo.FileName}",
LogLevel.Debug);
_logger.LogDebug("Processing static data for file {FileName}", partialData.BasicInfo.FileName);
var fileContent = await File.ReadAllTextAsync(partialData.BasicInfo.FileName);
var dataDump = _jsonSerializer.Deserialize<RootData>(fileContent);
if (dataDump == null)
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance()
.Log($"Failed to deserialize data from file {partialData.BasicInfo.FileName}",
LogLevel.Error);
_logger.LogError("Failed to deserialize data from file {FileName}", partialData.BasicInfo.FileName);
return;
}
var mapId = dataDump.Data.LocationLoot.Id.ToLower();
// Because we may use multiple version dumps for map data, merge the static loot between dumps
if (!staticContainers.TryGetValue(mapId, out var mapStaticLoot))
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance()
.Log($"Doing first time process for map {mapId} of real static data",
LogLevel.Info);
_logger.LogInformation("Doing first time process for map {MapId} of real static data", mapId);
staticContainers[mapId] = new MapStaticLoot
{
@ -197,29 +160,19 @@ public class MultithreadSteppedDumpProcessor(
mapStaticLoot.StaticForced.AddRange(newStaticForced);
}
// Takes care of finding how many "dynamic static containers" we have on the map
ConcurrentDictionary<Template, int> mapAggregatedDataDict;
// Init dict if map key doesnt exist
if (!mapStaticContainersAggregated.TryGetValue(mapId, out mapAggregatedDataDict))
if (!mapStaticContainersAggregated.TryGetValue(mapId, out ConcurrentDictionary<Template, int> mapAggregatedDataDict))
{
mapAggregatedDataDict = new ConcurrentDictionary<Template, int>();
mapStaticContainersAggregated.TryAdd(mapId, mapAggregatedDataDict);
}
// Only process the dump file if the date is higher (after) the configuration date
if (!DumpWasMadeAfterConfigThresholdDate(partialData))
{
return;
}
// Keep track of how many dumps we have for each map
IncrementMapCounterDictionaryValue(mapDumpCounter, mapId);
var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList
.TryGetValue(mapId, out var ignoreListForMap);
foreach (var dynamicStaticContainer in _staticContainersProcessor.CreateDynamicStaticContainers(
@ -227,11 +180,9 @@ public class MultithreadSteppedDumpProcessor(
{
if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id))
{
// Skip adding containers to aggregated data if container id is in ignore list
continue;
}
// Increment times container seen in dump by 1
if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1))
{
mapAggregatedDataDict[dynamicStaticContainer] += 1;
@ -253,7 +204,6 @@ public class MultithreadSteppedDumpProcessor(
{
if (!mapDumpCounter.TryAdd(mapName, 1))
{
// Dict has map, increment count by 1
mapDumpCounter[mapName] += 1;
}
}
@ -264,7 +214,7 @@ public class MultithreadSteppedDumpProcessor(
return Math.Round((double)((decimal)td.Value / (decimal)mapDumpCounter[mapName]), 2);
}
private static DumpProcessData GetDumpProcessData(List<PartialData> dumps)
private DumpProcessData GetDumpProcessData(List<PartialData> dumps)
{
var dumpProcessData = new DumpProcessData();
@ -274,15 +224,10 @@ public class MultithreadSteppedDumpProcessor(
{
var mapName = tuple.Key;
var partialFileMetaData = tuple.ToList();
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log(
$"Processing map {mapName}, total dump data to process: {partialFileMetaData.Count}",
LogLevel.Info
);
_logger.LogInformation("Processing map {MapName}, total dump data to process: {Count}", mapName, partialFileMetaData.Count);
dumpProcessData.MapCounts[mapName] = partialFileMetaData.Count;
var lockObjectContainerCounts = new object();
var lockObjectCounts = new object();
var looseLootCounts = new LooseLootCounts();
@ -290,11 +235,6 @@ public class MultithreadSteppedDumpProcessor(
var dictionaryCounts = new FlatKeyableDictionary<string, int>();
looseLootCounts.Counts = dictionaryCounts.GetKey();
/*
var dictionaryItemCounts = new FlatKeyableDictionary<string, List<string>>();
counts.Items = dictionaryItemCounts.GetKey();
*/
var lockObjectDictionaryItemProperties = new object();
var dictionaryItemProperties = new FlatKeyableDictionary<string, FlatKeyableList<Template>>();
@ -305,13 +245,11 @@ public class MultithreadSteppedDumpProcessor(
BlockingCollection<PartialData> _partialDataToProcess = new();
// add the items to the queue
foreach (var partialData in partialFileMetaData)
{
_partialDataToProcess.Add(partialData);
}
// Call GC before running threads
partialFileMetaData = null;
tuple = null;
GCHandler.Collect();
@ -337,11 +275,7 @@ public class MultithreadSteppedDumpProcessor(
_dataStorage.Store(dictionaryCounts);
dictionaryCounts = null;
GCHandler.Collect();
/*
DataStorageFactory.GetInstance().Store(dictionaryItemCounts);
dictionaryItemCounts = null;
GC.Collect();
*/
_dataStorage.Store(actualDictionaryItemProperties);
actualDictionaryItemProperties = null;
GCHandler.Collect();
@ -352,7 +286,7 @@ public class MultithreadSteppedDumpProcessor(
return dumpProcessData;
}
private static void ProcessPartialData(PartialData partialDataToProcess,
private void ProcessPartialData(PartialData partialDataToProcess,
object lockObjectContainerCounts,
DumpProcessData dumpProcessData, string mapName, object lockObjectDictionaryCounts,
FlatKeyableDictionary<string, int>? dictionaryCounts, object lockObjectDictionaryItemProperties,
@ -364,7 +298,6 @@ public class MultithreadSteppedDumpProcessor(
{
var dumpData = _dataStorage.GetItem<ParsedDump>(partialDataToProcess.ParsedDumpKey);
// Static containers
lock (lockObjectContainerCounts)
{
if (!dumpProcessData.ContainerCounts.ContainsKey(mapName))
@ -378,7 +311,6 @@ public class MultithreadSteppedDumpProcessor(
}
}
// Loose loot into ids on files
var loadedDictionary =
_dataStorage
.GetItem<SubdivisionedKeyableDictionary<string, List<Template>>>(
@ -416,11 +348,7 @@ public class MultithreadSteppedDumpProcessor(
}
catch (Exception e)
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log(
$"ERROR OCCURRED:{e.Message}\n{e.StackTrace}",
LogLevel.Error
);
_logger.LogError("ERROR OCCURRED: {Message}\n{StackTrace}", e.Message, e.StackTrace);
}
}
}

51
source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessor.cs
View File

@ -1,35 +1,41 @@
using LootDumpProcessor.Logger;
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Processor.FileProcessor;
using LootDumpProcessor.Process.Processor.v2.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
using LootDumpProcessor.Storage;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.FileProcessor;
public class FileProcessor(IStaticLootProcessor staticLootProcessor, ILooseLootProcessor looseLootProcessor)
: IFileProcessor
public class FileProcessor : IFileProcessor
{
private readonly IStaticLootProcessor _staticLootProcessor =
staticLootProcessor ?? throw new ArgumentNullException(nameof(staticLootProcessor));
private readonly IStaticLootProcessor _staticLootProcessor;
private readonly ILooseLootProcessor _looseLootProcessor;
private readonly ILogger<FileProcessor> _logger;
private readonly ILooseLootProcessor _looseLootProcessor =
looseLootProcessor ?? throw new ArgumentNullException(nameof(looseLootProcessor));
public FileProcessor(
IStaticLootProcessor staticLootProcessor,
ILooseLootProcessor looseLootProcessor,
ILogger<FileProcessor> logger)
{
_staticLootProcessor = staticLootProcessor
?? throw new ArgumentNullException(nameof(staticLootProcessor));
_looseLootProcessor = looseLootProcessor
?? throw new ArgumentNullException(nameof(looseLootProcessor));
_logger = logger
?? throw new ArgumentNullException(nameof(logger));
}
public PartialData Process(BasicInfo parsedData)
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"Processing file {parsedData.FileName}...", LogLevel.Debug);
_logger.LogDebug("Processing file {FileName}...", parsedData.FileName);
List<Template> looseLoot = new List<Template>();
List<Template> staticLoot = new List<Template>();
var looseLoot = new List<Template>();
var staticLoot = new List<Template>();
foreach (var item in parsedData.Data.Data.LocationLoot.Loot)
{
if (item.IsContainer ?? false)
staticLoot.Add(item);
else
looseLoot.Add(item);
if (item.IsContainer ?? false) staticLoot.Add(item);
else looseLoot.Add(item);
}
parsedData.Data = null;
@ -47,18 +53,17 @@ public class FileProcessor(IStaticLootProcessor staticLootProcessor, ILooseLootP
if (!DataStorageFactory.GetInstance().Exists(dumpData.GetKey()))
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log(
$"Cached not found for {string.Join("/", dumpData.GetKey().GetLookupIndex())} processing.",
LogLevel.Debug
_logger.LogDebug(
"Cache not found for {LookupIndex} processing.",
string.Join("/", dumpData.GetKey().GetLookupIndex())
);
dumpData.Containers = _staticLootProcessor.PreProcessStaticLoot(staticLoot);
dumpData.LooseLoot = _looseLootProcessor.PreProcessLooseLoot(looseLoot);
DataStorageFactory.GetInstance().Store(dumpData);
}
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"File {parsedData.FileName} finished processing!", LogLevel.Debug);
_logger.LogDebug("File {FileName} finished processing!", parsedData.FileName);
return data;
}
}

9
source/LootDumpProcessor/Process/Processor/v2/AmmoProcessor/AmmoProcessor.cs
View File

@ -4,17 +4,20 @@ using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
public class AmmoProcessor(ILogger<AmmoProcessor> logger) : IAmmoProcessor
public class AmmoProcessor(ILogger<AmmoProcessor> logger, ITarkovItemsProvider tarkovItemsProvider) : IAmmoProcessor
{
private readonly ILogger<AmmoProcessor> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
private readonly ITarkovItemsProvider _tarkovItemsProvider =
tarkovItemsProvider ?? throw new ArgumentNullException(nameof(tarkovItemsProvider));
public IReadOnlyDictionary<string, List<AmmoDistribution>> CreateAmmoDistribution(
string mapId,
List<PreProcessedStaticLoot> containers)
{
var ammoTemplates = containers
.SelectMany(container => container.Items)
.Where(item => LootDumpProcessorContext.GetTarkovItems().IsBaseClass(item.Tpl, BaseClasses.Ammo))
.Where(item => _tarkovItemsProvider.IsBaseClass(item.Tpl, BaseClasses.Ammo))
.Select(item => item.Tpl)
.ToList();
@ -22,7 +25,7 @@ public class AmmoProcessor(ILogger<AmmoProcessor> logger) : IAmmoProcessor
.GroupBy(tpl => tpl)
.Select(group => new CaliberTemplateCount
{
Caliber = LootDumpProcessorContext.GetTarkovItems().AmmoCaliber(group.Key),
Caliber = _tarkovItemsProvider.AmmoCaliber(group.Key),
Template = group.Key,
Count = group.Count()
})

42
source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/LooseLootProcessor.cs
View File

@ -9,11 +9,23 @@ using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
{
public class LooseLootProcessor(ILogger<LooseLootProcessor> logger, IDataStorage dataStorage)
public class LooseLootProcessor(
ILogger<LooseLootProcessor> logger, IDataStorage dataStorage, ITarkovItemsProvider tarkovItemsProvider,
IComposedKeyGenerator composedKeyGenerator
)
: ILooseLootProcessor
{
private readonly ILogger<LooseLootProcessor> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
private readonly IDataStorage _dataStorage = dataStorage ?? throw new ArgumentNullException(nameof(dataStorage));
private readonly ILogger<LooseLootProcessor>
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
private readonly IDataStorage
_dataStorage = dataStorage ?? throw new ArgumentNullException(nameof(dataStorage));
private readonly ITarkovItemsProvider _tarkovItemsProvider =
tarkovItemsProvider ?? throw new ArgumentNullException(nameof(tarkovItemsProvider));
private readonly IComposedKeyGenerator _composedKeyGenerator =
composedKeyGenerator ?? throw new ArgumentNullException(nameof(composedKeyGenerator));
public PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseLoot)
{
@ -35,7 +47,8 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
if (!uniqueLootIds.ContainsKey(sanitizedId))
{
uniqueLootIds[sanitizedId] = template.Id;
preProcessedLoot.Counts[sanitizedId] = preProcessedLoot.Counts.TryGetValue(sanitizedId, out var count)
preProcessedLoot.Counts[sanitizedId] =
preProcessedLoot.Counts.TryGetValue(sanitizedId, out var count)
? count + 1
: 1;
}
@ -84,12 +97,14 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
looseLootDistribution.SpawnPointCount = new SpawnPointCount
{
Mean = CalculateMean(looseLootCountsItem.MapSpawnpointCount.Select(Convert.ToDouble).ToList()),
Std = CalculateStandardDeviation(looseLootCountsItem.MapSpawnpointCount.Select(Convert.ToDouble).ToList())
Std = CalculateStandardDeviation(looseLootCountsItem.MapSpawnpointCount.Select(Convert.ToDouble)
.ToList())
};
looseLootDistribution.SpawnPointsForced = new List<SpawnPointsForced>();
looseLootDistribution.SpawnPoints = new List<SpawnPoint>();
var itemProperties = _dataStorage.GetItem<FlatKeyableDictionary<string, IKey>>(looseLootCountsItem.ItemProperties);
var itemProperties =
_dataStorage.GetItem<FlatKeyableDictionary<string, IKey>>(looseLootCountsItem.ItemProperties);
foreach (var (spawnPoint, itemListKey) in itemProperties)
{
var itemCounts = new Dictionary<ComposedKey, int>();
@ -97,7 +112,7 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
foreach (var template in savedTemplates)
{
var composedKey = new ComposedKey(template);
var composedKey = _composedKeyGenerator.Generate(template.Items);
if (!itemCounts.TryAdd(composedKey, 1))
itemCounts[composedKey]++;
}
@ -122,8 +137,9 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
}).ToList();
if (itemDistributions.Count == 1 &&
(LootDumpProcessorContext.GetTarkovItems().IsQuestItem(itemDistributions[0].ComposedKey?.FirstItem?.Tpl) ||
LootDumpProcessorContext.GetForcedLooseItems()[mapId].Contains(itemDistributions[0].ComposedKey?.FirstItem?.Tpl)))
(_tarkovItemsProvider.IsQuestItem(itemDistributions[0].ComposedKey?.FirstItem?.Tpl) ||
LootDumpProcessorContext.GetForcedLooseItems()[mapId]
.Contains(itemDistributions[0].ComposedKey?.FirstItem?.Tpl)))
{
var forcedSpawnPoint = new SpawnPointsForced
{
@ -162,7 +178,7 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
templateCopy.Items = new List<Item>();
var groupedByKey = spawnPoints
.GroupBy(t => new ComposedKey(t))
.GroupBy(t => _composedKeyGenerator.Generate(t.Items))
.ToDictionary(g => g.Key, g => g.ToList());
foreach (var distribution in itemDistributions)
@ -198,8 +214,10 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
.OrderBy(x => x.Template.Id)
.ToList();
var configuredForcedTemplates = new HashSet<string>(LootDumpProcessorContext.GetForcedLooseItems()[mapId].Select(item => item));
var foundForcedTemplates = new HashSet<string>(looseLootDistribution.SpawnPointsForced.Select(fp => fp.Template.Items[0].Tpl));
var configuredForcedTemplates =
new HashSet<string>(LootDumpProcessorContext.GetForcedLooseItems()[mapId].Select(item => item));
var foundForcedTemplates =
new HashSet<string>(looseLootDistribution.SpawnPointsForced.Select(fp => fp.Template.Items[0].Tpl));
foreach (var expectedTpl in configuredForcedTemplates)
{

114
source/LootDumpProcessor/Process/QueuePipeline.cs
View File

@ -1,5 +1,4 @@
using System.Collections.Concurrent;
using LootDumpProcessor.Logger;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Process.Collector;
using LootDumpProcessor.Process.Processor.DumpProcessor;
@ -11,10 +10,15 @@ using LootDumpProcessor.Process.Writer;
using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process;
public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProcessor) : IPipeline
public class QueuePipeline(
IFileProcessor fileProcessor, IDumpProcessor dumpProcessor, ILogger<QueuePipeline> logger,
IPreProcessReader preProcessReader, IFileFilter fileFilter, IIntakeReader intakeReader
)
: IPipeline
{
private readonly IFileProcessor _fileProcessor =
fileProcessor ?? throw new ArgumentNullException(nameof(fileProcessor));
@ -22,24 +26,21 @@ public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProc
private readonly IDumpProcessor _dumpProcessor =
dumpProcessor ?? throw new ArgumentNullException(nameof(dumpProcessor));
private readonly ILogger<QueuePipeline> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
private readonly IPreProcessReader _preProcessReader =
preProcessReader ?? throw new ArgumentNullException(nameof(preProcessReader));
private readonly IFileFilter _fileFilter = fileFilter ?? throw new ArgumentNullException(nameof(fileFilter));
private readonly IIntakeReader
_intakeReader = intakeReader ?? throw new ArgumentNullException(nameof(intakeReader));
private readonly List<string> _filesToRename = new();
private readonly BlockingCollection<string> _filesToProcess = new();
private readonly List<string> _mapNames = LootDumpProcessorContext.GetConfig().MapsToProcess;
private static readonly Dictionary<string, IPreProcessReader> _preProcessReaders;
static QueuePipeline()
{
_preProcessReaders = LootDumpProcessorContext.GetConfig()
.ReaderConfig
.PreProcessorConfig
?.PreProcessors
?.ToDictionary(
t => PreProcessReaderFactory.GetInstance(t).GetHandleExtension().ToLower(),
PreProcessReaderFactory.GetInstance
) ?? new Dictionary<string, IPreProcessReader>();
}
public async Task DoProcess()
{
@ -47,10 +48,7 @@ public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProc
var collector = CollectorFactory.GetInstance();
collector.Setup();
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
{
LoggerFactory.GetInstance().Log("Gathering files to begin processing", LogLevel.Info);
}
_logger.LogInformation("Gathering files to begin processing");
try
{
@ -62,11 +60,7 @@ public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProc
}
finally
{
// use dispose on the preprocessreaders to eliminate any temporary files generated
foreach (var (_, value) in _preProcessReaders)
{
value.Dispose();
}
_preProcessReader.Dispose();
}
}
@ -95,36 +89,30 @@ public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProc
throw new Exception("No files matched accepted extension types in configs");
}
var fileFilters = GetFileFilters() ?? new Dictionary<string, IFileFilter>();
while (queuedFilesToProcess.TryDequeue(out var file))
{
var extensionFull = Path.GetExtension(file);
if (extensionFull.Length > 1)
{
var extension = extensionFull[1..].ToLower();
// if there is a preprocessor, call it and preprocess the file, then add them to the queue
if (_preProcessReaders.TryGetValue(extension, out var preProcessor))
{
// if the preprocessor found something new to process or generated new files, add them to the queue
if (preProcessor.TryPreProcess(file, out var outputFiles, out var outputDirectories))
if (extensionFull == "7z" &&
_preProcessReader.TryPreProcess(file, out var outputFiles, out var outputDirectories))
{
// all new directories need to be processed as well
GetFileQueue(outputDirectories).ToList().ForEach(queuedFilesToProcess.Enqueue);
// all output files need to be queued as well
outputFiles.ForEach(queuedFilesToProcess.Enqueue);
}
}
else
{
// if there is no preprocessor for the file, means its ready to filter or accept
if (fileFilters.TryGetValue(extension, out var filter))
{
if (filter.Accept(file))
if (_fileFilter.Accept(file))
{
gatheredFiles.Add(file);
}
}
else
{
gatheredFiles.Add(file);
@ -134,6 +122,7 @@ public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProc
else
{
// Handle invalid extension
_logger.LogWarning("File '{File}' has an invalid extension.", file);
}
}
@ -145,11 +134,6 @@ public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProc
var queuedPathsToProcess = new Queue<string>();
var queuedFilesToProcess = new Queue<string>();
// Accepted file extensions on raw files
var acceptedFileExtension = LootDumpProcessorContext.GetConfig()
.ReaderConfig
.AcceptedFileExtensions
.Select(ex => ex.ToLower());
inputPath.ForEach(p => queuedPathsToProcess.Enqueue(p));
while (queuedPathsToProcess.TryDequeue(out var path))
@ -172,28 +156,14 @@ public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProc
var files = Directory.GetFiles(path);
foreach (var file in files)
{
if (acceptedFileExtension.Contains(Path.GetExtension(file)[1..].ToLower()))
{
queuedFilesToProcess.Enqueue(file);
}
}
}
return queuedFilesToProcess;
}
private Dictionary<string, IFileFilter>? GetFileFilters()
{
return LootDumpProcessorContext.GetConfig()
.ReaderConfig
.FileFilters
?.ToDictionary(
t => FileFilterFactory.GetInstance(t).GetExtension(),
FileFilterFactory.GetInstance
);
}
private void ProcessFilesFromDumpsPerMap(ICollector collector, string mapName)
{
// Gather all files, sort them by date descending and then add them into the processing queue
@ -204,36 +174,24 @@ public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProc
}
).ToList().ForEach(f => _filesToProcess.Add(f));
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
{
LoggerFactory.GetInstance().Log("Files sorted and ready to begin pre-processing", LogLevel.Info);
}
_logger.LogInformation("Files sorted and ready to begin pre-processing");
Parallel.ForEach(_filesToProcess, file =>
{
try
{
var reader = IntakeReaderFactory.GetInstance();
if (reader.Read(file, out var basicInfo))
if (_intakeReader.Read(file, out var basicInfo))
{
collector.Hold(_fileProcessor.Process(basicInfo));
}
}
catch (Exception e)
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
{
LoggerFactory.GetInstance().Log(
$"Error occurred while processing file {file}\n{e.Message}\n{e.StackTrace}",
LogLevel.Error);
}
_logger.LogError(e, "Error occurred while processing file {File}", file);
}
});
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
{
LoggerFactory.GetInstance().Log("Pre-processing finished", LogLevel.Info);
}
_logger.LogInformation("Pre-processing finished");
// Single writer instance to collect results
var writer = WriterFactory.GetInstance();
@ -246,7 +204,7 @@ public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProc
}
/// <summary>
/// Adds map name to file if they dont have it already.
/// Adds map name to file if they don't have it already.
/// </summary>
/// <param name="threads">Number of threads to use</param>
private async Task FixFilesFromDumps()
@ -275,18 +233,10 @@ public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProc
}
catch (Exception e)
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
{
LoggerFactory.GetInstance().Log(
$"Error occurred while processing file {file}\n{e.Message}\n{e.StackTrace}",
LogLevel.Error);
}
_logger.LogError(e, "Error occurred while processing file {File}", file);
}
});
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
{
LoggerFactory.GetInstance().Log("File-processing finished", LogLevel.Info);
}
_logger.LogInformation("File-processing finished");
}
}

25
source/LootDumpProcessor/Process/Reader/Filters/FileFilterFactory.cs
View File

@ -1,25 +0,0 @@
namespace LootDumpProcessor.Process.Reader.Filters;
public static class FileFilterFactory
{
private static readonly Dictionary<FileFilterTypes, IFileFilter> _fileFilters = new();
private static object lockObject = new();
public static IFileFilter GetInstance(FileFilterTypes type)
{
lock (lockObject)
{
if (!_fileFilters.TryGetValue(type, out var filter))
{
filter = type switch
{
FileFilterTypes.JsonDump => new JsonDumpFileFilter(),
_ => throw new ArgumentOutOfRangeException(nameof(type), type, null)
};
_fileFilters.Add(type, filter);
}
return filter;
}
}
}

6
source/LootDumpProcessor/Process/Reader/Filters/FileFilterTypes.cs
View File

@ -1,6 +0,0 @@
namespace LootDumpProcessor.Process.Reader.Filters;
public enum FileFilterTypes
{
JsonDump
}

17
source/LootDumpProcessor/Process/Reader/Filters/JsonDumpFileFilter.cs
View File

@ -1,24 +1,23 @@
using System.Globalization;
using System.Text.RegularExpressions;
using LootDumpProcessor.Logger;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Reader.Filters;
public class JsonDumpFileFilter : IFileFilter
{
private static readonly Regex _fileNameDateRegex = new("([0-9]{4}(-[0-9]{2}){2}_((-){0,1}[0-9]{2}){3})");
private static readonly DateTime _parsedThresholdDate;
private readonly ILogger<JsonDumpFileFilter> _logger;
private readonly Regex _fileNameDateRegex = new("([0-9]{4}(-[0-9]{2}){2}_((-){0,1}[0-9]{2}){3})");
private readonly DateTime _parsedThresholdDate;
static JsonDumpFileFilter()
public JsonDumpFileFilter(ILogger<JsonDumpFileFilter> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
// Calculate parsed date from config threshold
if (string.IsNullOrEmpty(LootDumpProcessorContext.GetConfig().ReaderConfig.ThresholdDate))
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
LoggerFactory.GetInstance()
.Log($"ThresholdDate is null or empty in configs, defaulting to current day minus 30 days",
LogLevel.Warning);
_parsedThresholdDate = (DateTime.Now - TimeSpan.FromDays(30));
_logger.LogWarning("ThresholdDate is null or empty in configs, defaulting to current day minus 30 days");
_parsedThresholdDate = DateTime.Now - TimeSpan.FromDays(30);
}
else
{

28
source/LootDumpProcessor/Process/Reader/Intake/IntakeReaderFactory.cs
View File

@ -1,28 +0,0 @@
namespace LootDumpProcessor.Process.Reader.Intake;
public static class IntakeReaderFactory
{
private static readonly Dictionary<IntakeReaderTypes, IIntakeReader> Instances = new();
private static readonly object DictionaryLock = new();
public static IIntakeReader GetInstance()
{
var type = LootDumpProcessorContext.GetConfig().ReaderConfig.IntakeReaderConfig?.IntakeReaderType ??
IntakeReaderTypes.Json;
lock (DictionaryLock)
{
if (!Instances.TryGetValue(type, out var intakeReader))
{
intakeReader = type switch
{
IntakeReaderTypes.Json => new JsonFileIntakeReader(),
_ => throw new ArgumentOutOfRangeException(
"IntakeReaderType",
"Value was not defined on IntakeReaderConfig"
)
};
Instances.Add(type, intakeReader);
}
return intakeReader;
}
}
}

6
source/LootDumpProcessor/Process/Reader/Intake/IntakeReaderTypes.cs
View File

@ -1,6 +0,0 @@
namespace LootDumpProcessor.Process.Reader.Intake;
public enum IntakeReaderTypes
{
Json
}

81
source/LootDumpProcessor/Process/Reader/Intake/JsonFileIntakeReader.cs
View File

@ -1,81 +1,84 @@
using System.Collections.Concurrent;
using LootDumpProcessor.Logger;
using System.Text.Json;
using LootDumpProcessor;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Process.Reader.Intake;
using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Reader.Intake;
public class JsonFileIntakeReader : IIntakeReader
public class JsonFileIntakeReader(ILogger<JsonFileIntakeReader> logger) : IIntakeReader
{
private static readonly IJsonSerializer _jsonSerializer = JsonSerializerFactory.GetInstance();
private readonly ILogger<JsonFileIntakeReader> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
private static readonly HashSet<string>? _ignoredLocations =
LootDumpProcessorContext.GetConfig().ReaderConfig.IntakeReaderConfig?.IgnoredDumpLocations.ToHashSet();
private readonly HashSet<string>? _ignoredLocations = LootDumpProcessorContext.GetConfig()
.ReaderConfig.IntakeReaderConfig?.IgnoredDumpLocations.ToHashSet();
private static readonly ConcurrentDictionary<string, int> _totalMapDumpsCounter = new();
private readonly ConcurrentDictionary<string, int> _totalMapDumpsCounter = new();
public bool Read(string file, out BasicInfo basicInfo)
{
var fileData = File.ReadAllText(file);
if (fileData == null)
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"Couldnt parse date from file: {file}", LogLevel.Error);
basicInfo = null;
string? fileData;
try
{
fileData = File.ReadAllText(file);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to read file: {File}", file);
return false;
}
// If the file format changes it may screw up this date parser
if (string.IsNullOrWhiteSpace(fileData))
{
_logger.LogError("Could not parse data from file: {File}", file);
return false;
}
// If the file format changes, it may affect the date parser
if (!FileDateParser.TryParseFileDate(file, out var date))
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"Couldnt parse date from file: {file}", LogLevel.Error);
_logger.LogError("Could not parse date from file: {File}", file);
}
var fi = _jsonSerializer.Deserialize<RootData>(fileData);
var fi = JsonSerializer.Deserialize<RootData>(fileData);
if (fi?.Data?.LocationLoot?.Name != null && (!_ignoredLocations?.Contains(fi.Data.LocationLoot.Name) ?? true))
{
if (!_totalMapDumpsCounter.TryGetValue(fi.Data.LocationLoot.Name, out var counter))
{
counter = 0;
_totalMapDumpsCounter[fi.Data.LocationLoot.Name] = counter;
}
var mapName = fi.Data.LocationLoot.Name;
var mapId = fi.Data.LocationLoot.Id.ToLower();
if (counter < (LootDumpProcessorContext.GetConfig().ReaderConfig.IntakeReaderConfig?.MaxDumpsPerMap ?? 1500))
var counter = _totalMapDumpsCounter.AddOrUpdate(mapName, 0, (_, current) => current);
var maxDumpsPerMap = LootDumpProcessorContext.GetConfig()
.ReaderConfig.IntakeReaderConfig?.MaxDumpsPerMap ?? 1500;
if (counter < maxDumpsPerMap)
{
basicInfo = new BasicInfo
{
Map = fi.Data.LocationLoot.Id.ToLower(),
Map = mapId,
FileHash = ProcessorUtil.HashFile(fileData),
Data = fi,
Date = date ?? DateTime.MinValue,
FileName = file
};
_totalMapDumpsCounter[fi.Data.LocationLoot.Name] += 1;
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"File {file} fully read, returning data", LogLevel.Debug);
_totalMapDumpsCounter[mapName] += 1;
_logger.LogDebug("File {File} fully read, returning data", file);
return true;
}
// Map dump limit reached, exit
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"Ignoring file {file} as the file cap for map {fi.Data.LocationLoot.Id} has been reached", LogLevel.Debug);
basicInfo = null;
_logger.LogDebug("Ignoring file {File} as the file cap for map {MapId} has been reached", file, mapId);
return false;
}
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
LoggerFactory.GetInstance().Log($"File {file} was not eligible for dump data, it did not contain a location name or it was on ignored locations config", LogLevel.Warning);
basicInfo = null;
_logger.LogWarning(
"File {File} was not eligible for dump data; it did not contain a location name or it was on the ignored locations config",
file);
return false;
}
}

20
source/LootDumpProcessor/Process/Reader/PreProcess/AbstractPreProcessReader.cs
View File

@ -1,4 +1,4 @@
using LootDumpProcessor.Logger;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Reader.PreProcess;
@ -6,17 +6,15 @@ public abstract class AbstractPreProcessReader : IPreProcessReader
{
protected readonly string _tempFolder;
public AbstractPreProcessReader()
public AbstractPreProcessReader(ILogger logger)
{
var tempFolder = LootDumpProcessorContext.GetConfig().ReaderConfig.PreProcessorConfig?.PreProcessorTempFolder;
if (string.IsNullOrEmpty(tempFolder))
{
tempFolder = GetBaseDirectory();
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
LoggerFactory.GetInstance()
.Log(
$"No temp folder was assigned preProcessorTempFolder in PreProcessorConfig, defaulting to {tempFolder}",
LogLevel.Warning
logger.LogWarning(
"No temp folder was assigned preProcessorTempFolder in PreProcessorConfig, defaulting to {tempFolder}",
tempFolder
);
}
@ -34,16 +32,12 @@ public abstract class AbstractPreProcessReader : IPreProcessReader
public abstract string GetHandleExtension();
public abstract bool TryPreProcess(string file, out List<string> files, out List<string> directories);
protected string GetBaseDirectory()
{
return $@"{Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)}\SPT\tmp\PreProcessor";
}
protected string GetBaseDirectory() =>
$@"{Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)}\SPT\tmp\PreProcessor";
public void Dispose()
{
if (LootDumpProcessorContext.GetConfig().ReaderConfig.PreProcessorConfig?.CleanupTempFolderAfterProcess ?? true)
{
Directory.Delete(_tempFolder, true);
}
}
}

21
source/LootDumpProcessor/Process/Reader/PreProcess/PreProcessReaderFactory.cs
View File

@ -1,21 +0,0 @@
namespace LootDumpProcessor.Process.Reader.PreProcess;
public static class PreProcessReaderFactory
{
private static readonly Dictionary<PreProcessReaderTypes, IPreProcessReader> _proProcessReaders = new();
public static IPreProcessReader GetInstance(PreProcessReaderTypes type)
{
if (!_proProcessReaders.TryGetValue(type, out var preProcessReader))
{
preProcessReader = type switch
{
PreProcessReaderTypes.SevenZip => new SevenZipPreProcessReader(),
_ => throw new ArgumentOutOfRangeException(nameof(type), type, null)
};
_proProcessReaders.Add(type, preProcessReader);
}
return preProcessReader;
}
}

39
source/LootDumpProcessor/Process/Reader/PreProcess/SevenZipPreProcessReader.cs
View File

@ -1,10 +1,16 @@
using LootDumpProcessor.Logger;
using LootDumpProcessor.Process.Reader.PreProcess;
using SevenZip;
namespace LootDumpProcessor.Process.Reader.PreProcess;
using Microsoft.Extensions.Logging;
public class SevenZipPreProcessReader : AbstractPreProcessReader
{
private readonly ILogger<SevenZipPreProcessReader> _logger;
public SevenZipPreProcessReader(ILogger<SevenZipPreProcessReader> logger) : base(logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public override string GetHandleExtension() => "7z";
static SevenZipPreProcessReader()
@ -14,29 +20,32 @@ public class SevenZipPreProcessReader : AbstractPreProcessReader
public override bool TryPreProcess(string file, out List<string> files, out List<string> directories)
{
files = new List<string>();
directories = new List<string>();
var fileRaw = Path.GetFileNameWithoutExtension(file);
// SevenZip library doesnt like forward slashes for some reason
// SevenZip library doesn't handle forward slashes properly
var outPath = $"{_tempFolder}\\{fileRaw}".Replace("/", "\\");
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log(
$"Unzipping {file} into temp path {outPath}, this may take a while...",
LogLevel.Info);
_logger.LogInformation("Unzipping {File} into temp path {OutPath}, this may take a while...", file, outPath);
var extractor = new SevenZipExtractor(file);
// Only log process on debug mode
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
{
// Log progress in debug mode
extractor.Extracting += (_, args) =>
{
if (args.PercentDone % 10 == 0)
LoggerFactory.GetInstance().Log($"Unzip progress: {args.PercentDone}%", LogLevel.Debug);
};
{
_logger.LogDebug("Unzip progress: {PercentDone}%", args.PercentDone);
}
};
extractor.ExtractArchive(outPath);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log($"Finished unzipping {file} into temp path {outPath}", LogLevel.Info);
_logger.LogInformation("Finished unzipping {File} into temp path {OutPath}", file, outPath);
files = Directory.GetFiles(outPath).ToList();
directories = Directory.GetDirectories(outPath).ToList();
return true;
}
}

68
source/LootDumpProcessor/Process/TarkovItems.cs
View File

@ -1,68 +0,0 @@
using LootDumpProcessor.Logger;
using LootDumpProcessor.Model.Tarkov;
using LootDumpProcessor.Serializers.Json;
namespace LootDumpProcessor.Process;
public class TarkovItems(string items)
{
private static readonly IJsonSerializer _jsonSerializer = JsonSerializerFactory.GetInstance();
private readonly Dictionary<string, TemplateFileItem>? _items = _jsonSerializer.Deserialize<Dictionary<string, TemplateFileItem>>(File.ReadAllText(items));
public virtual bool IsBaseClass(string tpl, string baseclass_id)
{
if (_items == null)
throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
if (!_items.TryGetValue(tpl, out var item_template))
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"[IsBaseClass] Item template '{tpl}' with base class id '{baseclass_id}' was not found on the server items!", LogLevel.Error);
return false;
}
if (string.IsNullOrEmpty(item_template.Parent))
return false;
return item_template.Parent == baseclass_id || IsBaseClass(item_template.Parent, baseclass_id);
}
public virtual bool IsQuestItem(string tpl)
{
if (_items == null)
throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
if (!_items.TryGetValue(tpl, out var item_template))
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"[IsQuestItem] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
return false;
}
return item_template.Props.QuestItem;
}
public virtual string? MaxDurability(string tpl)
{
if (_items == null)
throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
if (!_items.TryGetValue(tpl, out var item_template))
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"[MaxDurability] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
return null;
}
return item_template.Props.MaxDurability?.ToString() ?? "";
}
public virtual string? AmmoCaliber(string tpl)
{
if (_items == null)
throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
if (!_items.TryGetValue(tpl, out var item_template))
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"[AmmoCaliber] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
return null;
}
return item_template.Props.Caliber;
}
}

113
source/LootDumpProcessor/Process/TarkovItemsProvider.cs Normal file
View File

@ -0,0 +1,113 @@
using System.Collections.Frozen;
using System.Text.Json;
using LootDumpProcessor.Model.Tarkov;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process
{
public class TarkovItemsProvider : ITarkovItemsProvider
{
private readonly ILogger<TarkovItemsProvider> _logger;
private readonly FrozenDictionary<string, TemplateFileItem>? _items;
private static readonly string ItemsFilePath = Path.Combine(
LootDumpProcessorContext.GetConfig().ServerLocation,
"project", "assets", "database", "templates", "items.json");
public TarkovItemsProvider(ILogger<TarkovItemsProvider> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
try
{
var jsonContent = File.ReadAllText(ItemsFilePath);
_items = (JsonSerializer.Deserialize<Dictionary<string, TemplateFileItem>>(jsonContent)
?? throw new InvalidOperationException()).ToFrozenDictionary();
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load server items from {ItemsPath}", ItemsFilePath);
throw new InvalidOperationException(
"The server items couldn't be found or loaded. Check server config is pointing to the correct place.",
ex);
}
}
public bool IsBaseClass(string tpl, string baseclassId)
{
if (_items == null)
{
_logger.LogError("The server items are null. Check server config is pointing to the correct place.");
throw new InvalidOperationException(
"The server items couldn't be found or loaded. Check server config is pointing to the correct place.");
}
if (!_items.TryGetValue(tpl, out var itemTemplate))
{
_logger.LogError(
"Item template '{Tpl}' with base class id '{BaseclassId}' was not found in the server items!", tpl,
baseclassId);
return false;
}
if (string.IsNullOrEmpty(itemTemplate.Parent))
return false;
return itemTemplate.Parent == baseclassId || IsBaseClass(itemTemplate.Parent, baseclassId);
}
public bool IsQuestItem(string tpl)
{
if (_items == null)
{
_logger.LogError("The server items are null. Check server config is pointing to the correct place.");
throw new InvalidOperationException(
"The server items couldn't be found or loaded. Check server config is pointing to the correct place.");
}
if (!_items.TryGetValue(tpl, out var itemTemplate))
{
_logger.LogError("Item template '{Tpl}' was not found in the server items!", tpl);
return false;
}
return itemTemplate.Props.QuestItem;
}
public string? MaxDurability(string tpl)
{
if (_items == null)
{
_logger.LogError("The server items are null. Check server config is pointing to the correct place.");
throw new InvalidOperationException(
"The server items couldn't be found or loaded. Check server config is pointing to the correct place.");
}
if (!_items.TryGetValue(tpl, out var itemTemplate))
{
_logger.LogError("Item template '{Tpl}' was not found in the server items!", tpl);
return null;
}
return itemTemplate.Props.MaxDurability?.ToString() ?? string.Empty;
}
public string? AmmoCaliber(string tpl)
{
if (_items == null)
{
_logger.LogError("The server items are null. Check server config is pointing to the correct place.");
throw new InvalidOperationException(
"The server items couldn't be found or loaded. Check server config is pointing to the correct place.");
}
if (!_items.TryGetValue(tpl, out var itemTemplate))
{
_logger.LogError("Item template '{Tpl}' was not found in the server items!", tpl);
return null;
}
return itemTemplate.Props.Caliber;
}
}
}

15
source/LootDumpProcessor/Program.cs
View File

@ -5,10 +5,12 @@ using LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
using LootDumpProcessor.Process.Processor.v2.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
using LootDumpProcessor.Process.Reader.Filters;
using LootDumpProcessor.Process.Reader.Intake;
using LootDumpProcessor.Process.Reader.PreProcess;
using LootDumpProcessor.Storage;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using LoggerFactory = LootDumpProcessor.Logger.LoggerFactory;
namespace LootDumpProcessor;
@ -24,8 +26,13 @@ public static class Program
services.AddTransient<IAmmoProcessor, AmmoProcessor>();
services.AddTransient<ILooseLootProcessor, LooseLootProcessor>();
services.AddSingleton<ITarkovItemsProvider, TarkovItemsProvider>();
services.AddSingleton<IDataStorage>(_ => DataStorageFactory.GetInstance());
services.AddTransient<IComposedKeyGenerator, ComposedKeyGenerator>();
services.AddTransient<IIntakeReader, JsonFileIntakeReader>();
services.AddTransient<IFileFilter, JsonDumpFileFilter>();
services.AddTransient<IPreProcessReader, SevenZipPreProcessReader>();
services.AddTransient<IFileProcessor, FileProcessor>();
services.AddTransient<IDumpProcessor, MultithreadSteppedDumpProcessor>();
services.AddTransient<IPipeline, QueuePipeline>();
@ -34,15 +41,11 @@ public static class Program
// Bootstrap the config before anything else, its required by the whole application to work
LootDumpProcessorContext.GetConfig();
// Some loggers may need a startup and stop mechanism
LoggerFactory.GetInstance().Setup();
// Setup Data storage
DataStorageFactory.GetInstance().Setup();
// startup the pipeline
var pipeline = serviceProvider.GetRequiredService<IPipeline>();
pipeline.DoProcess();
// stop loggers at the end
LoggerFactory.GetInstance().Stop();
await pipeline.Execute();
Thread.Sleep(10000);
}
}