spt/loot-dump-processor
spt/loot-dump-processor
0
0
Fork 0
mirror of https://github.com/sp-tarkov/loot-dump-processor.git synced 2025-02-12 17:10:45 -05:00
Code

Get rid of static and use dependency injection instead (#8)

Browse Source 
* Improved thread safety and async processing in dump processor components

* Removed unused _processedDumps field and simplified variable scope in QueuePipeline

* Refactored service registration into dedicated extension methods

* Added configuration binding and environment variables support

* Refactored collector initialization to use dependency injection

* Refactored data storage to use dependency injection

* Refactored configuration models to use records and added validation

* Refactored static loot configuration to use dependency injection

The changes include:
- Moved static weapon IDs and forced items from LootDumpProcessorContext to ForcedStatic record
- Added ForcedStatic configuration injection in StaticLootProcessor and StaticContainerProcessor
- Improved immutability by using read-only collections in ForcedStatic model
- Simplified LootDumpProcessorContext by removing unused methods

* Refactored configuration access to use dependency injection consistently

* Fixed ForcedStatic configuration

* Refactored forced items configuration to use async provider pattern

The changes introduce a new `IForcedItemsProvider` abstraction to handle loading and caching of forced static and loose loot configurations. This improves the code by:

1. Making configuration loading asynchronous
2. Implementing caching of loaded configurations
3. Centralizing forced items configuration access
4. Removing direct file system dependencies from processors
5. Improving testability through dependency injection

The change also updates related processors and interfaces to use async/await pattern consistently.

* Refactored loose loot processor to use async forced items provider

* Reorganized processor and service components into dedicated namespaces
This commit is contained in:
BlueXTX 2025-01-13 20:05:36 +03:00 committed by GitHub
parent 3cb1be1ec9
commit f2fd256aac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
49 changed files with 489 additions and 399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
source/LootDumpProcessor/GCHandler.cs
View File

@ -1,9 +0,0 @@
namespace LootDumpProcessor;
public static class GCHandler
{
public static void Collect()
{
if (LootDumpProcessorContext.GetConfig().ManualGarbageCollectionCalls) GC.Collect();
}
}

4
source/LootDumpProcessor/LootDumpProcessor.csproj
View File

@ -11,7 +11,11 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="JetBrains.Annotations" Version="2024.3.0"/>
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.0"/>
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.0"/>
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0"/>
<PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" Version="9.0.0"/>
<PackageReference Include="YamlDotNet" Version="13.0.0"/>
</ItemGroup>

80
source/LootDumpProcessor/LootDumpProcessorContext.cs
View File

@ -1,80 +0,0 @@
using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Serializers.Yaml;
namespace LootDumpProcessor;
public static class LootDumpProcessorContext
{
private static Config? _config;
private static readonly object _configLock = new();
private static ForcedStatic? _forcedStatic;
private static readonly object _forcedStaticLock = new();
private static HashSet<string>? _staticWeaponIds;
private static readonly object _staticWeaponIdsLock = new();
private static Dictionary<string, List<StaticForced>>? _forcedItems;
private static readonly object _forcedItemsLock = new();
private static Dictionary<string, HashSet<string>>? _forcedLoose;
private static readonly object _forcedLooseLock = new();
public static Config GetConfig()
{
lock (_configLock)
{
if (_config == null)
// This is the only instance where manual selection of the serializer is required
// after this, GetInstance() for the JsonSerializerFactory should used without
// parameters
_config = JsonSerializer.Deserialize<Config>(File.ReadAllText("./Config/config.json"),
JsonSerializerSettings.Default);
}
return _config;
}
public static ForcedStatic GetForcedStatic()
{
lock (_forcedStaticLock)
{
if (_forcedStatic == null)
_forcedStatic = Yaml.Deserializer
.Deserialize<ForcedStatic>(File.ReadAllText("./Config/forced_static.yaml"));
}
return _forcedStatic;
}
public static HashSet<string> GetStaticWeaponIds()
{
lock (_staticWeaponIdsLock)
{
if (_staticWeaponIds == null) _staticWeaponIds = GetForcedStatic().StaticWeaponIds.ToHashSet();
}
return _staticWeaponIds;
}
public static Dictionary<string, List<StaticForced>> GetForcedItems()
{
lock (_forcedItemsLock)
{
if (_forcedItems == null) _forcedItems = GetForcedStatic().ForcedItems;
}
return _forcedItems;
}
public static Dictionary<string, HashSet<string>> GetForcedLooseItems()
{
lock (_forcedLooseLock)
{
if (_forcedLoose == null)
_forcedLoose = Yaml.Deserializer.Deserialize<Dictionary<string, HashSet<string>>>(
File.ReadAllText("./Config/forced_loose.yaml"));
}
return _forcedLoose;
}
}

14
source/LootDumpProcessor/Model/Config/CollectorConfig.cs
View File

@ -1,10 +1,12 @@
using System.ComponentModel.DataAnnotations;
using JetBrains.Annotations;
using LootDumpProcessor.Process.Collector;
namespace LootDumpProcessor.Model.Config;
public class CollectorConfig
{
public CollectorType CollectorType { get; set; }
public int MaxEntitiesBeforeDumping { get; set; }
public string DumpLocation { get; set; }
}
[UsedImplicitly]
public record CollectorConfig(
[Required] CollectorType CollectorType,
[Required] int MaxEntitiesBeforeDumping,
[Required] string DumpLocation
);

26
source/LootDumpProcessor/Model/Config/Config.cs
View File

@ -1,15 +1,19 @@
using System.ComponentModel.DataAnnotations;
using JetBrains.Annotations;
namespace LootDumpProcessor.Model.Config;
public class Config
[UsedImplicitly]
public record Config
{
public string ServerLocation { get; set; } = string.Empty;
public bool ManualGarbageCollectionCalls { get; set; }
public DataStorageConfig DataStorageConfig { get; set; }
public ReaderConfig ReaderConfig { get; set; }
public ProcessorConfig ProcessorConfig { get; set; }
public DumpProcessorConfig DumpProcessorConfig { get; set; }
public WriterConfig WriterConfig { get; set; }
public CollectorConfig CollectorConfig { get; set; }
public Dictionary<string, string[]> ContainerIgnoreList { get; set; }
public List<string> MapsToProcess { get; set; }
[Required] public string ServerLocation { get; init; } = string.Empty;
[Required] public bool ManualGarbageCollectionCalls { get; init; }
[Required] public DataStorageConfig DataStorageConfig { get; init; } = null!;
[Required] public ReaderConfig ReaderConfig { get; init; } = null!;
[Required] public ProcessorConfig ProcessorConfig { get; init; } = null!;
[Required] public DumpProcessorConfig DumpProcessorConfig { get; init; } = null!;
[Required] public WriterConfig WriterConfig { get; init; } = null!;
[Required] public CollectorConfig CollectorConfig { get; init; } = null!;
[Required] public IReadOnlyDictionary<string, string[]> ContainerIgnoreList { get; init; } = null!;
[Required] public IReadOnlyList<string> MapsToProcess { get; init; } = null!;
}

12
source/LootDumpProcessor/Model/Config/DataStorageConfig.cs
View File

@ -1,10 +1,12 @@
using System.ComponentModel.DataAnnotations;
using JetBrains.Annotations;
using LootDumpProcessor.Storage;
namespace LootDumpProcessor.Model.Config;
public class DataStorageConfig
{
public DataStorageTypes DataStorageType { get; set; } = DataStorageTypes.File;
public string? FileDataStorageTempLocation { get; set; }
}
[UsedImplicitly]
public record DataStorageConfig(
[Required] string FileDataStorageTempLocation,
[Required] DataStorageTypes DataStorageType = DataStorageTypes.File
);

12
source/LootDumpProcessor/Model/Config/DumpProcessorConfig.cs
View File

@ -1,10 +1,12 @@
using System.Text.Json.Serialization;
using System.ComponentModel.DataAnnotations;
using System.Text.Json.Serialization;
using JetBrains.Annotations;
using LootDumpProcessor.Serializers.Json.Converters;
namespace LootDumpProcessor.Model.Config;
public class DumpProcessorConfig
{
[JsonConverter(typeof(NetDateTimeConverter))] public DateTime SpawnContainerChanceIncludeAfterDate { get; set; }
}
[UsedImplicitly]
public record DumpProcessorConfig(
[Required] [property: JsonConverter(typeof(NetDateTimeConverter))] DateTime SpawnContainerChanceIncludeAfterDate
);

13
source/LootDumpProcessor/Model/Config/ForcedStatic.cs
View File

@ -1,11 +1,14 @@
using System.Collections.Frozen;
using JetBrains.Annotations;
using LootDumpProcessor.Model.Output.StaticContainer;
using YamlDotNet.Serialization;
namespace LootDumpProcessor.Model.Config;
public class ForcedStatic
[UsedImplicitly]
public class ForcedStatic(
IReadOnlyList<string> staticWeaponIds, FrozenDictionary<string, IReadOnlyList<StaticForced>> forcedItems
)
{
[YamlMember(Alias = "static_weapon_ids")] public List<string> StaticWeaponIds { get; set; }
[YamlMember(Alias = "forced_items")] public Dictionary<string, List<StaticForced>> ForcedItems { get; set; }
public readonly IReadOnlyList<string> StaticWeaponIds = staticWeaponIds;
public readonly FrozenDictionary<string, IReadOnlyList<StaticForced>> ForcedItems = forcedItems;
}

13
source/LootDumpProcessor/Model/Config/ForcedStaticDto.cs Normal file
View File

@ -0,0 +1,13 @@
using JetBrains.Annotations;
using LootDumpProcessor.Model.Output.StaticContainer;
using YamlDotNet.Serialization;
namespace LootDumpProcessor.Model.Config;
[UsedImplicitly]
public class ForcedStaticDto
{
[YamlMember(Alias = "static_weapon_ids")] public List<string> StaticWeaponIds { get; set; }
[YamlMember(Alias = "forced_items")] public Dictionary<string, List<StaticForced>> ForcedItems { get; set; }
}

9
source/LootDumpProcessor/Model/Config/IntakeReaderConfig.cs
View File

@ -1,7 +1,6 @@
using JetBrains.Annotations;
namespace LootDumpProcessor.Model.Config;
public class IntakeReaderConfig
{
public int MaxDumpsPerMap { get; set; } = 1500;
public List<string> IgnoredDumpLocations { get; set; } = new();
}
[UsedImplicitly]
public record IntakeReaderConfig(IReadOnlyList<string> IgnoredDumpLocations, int MaxDumpsPerMap = 1500);

13
source/LootDumpProcessor/Model/Config/ProcessorConfig.cs
View File

@ -1,13 +1,6 @@
using System.Text.Json.Serialization;
using JetBrains.Annotations;
namespace LootDumpProcessor.Model.Config;
public class ProcessorConfig
{
[JsonPropertyName("spawnPointToleranceForForced")] public double SpawnPointToleranceForForced { get; set; } = 99D;
[JsonPropertyName("looseLootCountTolerancePercentage")]
public double LooseLootCountTolerancePercentage { get; set; } = 75D;
}
[UsedImplicitly]
public record ProcessorConfig(double SpawnPointToleranceForForced = 99, double LooseLootCountTolerancePercentage = 75);

17
source/LootDumpProcessor/Model/Config/ReaderConfig.cs
View File

@ -1,9 +1,12 @@
using System.ComponentModel.DataAnnotations;
using JetBrains.Annotations;
namespace LootDumpProcessor.Model.Config;
public class ReaderConfig
{
public IntakeReaderConfig? IntakeReaderConfig { get; set; }
public List<string>? DumpFilesLocation { get; set; }
public string? ThresholdDate { get; set; }
public bool ProcessSubFolders { get; set; }
}
[UsedImplicitly]
public record ReaderConfig(
[Required] IntakeReaderConfig IntakeReaderConfig,
[Required] IReadOnlyList<string> DumpFilesLocation,
string? ThresholdDate,
bool ProcessSubFolders = true
);

9
source/LootDumpProcessor/Model/Config/WriterConfig.cs
View File

@ -1,9 +1,8 @@
using System.Text.Json.Serialization;
using System.ComponentModel.DataAnnotations;
using JetBrains.Annotations;
namespace LootDumpProcessor.Model.Config;
public class WriterConfig
{
[JsonPropertyName("outputLocation")] public string? OutputLocation { get; set; }
}
[UsedImplicitly]
public record WriterConfig([Required] string OutputLocation);

18
source/LootDumpProcessor/Process/Collector/CollectorFactory.cs
View File

@ -1,18 +0,0 @@
namespace LootDumpProcessor.Process.Collector;
public static class CollectorFactory
{
private static ICollector? _collector;
public static ICollector GetInstance()
{
if (_collector == null)
_collector = LootDumpProcessorContext.GetConfig().CollectorConfig.CollectorType switch
{
CollectorType.Memory => new HashSetCollector(),
CollectorType.Dump => new DumpCollector(),
_ => throw new ArgumentOutOfRangeException()
};
return _collector;
}
}

26
source/LootDumpProcessor/Process/Collector/DumpCollector.cs
View File

@ -1,16 +1,18 @@
using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Serializers.Json;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Collector;
public class DumpCollector : ICollector
public class DumpCollector(IOptions<Config> config) : ICollector
{
private static readonly string DumpLocation =
$"{LootDumpProcessorContext.GetConfig().CollectorConfig.DumpLocation}/collector/";
private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
private readonly List<PartialData> processedDumps =
new(LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping + 50);
private string DumpLocation => Path.Combine(_config.CollectorConfig.DumpLocation, "collector");
private List<PartialData> ProcessedDumps => new(_config.CollectorConfig.MaxEntitiesBeforeDumping + 50);
private readonly object lockObject = new();
@ -25,13 +27,13 @@ public class DumpCollector : ICollector
{
lock (lockObject)
{
processedDumps.Add(parsedDump);
if (processedDumps.Count > LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping)
ProcessedDumps.Add(parsedDump);
if (ProcessedDumps.Count > _config.CollectorConfig.MaxEntitiesBeforeDumping)
{
var fileName = $"collector-{DateTime.Now.ToString("yyyyMMddHHmmssfffff")}.json";
File.WriteAllText($"{DumpLocation}{fileName}",
JsonSerializer.Serialize(processedDumps, JsonSerializerSettings.Default));
processedDumps.Clear();
JsonSerializer.Serialize(ProcessedDumps, JsonSerializerSettings.Default));
ProcessedDumps.Clear();
}
}
}
@ -39,10 +41,10 @@ public class DumpCollector : ICollector
public List<PartialData> Retrieve()
{
foreach (var file in Directory.GetFiles(DumpLocation))
processedDumps.AddRange(JsonSerializer
ProcessedDumps.AddRange(JsonSerializer
.Deserialize<List<PartialData>>(File.ReadAllText(file), JsonSerializerSettings.Default));
return processedDumps;
return ProcessedDumps;
}
public void Clear()
@ -51,7 +53,7 @@ public class DumpCollector : ICollector
{
foreach (var file in Directory.GetFiles(DumpLocation)) File.Delete(file);
processedDumps.Clear();
ProcessedDumps.Clear();
}
}
}

6
source/LootDumpProcessor/Process/IKeyGenerator.cs
View File

@ -1,6 +0,0 @@
namespace LootDumpProcessor.Process;
public interface IKeyGenerator
{
string Generate();
}

3
source/LootDumpProcessor/Process/Processor/v2/AmmoProcessor/AmmoProcessor.cs → source/LootDumpProcessor/Process/Processor/AmmoProcessor/AmmoProcessor.cs
View File

@ -1,8 +1,9 @@
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Services.TarkovItemsProvider;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
namespace LootDumpProcessor.Process.Processor.AmmoProcessor;
public class AmmoProcessor(ILogger<AmmoProcessor> logger, ITarkovItemsProvider tarkovItemsProvider) : IAmmoProcessor
{

2
source/LootDumpProcessor/Process/Processor/v2/AmmoProcessor/IAmmoProcessor.cs → source/LootDumpProcessor/Process/Processor/AmmoProcessor/IAmmoProcessor.cs
View File

@ -1,7 +1,7 @@
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
namespace LootDumpProcessor.Process.Processor.AmmoProcessor;
public interface IAmmoProcessor
{

48
source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
View File

@ -1,20 +1,23 @@
using System.Collections.Concurrent;
using System.Text.Json;
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
using LootDumpProcessor.Process.Processor.v2.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
using LootDumpProcessor.Process.Processor.AmmoProcessor;
using LootDumpProcessor.Process.Processor.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.StaticContainersProcessor;
using LootDumpProcessor.Process.Processor.StaticLootProcessor;
using LootDumpProcessor.Process.Services.KeyGenerator;
using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage.Collections;
using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Processor.DumpProcessor;
@ -23,7 +26,8 @@ public class MultithreadSteppedDumpProcessor(
IStaticContainersProcessor staticContainersProcessor,
IAmmoProcessor ammoProcessor,
ILooseLootProcessor looseLootProcessor,
ILogger<MultithreadSteppedDumpProcessor> logger, IKeyGenerator keyGenerator
ILogger<MultithreadSteppedDumpProcessor> logger, IKeyGenerator keyGenerator, IDataStorage dataStorage,
IOptions<Config> config
)
: IDumpProcessor
{
@ -45,7 +49,9 @@ public class MultithreadSteppedDumpProcessor(
private readonly IKeyGenerator
_keyGenerator = keyGenerator ?? throw new ArgumentNullException(nameof(keyGenerator));
private static readonly IDataStorage _dataStorage = DataStorageFactory.GetInstance();
private readonly IDataStorage _dataStorage = dataStorage ?? throw new ArgumentNullException(nameof(dataStorage));
private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
public async Task<Dictionary<OutputFileType, object>> ProcessDumps(List<PartialData> dumps)
{
@ -111,12 +117,12 @@ public class MultithreadSteppedDumpProcessor(
_logger.LogInformation("Processing loose loot distribution");
var looseLoot = new ConcurrentDictionary<string, LooseLootRoot>();
Parallel.ForEach(dumpProcessData.MapCounts.Keys, parallelOptions, mapId =>
await Parallel.ForEachAsync(dumpProcessData.MapCounts.Keys, parallelOptions, async (mapId, _) =>
{
var mapCount = dumpProcessData.MapCounts[mapId];
var looseLootCount = dumpProcessData.LooseLootCounts[mapId];
var looseLootDistribution =
_looseLootProcessor.CreateLooseLootDistribution(mapId, mapCount, looseLootCount);
await _looseLootProcessor.CreateLooseLootDistribution(mapId, mapCount, looseLootCount);
looseLoot[mapId] = looseLootDistribution;
});
_logger.LogInformation("Collecting loose loot distribution information");
@ -159,8 +165,7 @@ public class MultithreadSteppedDumpProcessor(
}
else
{
var mapStaticContainers =
_staticContainersProcessor.CreateStaticWeaponsAndForcedContainers(dataDump);
var mapStaticContainers = await _staticContainersProcessor.CreateStaticWeaponsAndForcedContainers(dataDump);
var newStaticWeapons = mapStaticContainers.StaticWeapons.Where(x =>
!mapStaticLoot.StaticWeapons.Exists(y => y.Id == x.Id));
@ -172,7 +177,7 @@ public class MultithreadSteppedDumpProcessor(
}
if (!mapStaticContainersAggregated.TryGetValue(mapId,
out ConcurrentDictionary<Template, int> mapAggregatedDataDict))
out var mapAggregatedDataDict))
{
mapAggregatedDataDict = new ConcurrentDictionary<Template, int>();
mapStaticContainersAggregated.TryAdd(mapId, mapAggregatedDataDict);
@ -182,9 +187,9 @@ public class MultithreadSteppedDumpProcessor(
IncrementMapCounterDictionaryValue(mapDumpCounter, mapId);
var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList
var containerIgnoreListExists = _config.ContainerIgnoreList
.TryGetValue(mapId, out var ignoreListForMap);
foreach (var dynamicStaticContainer in _staticContainersProcessor.CreateDynamicStaticContainers(
foreach (var dynamicStaticContainer in await _staticContainersProcessor.CreateDynamicStaticContainers(
dataDump))
{
if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id)) continue;
@ -193,16 +198,17 @@ public class MultithreadSteppedDumpProcessor(
mapAggregatedDataDict[dynamicStaticContainer] += 1;
}
GCHandler.Collect();
if (_config.ManualGarbageCollectionCalls) GC.Collect();
}
private static bool DumpWasMadeAfterConfigThresholdDate(PartialData dataDump) =>
private bool DumpWasMadeAfterConfigThresholdDate(PartialData dataDump) =>
FileDateParser.TryParseFileDate(dataDump.BasicInfo.FileName, out var fileDate) &&
fileDate.HasValue &&
fileDate.Value > LootDumpProcessorContext.GetConfig().DumpProcessorConfig
fileDate.Value > _config.DumpProcessorConfig
.SpawnContainerChanceIncludeAfterDate;
private static void IncrementMapCounterDictionaryValue(ConcurrentDictionary<string, int> mapDumpCounter, string mapName)
private static void IncrementMapCounterDictionaryValue(ConcurrentDictionary<string, int> mapDumpCounter,
string mapName)
{
if (!mapDumpCounter.TryAdd(mapName, 1)) mapDumpCounter[mapName] += 1;
}
@ -247,7 +253,7 @@ public class MultithreadSteppedDumpProcessor(
partialFileMetaData = null;
tuple = null;
GCHandler.Collect();
if (_config.ManualGarbageCollectionCalls) GC.Collect();
var parallelOptions = new ParallelOptions
{
@ -266,14 +272,14 @@ public class MultithreadSteppedDumpProcessor(
_dataStorage.Store(dictionaryCounts);
dictionaryCounts = null;
GCHandler.Collect();
if (_config.ManualGarbageCollectionCalls) GC.Collect();
_dataStorage.Store(actualDictionaryItemProperties);
actualDictionaryItemProperties = null;
GCHandler.Collect();
if (_config.ManualGarbageCollectionCalls) GC.Collect();
_dataStorage.Store(looseLootCounts);
looseLootCounts = null;
GCHandler.Collect();
if (_config.ManualGarbageCollectionCalls) GC.Collect();
});
return dumpProcessData;
}

43
source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessor.cs
View File

@ -1,32 +1,33 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Processor.v2.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
using LootDumpProcessor.Process.Processor.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.StaticLootProcessor;
using LootDumpProcessor.Storage;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.FileProcessor;
public class FileProcessor : IFileProcessor
{
private readonly IStaticLootProcessor _staticLootProcessor;
private readonly ILooseLootProcessor _looseLootProcessor;
private readonly ILogger<FileProcessor> _logger;
public FileProcessor(
public class FileProcessor(
IStaticLootProcessor staticLootProcessor,
ILooseLootProcessor looseLootProcessor,
ILogger<FileProcessor> logger)
ILogger<FileProcessor> logger, IDataStorage dataStorage
)
: IFileProcessor
{
_staticLootProcessor = staticLootProcessor
?? throw new ArgumentNullException(nameof(staticLootProcessor));
_looseLootProcessor = looseLootProcessor
?? throw new ArgumentNullException(nameof(looseLootProcessor));
_logger = logger
?? throw new ArgumentNullException(nameof(logger));
}
private readonly IStaticLootProcessor _staticLootProcessor = staticLootProcessor
?? throw new ArgumentNullException(
nameof(staticLootProcessor));
public PartialData Process(BasicInfo parsedData)
private readonly ILooseLootProcessor _looseLootProcessor = looseLootProcessor
?? throw new ArgumentNullException(
nameof(looseLootProcessor));
private readonly ILogger<FileProcessor> _logger = logger
?? throw new ArgumentNullException(nameof(logger));
private readonly IDataStorage _dataStorage = dataStorage ?? throw new ArgumentNullException(nameof(dataStorage));
public async Task<PartialData> Process(BasicInfo parsedData)
{
_logger.LogDebug("Processing file {FileName}...", parsedData.FileName);
@ -50,16 +51,16 @@ public class FileProcessor : IFileProcessor
ParsedDumpKey = (AbstractKey)dumpData.GetKey()
};
if (!DataStorageFactory.GetInstance().Exists(dumpData.GetKey()))
if (!_dataStorage.Exists(dumpData.GetKey()))
{
_logger.LogDebug(
"Cache not found for {LookupIndex} processing.",
string.Join("/", dumpData.GetKey().GetLookupIndex())
);
dumpData.Containers = _staticLootProcessor.PreProcessStaticLoot(staticLoot);
dumpData.Containers = await _staticLootProcessor.PreProcessStaticLoot(staticLoot);
dumpData.LooseLoot = _looseLootProcessor.PreProcessLooseLoot(looseLoot);
DataStorageFactory.GetInstance().Store(dumpData);
dataStorage.Store(dumpData);
}
_logger.LogDebug("File {FileName} finished processing!", parsedData.FileName);

2
source/LootDumpProcessor/Process/Processor/FileProcessor/IFileProcessor.cs
View File

@ -4,5 +4,5 @@ namespace LootDumpProcessor.Process.Processor.FileProcessor;
public interface IFileProcessor
{
PartialData Process(BasicInfo parsedData);
Task<PartialData> Process(BasicInfo parsedData);
}

8
source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/ILooseLootProcessor.cs → source/LootDumpProcessor/Process/Processor/LooseLootProcessor/ILooseLootProcessor.cs
View File

@ -3,15 +3,13 @@ using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Storage;
namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor;
namespace LootDumpProcessor.Process.Processor.LooseLootProcessor;
public interface ILooseLootProcessor
{
PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseLoot);
LooseLootRoot CreateLooseLootDistribution(
string mapId,
Task<LooseLootRoot> CreateLooseLootDistribution(string mapId,
int mapCount,
IKey looseLootCountKey
);
IKey looseLootCountKey);
}

34
source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/LooseLootProcessor.cs → source/LootDumpProcessor/Process/Processor/LooseLootProcessor/LooseLootProcessor.cs
View File

@ -1,17 +1,24 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Services.ComposedKeyGenerator;
using LootDumpProcessor.Process.Services.ForcedItemsProvider;
using LootDumpProcessor.Process.Services.KeyGenerator;
using LootDumpProcessor.Process.Services.TarkovItemsProvider;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage.Collections;
using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor;
namespace LootDumpProcessor.Process.Processor.LooseLootProcessor;
public class LooseLootProcessor(
ILogger<LooseLootProcessor> logger, IDataStorage dataStorage, ITarkovItemsProvider tarkovItemsProvider,
IComposedKeyGenerator composedKeyGenerator, IKeyGenerator keyGenerator
IComposedKeyGenerator composedKeyGenerator, IKeyGenerator keyGenerator, IOptions<Config> config,
IForcedItemsProvider forcedItemsProvider
)
: ILooseLootProcessor
{
@ -30,6 +37,11 @@ public class LooseLootProcessor(
private readonly IKeyGenerator
_keyGenerator = keyGenerator ?? throw new ArgumentNullException(nameof(keyGenerator));
private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
private readonly IForcedItemsProvider _forcedItemsProvider =
forcedItemsProvider ?? throw new ArgumentNullException(nameof(forcedItemsProvider));
public PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseLoot)
{
var preProcessedLoot = new PreProcessedLooseLoot
@ -70,25 +82,23 @@ public class LooseLootProcessor(
return preProcessedLoot;
}
public LooseLootRoot CreateLooseLootDistribution(
string mapId,
public async Task<LooseLootRoot> CreateLooseLootDistribution(string mapId,
int mapCount,
IKey looseLootCountKey
)
IKey looseLootCountKey)
{
var config = LootDumpProcessorContext.GetConfig();
var spawnPointTolerance = config.ProcessorConfig.SpawnPointToleranceForForced / 100.0;
var spawnPointTolerance = _config.ProcessorConfig.SpawnPointToleranceForForced / 100.0;
var looseLootDistribution = new LooseLootRoot();
var probabilities = new Dictionary<string, double>();
var looseLootCountsItem = _dataStorage.GetItem<LooseLootCounts>(looseLootCountKey);
var forcedLooseItems = await _forcedItemsProvider.GetForcedLooseItems();
var counts = _dataStorage.GetItem<FlatKeyableDictionary<string, int>>(looseLootCountsItem.Counts);
foreach (var (itemId, count) in counts) probabilities[itemId] = (double)count / mapCount;
var spawnPointCount = looseLootCountsItem.MapSpawnpointCount.Select(Convert.ToDouble).ToList();
var initialMean = CalculateMean(spawnPointCount);
var tolerancePercentage = config.ProcessorConfig.LooseLootCountTolerancePercentage / 100.0;
var tolerancePercentage = _config.ProcessorConfig.LooseLootCountTolerancePercentage / 100.0;
var highThreshold = initialMean * (1 + tolerancePercentage);
looseLootCountsItem.MapSpawnpointCount = looseLootCountsItem.MapSpawnpointCount
@ -137,7 +147,7 @@ public class LooseLootProcessor(
if (itemDistributions.Count == 1 &&
(_tarkovItemsProvider.IsQuestItem(itemDistributions[0].ComposedKey?.FirstItem?.Tpl) ||
LootDumpProcessorContext.GetForcedLooseItems()[mapId]
forcedLooseItems[mapId]
.Contains(itemDistributions[0].ComposedKey?.FirstItem?.Tpl)))
{
var forcedSpawnPoint = new SpawnPointsForced
@ -160,7 +170,7 @@ public class LooseLootProcessor(
_logger.LogWarning(
"Item: {ItemId} has > {Tolerance}% spawn chance in spawn point: {LocationId} but isn't in forced loot, adding to forced",
templateCopy.Id,
config.ProcessorConfig.SpawnPointToleranceForForced,
_config.ProcessorConfig.SpawnPointToleranceForForced,
forcedSpawnPoint.LocationId
);
}
@ -210,7 +220,7 @@ public class LooseLootProcessor(
.ToList();
var configuredForcedTemplates =
new HashSet<string>(LootDumpProcessorContext.GetForcedLooseItems()[mapId].Select(item => item));
new HashSet<string>(forcedLooseItems[mapId].Select(item => item));
var foundForcedTemplates =
new HashSet<string>(looseLootDistribution.SpawnPointsForced.Select(fp => fp.Template.Items[0].Tpl));

11
source/LootDumpProcessor/Process/Processor/StaticContainersProcessor/IStaticContainersProcessor.cs Normal file
View File

@ -0,0 +1,11 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output.StaticContainer;
namespace LootDumpProcessor.Process.Processor.StaticContainersProcessor;
public interface IStaticContainersProcessor
{
Task<MapStaticLoot> CreateStaticWeaponsAndForcedContainers(RootData rawMapDump);
Task<IReadOnlyList<Template>> CreateDynamicStaticContainers(RootData rawMapDump);
}

34
source/LootDumpProcessor/Process/Processor/v2/StaticContainersProcessor/StaticContainerProcessor.cs → source/LootDumpProcessor/Process/Processor/StaticContainersProcessor/StaticContainerProcessor.cs
View File

@ -1,21 +1,24 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Process.Services.ForcedItemsProvider;
using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
namespace LootDumpProcessor.Process.Processor.StaticContainersProcessor;
public class StaticContainersProcessor : IStaticContainersProcessor
public class StaticContainersProcessor(
ILogger<StaticContainersProcessor> logger, IForcedItemsProvider forcedItemsProvider
)
: IStaticContainersProcessor
{
private readonly ILogger<StaticContainersProcessor> _logger;
private readonly ILogger<StaticContainersProcessor> _logger =
logger ?? throw new ArgumentNullException(nameof(logger));
public StaticContainersProcessor(ILogger<StaticContainersProcessor> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
private readonly IForcedItemsProvider _forcedItemsProvider =
forcedItemsProvider ?? throw new ArgumentNullException(nameof(forcedItemsProvider));
public MapStaticLoot CreateStaticWeaponsAndForcedContainers(RootData rawMapDump)
public async Task<MapStaticLoot> CreateStaticWeaponsAndForcedContainers(RootData rawMapDump)
{
var locationLoot = rawMapDump.Data.LocationLoot;
var mapId = locationLoot.Id.ToLower();
@ -25,6 +28,7 @@ public class StaticContainersProcessor : IStaticContainersProcessor
.ToList();
var staticWeapons = new List<Template>();
var forcedStatic = await _forcedItemsProvider.GetForcedStatic();
foreach (var lootPosition in staticLootPositions)
{
@ -36,7 +40,7 @@ public class StaticContainersProcessor : IStaticContainersProcessor
var firstItemTpl = lootPosition.Items.First().Tpl;
if (!LootDumpProcessorContext.GetStaticWeaponIds().Contains(firstItemTpl))
if (!forcedStatic.StaticWeaponIds.Contains(firstItemTpl))
continue;
var copiedLoot = ProcessorUtil.Copy(lootPosition);
@ -44,26 +48,28 @@ public class StaticContainersProcessor : IStaticContainersProcessor
_logger.LogDebug("Added static weapon with ID {WeaponId} to Map {MapId}.", copiedLoot.Id, mapId);
}
var forcedStaticItems = LootDumpProcessorContext.GetForcedItems()
.TryGetValue(mapId, out List<StaticForced>? forcedItems)
var forcedStaticItems = forcedStatic.ForcedItems
.TryGetValue(mapId, out var forcedItems)
? forcedItems
: new List<StaticForced>();
var mapStaticLoot = new MapStaticLoot
{
StaticWeapons = staticWeapons,
StaticForced = forcedStaticItems
StaticForced = forcedStaticItems.ToList()
};
_logger.LogInformation("Created static weapons and forced containers for Map {MapId}.", mapId);
return mapStaticLoot;
}
public IReadOnlyList<Template> CreateDynamicStaticContainers(RootData rawMapDump)
public async Task<IReadOnlyList<Template>> CreateDynamicStaticContainers(RootData rawMapDump)
{
var forcedStatic = await _forcedItemsProvider.GetForcedStatic();
var dynamicContainers = rawMapDump.Data.LocationLoot.Loot
.Where(loot => loot.IsContainer &&
!LootDumpProcessorContext.GetStaticWeaponIds().Contains(loot.Items.FirstOrDefault()?.Tpl))
!forcedStatic.StaticWeaponIds.Contains(loot.Items.FirstOrDefault()?.Tpl))
.ToList();
foreach (var container in dynamicContainers)

4
source/LootDumpProcessor/Process/Processor/v2/StaticLootProcessor/IStaticLootProcessor.cs → source/LootDumpProcessor/Process/Processor/StaticLootProcessor/IStaticLootProcessor.cs
View File

@ -2,11 +2,11 @@
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
namespace LootDumpProcessor.Process.Processor.StaticLootProcessor;
public interface IStaticLootProcessor
{
IReadOnlyList<PreProcessedStaticLoot> PreProcessStaticLoot(IReadOnlyList<Template> staticLoot);
Task<IReadOnlyList<PreProcessedStaticLoot>> PreProcessStaticLoot(IReadOnlyList<Template> staticLoot);
IReadOnlyDictionary<string, StaticItemDistribution> CreateStaticLootDistribution(
string mapName,

14
source/LootDumpProcessor/Process/Processor/v2/StaticLootProcessor/StaticLootProcessor.cs → source/LootDumpProcessor/Process/Processor/StaticLootProcessor/StaticLootProcessor.cs
View File

@ -1,16 +1,21 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Services.ForcedItemsProvider;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
namespace LootDumpProcessor.Process.Processor.StaticLootProcessor;
public class StaticLootProcessor(ILogger<StaticLootProcessor> logger) : IStaticLootProcessor
public class StaticLootProcessor(ILogger<StaticLootProcessor> logger, IForcedItemsProvider forcedItemsProvider)
: IStaticLootProcessor
{
private readonly ILogger<StaticLootProcessor> _logger =
logger ?? throw new ArgumentNullException(nameof(logger));
public IReadOnlyList<PreProcessedStaticLoot> PreProcessStaticLoot(IReadOnlyList<Template> staticLoot)
private readonly IForcedItemsProvider _forcedItemsProvider =
forcedItemsProvider ?? throw new ArgumentNullException(nameof(forcedItemsProvider));
public async Task<IReadOnlyList<PreProcessedStaticLoot>> PreProcessStaticLoot(IReadOnlyList<Template> staticLoot)
{
var nonWeaponContainers = new List<PreProcessedStaticLoot>();
@ -24,7 +29,8 @@ public class StaticLootProcessor(ILogger<StaticLootProcessor> logger) : IStaticL
var firstItemTpl = lootSpawn.Items[0].Tpl;
if (!LootDumpProcessorContext.GetStaticWeaponIds().Contains(firstItemTpl))
var forcedStatic = await _forcedItemsProvider.GetForcedStatic();
if (!forcedStatic.StaticWeaponIds.Contains(firstItemTpl))
{
nonWeaponContainers.Add(new PreProcessedStaticLoot
{

11
source/LootDumpProcessor/Process/Processor/v2/StaticContainersProcessor/IStaticContainersProcessor.cs
View File

@ -1,11 +0,0 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output.StaticContainer;
namespace LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
public interface IStaticContainersProcessor
{
MapStaticLoot CreateStaticWeaponsAndForcedContainers(RootData rawMapDump);
IReadOnlyList<Template> CreateDynamicStaticContainers(RootData rawMapDump);
}

40
source/LootDumpProcessor/Process/QueuePipeline.cs
View File

@ -1,6 +1,7 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Process.Collector;
using LootDumpProcessor.Process.Processor.DumpProcessor;
@ -12,12 +13,13 @@ using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process;
public class QueuePipeline(
IFileProcessor fileProcessor, IDumpProcessor dumpProcessor, ILogger<QueuePipeline> logger, IFileFilter fileFilter,
IIntakeReader intakeReader
IIntakeReader intakeReader, ICollector collector, IDataStorage dataStorage, IOptions<Config> config, IWriter writer
)
: IPipeline
{
@ -34,25 +36,32 @@ public class QueuePipeline(
private readonly IIntakeReader
_intakeReader = intakeReader ?? throw new ArgumentNullException(nameof(intakeReader));
private readonly ICollector _collector = collector ?? throw new ArgumentNullException(nameof(collector));
private readonly IDataStorage _dataStorage = dataStorage ?? throw new ArgumentNullException(nameof(dataStorage));
private readonly IWriter _writer = writer ?? throw new ArgumentNullException(nameof(writer));
private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
private readonly List<string> _filesToRename = new();
private readonly BlockingCollection<string> _filesToProcess = new();
private readonly List<string> _mapNames = LootDumpProcessorContext.GetConfig().MapsToProcess;
private IReadOnlyList<string> MapNames => _config.MapsToProcess;
public async Task Execute()
{
var stopwatch = Stopwatch.StartNew();
// Single collector instance to collect results
var collector = CollectorFactory.GetInstance();
collector.Setup();
_collector.Setup();
_logger.LogInformation("Gathering files to begin processing");
try
{
await FixFilesFromDumps();
foreach (var mapName in _mapNames) await ProcessFilesFromDumpsPerMap(collector, mapName);
foreach (var mapName in MapNames) await ProcessFilesFromDumpsPerMap(_collector, mapName);
}
finally
{
@ -64,7 +73,7 @@ public class QueuePipeline(
private List<string> GatherFiles()
{
// Read locations
var inputPath = LootDumpProcessorContext.GetConfig().ReaderConfig.DumpFilesLocation;
var inputPath = _config.ReaderConfig.DumpFilesLocation;
if (inputPath == null || inputPath.Count == 0)
throw new Exception("Reader dumpFilesLocations must be set to a valid value");
@ -101,12 +110,12 @@ public class QueuePipeline(
return gatheredFiles;
}
private Queue<string> GetFileQueue(List<string> inputPath)
private Queue<string> GetFileQueue(IReadOnlyList<string> inputPath)
{
var queuedPathsToProcess = new Queue<string>();
var queuedFilesToProcess = new Queue<string>();
inputPath.ForEach(p => queuedPathsToProcess.Enqueue(p));
foreach (var path in inputPath) queuedPathsToProcess.Enqueue(path);
while (queuedPathsToProcess.TryDequeue(out var path))
{
@ -114,7 +123,7 @@ public class QueuePipeline(
if (!Directory.Exists(path)) throw new Exception($"Input directory \"{path}\" could not be found");
// If we should process subfolder, queue them up as well
if (LootDumpProcessorContext.GetConfig().ReaderConfig.ProcessSubFolders)
if (_config.ReaderConfig.ProcessSubFolders)
foreach (var directory in Directory.GetDirectories(path))
queuedPathsToProcess.Enqueue(directory);
@ -138,13 +147,13 @@ public class QueuePipeline(
_logger.LogInformation("Files sorted and ready to begin pre-processing");
Parallel.ForEach(_filesToProcess, file =>
await Parallel.ForEachAsync(_filesToProcess, async (file, _) =>
{
try
{
if (!_intakeReader.Read(file, out var basicInfo)) return;
var partialData = _fileProcessor.Process(basicInfo);
var partialData = await _fileProcessor.Process(basicInfo);
collector.Hold(partialData);
}
catch (Exception e)
@ -156,15 +165,14 @@ public class QueuePipeline(
_logger.LogInformation("Pre-processing finished");
// Single writer instance to collect results
var writer = WriterFactory.GetInstance();
// Single collector instance to collect results
var partialData = collector.Retrieve();
var processedDumps = await _dumpProcessor.ProcessDumps(partialData);
writer.WriteAll(processedDumps);
_writer.WriteAll(processedDumps);
// clear collector and datastorage as we process per map now
collector.Clear();
DataStorageFactory.GetInstance().Clear();
_dataStorage.Clear();
}
/// <summary>
@ -172,7 +180,7 @@ public class QueuePipeline(
/// </summary>
private async Task FixFilesFromDumps()
{
var inputPath = LootDumpProcessorContext.GetConfig().ReaderConfig.DumpFilesLocation;
var inputPath = _config.ReaderConfig.DumpFilesLocation;
if (inputPath == null || inputPath.Count == 0)
throw new Exception("Reader dumpFilesLocations must be set to a valid value");
@ -181,7 +189,7 @@ public class QueuePipeline(
await Parallel.ForEachAsync(_filesToRename, async (file, cancellationToken) =>
{
if (_mapNames.Any(file.Contains)) return;
if (MapNames.Any(file.Contains)) return;
try
{

10
source/LootDumpProcessor/Process/Reader/Filters/JsonDumpFileFilter.cs
View File

@ -1,6 +1,8 @@
using System.Globalization;
using System.Text.RegularExpressions;
using LootDumpProcessor.Model.Config;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Reader.Filters;
@ -9,12 +11,14 @@ public class JsonDumpFileFilter : IFileFilter
private readonly ILogger<JsonDumpFileFilter> _logger;
private readonly Regex _fileNameDateRegex = new("([0-9]{4}(-[0-9]{2}){2}_((-){0,1}[0-9]{2}){3})");
private readonly DateTime _parsedThresholdDate;
private readonly Config _config;
public JsonDumpFileFilter(ILogger<JsonDumpFileFilter> logger)
public JsonDumpFileFilter(ILogger<JsonDumpFileFilter> logger, IOptions<Config> config)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
// Calculate parsed date from config threshold
if (string.IsNullOrEmpty(LootDumpProcessorContext.GetConfig().ReaderConfig.ThresholdDate))
if (string.IsNullOrEmpty(_config.ReaderConfig.ThresholdDate))
{
_logger.LogWarning("ThresholdDate is null or empty in configs, defaulting to current day minus 30 days");
_parsedThresholdDate = DateTime.Now - TimeSpan.FromDays(30);
@ -22,7 +26,7 @@ public class JsonDumpFileFilter : IFileFilter
else
{
_parsedThresholdDate = DateTime.ParseExact(
LootDumpProcessorContext.GetConfig().ReaderConfig.ThresholdDate,
_config.ReaderConfig.ThresholdDate,
"yyyy-MM-dd",
CultureInfo.InvariantCulture
);

14
source/LootDumpProcessor/Process/Reader/Intake/JsonFileIntakeReader.cs
View File

@ -1,19 +1,23 @@
using System.Collections.Concurrent;
using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Reader.Intake;
public class JsonFileIntakeReader(ILogger<JsonFileIntakeReader> logger) : IIntakeReader
public class JsonFileIntakeReader(ILogger<JsonFileIntakeReader> logger, IOptions<Config> config) : IIntakeReader
{
private readonly ILogger<JsonFileIntakeReader> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
private readonly HashSet<string>? _ignoredLocations = LootDumpProcessorContext.GetConfig()
.ReaderConfig.IntakeReaderConfig?.IgnoredDumpLocations.ToHashSet();
private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
private HashSet<string> IgnoredLocations => _config
.ReaderConfig.IntakeReaderConfig.IgnoredDumpLocations.ToHashSet();
private readonly ConcurrentDictionary<string, int> _totalMapDumpsCounter = new();
@ -43,14 +47,14 @@ public class JsonFileIntakeReader(ILogger<JsonFileIntakeReader> logger) : IIntak
_logger.LogError("Could not parse date from file: {File}", file);
var fi = JsonSerializer.Deserialize<RootData>(fileData, JsonSerializerSettings.Default);
if (fi?.Data.LocationLoot.Name != null && (!_ignoredLocations?.Contains(fi.Data.LocationLoot.Name) ?? true))
if (fi?.Data.LocationLoot.Name != null && (!IgnoredLocations?.Contains(fi.Data.LocationLoot.Name) ?? true))
{
var mapName = fi.Data.LocationLoot.Name;
var mapId = fi.Data.LocationLoot.Id.ToLower();
var counter = _totalMapDumpsCounter.AddOrUpdate(mapName, 0, (_, current) => current);
var maxDumpsPerMap = LootDumpProcessorContext.GetConfig()
var maxDumpsPerMap = _config
.ReaderConfig.IntakeReaderConfig?.MaxDumpsPerMap ?? 1500;
if (counter < maxDumpsPerMap)

4
source/LootDumpProcessor/Process/ComposedKeyGenerator.cs → source/LootDumpProcessor/Process/Services/ComposedKeyGenerator/ComposedKeyGenerator.cs
View File

@ -1,8 +1,10 @@
using System.Globalization;
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Services.KeyGenerator;
using LootDumpProcessor.Process.Services.TarkovItemsProvider;
namespace LootDumpProcessor.Process;
namespace LootDumpProcessor.Process.Services.ComposedKeyGenerator;
public class ComposedKeyGenerator(ITarkovItemsProvider tarkovItemsProvider, IKeyGenerator keyGenerator)
: IComposedKeyGenerator

2
source/LootDumpProcessor/Process/IComposedKeyGenerator.cs → source/LootDumpProcessor/Process/Services/ComposedKeyGenerator/IComposedKeyGenerator.cs
View File

@ -1,6 +1,6 @@
using LootDumpProcessor.Model;
namespace LootDumpProcessor.Process;
namespace LootDumpProcessor.Process.Services.ComposedKeyGenerator;
public interface IComposedKeyGenerator
{

56
source/LootDumpProcessor/Process/Services/ForcedItemsProvider/ForcedItemsProvider.cs Normal file
View File

@ -0,0 +1,56 @@
using System.Collections.Frozen;
using System.Collections.Immutable;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Serializers.Yaml;
namespace LootDumpProcessor.Process.Services.ForcedItemsProvider;
public class ForcedItemsProvider : IForcedItemsProvider
{
private static readonly string ForcedStaticPath = Path.Combine("Config", "forced_static.yaml");
private static readonly string ForcedLooseLootPath = Path.Combine("Config", "forced_loose.yaml");
private ForcedStatic? _forcedStatic;
private FrozenDictionary<string, ImmutableHashSet<string>>? _forcedLoose;
public async Task<ForcedStatic> GetForcedStatic()
{
if (_forcedStatic is not null) return _forcedStatic;
_forcedStatic = await ReadForcedStatic();
return _forcedStatic;
}
private async Task<ForcedStatic> ReadForcedStatic()
{
var forcedStaticContent = await File.ReadAllTextAsync(ForcedStaticPath);
// Workaround needed because YamlDotNet cannot deserialize properly
var forcedStaticDto = Yaml.Deserializer.Deserialize<ForcedStaticDto>(forcedStaticContent);
var forcedStatic = new ForcedStatic(
forcedStaticDto.StaticWeaponIds.AsReadOnly(),
forcedStaticDto.ForcedItems.ToFrozenDictionary(
kvp => kvp.Key,
IReadOnlyList<StaticForced> (kvp) => kvp.Value.AsReadOnly()
));
return forcedStatic;
}
public async Task<FrozenDictionary<string, ImmutableHashSet<string>>> GetForcedLooseItems()
{
if (_forcedLoose is not null) return _forcedLoose;
_forcedLoose = await ReadForcedLooseItems();
return _forcedLoose;
}
private async Task<FrozenDictionary<string, ImmutableHashSet<string>>> ReadForcedLooseItems()
{
var forcedLooseContent = await File.ReadAllTextAsync(ForcedLooseLootPath);
var forcedLooseLoot = Yaml.Deserializer.Deserialize<Dictionary<string, HashSet<string>>>(forcedLooseContent);
return forcedLooseLoot.ToFrozenDictionary(
pair => pair.Key,
pair => ImmutableHashSet.CreateRange(pair.Value)
);
}
}

11
source/LootDumpProcessor/Process/Services/ForcedItemsProvider/IForcedItemsProvider.cs Normal file
View File

@ -0,0 +1,11 @@
using System.Collections.Frozen;
using System.Collections.Immutable;
using LootDumpProcessor.Model.Config;
namespace LootDumpProcessor.Process.Services.ForcedItemsProvider;
public interface IForcedItemsProvider
{
Task<ForcedStatic> GetForcedStatic();
Task<FrozenDictionary<string, ImmutableHashSet<string>>> GetForcedLooseItems();
}

6
source/LootDumpProcessor/Process/Services/KeyGenerator/IKeyGenerator.cs Normal file
View File

@ -0,0 +1,6 @@
namespace LootDumpProcessor.Process.Services.KeyGenerator;
public interface IKeyGenerator
{
string Generate();
}

2
source/LootDumpProcessor/Process/NumericKeyGenerator.cs → source/LootDumpProcessor/Process/Services/KeyGenerator/NumericKeyGenerator.cs
View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process;
namespace LootDumpProcessor.Process.Services.KeyGenerator;
public class NumericKeyGenerator : IKeyGenerator
{

2
source/LootDumpProcessor/Process/ITarkovItemsProvider.cs → source/LootDumpProcessor/Process/Services/TarkovItemsProvider/ITarkovItemsProvider.cs
View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process;
namespace LootDumpProcessor.Process.Services.TarkovItemsProvider;
public interface ITarkovItemsProvider
{

12
source/LootDumpProcessor/Process/TarkovItemsProvider.cs → source/LootDumpProcessor/Process/Services/TarkovItemsProvider/TarkovItemsProvider.cs
View File

@ -1,22 +1,26 @@
using System.Collections.Frozen;
using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Tarkov;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process;
namespace LootDumpProcessor.Process.Services.TarkovItemsProvider;
public class TarkovItemsProvider : ITarkovItemsProvider
{
private readonly ILogger<TarkovItemsProvider> _logger;
private readonly FrozenDictionary<string, TemplateFileItem>? _items;
private readonly Config _config;
private static readonly string ItemsFilePath = Path.Combine(
LootDumpProcessorContext.GetConfig().ServerLocation,
private string ItemsFilePath => Path.Combine(
_config.ServerLocation,
"project", "assets", "database", "templates", "items.json");
public TarkovItemsProvider(ILogger<TarkovItemsProvider> logger)
public TarkovItemsProvider(ILogger<TarkovItemsProvider> logger, IOptions<Config> config)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
try
{

9
source/LootDumpProcessor/Process/Writer/FileWriter.cs
View File

@ -1,18 +1,21 @@
using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Serializers.Json;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Writer;
public class FileWriter : IWriter
{
private static readonly string _outputPath;
private readonly string _outputPath;
static FileWriter()
public FileWriter(IOptions<Config> config)
{
var path = LootDumpProcessorContext.GetConfig().WriterConfig.OutputLocation;
var config1 = (config ?? throw new ArgumentNullException(nameof(config))).Value;
var path = config1.WriterConfig.OutputLocation;
if (string.IsNullOrEmpty(path))
throw new Exception("Output directory must be set in WriterConfigs");

8
source/LootDumpProcessor/Process/Writer/WriterFactory.cs
View File

@ -1,8 +0,0 @@
namespace LootDumpProcessor.Process.Writer;
public static class WriterFactory
{
public static IWriter GetInstance() =>
// implement actual factory someday
new FileWriter();
}

34
source/LootDumpProcessor/Program.cs
View File

@ -1,15 +1,5 @@
using LootDumpProcessor.Process;
using LootDumpProcessor.Process.Processor.DumpProcessor;
using LootDumpProcessor.Process.Processor.FileProcessor;
using LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
using LootDumpProcessor.Process.Processor.v2.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
using LootDumpProcessor.Process.Reader.Filters;
using LootDumpProcessor.Process.Reader.Intake;
using LootDumpProcessor.Storage;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor;
@ -18,33 +8,11 @@ public static class Program
public static async Task Main()
{
var services = new ServiceCollection();
RegisterServices(services);
services.AddLootDumpProcessor();
await using var serviceProvider = services.BuildServiceProvider();
// startup the pipeline
var pipeline = serviceProvider.GetRequiredService<IPipeline>();
await pipeline.Execute();
}
private static void RegisterServices(ServiceCollection services)
{
services.AddLogging(configure => configure.AddConsole());
services.AddTransient<IStaticLootProcessor, StaticLootProcessor>();
services.AddTransient<IStaticContainersProcessor, StaticContainersProcessor>();
services.AddTransient<IAmmoProcessor, AmmoProcessor>();
services.AddTransient<ILooseLootProcessor, LooseLootProcessor>();
services.AddSingleton<ITarkovItemsProvider, TarkovItemsProvider>();
services.AddSingleton<IDataStorage>(_ => DataStorageFactory.GetInstance());
services.AddSingleton<IKeyGenerator, NumericKeyGenerator>();
services.AddTransient<IComposedKeyGenerator, ComposedKeyGenerator>();
services.AddTransient<IIntakeReader, JsonFileIntakeReader>();
services.AddTransient<IFileFilter, JsonDumpFileFilter>();
services.AddTransient<IFileProcessor, FileProcessor>();
services.AddTransient<IDumpProcessor, MultithreadSteppedDumpProcessor>();
services.AddTransient<IPipeline, QueuePipeline>();
}
}

103
source/LootDumpProcessor/ServiceCollectionExtensions.cs Normal file
View File

@ -0,0 +1,103 @@
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Process;
using LootDumpProcessor.Process.Collector;
using LootDumpProcessor.Process.Processor.AmmoProcessor;
using LootDumpProcessor.Process.Processor.DumpProcessor;
using LootDumpProcessor.Process.Processor.FileProcessor;
using LootDumpProcessor.Process.Processor.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.StaticContainersProcessor;
using LootDumpProcessor.Process.Processor.StaticLootProcessor;
using LootDumpProcessor.Process.Reader.Filters;
using LootDumpProcessor.Process.Reader.Intake;
using LootDumpProcessor.Process.Services.ComposedKeyGenerator;
using LootDumpProcessor.Process.Services.ForcedItemsProvider;
using LootDumpProcessor.Process.Services.KeyGenerator;
using LootDumpProcessor.Process.Services.TarkovItemsProvider;
using LootDumpProcessor.Process.Writer;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage.Implementations.File;
using LootDumpProcessor.Storage.Implementations.Memory;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor;
public static class ServiceCollectionExtensions
{
public static void AddLootDumpProcessor(this ServiceCollection services)
{
services.AddLogging(configure => configure.AddConsole());
AddConfiguration(services);
AddCollector(services);
AddDataStorage(services);
RegisterProcessors(services);
services.AddSingleton<StoreHandlerFactory>();
services.AddSingleton<IForcedItemsProvider, ForcedItemsProvider>();
services.AddSingleton<ITarkovItemsProvider, TarkovItemsProvider>();
services.AddSingleton<IKeyGenerator, NumericKeyGenerator>();
services.AddTransient<IComposedKeyGenerator, ComposedKeyGenerator>();
services.AddTransient<IWriter, FileWriter>();
services.AddTransient<IIntakeReader, JsonFileIntakeReader>();
services.AddTransient<IFileFilter, JsonDumpFileFilter>();
services.AddTransient<IFileProcessor, FileProcessor>();
services.AddTransient<IDumpProcessor, MultithreadSteppedDumpProcessor>();
services.AddTransient<IPipeline, QueuePipeline>();
}
private static void AddConfiguration(IServiceCollection services)
{
const string configPath = "Config/config.json";
var configuration = new ConfigurationBuilder()
.AddJsonFile(configPath)
.AddEnvironmentVariables()
.Build();
services.AddOptions<Config>()
.Bind(configuration)
.ValidateDataAnnotations()
.ValidateOnStart();
}
private static void RegisterProcessors(IServiceCollection services)
{
services.AddTransient<IStaticLootProcessor, StaticLootProcessor>();
services.AddTransient<IStaticContainersProcessor, StaticContainersProcessor>();
services.AddTransient<IAmmoProcessor, AmmoProcessor>();
services.AddTransient<ILooseLootProcessor, LooseLootProcessor>();
}
private static void AddCollector(IServiceCollection services)
{
services.AddSingleton<ICollector>(provider =>
{
var config = provider.GetRequiredService<IOptions<Config>>();
var collectorType = config.Value.CollectorConfig.CollectorType;
return collectorType switch
{
CollectorType.Memory => new HashSetCollector(),
CollectorType.Dump => new DumpCollector(config),
_ => throw new ArgumentOutOfRangeException($"CollectorType '{collectorType} is not supported'")
};
});
}
private static void AddDataStorage(IServiceCollection services)
{
services.AddSingleton<IDataStorage>(provider =>
{
var config = provider.GetRequiredService<IOptions<Config>>().Value;
var dataStorageType = config.DataStorageConfig.DataStorageType;
return dataStorageType switch
{
DataStorageTypes.File => new FileDataStorage(provider.GetRequiredService<StoreHandlerFactory>()),
DataStorageTypes.Memory => new MemoryDataStorage(),
_ => throw new ArgumentOutOfRangeException($"DataStorageType '{dataStorageType} is not supported'")
};
});
}
}

32
source/LootDumpProcessor/Storage/DataStorageFactory.cs
View File

@ -1,32 +0,0 @@
using System.Collections.Concurrent;
using LootDumpProcessor.Storage.Implementations.File;
using LootDumpProcessor.Storage.Implementations.Memory;
namespace LootDumpProcessor.Storage;
public static class DataStorageFactory
{
private static readonly ConcurrentDictionary<DataStorageTypes, IDataStorage> DataStorage = new();
/**
* Requires LootDumpProcessorContext to be initialized before using
*/
public static IDataStorage GetInstance() =>
GetInstance(LootDumpProcessorContext.GetConfig().DataStorageConfig.DataStorageType);
private static IDataStorage GetInstance(DataStorageTypes type)
{
if (DataStorage.TryGetValue(type, out var dataStorage)) return dataStorage;
dataStorage = type switch
{
DataStorageTypes.File => new FileDataStorage(),
DataStorageTypes.Memory => new MemoryDataStorage(),
_ => throw new ArgumentOutOfRangeException(nameof(type), type, null)
};
DataStorage.TryAdd(type, dataStorage);
return dataStorage;
}
}

11
source/LootDumpProcessor/Storage/Implementations/File/FileDataStorage.cs
View File

@ -1,16 +1,19 @@
namespace LootDumpProcessor.Storage.Implementations.File;
public class FileDataStorage : IDataStorage
public class FileDataStorage(StoreHandlerFactory storeHandlerFactory) : IDataStorage
{
private readonly StoreHandlerFactory _storeHandlerFactory =
storeHandlerFactory ?? throw new ArgumentNullException(nameof(storeHandlerFactory));
public void Store<TEntity>(TEntity entity) where TEntity : IKeyable
{
StoreHandlerFactory.GetInstance(entity.GetKey().GetKeyType()).Store(entity);
_storeHandlerFactory.GetInstance(entity.GetKey().GetKeyType()).Store(entity);
}
public bool Exists(IKey key) => StoreHandlerFactory.GetInstance(key.GetKeyType()).Exists(key);
public bool Exists(IKey key) => _storeHandlerFactory.GetInstance(key.GetKeyType()).Exists(key);
public T GetItem<T>(IKey key) where T : IKeyable =>
StoreHandlerFactory.GetInstance(key.GetKeyType()).Retrieve<T>(key);
_storeHandlerFactory.GetInstance(key.GetKeyType()).Retrieve<T>(key);
public void Clear()
{

10
source/LootDumpProcessor/Storage/Implementations/File/Handlers/AbstractStoreHandler.cs
View File

@ -1,10 +1,14 @@
using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Serializers.Json;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Storage.Implementations.File.Handlers;
public abstract class AbstractStoreHandler : IStoreHandler
public abstract class AbstractStoreHandler(IOptions<Config> config) : IStoreHandler
{
private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
public void Store<TEntity>(TEntity entity, bool failIfDuplicate = true) where TEntity : IKeyable
{
var locationWithFile = GetLocation(entity.GetKey());
@ -34,8 +38,8 @@ public abstract class AbstractStoreHandler : IStoreHandler
protected virtual string GetBaseLocation()
{
var location =
string.IsNullOrEmpty(LootDumpProcessorContext.GetConfig().DataStorageConfig.FileDataStorageTempLocation)
? LootDumpProcessorContext.GetConfig().DataStorageConfig.FileDataStorageTempLocation
string.IsNullOrEmpty(_config.DataStorageConfig.FileDataStorageTempLocation)
? _config.DataStorageConfig.FileDataStorageTempLocation
: Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
return $"{location}/SPT/tmp/LootGen";

5
source/LootDumpProcessor/Storage/Implementations/File/Handlers/FlatStoreHandler.cs
View File

@ -1,6 +1,9 @@
using LootDumpProcessor.Model.Config;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Storage.Implementations.File.Handlers;
public class FlatStoreHandler : AbstractStoreHandler
public class FlatStoreHandler(IOptions<Config> config) : AbstractStoreHandler(config)
{
protected override string GetLocation(IKey key)
{

5
source/LootDumpProcessor/Storage/Implementations/File/Handlers/SubdivisionedStoreHandler.cs
View File

@ -1,6 +1,9 @@
using LootDumpProcessor.Model.Config;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Storage.Implementations.File.Handlers;
public class SubdivisionedStoreHandler : AbstractStoreHandler
public class SubdivisionedStoreHandler(IOptions<Config> config) : AbstractStoreHandler(config)
{
protected override string GetLocation(IKey key)
{

15
source/LootDumpProcessor/Storage/Implementations/File/StoreHandlerFactory.cs
View File

@ -1,20 +1,27 @@
using System.Collections.Concurrent;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Storage.Implementations.File.Handlers;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Storage.Implementations.File;
public class StoreHandlerFactory
public class StoreHandlerFactory(IServiceProvider serviceProvider)
{
private readonly IServiceProvider _serviceProvider =
serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
private static readonly ConcurrentDictionary<KeyType, IStoreHandler> Handlers = new();
public static IStoreHandler GetInstance(KeyType type)
public IStoreHandler GetInstance(KeyType type)
{
if (Handlers.TryGetValue(type, out var handler)) return handler;
var config = _serviceProvider.GetRequiredService<IOptions<Config>>();
handler = type switch
{
KeyType.Unique => new FlatStoreHandler(),
KeyType.Subdivisioned => new SubdivisionedStoreHandler(),
KeyType.Unique => new FlatStoreHandler(config),
KeyType.Subdivisioned => new SubdivisionedStoreHandler(config),
_ => throw new ArgumentOutOfRangeException(nameof(type), type, null)
};
Handlers.TryAdd(type, handler);