0
0
mirror of https://github.com/sp-tarkov/loot-dump-processor.git synced 2025-02-12 16:50:44 -05:00

Get rid of static and use dependency injection instead (#8)

* Improved thread safety and async processing in dump processor components

* Removed unused _processedDumps field and simplified variable scope in QueuePipeline

* Refactored service registration into dedicated extension methods

* Added configuration binding and environment variables support

* Refactored collector initialization to use dependency injection

* Refactored data storage to use dependency injection

* Refactored configuration models to use records and added validation

* Refactored static loot configuration to use dependency injection

The changes include:
- Moved static weapon IDs and forced items from LootDumpProcessorContext to ForcedStatic record
- Added ForcedStatic configuration injection in StaticLootProcessor and StaticContainerProcessor
- Improved immutability by using read-only collections in ForcedStatic model
- Simplified LootDumpProcessorContext by removing unused methods

* Refactored configuration access to use dependency injection consistently

* Fixed ForcedStatic configuration

* Refactored forced items configuration to use async provider pattern

The changes introduce a new `IForcedItemsProvider` abstraction to handle loading and caching of forced static and loose loot configurations. This improves the code by:

1. Making configuration loading asynchronous
2. Implementing caching of loaded configurations
3. Centralizing forced items configuration access
4. Removing direct file system dependencies from processors
5. Improving testability through dependency injection

The change also updates related processors and interfaces to use async/await pattern consistently.

* Refactored loose loot processor to use async forced items provider

* Reorganized processor and service components into dedicated namespaces
This commit is contained in:
BlueXTX 2025-01-13 20:05:36 +03:00 committed by GitHub
parent 3cb1be1ec9
commit f2fd256aac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
49 changed files with 489 additions and 399 deletions

View File

@ -1,9 +0,0 @@
namespace LootDumpProcessor;
public static class GCHandler
{
public static void Collect()
{
if (LootDumpProcessorContext.GetConfig().ManualGarbageCollectionCalls) GC.Collect();
}
}

View File

@ -11,7 +11,11 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="JetBrains.Annotations" Version="2024.3.0"/>
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.0"/>
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.0"/>
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0"/> <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0"/>
<PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" Version="9.0.0"/>
<PackageReference Include="YamlDotNet" Version="13.0.0"/> <PackageReference Include="YamlDotNet" Version="13.0.0"/>
</ItemGroup> </ItemGroup>

View File

@ -1,80 +0,0 @@
using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Serializers.Yaml;
namespace LootDumpProcessor;
public static class LootDumpProcessorContext
{
private static Config? _config;
private static readonly object _configLock = new();
private static ForcedStatic? _forcedStatic;
private static readonly object _forcedStaticLock = new();
private static HashSet<string>? _staticWeaponIds;
private static readonly object _staticWeaponIdsLock = new();
private static Dictionary<string, List<StaticForced>>? _forcedItems;
private static readonly object _forcedItemsLock = new();
private static Dictionary<string, HashSet<string>>? _forcedLoose;
private static readonly object _forcedLooseLock = new();
public static Config GetConfig()
{
lock (_configLock)
{
if (_config == null)
// This is the only instance where manual selection of the serializer is required
// after this, GetInstance() for the JsonSerializerFactory should used without
// parameters
_config = JsonSerializer.Deserialize<Config>(File.ReadAllText("./Config/config.json"),
JsonSerializerSettings.Default);
}
return _config;
}
public static ForcedStatic GetForcedStatic()
{
lock (_forcedStaticLock)
{
if (_forcedStatic == null)
_forcedStatic = Yaml.Deserializer
.Deserialize<ForcedStatic>(File.ReadAllText("./Config/forced_static.yaml"));
}
return _forcedStatic;
}
public static HashSet<string> GetStaticWeaponIds()
{
lock (_staticWeaponIdsLock)
{
if (_staticWeaponIds == null) _staticWeaponIds = GetForcedStatic().StaticWeaponIds.ToHashSet();
}
return _staticWeaponIds;
}
public static Dictionary<string, List<StaticForced>> GetForcedItems()
{
lock (_forcedItemsLock)
{
if (_forcedItems == null) _forcedItems = GetForcedStatic().ForcedItems;
}
return _forcedItems;
}
public static Dictionary<string, HashSet<string>> GetForcedLooseItems()
{
lock (_forcedLooseLock)
{
if (_forcedLoose == null)
_forcedLoose = Yaml.Deserializer.Deserialize<Dictionary<string, HashSet<string>>>(
File.ReadAllText("./Config/forced_loose.yaml"));
}
return _forcedLoose;
}
}

View File

@ -1,10 +1,12 @@
using System.ComponentModel.DataAnnotations;
using JetBrains.Annotations;
using LootDumpProcessor.Process.Collector; using LootDumpProcessor.Process.Collector;
namespace LootDumpProcessor.Model.Config; namespace LootDumpProcessor.Model.Config;
public class CollectorConfig [UsedImplicitly]
{ public record CollectorConfig(
public CollectorType CollectorType { get; set; } [Required] CollectorType CollectorType,
public int MaxEntitiesBeforeDumping { get; set; } [Required] int MaxEntitiesBeforeDumping,
public string DumpLocation { get; set; } [Required] string DumpLocation
} );

View File

@ -1,15 +1,19 @@
using System.ComponentModel.DataAnnotations;
using JetBrains.Annotations;
namespace LootDumpProcessor.Model.Config; namespace LootDumpProcessor.Model.Config;
public class Config [UsedImplicitly]
public record Config
{ {
public string ServerLocation { get; set; } = string.Empty; [Required] public string ServerLocation { get; init; } = string.Empty;
public bool ManualGarbageCollectionCalls { get; set; } [Required] public bool ManualGarbageCollectionCalls { get; init; }
public DataStorageConfig DataStorageConfig { get; set; } [Required] public DataStorageConfig DataStorageConfig { get; init; } = null!;
public ReaderConfig ReaderConfig { get; set; } [Required] public ReaderConfig ReaderConfig { get; init; } = null!;
public ProcessorConfig ProcessorConfig { get; set; } [Required] public ProcessorConfig ProcessorConfig { get; init; } = null!;
public DumpProcessorConfig DumpProcessorConfig { get; set; } [Required] public DumpProcessorConfig DumpProcessorConfig { get; init; } = null!;
public WriterConfig WriterConfig { get; set; } [Required] public WriterConfig WriterConfig { get; init; } = null!;
public CollectorConfig CollectorConfig { get; set; } [Required] public CollectorConfig CollectorConfig { get; init; } = null!;
public Dictionary<string, string[]> ContainerIgnoreList { get; set; } [Required] public IReadOnlyDictionary<string, string[]> ContainerIgnoreList { get; init; } = null!;
public List<string> MapsToProcess { get; set; } [Required] public IReadOnlyList<string> MapsToProcess { get; init; } = null!;
} }

View File

@ -1,10 +1,12 @@
using System.ComponentModel.DataAnnotations;
using JetBrains.Annotations;
using LootDumpProcessor.Storage; using LootDumpProcessor.Storage;
namespace LootDumpProcessor.Model.Config; namespace LootDumpProcessor.Model.Config;
public class DataStorageConfig [UsedImplicitly]
{ public record DataStorageConfig(
public DataStorageTypes DataStorageType { get; set; } = DataStorageTypes.File; [Required] string FileDataStorageTempLocation,
public string? FileDataStorageTempLocation { get; set; } [Required] DataStorageTypes DataStorageType = DataStorageTypes.File
} );

View File

@ -1,10 +1,12 @@
using System.Text.Json.Serialization; using System.ComponentModel.DataAnnotations;
using System.Text.Json.Serialization;
using JetBrains.Annotations;
using LootDumpProcessor.Serializers.Json.Converters; using LootDumpProcessor.Serializers.Json.Converters;
namespace LootDumpProcessor.Model.Config; namespace LootDumpProcessor.Model.Config;
public class DumpProcessorConfig [UsedImplicitly]
{ public record DumpProcessorConfig(
[JsonConverter(typeof(NetDateTimeConverter))] public DateTime SpawnContainerChanceIncludeAfterDate { get; set; } [Required] [property: JsonConverter(typeof(NetDateTimeConverter))] DateTime SpawnContainerChanceIncludeAfterDate
} );

View File

@ -1,11 +1,14 @@
using System.Collections.Frozen;
using JetBrains.Annotations;
using LootDumpProcessor.Model.Output.StaticContainer; using LootDumpProcessor.Model.Output.StaticContainer;
using YamlDotNet.Serialization;
namespace LootDumpProcessor.Model.Config; namespace LootDumpProcessor.Model.Config;
public class ForcedStatic [UsedImplicitly]
public class ForcedStatic(
IReadOnlyList<string> staticWeaponIds, FrozenDictionary<string, IReadOnlyList<StaticForced>> forcedItems
)
{ {
[YamlMember(Alias = "static_weapon_ids")] public List<string> StaticWeaponIds { get; set; } public readonly IReadOnlyList<string> StaticWeaponIds = staticWeaponIds;
public readonly FrozenDictionary<string, IReadOnlyList<StaticForced>> ForcedItems = forcedItems;
[YamlMember(Alias = "forced_items")] public Dictionary<string, List<StaticForced>> ForcedItems { get; set; }
} }

View File

@ -0,0 +1,13 @@
using JetBrains.Annotations;
using LootDumpProcessor.Model.Output.StaticContainer;
using YamlDotNet.Serialization;
namespace LootDumpProcessor.Model.Config;
[UsedImplicitly]
public class ForcedStaticDto
{
[YamlMember(Alias = "static_weapon_ids")] public List<string> StaticWeaponIds { get; set; }
[YamlMember(Alias = "forced_items")] public Dictionary<string, List<StaticForced>> ForcedItems { get; set; }
}

View File

@ -1,7 +1,6 @@
using JetBrains.Annotations;
namespace LootDumpProcessor.Model.Config; namespace LootDumpProcessor.Model.Config;
public class IntakeReaderConfig [UsedImplicitly]
{ public record IntakeReaderConfig(IReadOnlyList<string> IgnoredDumpLocations, int MaxDumpsPerMap = 1500);
public int MaxDumpsPerMap { get; set; } = 1500;
public List<string> IgnoredDumpLocations { get; set; } = new();
}

View File

@ -1,13 +1,6 @@
using System.Text.Json.Serialization; using JetBrains.Annotations;
namespace LootDumpProcessor.Model.Config; namespace LootDumpProcessor.Model.Config;
public class ProcessorConfig [UsedImplicitly]
{ public record ProcessorConfig(double SpawnPointToleranceForForced = 99, double LooseLootCountTolerancePercentage = 75);
[JsonPropertyName("spawnPointToleranceForForced")] public double SpawnPointToleranceForForced { get; set; } = 99D;
[JsonPropertyName("looseLootCountTolerancePercentage")]
public double LooseLootCountTolerancePercentage { get; set; } = 75D;
}

View File

@ -1,9 +1,12 @@
using System.ComponentModel.DataAnnotations;
using JetBrains.Annotations;
namespace LootDumpProcessor.Model.Config; namespace LootDumpProcessor.Model.Config;
public class ReaderConfig [UsedImplicitly]
{ public record ReaderConfig(
public IntakeReaderConfig? IntakeReaderConfig { get; set; } [Required] IntakeReaderConfig IntakeReaderConfig,
public List<string>? DumpFilesLocation { get; set; } [Required] IReadOnlyList<string> DumpFilesLocation,
public string? ThresholdDate { get; set; } string? ThresholdDate,
public bool ProcessSubFolders { get; set; } bool ProcessSubFolders = true
} );

View File

@ -1,9 +1,8 @@
using System.Text.Json.Serialization; using System.ComponentModel.DataAnnotations;
using JetBrains.Annotations;
namespace LootDumpProcessor.Model.Config; namespace LootDumpProcessor.Model.Config;
public class WriterConfig [UsedImplicitly]
{ public record WriterConfig([Required] string OutputLocation);
[JsonPropertyName("outputLocation")] public string? OutputLocation { get; set; }
}

View File

@ -1,18 +0,0 @@
namespace LootDumpProcessor.Process.Collector;
public static class CollectorFactory
{
private static ICollector? _collector;
public static ICollector GetInstance()
{
if (_collector == null)
_collector = LootDumpProcessorContext.GetConfig().CollectorConfig.CollectorType switch
{
CollectorType.Memory => new HashSetCollector(),
CollectorType.Dump => new DumpCollector(),
_ => throw new ArgumentOutOfRangeException()
};
return _collector;
}
}

View File

@ -1,16 +1,18 @@
using System.Text.Json; using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Serializers.Json; using LootDumpProcessor.Serializers.Json;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Collector; namespace LootDumpProcessor.Process.Collector;
public class DumpCollector : ICollector public class DumpCollector(IOptions<Config> config) : ICollector
{ {
private static readonly string DumpLocation = private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
$"{LootDumpProcessorContext.GetConfig().CollectorConfig.DumpLocation}/collector/";
private readonly List<PartialData> processedDumps = private string DumpLocation => Path.Combine(_config.CollectorConfig.DumpLocation, "collector");
new(LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping + 50);
private List<PartialData> ProcessedDumps => new(_config.CollectorConfig.MaxEntitiesBeforeDumping + 50);
private readonly object lockObject = new(); private readonly object lockObject = new();
@ -25,13 +27,13 @@ public class DumpCollector : ICollector
{ {
lock (lockObject) lock (lockObject)
{ {
processedDumps.Add(parsedDump); ProcessedDumps.Add(parsedDump);
if (processedDumps.Count > LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping) if (ProcessedDumps.Count > _config.CollectorConfig.MaxEntitiesBeforeDumping)
{ {
var fileName = $"collector-{DateTime.Now.ToString("yyyyMMddHHmmssfffff")}.json"; var fileName = $"collector-{DateTime.Now.ToString("yyyyMMddHHmmssfffff")}.json";
File.WriteAllText($"{DumpLocation}{fileName}", File.WriteAllText($"{DumpLocation}{fileName}",
JsonSerializer.Serialize(processedDumps, JsonSerializerSettings.Default)); JsonSerializer.Serialize(ProcessedDumps, JsonSerializerSettings.Default));
processedDumps.Clear(); ProcessedDumps.Clear();
} }
} }
} }
@ -39,10 +41,10 @@ public class DumpCollector : ICollector
public List<PartialData> Retrieve() public List<PartialData> Retrieve()
{ {
foreach (var file in Directory.GetFiles(DumpLocation)) foreach (var file in Directory.GetFiles(DumpLocation))
processedDumps.AddRange(JsonSerializer ProcessedDumps.AddRange(JsonSerializer
.Deserialize<List<PartialData>>(File.ReadAllText(file), JsonSerializerSettings.Default)); .Deserialize<List<PartialData>>(File.ReadAllText(file), JsonSerializerSettings.Default));
return processedDumps; return ProcessedDumps;
} }
public void Clear() public void Clear()
@ -51,7 +53,7 @@ public class DumpCollector : ICollector
{ {
foreach (var file in Directory.GetFiles(DumpLocation)) File.Delete(file); foreach (var file in Directory.GetFiles(DumpLocation)) File.Delete(file);
processedDumps.Clear(); ProcessedDumps.Clear();
} }
} }
} }

View File

@ -1,6 +0,0 @@
namespace LootDumpProcessor.Process;
public interface IKeyGenerator
{
string Generate();
}

View File

@ -1,8 +1,9 @@
using LootDumpProcessor.Model.Output; using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Services.TarkovItemsProvider;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.v2.AmmoProcessor; namespace LootDumpProcessor.Process.Processor.AmmoProcessor;
public class AmmoProcessor(ILogger<AmmoProcessor> logger, ITarkovItemsProvider tarkovItemsProvider) : IAmmoProcessor public class AmmoProcessor(ILogger<AmmoProcessor> logger, ITarkovItemsProvider tarkovItemsProvider) : IAmmoProcessor
{ {

View File

@ -1,7 +1,7 @@
using LootDumpProcessor.Model.Output; using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Process.Processor.v2.AmmoProcessor; namespace LootDumpProcessor.Process.Processor.AmmoProcessor;
public interface IAmmoProcessor public interface IAmmoProcessor
{ {

View File

@ -1,20 +1,23 @@
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Text.Json; using System.Text.Json;
using LootDumpProcessor.Model; using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Input; using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output; using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot; using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Output.StaticContainer; using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Processor.v2.AmmoProcessor; using LootDumpProcessor.Process.Processor.AmmoProcessor;
using LootDumpProcessor.Process.Processor.v2.LooseLootProcessor; using LootDumpProcessor.Process.Processor.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor; using LootDumpProcessor.Process.Processor.StaticContainersProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor; using LootDumpProcessor.Process.Processor.StaticLootProcessor;
using LootDumpProcessor.Process.Services.KeyGenerator;
using LootDumpProcessor.Serializers.Json; using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Storage; using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage.Collections; using LootDumpProcessor.Storage.Collections;
using LootDumpProcessor.Utils; using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Processor.DumpProcessor; namespace LootDumpProcessor.Process.Processor.DumpProcessor;
@ -23,7 +26,8 @@ public class MultithreadSteppedDumpProcessor(
IStaticContainersProcessor staticContainersProcessor, IStaticContainersProcessor staticContainersProcessor,
IAmmoProcessor ammoProcessor, IAmmoProcessor ammoProcessor,
ILooseLootProcessor looseLootProcessor, ILooseLootProcessor looseLootProcessor,
ILogger<MultithreadSteppedDumpProcessor> logger, IKeyGenerator keyGenerator ILogger<MultithreadSteppedDumpProcessor> logger, IKeyGenerator keyGenerator, IDataStorage dataStorage,
IOptions<Config> config
) )
: IDumpProcessor : IDumpProcessor
{ {
@ -45,8 +49,10 @@ public class MultithreadSteppedDumpProcessor(
private readonly IKeyGenerator private readonly IKeyGenerator
_keyGenerator = keyGenerator ?? throw new ArgumentNullException(nameof(keyGenerator)); _keyGenerator = keyGenerator ?? throw new ArgumentNullException(nameof(keyGenerator));
private static readonly IDataStorage _dataStorage = DataStorageFactory.GetInstance(); private readonly IDataStorage _dataStorage = dataStorage ?? throw new ArgumentNullException(nameof(dataStorage));
private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
public async Task<Dictionary<OutputFileType, object>> ProcessDumps(List<PartialData> dumps) public async Task<Dictionary<OutputFileType, object>> ProcessDumps(List<PartialData> dumps)
{ {
_logger.LogInformation("Starting final dump processing"); _logger.LogInformation("Starting final dump processing");
@ -111,12 +117,12 @@ public class MultithreadSteppedDumpProcessor(
_logger.LogInformation("Processing loose loot distribution"); _logger.LogInformation("Processing loose loot distribution");
var looseLoot = new ConcurrentDictionary<string, LooseLootRoot>(); var looseLoot = new ConcurrentDictionary<string, LooseLootRoot>();
Parallel.ForEach(dumpProcessData.MapCounts.Keys, parallelOptions, mapId => await Parallel.ForEachAsync(dumpProcessData.MapCounts.Keys, parallelOptions, async (mapId, _) =>
{ {
var mapCount = dumpProcessData.MapCounts[mapId]; var mapCount = dumpProcessData.MapCounts[mapId];
var looseLootCount = dumpProcessData.LooseLootCounts[mapId]; var looseLootCount = dumpProcessData.LooseLootCounts[mapId];
var looseLootDistribution = var looseLootDistribution =
_looseLootProcessor.CreateLooseLootDistribution(mapId, mapCount, looseLootCount); await _looseLootProcessor.CreateLooseLootDistribution(mapId, mapCount, looseLootCount);
looseLoot[mapId] = looseLootDistribution; looseLoot[mapId] = looseLootDistribution;
}); });
_logger.LogInformation("Collecting loose loot distribution information"); _logger.LogInformation("Collecting loose loot distribution information");
@ -159,8 +165,7 @@ public class MultithreadSteppedDumpProcessor(
} }
else else
{ {
var mapStaticContainers = var mapStaticContainers = await _staticContainersProcessor.CreateStaticWeaponsAndForcedContainers(dataDump);
_staticContainersProcessor.CreateStaticWeaponsAndForcedContainers(dataDump);
var newStaticWeapons = mapStaticContainers.StaticWeapons.Where(x => var newStaticWeapons = mapStaticContainers.StaticWeapons.Where(x =>
!mapStaticLoot.StaticWeapons.Exists(y => y.Id == x.Id)); !mapStaticLoot.StaticWeapons.Exists(y => y.Id == x.Id));
@ -172,7 +177,7 @@ public class MultithreadSteppedDumpProcessor(
} }
if (!mapStaticContainersAggregated.TryGetValue(mapId, if (!mapStaticContainersAggregated.TryGetValue(mapId,
out ConcurrentDictionary<Template, int> mapAggregatedDataDict)) out var mapAggregatedDataDict))
{ {
mapAggregatedDataDict = new ConcurrentDictionary<Template, int>(); mapAggregatedDataDict = new ConcurrentDictionary<Template, int>();
mapStaticContainersAggregated.TryAdd(mapId, mapAggregatedDataDict); mapStaticContainersAggregated.TryAdd(mapId, mapAggregatedDataDict);
@ -182,9 +187,9 @@ public class MultithreadSteppedDumpProcessor(
IncrementMapCounterDictionaryValue(mapDumpCounter, mapId); IncrementMapCounterDictionaryValue(mapDumpCounter, mapId);
var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList var containerIgnoreListExists = _config.ContainerIgnoreList
.TryGetValue(mapId, out var ignoreListForMap); .TryGetValue(mapId, out var ignoreListForMap);
foreach (var dynamicStaticContainer in _staticContainersProcessor.CreateDynamicStaticContainers( foreach (var dynamicStaticContainer in await _staticContainersProcessor.CreateDynamicStaticContainers(
dataDump)) dataDump))
{ {
if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id)) continue; if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id)) continue;
@ -193,16 +198,17 @@ public class MultithreadSteppedDumpProcessor(
mapAggregatedDataDict[dynamicStaticContainer] += 1; mapAggregatedDataDict[dynamicStaticContainer] += 1;
} }
GCHandler.Collect(); if (_config.ManualGarbageCollectionCalls) GC.Collect();
} }
private static bool DumpWasMadeAfterConfigThresholdDate(PartialData dataDump) => private bool DumpWasMadeAfterConfigThresholdDate(PartialData dataDump) =>
FileDateParser.TryParseFileDate(dataDump.BasicInfo.FileName, out var fileDate) && FileDateParser.TryParseFileDate(dataDump.BasicInfo.FileName, out var fileDate) &&
fileDate.HasValue && fileDate.HasValue &&
fileDate.Value > LootDumpProcessorContext.GetConfig().DumpProcessorConfig fileDate.Value > _config.DumpProcessorConfig
.SpawnContainerChanceIncludeAfterDate; .SpawnContainerChanceIncludeAfterDate;
private static void IncrementMapCounterDictionaryValue(ConcurrentDictionary<string, int> mapDumpCounter, string mapName) private static void IncrementMapCounterDictionaryValue(ConcurrentDictionary<string, int> mapDumpCounter,
string mapName)
{ {
if (!mapDumpCounter.TryAdd(mapName, 1)) mapDumpCounter[mapName] += 1; if (!mapDumpCounter.TryAdd(mapName, 1)) mapDumpCounter[mapName] += 1;
} }
@ -247,7 +253,7 @@ public class MultithreadSteppedDumpProcessor(
partialFileMetaData = null; partialFileMetaData = null;
tuple = null; tuple = null;
GCHandler.Collect(); if (_config.ManualGarbageCollectionCalls) GC.Collect();
var parallelOptions = new ParallelOptions var parallelOptions = new ParallelOptions
{ {
@ -266,14 +272,14 @@ public class MultithreadSteppedDumpProcessor(
_dataStorage.Store(dictionaryCounts); _dataStorage.Store(dictionaryCounts);
dictionaryCounts = null; dictionaryCounts = null;
GCHandler.Collect(); if (_config.ManualGarbageCollectionCalls) GC.Collect();
_dataStorage.Store(actualDictionaryItemProperties); _dataStorage.Store(actualDictionaryItemProperties);
actualDictionaryItemProperties = null; actualDictionaryItemProperties = null;
GCHandler.Collect(); if (_config.ManualGarbageCollectionCalls) GC.Collect();
_dataStorage.Store(looseLootCounts); _dataStorage.Store(looseLootCounts);
looseLootCounts = null; looseLootCounts = null;
GCHandler.Collect(); if (_config.ManualGarbageCollectionCalls) GC.Collect();
}); });
return dumpProcessData; return dumpProcessData;
} }

View File

@ -1,32 +1,33 @@
using LootDumpProcessor.Model; using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Processor.v2.LooseLootProcessor; using LootDumpProcessor.Process.Processor.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor; using LootDumpProcessor.Process.Processor.StaticLootProcessor;
using LootDumpProcessor.Storage; using LootDumpProcessor.Storage;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.FileProcessor; namespace LootDumpProcessor.Process.Processor.FileProcessor;
public class FileProcessor : IFileProcessor public class FileProcessor(
IStaticLootProcessor staticLootProcessor,
ILooseLootProcessor looseLootProcessor,
ILogger<FileProcessor> logger, IDataStorage dataStorage
)
: IFileProcessor
{ {
private readonly IStaticLootProcessor _staticLootProcessor; private readonly IStaticLootProcessor _staticLootProcessor = staticLootProcessor
private readonly ILooseLootProcessor _looseLootProcessor; ?? throw new ArgumentNullException(
private readonly ILogger<FileProcessor> _logger; nameof(staticLootProcessor));
public FileProcessor( private readonly ILooseLootProcessor _looseLootProcessor = looseLootProcessor
IStaticLootProcessor staticLootProcessor, ?? throw new ArgumentNullException(
ILooseLootProcessor looseLootProcessor, nameof(looseLootProcessor));
ILogger<FileProcessor> logger)
{
_staticLootProcessor = staticLootProcessor
?? throw new ArgumentNullException(nameof(staticLootProcessor));
_looseLootProcessor = looseLootProcessor
?? throw new ArgumentNullException(nameof(looseLootProcessor));
_logger = logger
?? throw new ArgumentNullException(nameof(logger));
}
public PartialData Process(BasicInfo parsedData) private readonly ILogger<FileProcessor> _logger = logger
?? throw new ArgumentNullException(nameof(logger));
private readonly IDataStorage _dataStorage = dataStorage ?? throw new ArgumentNullException(nameof(dataStorage));
public async Task<PartialData> Process(BasicInfo parsedData)
{ {
_logger.LogDebug("Processing file {FileName}...", parsedData.FileName); _logger.LogDebug("Processing file {FileName}...", parsedData.FileName);
@ -50,16 +51,16 @@ public class FileProcessor : IFileProcessor
ParsedDumpKey = (AbstractKey)dumpData.GetKey() ParsedDumpKey = (AbstractKey)dumpData.GetKey()
}; };
if (!DataStorageFactory.GetInstance().Exists(dumpData.GetKey())) if (!_dataStorage.Exists(dumpData.GetKey()))
{ {
_logger.LogDebug( _logger.LogDebug(
"Cache not found for {LookupIndex} processing.", "Cache not found for {LookupIndex} processing.",
string.Join("/", dumpData.GetKey().GetLookupIndex()) string.Join("/", dumpData.GetKey().GetLookupIndex())
); );
dumpData.Containers = _staticLootProcessor.PreProcessStaticLoot(staticLoot); dumpData.Containers = await _staticLootProcessor.PreProcessStaticLoot(staticLoot);
dumpData.LooseLoot = _looseLootProcessor.PreProcessLooseLoot(looseLoot); dumpData.LooseLoot = _looseLootProcessor.PreProcessLooseLoot(looseLoot);
DataStorageFactory.GetInstance().Store(dumpData); dataStorage.Store(dumpData);
} }
_logger.LogDebug("File {FileName} finished processing!", parsedData.FileName); _logger.LogDebug("File {FileName} finished processing!", parsedData.FileName);

View File

@ -4,5 +4,5 @@ namespace LootDumpProcessor.Process.Processor.FileProcessor;
public interface IFileProcessor public interface IFileProcessor
{ {
PartialData Process(BasicInfo parsedData); Task<PartialData> Process(BasicInfo parsedData);
} }

View File

@ -3,15 +3,13 @@ using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Storage; using LootDumpProcessor.Storage;
namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor; namespace LootDumpProcessor.Process.Processor.LooseLootProcessor;
public interface ILooseLootProcessor public interface ILooseLootProcessor
{ {
PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseLoot); PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseLoot);
LooseLootRoot CreateLooseLootDistribution( Task<LooseLootRoot> CreateLooseLootDistribution(string mapId,
string mapId,
int mapCount, int mapCount,
IKey looseLootCountKey IKey looseLootCountKey);
);
} }

View File

@ -1,17 +1,24 @@
using LootDumpProcessor.Model; using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Output; using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot; using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Services.ComposedKeyGenerator;
using LootDumpProcessor.Process.Services.ForcedItemsProvider;
using LootDumpProcessor.Process.Services.KeyGenerator;
using LootDumpProcessor.Process.Services.TarkovItemsProvider;
using LootDumpProcessor.Storage; using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage.Collections; using LootDumpProcessor.Storage.Collections;
using LootDumpProcessor.Utils; using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor; namespace LootDumpProcessor.Process.Processor.LooseLootProcessor;
public class LooseLootProcessor( public class LooseLootProcessor(
ILogger<LooseLootProcessor> logger, IDataStorage dataStorage, ITarkovItemsProvider tarkovItemsProvider, ILogger<LooseLootProcessor> logger, IDataStorage dataStorage, ITarkovItemsProvider tarkovItemsProvider,
IComposedKeyGenerator composedKeyGenerator, IKeyGenerator keyGenerator IComposedKeyGenerator composedKeyGenerator, IKeyGenerator keyGenerator, IOptions<Config> config,
IForcedItemsProvider forcedItemsProvider
) )
: ILooseLootProcessor : ILooseLootProcessor
{ {
@ -30,6 +37,11 @@ public class LooseLootProcessor(
private readonly IKeyGenerator private readonly IKeyGenerator
_keyGenerator = keyGenerator ?? throw new ArgumentNullException(nameof(keyGenerator)); _keyGenerator = keyGenerator ?? throw new ArgumentNullException(nameof(keyGenerator));
private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
private readonly IForcedItemsProvider _forcedItemsProvider =
forcedItemsProvider ?? throw new ArgumentNullException(nameof(forcedItemsProvider));
public PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseLoot) public PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseLoot)
{ {
var preProcessedLoot = new PreProcessedLooseLoot var preProcessedLoot = new PreProcessedLooseLoot
@ -70,25 +82,23 @@ public class LooseLootProcessor(
return preProcessedLoot; return preProcessedLoot;
} }
public LooseLootRoot CreateLooseLootDistribution( public async Task<LooseLootRoot> CreateLooseLootDistribution(string mapId,
string mapId,
int mapCount, int mapCount,
IKey looseLootCountKey IKey looseLootCountKey)
)
{ {
var config = LootDumpProcessorContext.GetConfig(); var spawnPointTolerance = _config.ProcessorConfig.SpawnPointToleranceForForced / 100.0;
var spawnPointTolerance = config.ProcessorConfig.SpawnPointToleranceForForced / 100.0;
var looseLootDistribution = new LooseLootRoot(); var looseLootDistribution = new LooseLootRoot();
var probabilities = new Dictionary<string, double>(); var probabilities = new Dictionary<string, double>();
var looseLootCountsItem = _dataStorage.GetItem<LooseLootCounts>(looseLootCountKey); var looseLootCountsItem = _dataStorage.GetItem<LooseLootCounts>(looseLootCountKey);
var forcedLooseItems = await _forcedItemsProvider.GetForcedLooseItems();
var counts = _dataStorage.GetItem<FlatKeyableDictionary<string, int>>(looseLootCountsItem.Counts); var counts = _dataStorage.GetItem<FlatKeyableDictionary<string, int>>(looseLootCountsItem.Counts);
foreach (var (itemId, count) in counts) probabilities[itemId] = (double)count / mapCount; foreach (var (itemId, count) in counts) probabilities[itemId] = (double)count / mapCount;
var spawnPointCount = looseLootCountsItem.MapSpawnpointCount.Select(Convert.ToDouble).ToList(); var spawnPointCount = looseLootCountsItem.MapSpawnpointCount.Select(Convert.ToDouble).ToList();
var initialMean = CalculateMean(spawnPointCount); var initialMean = CalculateMean(spawnPointCount);
var tolerancePercentage = config.ProcessorConfig.LooseLootCountTolerancePercentage / 100.0; var tolerancePercentage = _config.ProcessorConfig.LooseLootCountTolerancePercentage / 100.0;
var highThreshold = initialMean * (1 + tolerancePercentage); var highThreshold = initialMean * (1 + tolerancePercentage);
looseLootCountsItem.MapSpawnpointCount = looseLootCountsItem.MapSpawnpointCount looseLootCountsItem.MapSpawnpointCount = looseLootCountsItem.MapSpawnpointCount
@ -137,7 +147,7 @@ public class LooseLootProcessor(
if (itemDistributions.Count == 1 && if (itemDistributions.Count == 1 &&
(_tarkovItemsProvider.IsQuestItem(itemDistributions[0].ComposedKey?.FirstItem?.Tpl) || (_tarkovItemsProvider.IsQuestItem(itemDistributions[0].ComposedKey?.FirstItem?.Tpl) ||
LootDumpProcessorContext.GetForcedLooseItems()[mapId] forcedLooseItems[mapId]
.Contains(itemDistributions[0].ComposedKey?.FirstItem?.Tpl))) .Contains(itemDistributions[0].ComposedKey?.FirstItem?.Tpl)))
{ {
var forcedSpawnPoint = new SpawnPointsForced var forcedSpawnPoint = new SpawnPointsForced
@ -160,7 +170,7 @@ public class LooseLootProcessor(
_logger.LogWarning( _logger.LogWarning(
"Item: {ItemId} has > {Tolerance}% spawn chance in spawn point: {LocationId} but isn't in forced loot, adding to forced", "Item: {ItemId} has > {Tolerance}% spawn chance in spawn point: {LocationId} but isn't in forced loot, adding to forced",
templateCopy.Id, templateCopy.Id,
config.ProcessorConfig.SpawnPointToleranceForForced, _config.ProcessorConfig.SpawnPointToleranceForForced,
forcedSpawnPoint.LocationId forcedSpawnPoint.LocationId
); );
} }
@ -210,7 +220,7 @@ public class LooseLootProcessor(
.ToList(); .ToList();
var configuredForcedTemplates = var configuredForcedTemplates =
new HashSet<string>(LootDumpProcessorContext.GetForcedLooseItems()[mapId].Select(item => item)); new HashSet<string>(forcedLooseItems[mapId].Select(item => item));
var foundForcedTemplates = var foundForcedTemplates =
new HashSet<string>(looseLootDistribution.SpawnPointsForced.Select(fp => fp.Template.Items[0].Tpl)); new HashSet<string>(looseLootDistribution.SpawnPointsForced.Select(fp => fp.Template.Items[0].Tpl));

View File

@ -0,0 +1,11 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output.StaticContainer;
namespace LootDumpProcessor.Process.Processor.StaticContainersProcessor;
public interface IStaticContainersProcessor
{
Task<MapStaticLoot> CreateStaticWeaponsAndForcedContainers(RootData rawMapDump);
Task<IReadOnlyList<Template>> CreateDynamicStaticContainers(RootData rawMapDump);
}

View File

@ -1,21 +1,24 @@
using LootDumpProcessor.Model; using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input; using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output.StaticContainer; using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Process.Services.ForcedItemsProvider;
using LootDumpProcessor.Utils; using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor; namespace LootDumpProcessor.Process.Processor.StaticContainersProcessor;
public class StaticContainersProcessor : IStaticContainersProcessor public class StaticContainersProcessor(
ILogger<StaticContainersProcessor> logger, IForcedItemsProvider forcedItemsProvider
)
: IStaticContainersProcessor
{ {
private readonly ILogger<StaticContainersProcessor> _logger; private readonly ILogger<StaticContainersProcessor> _logger =
logger ?? throw new ArgumentNullException(nameof(logger));
public StaticContainersProcessor(ILogger<StaticContainersProcessor> logger) private readonly IForcedItemsProvider _forcedItemsProvider =
{ forcedItemsProvider ?? throw new ArgumentNullException(nameof(forcedItemsProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public MapStaticLoot CreateStaticWeaponsAndForcedContainers(RootData rawMapDump) public async Task<MapStaticLoot> CreateStaticWeaponsAndForcedContainers(RootData rawMapDump)
{ {
var locationLoot = rawMapDump.Data.LocationLoot; var locationLoot = rawMapDump.Data.LocationLoot;
var mapId = locationLoot.Id.ToLower(); var mapId = locationLoot.Id.ToLower();
@ -25,6 +28,7 @@ public class StaticContainersProcessor : IStaticContainersProcessor
.ToList(); .ToList();
var staticWeapons = new List<Template>(); var staticWeapons = new List<Template>();
var forcedStatic = await _forcedItemsProvider.GetForcedStatic();
foreach (var lootPosition in staticLootPositions) foreach (var lootPosition in staticLootPositions)
{ {
@ -36,7 +40,7 @@ public class StaticContainersProcessor : IStaticContainersProcessor
var firstItemTpl = lootPosition.Items.First().Tpl; var firstItemTpl = lootPosition.Items.First().Tpl;
if (!LootDumpProcessorContext.GetStaticWeaponIds().Contains(firstItemTpl)) if (!forcedStatic.StaticWeaponIds.Contains(firstItemTpl))
continue; continue;
var copiedLoot = ProcessorUtil.Copy(lootPosition); var copiedLoot = ProcessorUtil.Copy(lootPosition);
@ -44,26 +48,28 @@ public class StaticContainersProcessor : IStaticContainersProcessor
_logger.LogDebug("Added static weapon with ID {WeaponId} to Map {MapId}.", copiedLoot.Id, mapId); _logger.LogDebug("Added static weapon with ID {WeaponId} to Map {MapId}.", copiedLoot.Id, mapId);
} }
var forcedStaticItems = LootDumpProcessorContext.GetForcedItems() var forcedStaticItems = forcedStatic.ForcedItems
.TryGetValue(mapId, out List<StaticForced>? forcedItems) .TryGetValue(mapId, out var forcedItems)
? forcedItems ? forcedItems
: new List<StaticForced>(); : new List<StaticForced>();
var mapStaticLoot = new MapStaticLoot var mapStaticLoot = new MapStaticLoot
{ {
StaticWeapons = staticWeapons, StaticWeapons = staticWeapons,
StaticForced = forcedStaticItems StaticForced = forcedStaticItems.ToList()
}; };
_logger.LogInformation("Created static weapons and forced containers for Map {MapId}.", mapId); _logger.LogInformation("Created static weapons and forced containers for Map {MapId}.", mapId);
return mapStaticLoot; return mapStaticLoot;
} }
public IReadOnlyList<Template> CreateDynamicStaticContainers(RootData rawMapDump) public async Task<IReadOnlyList<Template>> CreateDynamicStaticContainers(RootData rawMapDump)
{ {
var forcedStatic = await _forcedItemsProvider.GetForcedStatic();
var dynamicContainers = rawMapDump.Data.LocationLoot.Loot var dynamicContainers = rawMapDump.Data.LocationLoot.Loot
.Where(loot => loot.IsContainer && .Where(loot => loot.IsContainer &&
!LootDumpProcessorContext.GetStaticWeaponIds().Contains(loot.Items.FirstOrDefault()?.Tpl)) !forcedStatic.StaticWeaponIds.Contains(loot.Items.FirstOrDefault()?.Tpl))
.ToList(); .ToList();
foreach (var container in dynamicContainers) foreach (var container in dynamicContainers)

View File

@ -2,11 +2,11 @@
using LootDumpProcessor.Model.Output; using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Process.Processor.v2.StaticLootProcessor; namespace LootDumpProcessor.Process.Processor.StaticLootProcessor;
public interface IStaticLootProcessor public interface IStaticLootProcessor
{ {
IReadOnlyList<PreProcessedStaticLoot> PreProcessStaticLoot(IReadOnlyList<Template> staticLoot); Task<IReadOnlyList<PreProcessedStaticLoot>> PreProcessStaticLoot(IReadOnlyList<Template> staticLoot);
IReadOnlyDictionary<string, StaticItemDistribution> CreateStaticLootDistribution( IReadOnlyDictionary<string, StaticItemDistribution> CreateStaticLootDistribution(
string mapName, string mapName,

View File

@ -1,16 +1,21 @@
using LootDumpProcessor.Model; using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Output; using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Services.ForcedItemsProvider;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.v2.StaticLootProcessor; namespace LootDumpProcessor.Process.Processor.StaticLootProcessor;
public class StaticLootProcessor(ILogger<StaticLootProcessor> logger) : IStaticLootProcessor public class StaticLootProcessor(ILogger<StaticLootProcessor> logger, IForcedItemsProvider forcedItemsProvider)
: IStaticLootProcessor
{ {
private readonly ILogger<StaticLootProcessor> _logger = private readonly ILogger<StaticLootProcessor> _logger =
logger ?? throw new ArgumentNullException(nameof(logger)); logger ?? throw new ArgumentNullException(nameof(logger));
public IReadOnlyList<PreProcessedStaticLoot> PreProcessStaticLoot(IReadOnlyList<Template> staticLoot) private readonly IForcedItemsProvider _forcedItemsProvider =
forcedItemsProvider ?? throw new ArgumentNullException(nameof(forcedItemsProvider));
public async Task<IReadOnlyList<PreProcessedStaticLoot>> PreProcessStaticLoot(IReadOnlyList<Template> staticLoot)
{ {
var nonWeaponContainers = new List<PreProcessedStaticLoot>(); var nonWeaponContainers = new List<PreProcessedStaticLoot>();
@ -24,7 +29,8 @@ public class StaticLootProcessor(ILogger<StaticLootProcessor> logger) : IStaticL
var firstItemTpl = lootSpawn.Items[0].Tpl; var firstItemTpl = lootSpawn.Items[0].Tpl;
if (!LootDumpProcessorContext.GetStaticWeaponIds().Contains(firstItemTpl)) var forcedStatic = await _forcedItemsProvider.GetForcedStatic();
if (!forcedStatic.StaticWeaponIds.Contains(firstItemTpl))
{ {
nonWeaponContainers.Add(new PreProcessedStaticLoot nonWeaponContainers.Add(new PreProcessedStaticLoot
{ {

View File

@ -1,11 +0,0 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output.StaticContainer;
namespace LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
public interface IStaticContainersProcessor
{
MapStaticLoot CreateStaticWeaponsAndForcedContainers(RootData rawMapDump);
IReadOnlyList<Template> CreateDynamicStaticContainers(RootData rawMapDump);
}

View File

@ -1,6 +1,7 @@
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Diagnostics; using System.Diagnostics;
using System.Text.Json; using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Input; using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Process.Collector; using LootDumpProcessor.Process.Collector;
using LootDumpProcessor.Process.Processor.DumpProcessor; using LootDumpProcessor.Process.Processor.DumpProcessor;
@ -12,12 +13,13 @@ using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Storage; using LootDumpProcessor.Storage;
using LootDumpProcessor.Utils; using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process; namespace LootDumpProcessor.Process;
public class QueuePipeline( public class QueuePipeline(
IFileProcessor fileProcessor, IDumpProcessor dumpProcessor, ILogger<QueuePipeline> logger, IFileFilter fileFilter, IFileProcessor fileProcessor, IDumpProcessor dumpProcessor, ILogger<QueuePipeline> logger, IFileFilter fileFilter,
IIntakeReader intakeReader IIntakeReader intakeReader, ICollector collector, IDataStorage dataStorage, IOptions<Config> config, IWriter writer
) )
: IPipeline : IPipeline
{ {
@ -34,25 +36,32 @@ public class QueuePipeline(
private readonly IIntakeReader private readonly IIntakeReader
_intakeReader = intakeReader ?? throw new ArgumentNullException(nameof(intakeReader)); _intakeReader = intakeReader ?? throw new ArgumentNullException(nameof(intakeReader));
private readonly ICollector _collector = collector ?? throw new ArgumentNullException(nameof(collector));
private readonly IDataStorage _dataStorage = dataStorage ?? throw new ArgumentNullException(nameof(dataStorage));
private readonly IWriter _writer = writer ?? throw new ArgumentNullException(nameof(writer));
private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
private readonly List<string> _filesToRename = new(); private readonly List<string> _filesToRename = new();
private readonly BlockingCollection<string> _filesToProcess = new(); private readonly BlockingCollection<string> _filesToProcess = new();
private readonly List<string> _mapNames = LootDumpProcessorContext.GetConfig().MapsToProcess; private IReadOnlyList<string> MapNames => _config.MapsToProcess;
public async Task Execute() public async Task Execute()
{ {
var stopwatch = Stopwatch.StartNew(); var stopwatch = Stopwatch.StartNew();
// Single collector instance to collect results // Single collector instance to collect results
var collector = CollectorFactory.GetInstance(); _collector.Setup();
collector.Setup();
_logger.LogInformation("Gathering files to begin processing"); _logger.LogInformation("Gathering files to begin processing");
try try
{ {
await FixFilesFromDumps(); await FixFilesFromDumps();
foreach (var mapName in _mapNames) await ProcessFilesFromDumpsPerMap(collector, mapName); foreach (var mapName in MapNames) await ProcessFilesFromDumpsPerMap(_collector, mapName);
} }
finally finally
{ {
@ -64,7 +73,7 @@ public class QueuePipeline(
private List<string> GatherFiles() private List<string> GatherFiles()
{ {
// Read locations // Read locations
var inputPath = LootDumpProcessorContext.GetConfig().ReaderConfig.DumpFilesLocation; var inputPath = _config.ReaderConfig.DumpFilesLocation;
if (inputPath == null || inputPath.Count == 0) if (inputPath == null || inputPath.Count == 0)
throw new Exception("Reader dumpFilesLocations must be set to a valid value"); throw new Exception("Reader dumpFilesLocations must be set to a valid value");
@ -101,12 +110,12 @@ public class QueuePipeline(
return gatheredFiles; return gatheredFiles;
} }
private Queue<string> GetFileQueue(List<string> inputPath) private Queue<string> GetFileQueue(IReadOnlyList<string> inputPath)
{ {
var queuedPathsToProcess = new Queue<string>(); var queuedPathsToProcess = new Queue<string>();
var queuedFilesToProcess = new Queue<string>(); var queuedFilesToProcess = new Queue<string>();
inputPath.ForEach(p => queuedPathsToProcess.Enqueue(p)); foreach (var path in inputPath) queuedPathsToProcess.Enqueue(path);
while (queuedPathsToProcess.TryDequeue(out var path)) while (queuedPathsToProcess.TryDequeue(out var path))
{ {
@ -114,7 +123,7 @@ public class QueuePipeline(
if (!Directory.Exists(path)) throw new Exception($"Input directory \"{path}\" could not be found"); if (!Directory.Exists(path)) throw new Exception($"Input directory \"{path}\" could not be found");
// If we should process subfolder, queue them up as well // If we should process subfolder, queue them up as well
if (LootDumpProcessorContext.GetConfig().ReaderConfig.ProcessSubFolders) if (_config.ReaderConfig.ProcessSubFolders)
foreach (var directory in Directory.GetDirectories(path)) foreach (var directory in Directory.GetDirectories(path))
queuedPathsToProcess.Enqueue(directory); queuedPathsToProcess.Enqueue(directory);
@ -138,13 +147,13 @@ public class QueuePipeline(
_logger.LogInformation("Files sorted and ready to begin pre-processing"); _logger.LogInformation("Files sorted and ready to begin pre-processing");
Parallel.ForEach(_filesToProcess, file => await Parallel.ForEachAsync(_filesToProcess, async (file, _) =>
{ {
try try
{ {
if (!_intakeReader.Read(file, out var basicInfo)) return; if (!_intakeReader.Read(file, out var basicInfo)) return;
var partialData = _fileProcessor.Process(basicInfo); var partialData = await _fileProcessor.Process(basicInfo);
collector.Hold(partialData); collector.Hold(partialData);
} }
catch (Exception e) catch (Exception e)
@ -156,15 +165,14 @@ public class QueuePipeline(
_logger.LogInformation("Pre-processing finished"); _logger.LogInformation("Pre-processing finished");
// Single writer instance to collect results // Single writer instance to collect results
var writer = WriterFactory.GetInstance();
// Single collector instance to collect results // Single collector instance to collect results
var partialData = collector.Retrieve(); var partialData = collector.Retrieve();
var processedDumps = await _dumpProcessor.ProcessDumps(partialData); var processedDumps = await _dumpProcessor.ProcessDumps(partialData);
writer.WriteAll(processedDumps); _writer.WriteAll(processedDumps);
// clear collector and datastorage as we process per map now // clear collector and datastorage as we process per map now
collector.Clear(); collector.Clear();
DataStorageFactory.GetInstance().Clear(); _dataStorage.Clear();
} }
/// <summary> /// <summary>
@ -172,7 +180,7 @@ public class QueuePipeline(
/// </summary> /// </summary>
private async Task FixFilesFromDumps() private async Task FixFilesFromDumps()
{ {
var inputPath = LootDumpProcessorContext.GetConfig().ReaderConfig.DumpFilesLocation; var inputPath = _config.ReaderConfig.DumpFilesLocation;
if (inputPath == null || inputPath.Count == 0) if (inputPath == null || inputPath.Count == 0)
throw new Exception("Reader dumpFilesLocations must be set to a valid value"); throw new Exception("Reader dumpFilesLocations must be set to a valid value");
@ -181,7 +189,7 @@ public class QueuePipeline(
await Parallel.ForEachAsync(_filesToRename, async (file, cancellationToken) => await Parallel.ForEachAsync(_filesToRename, async (file, cancellationToken) =>
{ {
if (_mapNames.Any(file.Contains)) return; if (MapNames.Any(file.Contains)) return;
try try
{ {

View File

@ -1,6 +1,8 @@
using System.Globalization; using System.Globalization;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using LootDumpProcessor.Model.Config;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Reader.Filters; namespace LootDumpProcessor.Process.Reader.Filters;
@ -9,12 +11,14 @@ public class JsonDumpFileFilter : IFileFilter
private readonly ILogger<JsonDumpFileFilter> _logger; private readonly ILogger<JsonDumpFileFilter> _logger;
private readonly Regex _fileNameDateRegex = new("([0-9]{4}(-[0-9]{2}){2}_((-){0,1}[0-9]{2}){3})"); private readonly Regex _fileNameDateRegex = new("([0-9]{4}(-[0-9]{2}){2}_((-){0,1}[0-9]{2}){3})");
private readonly DateTime _parsedThresholdDate; private readonly DateTime _parsedThresholdDate;
private readonly Config _config;
public JsonDumpFileFilter(ILogger<JsonDumpFileFilter> logger) public JsonDumpFileFilter(ILogger<JsonDumpFileFilter> logger, IOptions<Config> config)
{ {
_logger = logger ?? throw new ArgumentNullException(nameof(logger)); _logger = logger ?? throw new ArgumentNullException(nameof(logger));
_config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
// Calculate parsed date from config threshold // Calculate parsed date from config threshold
if (string.IsNullOrEmpty(LootDumpProcessorContext.GetConfig().ReaderConfig.ThresholdDate)) if (string.IsNullOrEmpty(_config.ReaderConfig.ThresholdDate))
{ {
_logger.LogWarning("ThresholdDate is null or empty in configs, defaulting to current day minus 30 days"); _logger.LogWarning("ThresholdDate is null or empty in configs, defaulting to current day minus 30 days");
_parsedThresholdDate = DateTime.Now - TimeSpan.FromDays(30); _parsedThresholdDate = DateTime.Now - TimeSpan.FromDays(30);
@ -22,7 +26,7 @@ public class JsonDumpFileFilter : IFileFilter
else else
{ {
_parsedThresholdDate = DateTime.ParseExact( _parsedThresholdDate = DateTime.ParseExact(
LootDumpProcessorContext.GetConfig().ReaderConfig.ThresholdDate, _config.ReaderConfig.ThresholdDate,
"yyyy-MM-dd", "yyyy-MM-dd",
CultureInfo.InvariantCulture CultureInfo.InvariantCulture
); );

View File

@ -1,19 +1,23 @@
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Text.Json; using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Input; using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Serializers.Json; using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Utils; using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Reader.Intake; namespace LootDumpProcessor.Process.Reader.Intake;
public class JsonFileIntakeReader(ILogger<JsonFileIntakeReader> logger) : IIntakeReader public class JsonFileIntakeReader(ILogger<JsonFileIntakeReader> logger, IOptions<Config> config) : IIntakeReader
{ {
private readonly ILogger<JsonFileIntakeReader> _logger = logger ?? throw new ArgumentNullException(nameof(logger)); private readonly ILogger<JsonFileIntakeReader> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
private readonly HashSet<string>? _ignoredLocations = LootDumpProcessorContext.GetConfig() private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
.ReaderConfig.IntakeReaderConfig?.IgnoredDumpLocations.ToHashSet();
private HashSet<string> IgnoredLocations => _config
.ReaderConfig.IntakeReaderConfig.IgnoredDumpLocations.ToHashSet();
private readonly ConcurrentDictionary<string, int> _totalMapDumpsCounter = new(); private readonly ConcurrentDictionary<string, int> _totalMapDumpsCounter = new();
@ -43,14 +47,14 @@ public class JsonFileIntakeReader(ILogger<JsonFileIntakeReader> logger) : IIntak
_logger.LogError("Could not parse date from file: {File}", file); _logger.LogError("Could not parse date from file: {File}", file);
var fi = JsonSerializer.Deserialize<RootData>(fileData, JsonSerializerSettings.Default); var fi = JsonSerializer.Deserialize<RootData>(fileData, JsonSerializerSettings.Default);
if (fi?.Data.LocationLoot.Name != null && (!_ignoredLocations?.Contains(fi.Data.LocationLoot.Name) ?? true)) if (fi?.Data.LocationLoot.Name != null && (!IgnoredLocations?.Contains(fi.Data.LocationLoot.Name) ?? true))
{ {
var mapName = fi.Data.LocationLoot.Name; var mapName = fi.Data.LocationLoot.Name;
var mapId = fi.Data.LocationLoot.Id.ToLower(); var mapId = fi.Data.LocationLoot.Id.ToLower();
var counter = _totalMapDumpsCounter.AddOrUpdate(mapName, 0, (_, current) => current); var counter = _totalMapDumpsCounter.AddOrUpdate(mapName, 0, (_, current) => current);
var maxDumpsPerMap = LootDumpProcessorContext.GetConfig() var maxDumpsPerMap = _config
.ReaderConfig.IntakeReaderConfig?.MaxDumpsPerMap ?? 1500; .ReaderConfig.IntakeReaderConfig?.MaxDumpsPerMap ?? 1500;
if (counter < maxDumpsPerMap) if (counter < maxDumpsPerMap)

View File

@ -1,8 +1,10 @@
using System.Globalization; using System.Globalization;
using LootDumpProcessor.Model; using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Services.KeyGenerator;
using LootDumpProcessor.Process.Services.TarkovItemsProvider;
namespace LootDumpProcessor.Process; namespace LootDumpProcessor.Process.Services.ComposedKeyGenerator;
public class ComposedKeyGenerator(ITarkovItemsProvider tarkovItemsProvider, IKeyGenerator keyGenerator) public class ComposedKeyGenerator(ITarkovItemsProvider tarkovItemsProvider, IKeyGenerator keyGenerator)
: IComposedKeyGenerator : IComposedKeyGenerator

View File

@ -1,6 +1,6 @@
using LootDumpProcessor.Model; using LootDumpProcessor.Model;
namespace LootDumpProcessor.Process; namespace LootDumpProcessor.Process.Services.ComposedKeyGenerator;
public interface IComposedKeyGenerator public interface IComposedKeyGenerator
{ {

View File

@ -0,0 +1,56 @@
using System.Collections.Frozen;
using System.Collections.Immutable;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Serializers.Yaml;
namespace LootDumpProcessor.Process.Services.ForcedItemsProvider;
public class ForcedItemsProvider : IForcedItemsProvider
{
private static readonly string ForcedStaticPath = Path.Combine("Config", "forced_static.yaml");
private static readonly string ForcedLooseLootPath = Path.Combine("Config", "forced_loose.yaml");
private ForcedStatic? _forcedStatic;
private FrozenDictionary<string, ImmutableHashSet<string>>? _forcedLoose;
public async Task<ForcedStatic> GetForcedStatic()
{
if (_forcedStatic is not null) return _forcedStatic;
_forcedStatic = await ReadForcedStatic();
return _forcedStatic;
}
private async Task<ForcedStatic> ReadForcedStatic()
{
var forcedStaticContent = await File.ReadAllTextAsync(ForcedStaticPath);
// Workaround needed because YamlDotNet cannot deserialize properly
var forcedStaticDto = Yaml.Deserializer.Deserialize<ForcedStaticDto>(forcedStaticContent);
var forcedStatic = new ForcedStatic(
forcedStaticDto.StaticWeaponIds.AsReadOnly(),
forcedStaticDto.ForcedItems.ToFrozenDictionary(
kvp => kvp.Key,
IReadOnlyList<StaticForced> (kvp) => kvp.Value.AsReadOnly()
));
return forcedStatic;
}
public async Task<FrozenDictionary<string, ImmutableHashSet<string>>> GetForcedLooseItems()
{
if (_forcedLoose is not null) return _forcedLoose;
_forcedLoose = await ReadForcedLooseItems();
return _forcedLoose;
}
private async Task<FrozenDictionary<string, ImmutableHashSet<string>>> ReadForcedLooseItems()
{
var forcedLooseContent = await File.ReadAllTextAsync(ForcedLooseLootPath);
var forcedLooseLoot = Yaml.Deserializer.Deserialize<Dictionary<string, HashSet<string>>>(forcedLooseContent);
return forcedLooseLoot.ToFrozenDictionary(
pair => pair.Key,
pair => ImmutableHashSet.CreateRange(pair.Value)
);
}
}

View File

@ -0,0 +1,11 @@
using System.Collections.Frozen;
using System.Collections.Immutable;
using LootDumpProcessor.Model.Config;
namespace LootDumpProcessor.Process.Services.ForcedItemsProvider;
public interface IForcedItemsProvider
{
Task<ForcedStatic> GetForcedStatic();
Task<FrozenDictionary<string, ImmutableHashSet<string>>> GetForcedLooseItems();
}

View File

@ -0,0 +1,6 @@
namespace LootDumpProcessor.Process.Services.KeyGenerator;
public interface IKeyGenerator
{
string Generate();
}

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process; namespace LootDumpProcessor.Process.Services.KeyGenerator;
public class NumericKeyGenerator : IKeyGenerator public class NumericKeyGenerator : IKeyGenerator
{ {

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process; namespace LootDumpProcessor.Process.Services.TarkovItemsProvider;
public interface ITarkovItemsProvider public interface ITarkovItemsProvider
{ {

View File

@ -1,22 +1,26 @@
using System.Collections.Frozen; using System.Collections.Frozen;
using System.Text.Json; using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Tarkov; using LootDumpProcessor.Model.Tarkov;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process; namespace LootDumpProcessor.Process.Services.TarkovItemsProvider;
public class TarkovItemsProvider : ITarkovItemsProvider public class TarkovItemsProvider : ITarkovItemsProvider
{ {
private readonly ILogger<TarkovItemsProvider> _logger; private readonly ILogger<TarkovItemsProvider> _logger;
private readonly FrozenDictionary<string, TemplateFileItem>? _items; private readonly FrozenDictionary<string, TemplateFileItem>? _items;
private readonly Config _config;
private static readonly string ItemsFilePath = Path.Combine( private string ItemsFilePath => Path.Combine(
LootDumpProcessorContext.GetConfig().ServerLocation, _config.ServerLocation,
"project", "assets", "database", "templates", "items.json"); "project", "assets", "database", "templates", "items.json");
public TarkovItemsProvider(ILogger<TarkovItemsProvider> logger) public TarkovItemsProvider(ILogger<TarkovItemsProvider> logger, IOptions<Config> config)
{ {
_logger = logger ?? throw new ArgumentNullException(nameof(logger)); _logger = logger ?? throw new ArgumentNullException(nameof(logger));
_config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
try try
{ {

View File

@ -1,18 +1,21 @@
using System.Text.Json; using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Output; using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot; using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Output.StaticContainer; using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Serializers.Json; using LootDumpProcessor.Serializers.Json;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Process.Writer; namespace LootDumpProcessor.Process.Writer;
public class FileWriter : IWriter public class FileWriter : IWriter
{ {
private static readonly string _outputPath; private readonly string _outputPath;
static FileWriter() public FileWriter(IOptions<Config> config)
{ {
var path = LootDumpProcessorContext.GetConfig().WriterConfig.OutputLocation; var config1 = (config ?? throw new ArgumentNullException(nameof(config))).Value;
var path = config1.WriterConfig.OutputLocation;
if (string.IsNullOrEmpty(path)) if (string.IsNullOrEmpty(path))
throw new Exception("Output directory must be set in WriterConfigs"); throw new Exception("Output directory must be set in WriterConfigs");

View File

@ -1,8 +0,0 @@
namespace LootDumpProcessor.Process.Writer;
public static class WriterFactory
{
public static IWriter GetInstance() =>
// implement actual factory someday
new FileWriter();
}

View File

@ -1,15 +1,5 @@
using LootDumpProcessor.Process; using LootDumpProcessor.Process;
using LootDumpProcessor.Process.Processor.DumpProcessor;
using LootDumpProcessor.Process.Processor.FileProcessor;
using LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
using LootDumpProcessor.Process.Processor.v2.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
using LootDumpProcessor.Process.Reader.Filters;
using LootDumpProcessor.Process.Reader.Intake;
using LootDumpProcessor.Storage;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor; namespace LootDumpProcessor;
@ -18,33 +8,11 @@ public static class Program
public static async Task Main() public static async Task Main()
{ {
var services = new ServiceCollection(); var services = new ServiceCollection();
RegisterServices(services); services.AddLootDumpProcessor();
await using var serviceProvider = services.BuildServiceProvider(); await using var serviceProvider = services.BuildServiceProvider();
// startup the pipeline
var pipeline = serviceProvider.GetRequiredService<IPipeline>(); var pipeline = serviceProvider.GetRequiredService<IPipeline>();
await pipeline.Execute(); await pipeline.Execute();
} }
private static void RegisterServices(ServiceCollection services)
{
services.AddLogging(configure => configure.AddConsole());
services.AddTransient<IStaticLootProcessor, StaticLootProcessor>();
services.AddTransient<IStaticContainersProcessor, StaticContainersProcessor>();
services.AddTransient<IAmmoProcessor, AmmoProcessor>();
services.AddTransient<ILooseLootProcessor, LooseLootProcessor>();
services.AddSingleton<ITarkovItemsProvider, TarkovItemsProvider>();
services.AddSingleton<IDataStorage>(_ => DataStorageFactory.GetInstance());
services.AddSingleton<IKeyGenerator, NumericKeyGenerator>();
services.AddTransient<IComposedKeyGenerator, ComposedKeyGenerator>();
services.AddTransient<IIntakeReader, JsonFileIntakeReader>();
services.AddTransient<IFileFilter, JsonDumpFileFilter>();
services.AddTransient<IFileProcessor, FileProcessor>();
services.AddTransient<IDumpProcessor, MultithreadSteppedDumpProcessor>();
services.AddTransient<IPipeline, QueuePipeline>();
}
} }

View File

@ -0,0 +1,103 @@
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Process;
using LootDumpProcessor.Process.Collector;
using LootDumpProcessor.Process.Processor.AmmoProcessor;
using LootDumpProcessor.Process.Processor.DumpProcessor;
using LootDumpProcessor.Process.Processor.FileProcessor;
using LootDumpProcessor.Process.Processor.LooseLootProcessor;
using LootDumpProcessor.Process.Processor.StaticContainersProcessor;
using LootDumpProcessor.Process.Processor.StaticLootProcessor;
using LootDumpProcessor.Process.Reader.Filters;
using LootDumpProcessor.Process.Reader.Intake;
using LootDumpProcessor.Process.Services.ComposedKeyGenerator;
using LootDumpProcessor.Process.Services.ForcedItemsProvider;
using LootDumpProcessor.Process.Services.KeyGenerator;
using LootDumpProcessor.Process.Services.TarkovItemsProvider;
using LootDumpProcessor.Process.Writer;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage.Implementations.File;
using LootDumpProcessor.Storage.Implementations.Memory;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor;
public static class ServiceCollectionExtensions
{
public static void AddLootDumpProcessor(this ServiceCollection services)
{
services.AddLogging(configure => configure.AddConsole());
AddConfiguration(services);
AddCollector(services);
AddDataStorage(services);
RegisterProcessors(services);
services.AddSingleton<StoreHandlerFactory>();
services.AddSingleton<IForcedItemsProvider, ForcedItemsProvider>();
services.AddSingleton<ITarkovItemsProvider, TarkovItemsProvider>();
services.AddSingleton<IKeyGenerator, NumericKeyGenerator>();
services.AddTransient<IComposedKeyGenerator, ComposedKeyGenerator>();
services.AddTransient<IWriter, FileWriter>();
services.AddTransient<IIntakeReader, JsonFileIntakeReader>();
services.AddTransient<IFileFilter, JsonDumpFileFilter>();
services.AddTransient<IFileProcessor, FileProcessor>();
services.AddTransient<IDumpProcessor, MultithreadSteppedDumpProcessor>();
services.AddTransient<IPipeline, QueuePipeline>();
}
private static void AddConfiguration(IServiceCollection services)
{
const string configPath = "Config/config.json";
var configuration = new ConfigurationBuilder()
.AddJsonFile(configPath)
.AddEnvironmentVariables()
.Build();
services.AddOptions<Config>()
.Bind(configuration)
.ValidateDataAnnotations()
.ValidateOnStart();
}
private static void RegisterProcessors(IServiceCollection services)
{
services.AddTransient<IStaticLootProcessor, StaticLootProcessor>();
services.AddTransient<IStaticContainersProcessor, StaticContainersProcessor>();
services.AddTransient<IAmmoProcessor, AmmoProcessor>();
services.AddTransient<ILooseLootProcessor, LooseLootProcessor>();
}
private static void AddCollector(IServiceCollection services)
{
services.AddSingleton<ICollector>(provider =>
{
var config = provider.GetRequiredService<IOptions<Config>>();
var collectorType = config.Value.CollectorConfig.CollectorType;
return collectorType switch
{
CollectorType.Memory => new HashSetCollector(),
CollectorType.Dump => new DumpCollector(config),
_ => throw new ArgumentOutOfRangeException($"CollectorType '{collectorType} is not supported'")
};
});
}
private static void AddDataStorage(IServiceCollection services)
{
services.AddSingleton<IDataStorage>(provider =>
{
var config = provider.GetRequiredService<IOptions<Config>>().Value;
var dataStorageType = config.DataStorageConfig.DataStorageType;
return dataStorageType switch
{
DataStorageTypes.File => new FileDataStorage(provider.GetRequiredService<StoreHandlerFactory>()),
DataStorageTypes.Memory => new MemoryDataStorage(),
_ => throw new ArgumentOutOfRangeException($"DataStorageType '{dataStorageType} is not supported'")
};
});
}
}

View File

@ -1,32 +0,0 @@
using System.Collections.Concurrent;
using LootDumpProcessor.Storage.Implementations.File;
using LootDumpProcessor.Storage.Implementations.Memory;
namespace LootDumpProcessor.Storage;
public static class DataStorageFactory
{
private static readonly ConcurrentDictionary<DataStorageTypes, IDataStorage> DataStorage = new();
/**
* Requires LootDumpProcessorContext to be initialized before using
*/
public static IDataStorage GetInstance() =>
GetInstance(LootDumpProcessorContext.GetConfig().DataStorageConfig.DataStorageType);
private static IDataStorage GetInstance(DataStorageTypes type)
{
if (DataStorage.TryGetValue(type, out var dataStorage)) return dataStorage;
dataStorage = type switch
{
DataStorageTypes.File => new FileDataStorage(),
DataStorageTypes.Memory => new MemoryDataStorage(),
_ => throw new ArgumentOutOfRangeException(nameof(type), type, null)
};
DataStorage.TryAdd(type, dataStorage);
return dataStorage;
}
}

View File

@ -1,16 +1,19 @@
namespace LootDumpProcessor.Storage.Implementations.File; namespace LootDumpProcessor.Storage.Implementations.File;
public class FileDataStorage : IDataStorage public class FileDataStorage(StoreHandlerFactory storeHandlerFactory) : IDataStorage
{ {
private readonly StoreHandlerFactory _storeHandlerFactory =
storeHandlerFactory ?? throw new ArgumentNullException(nameof(storeHandlerFactory));
public void Store<TEntity>(TEntity entity) where TEntity : IKeyable public void Store<TEntity>(TEntity entity) where TEntity : IKeyable
{ {
StoreHandlerFactory.GetInstance(entity.GetKey().GetKeyType()).Store(entity); _storeHandlerFactory.GetInstance(entity.GetKey().GetKeyType()).Store(entity);
} }
public bool Exists(IKey key) => StoreHandlerFactory.GetInstance(key.GetKeyType()).Exists(key); public bool Exists(IKey key) => _storeHandlerFactory.GetInstance(key.GetKeyType()).Exists(key);
public T GetItem<T>(IKey key) where T : IKeyable => public T GetItem<T>(IKey key) where T : IKeyable =>
StoreHandlerFactory.GetInstance(key.GetKeyType()).Retrieve<T>(key); _storeHandlerFactory.GetInstance(key.GetKeyType()).Retrieve<T>(key);
public void Clear() public void Clear()
{ {

View File

@ -1,10 +1,14 @@
using System.Text.Json; using System.Text.Json;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Serializers.Json; using LootDumpProcessor.Serializers.Json;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Storage.Implementations.File.Handlers; namespace LootDumpProcessor.Storage.Implementations.File.Handlers;
public abstract class AbstractStoreHandler : IStoreHandler public abstract class AbstractStoreHandler(IOptions<Config> config) : IStoreHandler
{ {
private readonly Config _config = (config ?? throw new ArgumentNullException(nameof(config))).Value;
public void Store<TEntity>(TEntity entity, bool failIfDuplicate = true) where TEntity : IKeyable public void Store<TEntity>(TEntity entity, bool failIfDuplicate = true) where TEntity : IKeyable
{ {
var locationWithFile = GetLocation(entity.GetKey()); var locationWithFile = GetLocation(entity.GetKey());
@ -34,8 +38,8 @@ public abstract class AbstractStoreHandler : IStoreHandler
protected virtual string GetBaseLocation() protected virtual string GetBaseLocation()
{ {
var location = var location =
string.IsNullOrEmpty(LootDumpProcessorContext.GetConfig().DataStorageConfig.FileDataStorageTempLocation) string.IsNullOrEmpty(_config.DataStorageConfig.FileDataStorageTempLocation)
? LootDumpProcessorContext.GetConfig().DataStorageConfig.FileDataStorageTempLocation ? _config.DataStorageConfig.FileDataStorageTempLocation
: Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); : Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
return $"{location}/SPT/tmp/LootGen"; return $"{location}/SPT/tmp/LootGen";

View File

@ -1,6 +1,9 @@
using LootDumpProcessor.Model.Config;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Storage.Implementations.File.Handlers; namespace LootDumpProcessor.Storage.Implementations.File.Handlers;
public class FlatStoreHandler : AbstractStoreHandler public class FlatStoreHandler(IOptions<Config> config) : AbstractStoreHandler(config)
{ {
protected override string GetLocation(IKey key) protected override string GetLocation(IKey key)
{ {

View File

@ -1,6 +1,9 @@
using LootDumpProcessor.Model.Config;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Storage.Implementations.File.Handlers; namespace LootDumpProcessor.Storage.Implementations.File.Handlers;
public class SubdivisionedStoreHandler : AbstractStoreHandler public class SubdivisionedStoreHandler(IOptions<Config> config) : AbstractStoreHandler(config)
{ {
protected override string GetLocation(IKey key) protected override string GetLocation(IKey key)
{ {

View File

@ -1,20 +1,27 @@
using System.Collections.Concurrent; using System.Collections.Concurrent;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Storage.Implementations.File.Handlers; using LootDumpProcessor.Storage.Implementations.File.Handlers;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
namespace LootDumpProcessor.Storage.Implementations.File; namespace LootDumpProcessor.Storage.Implementations.File;
public class StoreHandlerFactory public class StoreHandlerFactory(IServiceProvider serviceProvider)
{ {
private readonly IServiceProvider _serviceProvider =
serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
private static readonly ConcurrentDictionary<KeyType, IStoreHandler> Handlers = new(); private static readonly ConcurrentDictionary<KeyType, IStoreHandler> Handlers = new();
public static IStoreHandler GetInstance(KeyType type) public IStoreHandler GetInstance(KeyType type)
{ {
if (Handlers.TryGetValue(type, out var handler)) return handler; if (Handlers.TryGetValue(type, out var handler)) return handler;
var config = _serviceProvider.GetRequiredService<IOptions<Config>>();
handler = type switch handler = type switch
{ {
KeyType.Unique => new FlatStoreHandler(), KeyType.Unique => new FlatStoreHandler(config),
KeyType.Subdivisioned => new SubdivisionedStoreHandler(), KeyType.Subdivisioned => new SubdivisionedStoreHandler(config),
_ => throw new ArgumentOutOfRangeException(nameof(type), type, null) _ => throw new ArgumentOutOfRangeException(nameof(type), type, null)
}; };
Handlers.TryAdd(type, handler); Handlers.TryAdd(type, handler);