spt/loot-dump-processor
spt/loot-dump-processor
0
0
Fork 0
mirror of https://github.com/sp-tarkov/loot-dump-processor.git synced 2025-02-13 02:50:45 -05:00
Code

Refactored processors to use dependency injection and improved immutability

Browse Source
This commit is contained in:
bluextx 2025-01-11 06:54:59 +03:00
parent 8cc4340340
commit 9be5d6e342
19 changed files with 398 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
source/LootDumpProcessor/LootDumpProcessor.csproj
View File

@ -12,6 +12,8 @@
<ItemGroup>
<PackageReference Include="7z.Libs" Version="21.7.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
<PackageReference Include="NumSharp" Version="0.30.0" />
<PackageReference Include="SevenZipSharp.Interop" Version="19.1.0" />

4
source/LootDumpProcessor/Model/Output/StaticItemDistribution.cs
View File

@ -7,10 +7,10 @@ namespace LootDumpProcessor.Model.Output
{
[JsonProperty("itemcountDistribution", NullValueHandling = NullValueHandling.Ignore)]
[JsonPropertyName("itemcountDistribution")]
public List<ItemCountDistribution>? ItemCountDistribution { get; set; }
public IReadOnlyList<ItemCountDistribution>? ItemCountDistribution { get; set; }
[JsonProperty("itemDistribution", NullValueHandling = NullValueHandling.Ignore)]
[JsonPropertyName("itemDistribution")]
public List<StaticDistribution>? ItemDistribution { get; set; }
public IReadOnlyList<StaticDistribution>? ItemDistribution { get; set; }
}
}

2
source/LootDumpProcessor/Model/Processing/ParsedDump.cs
View File

@ -8,7 +8,7 @@ public class ParsedDump : IKeyable
private static readonly Regex _hashRegex = new("([^a-zA-Z0-9])");
public BasicInfo BasicInfo { get; set; }
public PreProcessedLooseLoot LooseLoot { get; set; }
public List<PreProcessedStaticLoot> Containers { get; set; }
public IReadOnlyList<PreProcessedStaticLoot> Containers { get; set; }
public override bool Equals(object? obj)
{

10
source/LootDumpProcessor/Process/PipelineFactory.cs
View File

@ -1,10 +0,0 @@
namespace LootDumpProcessor.Process;
public static class PipelineFactory
{
public static IPipeline GetInstance()
{
// implement actual factory at some point
return new QueuePipeline();
}
}

12
source/LootDumpProcessor/Process/Processor/DumpProcessor/DumpProcessorFactory.cs
View File

@ -1,12 +0,0 @@
namespace LootDumpProcessor.Process.Processor.DumpProcessor;
public static class DumpProcessorFactory
{
public static IDumpProcessor GetInstance()
{
// Implement real factory
return new MultithreadSteppedDumpProcessor();
}
}

60
source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
View File

@ -4,6 +4,9 @@ using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage.Collections;
@ -11,8 +14,20 @@ using LootDumpProcessor.Utils;
namespace LootDumpProcessor.Process.Processor.DumpProcessor;
public class MultithreadSteppedDumpProcessor : IDumpProcessor
public class MultithreadSteppedDumpProcessor(
IStaticLootProcessor staticLootProcessor, IStaticContainersProcessor staticContainersProcessor,
IAmmoProcessor ammoProcessor
) : IDumpProcessor
{
private readonly IStaticLootProcessor _staticLootProcessor =
staticLootProcessor ?? throw new ArgumentNullException(nameof(staticLootProcessor));
private readonly IStaticContainersProcessor _staticContainersProcessor =
staticContainersProcessor ?? throw new ArgumentNullException(nameof(staticContainersProcessor));
private readonly IAmmoProcessor _ammoProcessor =
ammoProcessor ?? throw new ArgumentNullException(nameof(ammoProcessor));
private static IJsonSerializer _jsonSerializer = JsonSerializerFactory.GetInstance();
private static readonly List<Task> Runners = new();
@ -51,14 +66,17 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
Task.Factory.StartNew(() =>
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", LogLevel.Debug);
LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}",
LogLevel.Debug);
var dataDump = _jsonSerializer.Deserialize<RootData>(File.ReadAllText(dumped.BasicInfo.FileName));
if (dataDump == null)
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"Failed to deserialize data from file {dumped.BasicInfo.FileName}", LogLevel.Error);
LoggerFactory.GetInstance()
.Log($"Failed to deserialize data from file {dumped.BasicInfo.FileName}",
LogLevel.Error);
return; // Skip processing this dump
}
@ -70,7 +88,9 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
if (!staticContainers.TryGetValue(mapId, out var mapStaticLoot))
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log($"Doing first time process for map {mapId} of real static data", LogLevel.Info);
LoggerFactory.GetInstance()
.Log($"Doing first time process for map {mapId} of real static data",
LogLevel.Info);
staticContainers[mapId] = new MapStaticLoot
{
@ -82,10 +102,13 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
{
// .Item1 = map name
// .Item2 = force/weapon static arrays
var mapStaticContainers = StaticLootProcessor.CreateStaticWeaponsAndStaticForcedContainers(dataDump);
var mapStaticContainers =
_staticContainersProcessor.CreateStaticWeaponsAndForcedContainers(dataDump);
var newStaticWeapons = mapStaticContainers.Item2.StaticWeapons.Where(x => !mapStaticLoot.StaticWeapons.Exists(y => y.Id == x.Id));
var newStaticForced = mapStaticContainers.Item2.StaticForced.Where(x => !mapStaticLoot.StaticForced.Exists(y => y.ContainerId == x.ContainerId));
var newStaticWeapons = mapStaticContainers.Item2.StaticWeapons.Where(x =>
!mapStaticLoot.StaticWeapons.Exists(y => y.Id == x.Id));
var newStaticForced = mapStaticContainers.Item2.StaticForced.Where(x =>
!mapStaticLoot.StaticForced.Exists(y => y.ContainerId == x.ContainerId));
mapStaticLoot.StaticWeapons.AddRange(newStaticWeapons);
mapStaticLoot.StaticForced.AddRange(newStaticForced);
@ -116,8 +139,10 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
IncrementMapCounterDictionaryValue(mapDumpCounter, mapId);
}
var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(mapId, out string[]? ignoreListForMap);
foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(dataDump))
var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList
.TryGetValue(mapId, out string[]? ignoreListForMap);
foreach (var dynamicStaticContainer in _staticContainersProcessor.CreateDynamicStaticContainers(
dataDump))
{
lock (mapStaticContainersAggregatedLock)
{
@ -153,7 +178,9 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) // kv.Key = map name
}
).ToList()
).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); // Hydrate staticContainers.StaticContainers
).ToList()
.ForEach(kv =>
staticContainers[kv.Key].StaticContainers = kv.Value); // Hydrate staticContainers.StaticContainers
// Static containers
output.Add(OutputFileType.StaticContainer, staticContainers);
@ -162,7 +189,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
// Ammo distribution
output.Add(
OutputFileType.StaticAmmo,
StaticLootProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts)
_ammoProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts)
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
@ -170,7 +197,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
// Static loot distribution
output.Add(
OutputFileType.StaticLoot,
StaticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts, staticContainers)
_staticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts, staticContainers)
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
@ -210,7 +237,8 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
}
}
private static double GetStaticContainerProbability(string mapName, KeyValuePair<Template, int> td, Dictionary<string, int> mapDumpCounter)
private static double GetStaticContainerProbability(string mapName, KeyValuePair<Template, int> td,
Dictionary<string, int> mapDumpCounter)
{
return Math.Round((double)((decimal)td.Value / (decimal)mapDumpCounter[mapName]), 2);
}
@ -286,7 +314,8 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
{
if (!dumpProcessData.ContainerCounts.ContainsKey(mapName))
{
dumpProcessData.ContainerCounts.Add(mapName, dumpData.Containers);
dumpProcessData.ContainerCounts.Add(mapName,
dumpData.Containers.ToList());
}
else
{
@ -336,7 +365,8 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
lock (lockObjectCounts)
{
looseLootCounts.MapSpawnpointCount.Add(dumpData.LooseLoot.MapSpawnpointCount);
looseLootCounts.MapSpawnpointCount.Add(
dumpData.LooseLoot.MapSpawnpointCount);
}
}
catch (Exception e)

8
source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessor.cs
View File

@ -1,12 +1,15 @@
using LootDumpProcessor.Logger;
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
using LootDumpProcessor.Storage;
namespace LootDumpProcessor.Process.Processor.FileProcessor;
public class FileProcessor : IFileProcessor
public class FileProcessor(IStaticLootProcessor staticLootProcessor) : IFileProcessor
{
private readonly IStaticLootProcessor _staticLootProcessor = staticLootProcessor ?? throw new ArgumentNullException(nameof(staticLootProcessor));
public PartialData Process(BasicInfo parsedData)
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
@ -43,10 +46,11 @@ public class FileProcessor : IFileProcessor
$"Cached not found for {string.Join("/", dumpData.GetKey().GetLookupIndex())} processing.",
LogLevel.Debug
);
dumpData.Containers = StaticLootProcessor.PreProcessStaticLoot(staticLoot);
dumpData.Containers = _staticLootProcessor.PreProcessStaticLoot(staticLoot);
dumpData.LooseLoot = LooseLootProcessor.PreProcessLooseLoot(looseLoot);
DataStorageFactory.GetInstance().Store(dumpData);
}
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"File {parsedData.FileName} finished processing!", LogLevel.Debug);
return data;

13
source/LootDumpProcessor/Process/Processor/FileProcessor/FileProcessorFactory.cs
View File

@ -1,13 +0,0 @@
namespace LootDumpProcessor.Process.Processor.FileProcessor;
public static class FileProcessorFactory
{
private static IFileProcessor? _fileProcessor;
public static IFileProcessor GetInstance()
{
// TODO: implement actual factory someday
if (_fileProcessor == null)
_fileProcessor = new FileProcessor();
return _fileProcessor;
}
}

281
source/LootDumpProcessor/Process/Processor/StaticLootProcessor.cs
View File

@ -1,281 +0,0 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Utils;
using System.Collections.Generic;
namespace LootDumpProcessor.Process.Processor;
public static class StaticLootProcessor
{
public static List<PreProcessedStaticLoot> PreProcessStaticLoot(List<Template> staticloot)
{
var containers = new List<PreProcessedStaticLoot>();
foreach (var lootSpawnPosition in staticloot)
{
var tpl = lootSpawnPosition.Items[0].Tpl;
if (!LootDumpProcessorContext.GetStaticWeaponIds().Contains(tpl))
{
// Only add non-weapon static containers
containers.Add(new PreProcessedStaticLoot
{
Type = tpl,
ContainerId = lootSpawnPosition.Items[0].Id,
Items = lootSpawnPosition.Items.Skip(1).ToList()
});
}
}
return containers;
}
public static Tuple<string, MapStaticLoot> CreateStaticWeaponsAndStaticForcedContainers(RootData rawMapDump)
{
var mapName = rawMapDump.Data.LocationLoot.Name;
var mapId = rawMapDump.Data.LocationLoot.Id.ToLower();
var staticLootPositions = (from li in rawMapDump.Data.LocationLoot.Loot
where li.IsContainer ?? false
select li).ToList();
var staticWeapons = new List<Template>();
staticLootPositions = staticLootPositions.OrderBy(x => x.Id).ToList();
foreach (var staticLootPosition in staticLootPositions)
{
if (LootDumpProcessorContext.GetStaticWeaponIds().Contains(staticLootPosition.Items[0].Tpl))
{
staticWeapons.Add(ProcessorUtil.Copy(staticLootPosition));
}
}
var forcedStaticItems = LootDumpProcessorContext.GetForcedItems().ContainsKey(mapId)
? LootDumpProcessorContext.GetForcedItems()[mapId]
: new List<StaticForced>();
var mapStaticData = new MapStaticLoot
{
StaticWeapons = staticWeapons,
StaticForced = forcedStaticItems
};
return Tuple.Create(mapId, mapStaticData);
}
public static List<Template> CreateDynamicStaticContainers(RootData rawMapDump)
{
var data = (from li in rawMapDump.Data.LocationLoot.Loot
where (li.IsContainer ?? false) && (!LootDumpProcessorContext.GetStaticWeaponIds().Contains(li.Items[0].Tpl))
select li).ToList();
foreach (var item in data)
{
// remove all but first item from containers items
item.Items = new List<Item> { item.Items[0] };
}
return data;
}
/// <summary>
///
/// </summary>
/// <param name="container_counts"></param>
/// <returns>key = mapid / </returns>
public static Dictionary<string, Dictionary<string, List<AmmoDistribution>>> CreateAmmoDistribution(
Dictionary<string, List<PreProcessedStaticLoot>> container_counts
)
{
var allMapsAmmoDistro = new Dictionary<string, Dictionary<string, List<AmmoDistribution>>>();
foreach (var mapAndContainers in container_counts)
{
var mapid = mapAndContainers.Key;
var containers = mapAndContainers.Value;
var ammo = new List<string>();
foreach (var ci in containers)
{
ammo.AddRange(from item in ci.Items
where LootDumpProcessorContext.GetTarkovItems().IsBaseClass(item.Tpl, BaseClasses.Ammo)
select item.Tpl);
}
var ammo_counts = new List<CaliberTemplateCount>();
ammo_counts.AddRange(
ammo.GroupBy(a => a)
.Select(g => new CaliberTemplateCount
{
Caliber = LootDumpProcessorContext.GetTarkovItems().AmmoCaliber(g.Key),
Template = g.Key,
Count = g.Count()
})
);
ammo_counts = ammo_counts.OrderBy(x => x.Caliber).ToList();
var ammo_distribution = new Dictionary<string, List<AmmoDistribution>>();
foreach (var _tup_3 in ammo_counts.GroupBy(x => x.Caliber))
{
var k = _tup_3.Key;
var g = _tup_3.ToList();
ammo_distribution[k] = (from gi in g
select new AmmoDistribution
{
Tpl = gi.Template,
RelativeProbability = gi.Count
}).ToList();
}
allMapsAmmoDistro.TryAdd(mapid, ammo_distribution);
}
return allMapsAmmoDistro;
//var ammo = new List<string>();
//foreach (var ci in container_counts)
//{
// ammo.AddRange(from item in ci.Items
// where LootDumpProcessorContext.GetTarkovItems().IsBaseClass(item.Tpl, BaseClasses.Ammo)
// select item.Tpl);
//}
//var ammo_counts = new List<CaliberTemplateCount>();
//ammo_counts.AddRange(
// ammo.GroupBy(a => a)
// .Select(g => new CaliberTemplateCount
// {
// Caliber = LootDumpProcessorContext.GetTarkovItems().AmmoCaliber(g.Key),
// Template = g.Key,
// Count = g.Count()
// })
//);
//ammo_counts = ammo_counts.OrderBy(x => x.Caliber).ToList();
//var ammo_distribution = new Dictionary<string, List<AmmoDistribution>>();
//foreach (var _tup_3 in ammo_counts.GroupBy(x => x.Caliber))
//{
// var k = _tup_3.Key;
// var g = _tup_3.ToList();
// ammo_distribution[k] = (from gi in g
// select new AmmoDistribution
// {
// Tpl = gi.Template,
// RelativeProbability = gi.Count
// }).ToList();
//}
//return ammo_distribution;
}
/// <summary>
/// Dict key = map,
/// value = sub dit:
/// key = container Ids
/// value = items + counts
/// </summary>
public static Dictionary<string, Dictionary<string, StaticItemDistribution>> CreateStaticLootDistribution(
Dictionary<string, List<PreProcessedStaticLoot>> container_counts,
Dictionary<string, MapStaticLoot> staticContainers)
{
var allMapsStaticLootDisto = new Dictionary< string, Dictionary<string, StaticItemDistribution>>();
// Iterate over each map we have containers for
foreach (var mapContainersKvp in container_counts)
{
var mapName = mapContainersKvp.Key;
var containers = mapContainersKvp.Value;
var static_loot_distribution = new Dictionary<string, StaticItemDistribution>();
var uniqueContainerTypeIds = Enumerable.Distinct((from ci in containers
select ci.Type).ToList());
foreach (var typeId in uniqueContainerTypeIds)
{
var container_counts_selected = (from ci in containers
where ci.Type == typeId
select ci).ToList();
// Get array of all times a count of items was found in container
List<int> itemCountsInContainer = GetCountOfItemsInContainer(container_counts_selected);
// Create structure to hold item count + weight that it will be picked
// Group same counts together
static_loot_distribution[typeId] = new StaticItemDistribution();
static_loot_distribution[typeId].ItemCountDistribution = itemCountsInContainer.GroupBy(i => i)
.Select(g => new ItemCountDistribution
{
Count = g.Key,
RelativeProbability = g.Count()
}).ToList();
static_loot_distribution[typeId].ItemDistribution = CreateItemDistribution(container_counts_selected);
}
// Key = containers tpl, value = items + count weights
allMapsStaticLootDisto.TryAdd(mapName, static_loot_distribution);
}
return allMapsStaticLootDisto;
//var static_loot_distribution = new Dictionary<string, StaticItemDistribution>();
//var uniqueContainerTypeIds = Enumerable.Distinct((from ci in container_counts
// select ci.Type).ToList());
//foreach (var typeId in uniqueContainerTypeIds)
//{
// var container_counts_selected = (from ci in container_counts
// where ci.Type == typeId
// select ci).ToList();
// // Get array of all times a count of items was found in container
// List<int> itemCountsInContainer = GetCountOfItemsInContainer(container_counts_selected);
// // Create structure to hold item count + weight that it will be picked
// // Group same counts together
// static_loot_distribution[typeId] = new StaticItemDistribution();
// static_loot_distribution[typeId].ItemCountDistribution = itemCountsInContainer.GroupBy(i => i)
// .Select(g => new ItemCountDistribution
// {
// Count = g.Key,
// RelativeProbability = g.Count()
// }).ToList();
// static_loot_distribution[typeId].ItemDistribution = CreateItemDistribution(container_counts_selected);
//}
//// Key = containers tpl, value = items + count weights
//return static_loot_distribution;
}
private static List<StaticDistribution> CreateItemDistribution(List<PreProcessedStaticLoot> container_counts_selected)
{
// TODO: Change for different algo that splits items per parent once parentid = containerid, then compose
// TODO: key and finally create distribution based on composed Id instead
var itemsHitCounts = new Dictionary<string, int>();
foreach (var ci in container_counts_selected)
{
foreach (var cii in ci.Items.Where(cii => cii.ParentId == ci.ContainerId))
{
if (itemsHitCounts.ContainsKey(cii.Tpl))
itemsHitCounts[cii.Tpl] += 1;
else
itemsHitCounts[cii.Tpl] = 1;
}
}
// WIll create array of objects that have a tpl + relative probability weight value
return itemsHitCounts.Select(v => new StaticDistribution
{
Tpl = v.Key,
RelativeProbability = v.Value
}).ToList();
}
private static List<int> GetCountOfItemsInContainer(List<PreProcessedStaticLoot> container_counts_selected)
{
var itemCountsInContainer = new List<int>();
foreach (var containerWithItems in container_counts_selected)
{
// Only count item if its parent is the container, only root items are counted (not mod/attachment items)
itemCountsInContainer.Add((from cii in containerWithItems.Items
where cii.ParentId == containerWithItems.ContainerId
select cii).ToList().Count);
}
return itemCountsInContainer;
}
}

56
source/LootDumpProcessor/Process/Processor/v2/AmmoProcessor/AmmoProcessor.cs Normal file
View File

@ -0,0 +1,56 @@
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Processing;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.v2.AmmoProcessor
{
public class AmmoProcessor(ILogger<AmmoProcessor> logger) : IAmmoProcessor
{
private readonly ILogger<AmmoProcessor> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
public IReadOnlyDictionary<string, IReadOnlyDictionary<string, List<AmmoDistribution>>> CreateAmmoDistribution(
IReadOnlyDictionary<string, List<PreProcessedStaticLoot>> containerCounts)
{
var allMapsAmmoDistribution = new Dictionary<string, IReadOnlyDictionary<string, List<AmmoDistribution>>>();
foreach (var mapEntry in containerCounts)
{
var mapId = mapEntry.Key;
var containers = mapEntry.Value;
var ammoTemplates = containers
.SelectMany(container => container.Items)
.Where(item => LootDumpProcessorContext.GetTarkovItems().IsBaseClass(item.Tpl, BaseClasses.Ammo))
.Select(item => item.Tpl)
.ToList();
var caliberTemplateCounts = ammoTemplates
.GroupBy(tpl => tpl)
.Select(group => new CaliberTemplateCount
{
Caliber = LootDumpProcessorContext.GetTarkovItems().AmmoCaliber(group.Key),
Template = group.Key,
Count = group.Count()
})
.OrderBy(ctc => ctc.Caliber)
.ToList();
var ammoDistribution = caliberTemplateCounts
.GroupBy(ctc => ctc.Caliber)
.ToDictionary(
group => group.Key,
group => group.Select(ctc => new AmmoDistribution
{
Tpl = ctc.Template,
RelativeProbability = ctc.Count
}).ToList()
);
allMapsAmmoDistribution[mapId] = ammoDistribution;
_logger.LogInformation("Created ammo distribution for Map {MapId}.", mapId);
}
return allMapsAmmoDistribution;
}
}
}

10
source/LootDumpProcessor/Process/Processor/v2/AmmoProcessor/IAmmoProcessor.cs Normal file
View File

@ -0,0 +1,10 @@
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
public interface IAmmoProcessor
{
IReadOnlyDictionary<string, IReadOnlyDictionary<string, List<AmmoDistribution>>> CreateAmmoDistribution(
IReadOnlyDictionary<string, List<PreProcessedStaticLoot>> containerCounts);
}

11
source/LootDumpProcessor/Process/Processor/v2/StaticContainersProcessor/IStaticContainersProcessor.cs Normal file
View File

@ -0,0 +1,11 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output.StaticContainer;
namespace LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
public interface IStaticContainersProcessor
{
(string, MapStaticLoot) CreateStaticWeaponsAndForcedContainers(RootData rawMapDump);
IReadOnlyList<Template> CreateDynamicStaticContainers(RootData rawMapDump);
}

79
source/LootDumpProcessor/Process/Processor/v2/StaticContainersProcessor/StaticContainerProcessor.cs Normal file
View File

@ -0,0 +1,79 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Utils;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
public class StaticContainersProcessor(ILogger<StaticContainersProcessor> logger)
: IStaticContainersProcessor
{
private readonly ILogger<StaticContainersProcessor> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
public (string, MapStaticLoot) CreateStaticWeaponsAndForcedContainers(RootData rawMapDump)
{
var locationLoot = rawMapDump.Data.LocationLoot;
var mapId = locationLoot.Id.ToLower();
var staticLootPositions = locationLoot.Loot
.Where(loot => loot.IsContainer.GetValueOrDefault())
.OrderBy(loot => loot.Id)
.ToList();
var staticWeapons = new List<Template>();
foreach (var lootPosition in staticLootPositions)
{
if (lootPosition.Items == null || lootPosition.Items.Count == 0)
{
_logger.LogWarning("Loot position with ID {LootId} has no items.", lootPosition.Id);
continue;
}
var firstItemTpl = lootPosition.Items[0].Tpl;
if (!LootDumpProcessorContext.GetStaticWeaponIds().Contains(firstItemTpl)) continue;
var copiedLoot = ProcessorUtil.Copy(lootPosition);
staticWeapons.Add(copiedLoot);
_logger.LogDebug("Added static weapon with ID {WeaponId} to Map {MapId}.", copiedLoot.Id, mapId);
}
var forcedStaticItems = LootDumpProcessorContext.GetForcedItems().TryGetValue(mapId, out var forcedItems)
? forcedItems
: new List<StaticForced>();
var mapStaticLoot = new MapStaticLoot
{
StaticWeapons = staticWeapons,
StaticForced = forcedStaticItems
};
_logger.LogInformation("Created static weapons and forced containers for Map {MapId}.", mapId);
return (mapId, mapStaticLoot);
}
public IReadOnlyList<Template> CreateDynamicStaticContainers(RootData rawMapDump)
{
var dynamicContainers = rawMapDump.Data.LocationLoot.Loot
.Where(loot => loot.IsContainer.GetValueOrDefault() &&
!LootDumpProcessorContext.GetStaticWeaponIds().Contains(loot.Items.FirstOrDefault()?.Tpl))
.ToList();
foreach (var container in dynamicContainers)
{
if (container.Items == null || container.Items.Count == 0)
{
_logger.LogWarning("Dynamic container with ID {ContainerId} has no items.", container.Id);
continue;
}
var firstItem = container.Items.First();
container.Items = [firstItem];
_logger.LogDebug("Retained only the first item in dynamic container with ID {ContainerId}.", container.Id);
}
_logger.LogInformation("Created {Count} dynamic static containers.", dynamicContainers.Count);
return dynamicContainers;
}
}

15
source/LootDumpProcessor/Process/Processor/v2/StaticLootProcessor/IStaticLootProcessor.cs Normal file
View File

@ -0,0 +1,15 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
public interface IStaticLootProcessor
{
IReadOnlyList<PreProcessedStaticLoot> PreProcessStaticLoot(IReadOnlyList<Template> staticLoot);
IReadOnlyDictionary<string, IReadOnlyDictionary<string, StaticItemDistribution>> CreateStaticLootDistribution(
IReadOnlyDictionary<string, List<PreProcessedStaticLoot>> containerCounts,
IReadOnlyDictionary<string, MapStaticLoot> staticContainers);
}

117
source/LootDumpProcessor/Process/Processor/v2/StaticLootProcessor/StaticLootProcessor.cs Normal file
View File

@ -0,0 +1,117 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing;
using Microsoft.Extensions.Logging;
namespace LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
public class StaticLootProcessor(ILogger<StaticLootProcessor> logger) : IStaticLootProcessor
{
private readonly ILogger<StaticLootProcessor> _logger =
logger ?? throw new ArgumentNullException(nameof(logger));
public IReadOnlyList<PreProcessedStaticLoot> PreProcessStaticLoot(IReadOnlyList<Template> staticLoot)
{
var nonWeaponContainers = new List<PreProcessedStaticLoot>();
foreach (var lootSpawn in staticLoot)
{
if (lootSpawn.Items == null || lootSpawn.Items.Count == 0)
{
_logger.LogWarning("Loot spawn position with ID {LootId} has no items.", lootSpawn.Id);
continue;
}
var firstItemTpl = lootSpawn.Items[0].Tpl;
if (!LootDumpProcessorContext.GetStaticWeaponIds().Contains(firstItemTpl))
{
nonWeaponContainers.Add(new PreProcessedStaticLoot
{
Type = firstItemTpl,
ContainerId = lootSpawn.Items[0].Id,
Items = lootSpawn.Items.Skip(1).ToList()
});
_logger.LogDebug("Added non-weapon container with ID {ContainerId} and Type {Type}.",
lootSpawn.Items[0].Id, firstItemTpl);
}
}
_logger.LogInformation("Preprocessed {Count} static loot containers.", nonWeaponContainers.Count);
return nonWeaponContainers;
}
public IReadOnlyDictionary<string, IReadOnlyDictionary<string, StaticItemDistribution>>
CreateStaticLootDistribution(
IReadOnlyDictionary<string, List<PreProcessedStaticLoot>> containerCounts,
IReadOnlyDictionary<string, MapStaticLoot> staticContainers)
{
var allMapsStaticLootDistribution =
new Dictionary<string, IReadOnlyDictionary<string, StaticItemDistribution>>();
foreach (var (mapName, containers) in containerCounts)
{
var staticLootDistribution = new Dictionary<string, StaticItemDistribution>();
var uniqueContainerTypes = containers.Select(container => container.Type).Distinct();
foreach (var containerType in uniqueContainerTypes)
{
var selectedContainers = containers.Where(container => container.Type == containerType).ToArray();
var itemCounts = GetItemCountsInContainers(selectedContainers);
var itemDistribution = GenerateItemDistribution(selectedContainers);
staticLootDistribution[containerType] = new StaticItemDistribution
{
ItemCountDistribution = itemCounts
.GroupBy(count => count)
.Select(group => new ItemCountDistribution
{
Count = group.Key,
RelativeProbability = group.Count()
})
.ToList(),
ItemDistribution = itemDistribution
};
_logger.LogDebug(
"Processed static loot distribution for ContainerType {ContainerType} in Map {MapName}.",
containerType, mapName);
}
allMapsStaticLootDistribution[mapName] = staticLootDistribution;
_logger.LogInformation("Created static loot distribution for Map {MapName}.", mapName);
}
return allMapsStaticLootDistribution;
}
private static IReadOnlyList<int> GetItemCountsInContainers(IReadOnlyList<PreProcessedStaticLoot> selectedContainers)
{
return selectedContainers
.Select(container => container.Items.Count(item => item.ParentId == container.ContainerId))
.ToList();
}
private static IReadOnlyList<StaticDistribution> GenerateItemDistribution(
IReadOnlyList<PreProcessedStaticLoot> selectedContainers)
{
var itemHitCounts = new Dictionary<string, int>();
foreach (var container in selectedContainers)
{
foreach (var item in container.Items.Where(item => item.ParentId == container.ContainerId))
{
if (!itemHitCounts.TryAdd(item.Tpl, 1))
itemHitCounts[item.Tpl]++;
}
}
return itemHitCounts.Select(kv => new StaticDistribution
{
Tpl = kv.Key,
RelativeProbability = kv.Value
}).ToArray();
}
}

24
source/LootDumpProcessor/Process/QueuePipeline.cs
View File

@ -16,8 +16,14 @@ using LootDumpProcessor.Utils;
namespace LootDumpProcessor.Process;
public class QueuePipeline : IPipeline
public class QueuePipeline(IFileProcessor fileProcessor, IDumpProcessor dumpProcessor) : IPipeline
{
private readonly IFileProcessor _fileProcessor =
fileProcessor ?? throw new ArgumentNullException(nameof(fileProcessor));
private readonly IDumpProcessor _dumpProcessor =
dumpProcessor ?? throw new ArgumentNullException(nameof(dumpProcessor));
private static readonly BlockingCollection<string> _filesToRename = new();
private static readonly BlockingCollection<string> _filesToProcess = new();
private static readonly List<Task> Runners = new();
@ -227,10 +233,9 @@ public class QueuePipeline : IPipeline
try
{
var reader = IntakeReaderFactory.GetInstance();
var processor = FileProcessorFactory.GetInstance();
if (reader.Read(file, out var basicInfo))
{
collector.Hold(processor.Process(basicInfo));
collector.Hold(_fileProcessor.Process(basicInfo));
}
}
catch (Exception e)
@ -269,8 +274,7 @@ public class QueuePipeline : IPipeline
// Single writer instance to collect results
var writer = WriterFactory.GetInstance();
// Single collector instance to collect results
var dumpProcessor = DumpProcessorFactory.GetInstance();
writer.WriteAll(dumpProcessor.ProcessDumps(collector.Retrieve()));
writer.WriteAll(_dumpProcessor.ProcessDumps(collector.Retrieve()));
}
private void ProcessFilesFromDumpsPerMap(int threads, ICollector collector, string mapName)
@ -305,10 +309,9 @@ public class QueuePipeline : IPipeline
try
{
var reader = IntakeReaderFactory.GetInstance();
var processor = FileProcessorFactory.GetInstance();
if (reader.Read(file, out var basicInfo))
{
collector.Hold(processor.Process(basicInfo));
collector.Hold(_fileProcessor.Process(basicInfo));
}
}
catch (Exception e)
@ -347,8 +350,7 @@ public class QueuePipeline : IPipeline
// Single writer instance to collect results
var writer = WriterFactory.GetInstance();
// Single collector instance to collect results
var dumpProcessor = DumpProcessorFactory.GetInstance();
writer.WriteAll(dumpProcessor.ProcessDumps(collector.Retrieve()));
writer.WriteAll(_dumpProcessor.ProcessDumps(collector.Retrieve()));
// clear collector and datastorage as we process per map now
collector.Clear();
@ -413,7 +415,9 @@ public class QueuePipeline : IPipeline
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
{
LoggerFactory.GetInstance()
.Log($"one or more files are being processed. Waiting {LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs} ms", LogLevel.Info);
.Log(
$"one or more files are being processed. Waiting {LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs} ms",
LogLevel.Info);
}
Thread.Sleep(TimeSpan.FromMilliseconds(LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs));

4
source/LootDumpProcessor/Process/Writer/FileWriter.cs
View File

@ -66,7 +66,7 @@ public class FileWriter : IWriter
break;
case OutputFileType.StaticLoot:
var staticLootData = (Dictionary<string, Dictionary<string, StaticItemDistribution>>)data;
var staticLootData = (IReadOnlyDictionary<string, IReadOnlyDictionary<string, StaticItemDistribution>>)data;
foreach (var (key, value) in staticLootData)
{
if (!Directory.Exists($@"{_outputPath}\locations\{key}"))
@ -77,7 +77,7 @@ public class FileWriter : IWriter
break;
case OutputFileType.StaticAmmo:
var staticAmmo = (Dictionary<string, Dictionary<string, List<AmmoDistribution>>>)data;
var staticAmmo = (IReadOnlyDictionary<string, IReadOnlyDictionary<string, List<AmmoDistribution>>>)data;
foreach (var (key, value) in staticAmmo)
{
if (!Directory.Exists($@"{_outputPath}\locations\{key}"))

28
source/LootDumpProcessor/Program.cs
View File

@ -1,13 +1,34 @@
using LootDumpProcessor.Logger;
using LootDumpProcessor.Process;
using LootDumpProcessor.Process.Processor.DumpProcessor;
using LootDumpProcessor.Process.Processor.FileProcessor;
using LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
using LootDumpProcessor.Storage;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using LoggerFactory = LootDumpProcessor.Logger.LoggerFactory;
namespace LootDumpProcessor;
public static class Program
{
public static void Main()
public static async Task Main()
{
var services = new ServiceCollection();
services.AddLogging(configure => configure.AddConsole());
services.AddTransient<IStaticLootProcessor, StaticLootProcessor>();
services.AddTransient<IStaticContainersProcessor, StaticContainersProcessor>();
services.AddTransient<IAmmoProcessor, AmmoProcessor>();
services.AddTransient<StaticLootProcessor>();
services.AddTransient<IFileProcessor, FileProcessor>();
services.AddTransient<IDumpProcessor, MultithreadSteppedDumpProcessor>();
services.AddTransient<IPipeline, QueuePipeline>();
await using var serviceProvider = services.BuildServiceProvider();
// Bootstrap the config before anything else, its required by the whole application to work
LootDumpProcessorContext.GetConfig();
// Some loggers may need a startup and stop mechanism
@ -15,7 +36,8 @@ public static class Program
// Setup Data storage
DataStorageFactory.GetInstance().Setup();
// startup the pipeline
PipelineFactory.GetInstance().DoProcess();
var pipeline = serviceProvider.GetRequiredService<IPipeline>();
pipeline.DoProcess();
// stop loggers at the end
LoggerFactory.GetInstance().Stop();
Thread.Sleep(10000);

3
source/LootDumpProcessor/Serializers/Json/NetJsonSerializer.cs
View File

@ -15,7 +15,8 @@ public class NetJsonSerializer : IJsonSerializer
new NetJsonKeyConverter(),
new JsonStringEnumConverter(),
new NetDateTimeConverter()
}
},
WriteIndented = true
};
public string Serialize<T>(T obj)