spt/loot-dump-processor
spt/loot-dump-processor
0
0
Fork 0
mirror of https://github.com/sp-tarkov/loot-dump-processor.git synced 2025-02-12 16:50:44 -05:00
Code

Fix static loot output (#6)

Browse Source 
* Improved thread safety and async processing in dump processor components

* Removed unused _processedDumps field and simplified variable scope in QueuePipeline
This commit is contained in:
BlueXTX 2025-01-13 02:19:41 +03:00 committed by GitHub
parent a28df281a0
commit 3cb1be1ec9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 40 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
source/LootDumpProcessor/Model/Template.cs
View File

@ -5,38 +5,25 @@ using LootDumpProcessor.Utils;
namespace LootDumpProcessor.Model;
public class Template : IKeyable, ICloneable
public class Template(
string internalId, string? id, bool isContainer, bool? useGravity, bool? randomRotation,
Vector3? position, Vector3? rotation, bool? isGroupPosition, List<GroupPosition>? groupPositions,
bool? isAlwaysSpawn, string? root, List<Item> items
)
: IKeyable, ICloneable
{
[JsonIgnore] public string InternalId { get; }
[JsonPropertyName("Id")] public string Id { get; set; }
[JsonPropertyName("IsContainer")] public bool IsContainer { get; set; }
public bool UseGravity { get; set; }
public bool RandomRotation { get; set; }
[JsonPropertyName("Position")] public Vector3 Position { get; set; }
[JsonPropertyName("Rotation")] public Vector3 Rotation { get; set; }
[JsonPropertyName("IsGroupPosition")] public bool IsGroupPosition { get; set; }
[JsonPropertyName("GroupPositions")] public List<GroupPosition> GroupPositions { get; set; }
[JsonPropertyName("IsAlwaysSpawn")] public bool IsAlwaysSpawn { get; set; }
[JsonPropertyName("Root")] public string Root { get; set; }
[JsonPropertyName("Items")] public List<Item> Items { get; set; }
public Template(string internalId, string id, bool isContainer, bool useGravity, bool randomRotation,
Vector3 position, Vector3 rotation, bool isGroupPosition, List<GroupPosition> groupPositions,
bool isAlwaysSpawn, string root, List<Item> items)
{
this.InternalId = internalId;
Id = id;
IsContainer = isContainer;
UseGravity = useGravity;
RandomRotation = randomRotation;
Position = position;
Rotation = rotation;
IsGroupPosition = isGroupPosition;
GroupPositions = groupPositions;
IsAlwaysSpawn = isAlwaysSpawn;
Root = root;
Items = items;
}
[JsonIgnore] public string InternalId { get; } = internalId;
[JsonPropertyName("Id")] public string? Id { get; set; } = id;
[JsonPropertyName("IsContainer")] public bool IsContainer { get; set; } = isContainer;
public bool? UseGravity { get; set; } = useGravity;
public bool? RandomRotation { get; set; } = randomRotation;
[JsonPropertyName("Position")] public Vector3? Position { get; set; } = position;
[JsonPropertyName("Rotation")] public Vector3? Rotation { get; set; } = rotation;
[JsonPropertyName("IsGroupPosition")] public bool? IsGroupPosition { get; set; } = isGroupPosition;
[JsonPropertyName("GroupPositions")] public List<GroupPosition>? GroupPositions { get; set; } = groupPositions;
[JsonPropertyName("IsAlwaysSpawn")] public bool? IsAlwaysSpawn { get; set; } = isAlwaysSpawn;
[JsonPropertyName("Root")] public string? Root { get; set; } = root;
[JsonPropertyName("Items")] public List<Item> Items { get; set; } = items;
private bool Equals(Template other) => Id == other.Id;

2
source/LootDumpProcessor/Process/Processor/DumpProcessor/IDumpProcessor.cs
View File

@ -4,5 +4,5 @@ namespace LootDumpProcessor.Process.Processor.DumpProcessor;
public interface IDumpProcessor
{
Dictionary<OutputFileType, object> ProcessDumps(List<PartialData> dumps);
Task<Dictionary<OutputFileType, object>> ProcessDumps(List<PartialData> dumps);
}

19
source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
View File

@ -47,7 +47,7 @@ public class MultithreadSteppedDumpProcessor(
private static readonly IDataStorage _dataStorage = DataStorageFactory.GetInstance();
public Dictionary<OutputFileType, object> ProcessDumps(List<PartialData> dumps)
public async Task<Dictionary<OutputFileType, object>> ProcessDumps(List<PartialData> dumps)
{
_logger.LogInformation("Starting final dump processing");
var output = new Dictionary<OutputFileType, object>();
@ -65,8 +65,8 @@ public class MultithreadSteppedDumpProcessor(
{
MaxDegreeOfParallelism = Environment.ProcessorCount
};
Parallel.ForEachAsync(dumps, parallelOptions,
async (partialData, cancellationToken) =>
await Parallel.ForEachAsync(dumps, parallelOptions,
async (partialData, _) =>
await Process(partialData, staticContainers, mapStaticContainersAggregated, mapDumpCounter));
_logger.LogInformation("All static data processing threads finished");
@ -202,7 +202,7 @@ public class MultithreadSteppedDumpProcessor(
fileDate.Value > LootDumpProcessorContext.GetConfig().DumpProcessorConfig
.SpawnContainerChanceIncludeAfterDate;
private static void IncrementMapCounterDictionaryValue(IDictionary<string, int> mapDumpCounter, string mapName)
private static void IncrementMapCounterDictionaryValue(ConcurrentDictionary<string, int> mapDumpCounter, string mapName)
{
if (!mapDumpCounter.TryAdd(mapName, 1)) mapDumpCounter[mapName] += 1;
}
@ -305,13 +305,12 @@ public class MultithreadSteppedDumpProcessor(
);
foreach (var (uniqueKey, containerTemplate) in loadedDictionary)
{
var count = dumpData.LooseLoot.Counts[uniqueKey];
var isValueFound = dumpData.LooseLoot.Counts.TryGetValue(uniqueKey, out var count);
if (!isValueFound) _logger.LogError("Value for {UniqueKey} not found", uniqueKey);
lock (lockObjectDictionaryCounts)
{
if (dictionaryCounts.ContainsKey(uniqueKey))
if (!dictionaryCounts.TryAdd(uniqueKey, count))
dictionaryCounts[uniqueKey] += count;
else
dictionaryCounts[uniqueKey] = count;
}
lock (lockObjectDictionaryItemProperties)
@ -319,8 +318,8 @@ public class MultithreadSteppedDumpProcessor(
if (!dictionaryItemProperties.TryGetValue(uniqueKey, out var values))
{
values = new FlatKeyableList<Template>(_keyGenerator.Generate());
dictionaryItemProperties.Add(uniqueKey, values);
actualDictionaryItemProperties.Add(uniqueKey, values.GetKey());
dictionaryItemProperties.TryAdd(uniqueKey, values);
actualDictionaryItemProperties.TryAdd(uniqueKey, values.GetKey());
}
values.AddRange(containerTemplate);

8
source/LootDumpProcessor/Process/QueuePipeline.cs
View File

@ -52,7 +52,7 @@ public class QueuePipeline(
try
{
await FixFilesFromDumps();
foreach (var mapName in _mapNames) ProcessFilesFromDumpsPerMap(collector, mapName);
foreach (var mapName in _mapNames) await ProcessFilesFromDumpsPerMap(collector, mapName);
}
finally
{
@ -126,7 +126,7 @@ public class QueuePipeline(
return queuedFilesToProcess;
}
private void ProcessFilesFromDumpsPerMap(ICollector collector, string mapName)
private async Task ProcessFilesFromDumpsPerMap(ICollector collector, string mapName)
{
// Gather all files, sort them by date descending and then add them into the processing queue
GatherFiles().FindAll(f => f.ToLower().Contains($"{mapName}--")).OrderByDescending(f =>
@ -158,7 +158,9 @@ public class QueuePipeline(
// Single writer instance to collect results
var writer = WriterFactory.GetInstance();
// Single collector instance to collect results
writer.WriteAll(_dumpProcessor.ProcessDumps(collector.Retrieve()));
var partialData = collector.Retrieve();
var processedDumps = await _dumpProcessor.ProcessDumps(partialData);
writer.WriteAll(processedDumps);
// clear collector and datastorage as we process per map now
collector.Clear();

3
source/LootDumpProcessor/Serializers/Json/JsonSerializerSettings.cs
View File

@ -15,7 +15,6 @@ public static class JsonSerializerSettings
new NetJsonKeyConverter(),
new JsonStringEnumConverter(),
new NetDateTimeConverter()
},
WriteIndented = true
}
};
}

3
source/LootDumpProcessor/Storage/Collections/FlatKeyableDictionary.cs
View File

@ -1,8 +1,9 @@
using System.Collections.Concurrent;
using System.Text.Json.Serialization;
namespace LootDumpProcessor.Storage.Collections;
public class FlatKeyableDictionary<K, V> : Dictionary<K, V>, IKeyable
public class FlatKeyableDictionary<K, V> : ConcurrentDictionary<K, V>, IKeyable
{
[JsonPropertyName("__id__")] private string Id { get; set; }

7
source/LootDumpProcessor/Utils/ProcessorUtil.cs
View File

@ -7,11 +7,12 @@ namespace LootDumpProcessor.Utils;
public static class ProcessorUtil
{
public static string GetSaneId(this Template x) =>
$"({x.Position.X}, {x.Position.Y}, {x.Position.Z}, {Math.Round(x.Rotation.X, 3)}," +
$" {Math.Round(x.Rotation.Y, 3)}, {Math.Round(x.Rotation.Z, 3)}," +
$"({x.Position.GetValueOrDefault().X}, {x.Position.GetValueOrDefault().Y}, {x.Position.GetValueOrDefault().Z}, {Math.Round(x.Rotation.GetValueOrDefault().X, 3)}," +
$" {Math.Round(x.Rotation.GetValueOrDefault().Y, 3)}, {Math.Round(x.Rotation.GetValueOrDefault().Z, 3)}," +
$" {x.UseGravity}, {x.IsGroupPosition})";
public static string GetLocationId(this Template x) => $"({x.Position.X}, {x.Position.Y}, {x.Position.Z})";
public static string GetLocationId(this Template x) =>
$"({x.Position.GetValueOrDefault().X}, {x.Position.GetValueOrDefault().Y}, {x.Position.GetValueOrDefault().Z})";
public static T? Copy<T>(T? obj) where T : ICloneable
{