spt/loot-dump-processor
spt/loot-dump-processor
0
0
Fork 0
mirror of https://github.com/sp-tarkov/loot-dump-processor.git synced 2025-02-13 02:50:45 -05:00
Code

Refactored loose loot processor to process single map

Browse Source
This commit is contained in:
bluextx 2025-01-11 07:41:15 +03:00
parent 8366915048
commit 1a4003595a
3 changed files with 151 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
source/LootDumpProcessor/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
View File

@ -3,6 +3,7 @@ using LootDumpProcessor.Logger;
using LootDumpProcessor.Model; using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input; using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output; using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Output.StaticContainer; using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Processor.v2.AmmoProcessor; using LootDumpProcessor.Process.Processor.v2.AmmoProcessor;
@ -222,17 +223,22 @@ public class MultithreadSteppedDumpProcessor(
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Processing loose loot distribution", LogLevel.Info); LoggerFactory.GetInstance().Log("Processing loose loot distribution", LogLevel.Info);
// Loose loot distribution // Loose loot distribution
var looseLootDistribution = _looseLootProcessor.CreateLooseLootDistribution( var looseLoot = new ConcurrentDictionary<string, LooseLootRoot>();
dumpProcessData.MapCounts, Parallel.ForEach(dumpProcessData.MapCounts.Keys, mapId =>
dumpProcessData.LooseLootCounts {
); var mapCount = dumpProcessData.MapCounts[mapId];
var looseLootCount = dumpProcessData.LooseLootCounts[mapId];
var looseLootDistribution =
_looseLootProcessor.CreateLooseLootDistribution(mapId, mapCount, looseLootCount);
looseLoot[mapId] = looseLootDistribution;
});
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Collecting loose loot distribution information", LogLevel.Info); LoggerFactory.GetInstance().Log("Collecting loose loot distribution information", LogLevel.Info);
var loot = dumpProcessData.MapCounts var loot = dumpProcessData.MapCounts
.Select(mapCount => mapCount.Key) .Select(mapCount => mapCount.Key)
.ToDictionary(mi => mi, mi => looseLootDistribution[mi]); .ToDictionary(mi => mi, mi => looseLoot[mi]);
output.Add(OutputFileType.LooseLoot, loot); output.Add(OutputFileType.LooseLoot, loot);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))

7
source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/ILooseLootProcessor.cs
View File

@ -9,9 +9,10 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
{ {
PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseLoot); PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseLoot);
Dictionary<string, LooseLootRoot> CreateLooseLootDistribution( LooseLootRoot CreateLooseLootDistribution(
Dictionary<string, int> mapCounts, string mapId,
Dictionary<string, IKey> looseLootCounts int mapCount,
IKey looseLootCountKey
); );
} }
} }

278
source/LootDumpProcessor/Process/Processor/v2/LooseLootProcessor/LooseLootProcessor.cs
View File

@ -36,8 +36,8 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
if (!uniqueLootIds.ContainsKey(sanitizedId)) if (!uniqueLootIds.ContainsKey(sanitizedId))
{ {
uniqueLootIds[sanitizedId] = template.Id; uniqueLootIds[sanitizedId] = template.Id;
preProcessedLoot.Counts[sanitizedId] = preProcessedLoot.Counts.ContainsKey(sanitizedId) preProcessedLoot.Counts[sanitizedId] = preProcessedLoot.Counts.TryGetValue(sanitizedId, out var count)
? preProcessedLoot.Counts[sanitizedId] + 1 ? count + 1
: 1; : 1;
} }
@ -54,182 +54,174 @@ namespace LootDumpProcessor.Process.Processor.v2.LooseLootProcessor
return preProcessedLoot; return preProcessedLoot;
} }
public Dictionary<string, LooseLootRoot> CreateLooseLootDistribution( public LooseLootRoot CreateLooseLootDistribution(
Dictionary<string, int> mapCounts, string mapId,
Dictionary<string, IKey> looseLootCounts int mapCount,
IKey looseLootCountKey
) )
{ {
var config = LootDumpProcessorContext.GetConfig(); var config = LootDumpProcessorContext.GetConfig();
var spawnPointTolerance = config.ProcessorConfig.SpawnPointToleranceForForced / 100; var spawnPointTolerance = config.ProcessorConfig.SpawnPointToleranceForForced / 100;
var looseLootDistribution = new Dictionary<string, LooseLootRoot>(); var looseLootDistribution = new LooseLootRoot();
foreach (var map in mapCounts) var probabilities = new Dictionary<string, double>();
var looseLootCountsItem = _dataStorage.GetItem<LooseLootCounts>(looseLootCountKey);
var counts = _dataStorage.GetItem<FlatKeyableDictionary<string, int>>(looseLootCountsItem.Counts);
foreach (var (itemId, count) in counts)
{ {
var mapName = map.Key; probabilities[itemId] = (double)count / mapCount;
var mapCount = map.Value; }
var probabilities = new Dictionary<string, double>(); var spawnPointCount = looseLootCountsItem.MapSpawnpointCount.Select(Convert.ToDouble);
var looseLootCountsItem = _dataStorage.GetItem<LooseLootCounts>(looseLootCounts[mapName]); var initialMean = np.mean(np.array(spawnPointCount)).ToArray<double>().First();
var tolerancePercentage = config.ProcessorConfig.LooseLootCountTolerancePercentage / 100;
var highThreshold = initialMean * (1 + tolerancePercentage);
var counts = _dataStorage.GetItem<FlatKeyableDictionary<string, int>>(looseLootCountsItem.Counts); looseLootCountsItem.MapSpawnpointCount = looseLootCountsItem.MapSpawnpointCount
foreach (var (itemId, count) in counts) .Where(count => count <= highThreshold)
.ToList();
looseLootDistribution.SpawnPointCount = new SpawnPointCount
{
Mean = np.mean(np.array(looseLootCountsItem.MapSpawnpointCount)),
Std = np.std(np.array(looseLootCountsItem.MapSpawnpointCount))
};
looseLootDistribution.SpawnPointsForced = new List<SpawnPointsForced>();
looseLootDistribution.SpawnPoints = new List<SpawnPoint>();
var itemProperties = _dataStorage.GetItem<FlatKeyableDictionary<string, IKey>>(looseLootCountsItem.ItemProperties);
foreach (var (spawnPoint, itemListKey) in itemProperties)
{
var itemCounts = new Dictionary<ComposedKey, int>();
var savedTemplates = _dataStorage.GetItem<FlatKeyableList<Template>>(itemListKey);
foreach (var template in savedTemplates)
{ {
probabilities[itemId] = (double)count / mapCount; var composedKey = new ComposedKey(template);
if (!itemCounts.TryAdd(composedKey, 1))
itemCounts[composedKey]++;
} }
var spawnPointCount = looseLootCountsItem.MapSpawnpointCount.Select(Convert.ToDouble); var groupedTemplates = savedTemplates
var initialMean = np.mean(np.array(spawnPointCount)).ToArray<double>().First(); .Select(t => (t.GetSaneId(), t))
var tolerancePercentage = config.ProcessorConfig.LooseLootCountTolerancePercentage / 100; .GroupBy(g => g.Item1)
var highThreshold = initialMean * (1 + tolerancePercentage);
looseLootCountsItem.MapSpawnpointCount = looseLootCountsItem.MapSpawnpointCount
.Where(count => count <= highThreshold)
.ToList(); .ToList();
looseLootDistribution[mapName] = new LooseLootRoot if (groupedTemplates.Count > 1)
{ {
SpawnPointCount = new SpawnPointCount throw new Exception("Multiple sanitized IDs found for a single spawn point.");
{ }
Mean = np.mean(np.array(looseLootCountsItem.MapSpawnpointCount)),
Std = np.std(np.array(looseLootCountsItem.MapSpawnpointCount))
},
SpawnPointsForced = new List<SpawnPointsForced>(),
SpawnPoints = new List<SpawnPoint>()
};
var itemProperties = _dataStorage.GetItem<FlatKeyableDictionary<string, IKey>>(looseLootCountsItem.ItemProperties); var spawnPoints = groupedTemplates.First().Select(g => g.t).ToList();
foreach (var (spawnPoint, itemListKey) in itemProperties) var locationId = spawnPoints[0].GetLocationId();
var templateCopy = ProcessorUtil.Copy(spawnPoints[0]);
var itemDistributions = itemCounts.Select(kv => new ItemDistribution
{ {
var itemCounts = new Dictionary<ComposedKey, int>(); ComposedKey = kv.Key,
var savedTemplates = _dataStorage.GetItem<FlatKeyableList<Template>>(itemListKey); RelativeProbability = kv.Value
}).ToList();
foreach (var template in savedTemplates) if (itemDistributions.Count == 1 &&
(LootDumpProcessorContext.GetTarkovItems().IsQuestItem(itemDistributions[0].ComposedKey?.FirstItem?.Tpl) ||
LootDumpProcessorContext.GetForcedLooseItems()[mapId].Contains(itemDistributions[0].ComposedKey?.FirstItem?.Tpl)))
{
var forcedSpawnPoint = new SpawnPointsForced
{ {
var composedKey = new ComposedKey(template); LocationId = locationId,
if (!itemCounts.TryAdd(composedKey, 1)) Probability = probabilities[spawnPoint],
itemCounts[composedKey]++; Template = templateCopy
} };
looseLootDistribution.SpawnPointsForced.Add(forcedSpawnPoint);
var groupedTemplates = savedTemplates }
.Select(t => (t.GetSaneId(), t)) else if (probabilities[spawnPoint] > spawnPointTolerance)
.GroupBy(g => g.Item1) {
.ToList(); var forcedSpawnPoint = new SpawnPointsForced
if (groupedTemplates.Count > 1)
{ {
throw new Exception("Multiple sanitized IDs found for a single spawn point."); LocationId = locationId,
} Probability = probabilities[spawnPoint],
Template = templateCopy
var spawnPoints = groupedTemplates.First().Select(g => g.t).ToList(); };
var locationId = spawnPoints[0].GetLocationId(); looseLootDistribution.SpawnPointsForced.Add(forcedSpawnPoint);
var templateCopy = ProcessorUtil.Copy(spawnPoints[0]); _logger.LogWarning(
var itemDistributions = itemCounts.Select(kv => new ItemDistribution "Item: {ItemId} has > {Tolerance}% spawn chance in spawn point: {LocationId} but isn't in forced loot, adding to forced",
templateCopy.Id,
config.ProcessorConfig.SpawnPointToleranceForForced,
forcedSpawnPoint.LocationId
);
}
else
{
var spawnPointEntry = new SpawnPoint
{ {
ComposedKey = kv.Key, LocationId = locationId,
RelativeProbability = kv.Value Probability = probabilities[spawnPoint],
}).ToList(); Template = templateCopy,
ItemDistribution = itemDistributions
};
if (itemDistributions.Count == 1 && templateCopy.Items = new List<Item>();
(LootDumpProcessorContext.GetTarkovItems().IsQuestItem(itemDistributions[0].ComposedKey?.FirstItem?.Tpl) ||
LootDumpProcessorContext.GetForcedLooseItems()[mapName].Contains(itemDistributions[0].ComposedKey?.FirstItem?.Tpl))) var groupedByKey = spawnPoints
.GroupBy(t => new ComposedKey(t))
.ToDictionary(g => g.Key, g => g.ToList());
foreach (var distribution in itemDistributions)
{ {
var forcedSpawnPoint = new SpawnPointsForced if (groupedByKey.TryGetValue(distribution.ComposedKey, out var items))
{ {
LocationId = locationId, var itemList = items.First().Items;
Probability = probabilities[spawnPoint], var originalItem = itemList.Find(i => string.IsNullOrEmpty(i.ParentId));
Template = templateCopy var originalId = originalItem.Id;
}; originalItem.Id = distribution.ComposedKey.Key;
looseLootDistribution[mapName].SpawnPointsForced.Add(forcedSpawnPoint); foreach (var childItem in itemList.Where(i => i.ParentId == originalId))
}
else if (probabilities[spawnPoint] > spawnPointTolerance)
{
var forcedSpawnPoint = new SpawnPointsForced
{
LocationId = locationId,
Probability = probabilities[spawnPoint],
Template = templateCopy
};
looseLootDistribution[mapName].SpawnPointsForced.Add(forcedSpawnPoint);
_logger.LogWarning(
"Item: {ItemId} has > {Tolerance}% spawn chance in spawn point: {LocationId} but isn't in forced loot, adding to forced",
templateCopy.Id,
config.ProcessorConfig.SpawnPointToleranceForForced,
forcedSpawnPoint.LocationId
);
}
else
{
var spawnPointEntry = new SpawnPoint
{
LocationId = locationId,
Probability = probabilities[spawnPoint],
Template = templateCopy,
ItemDistribution = itemDistributions
};
templateCopy.Items = new List<Item>();
var groupedByKey = spawnPoints
.GroupBy(t => new ComposedKey(t))
.ToDictionary(g => g.Key, g => g.ToList());
foreach (var distribution in itemDistributions)
{
if (groupedByKey.TryGetValue(distribution.ComposedKey, out var items))
{ {
var itemList = items.First().Items; childItem.ParentId = originalItem.Id;
var originalItem = itemList.Find(i => string.IsNullOrEmpty(i.ParentId)); }
var originalId = originalItem.Id;
originalItem.Id = distribution.ComposedKey.Key;
foreach (var childItem in itemList.Where(i => i.ParentId == originalId))
{
childItem.ParentId = originalItem.Id;
}
templateCopy.Items.AddRange(itemList); templateCopy.Items.AddRange(itemList);
} }
else else
{ {
_logger.LogError( _logger.LogError(
"Item template {TemplateId} on loose loot distribution for spawn point {SpawnPointId} not found in spawn points.", "Item template {TemplateId} on loose loot distribution for spawn point {SpawnPointId} not found in spawn points.",
distribution.ComposedKey?.FirstItem?.Tpl, distribution.ComposedKey?.FirstItem?.Tpl,
templateCopy.Id templateCopy.Id
); );
}
} }
looseLootDistribution[mapName].SpawnPoints.Add(spawnPointEntry);
} }
looseLootDistribution.SpawnPoints.Add(spawnPointEntry);
} }
}
looseLootDistribution[mapName].SpawnPoints = looseLootDistribution[mapName].SpawnPoints looseLootDistribution.SpawnPoints = looseLootDistribution.SpawnPoints
.OrderBy(x => x.Template.Id) .OrderBy(x => x.Template.Id)
.ToList(); .ToList();
var configuredForcedTemplates = new HashSet<string>(LootDumpProcessorContext.GetForcedLooseItems()[mapName].Select(item => item)); var configuredForcedTemplates = new HashSet<string>(LootDumpProcessorContext.GetForcedLooseItems()[mapId].Select(item => item));
var foundForcedTemplates = new HashSet<string>(looseLootDistribution[mapName].SpawnPointsForced.Select(fp => fp.Template.Items[0].Tpl)); var foundForcedTemplates = new HashSet<string>(looseLootDistribution.SpawnPointsForced.Select(fp => fp.Template.Items[0].Tpl));
foreach (var expectedTpl in configuredForcedTemplates) foreach (var expectedTpl in configuredForcedTemplates)
{
if (!foundForcedTemplates.Contains(expectedTpl))
{ {
if (!foundForcedTemplates.Contains(expectedTpl)) _logger.LogError(
{ "Expected item: {ItemTpl} defined in forced_loose.yaml config not found in forced loot",
_logger.LogError( expectedTpl
"Expected item: {ItemTpl} defined in forced_loose.yaml config not found in forced loot", );
expectedTpl
);
}
} }
}
foreach (var foundTpl in foundForcedTemplates) foreach (var foundTpl in foundForcedTemplates)
{
if (!configuredForcedTemplates.Contains(foundTpl))
{ {
if (!configuredForcedTemplates.Contains(foundTpl)) _logger.LogWarning(
{ "Map: {MapName} Item: {ItemTpl} not defined in forced_loose.yaml config but was flagged as forced by code",
_logger.LogWarning( mapId,
"Map: {MapName} Item: {ItemTpl} not defined in forced_loose.yaml config but was flagged as forced by code", foundTpl
mapName, );
foundTpl
);
}
} }
} }