DrakiaXYZ/LootDumpProcessor
DrakiaXYZ/LootDumpProcessor
0
0
Fork 0
Code Pull Requests Activity

First attempt at splitting static loot per map, static ammo is disabled for now

Browse Source
This commit is contained in:
Dev 2024-04-22 21:01:02 +01:00
parent 892c7c548e
commit 8834775186
4 changed files with 180 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Model/Processing/DumpProcessData.cs
View File

@ -5,6 +5,6 @@ namespace LootDumpProcessor.Model.Processing;
public class DumpProcessData
{
public Dictionary<string, IKey> LooseLootCounts { get; set; } = new();
public List<PreProcessedStaticLoot> ContainerCounts { get; set; } = new();
public Dictionary<string, List<PreProcessedStaticLoot>> ContainerCounts { get; set; } = new();
public Dictionary<string, int> MapCounts { get; set; } = new();
}

109
Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs
View File

@ -67,6 +67,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
LoggerFactory.GetInstance().Log($"Doing first time process for map {mapName} of real static data", LogLevel.Info);
var mapStaticContainers = StaticLootProcessor.CreateStaticWeaponsAndStaticForcedContainers(dataDump);
// .Item1 = map name
// .Item2 = force/weapon static arrays
staticContainers[mapStaticContainers.Item1] = mapStaticContainers.Item2;
}
}
@ -84,28 +85,32 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
}
// Only process the dump file if the date is higher (after) the configuration date
if (DumpWasMadeAfterConfigThresholdDate(dumped))
if (!DumpWasMadeAfterConfigThresholdDate(dumped))
{
// Keep track of how many dumps we have for each map
lock (mapDumpCounterLock)
{
IncrementMapCounterDictionaryValue(mapDumpCounter, mapName);
}
return;
}
var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(dataDump.Data.Id.ToLower(), out string[]? ignoreListForMap);
foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(dataDump))
// Keep track of how many dumps we have for each map
lock (mapDumpCounterLock)
{
IncrementMapCounterDictionaryValue(mapDumpCounter, mapName);
}
var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(dataDump.Data.Id.ToLower(), out string[]? ignoreListForMap);
foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(dataDump))
{
lock (mapStaticContainersAggregatedLock)
{
lock (mapStaticContainersAggregatedLock)
if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id))
{
if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id))
{
// Skip adding containers to aggregated data if container id is in ignore list
continue;
}
// Skip adding containers to aggregated data if container id is in ignore list
continue;
}
// Increment times container seen in dump by 1
if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1))
mapAggregatedDataDict[dynamicStaticContainer] += 1;
// Increment times container seen in dump by 1
if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1))
{
mapAggregatedDataDict[dynamicStaticContainer] += 1;
}
}
}
@ -128,17 +133,17 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) // kv.Key = map name
}
).ToList()
).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value);
).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); // Hydrate staticContainers.StaticContainers
// Static containers
output.Add(OutputFileType.StaticContainer, staticContainers);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Processing ammo distribution", LogLevel.Info);
// Ammo distribution
output.Add(
OutputFileType.StaticAmmo,
StaticLootProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts)
);
//output.Add(
// OutputFileType.StaticAmmo,
// StaticLootProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts)
//);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Processing static loot distribution", LogLevel.Info);
@ -198,23 +203,23 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
.ToList()
.ForEach(tuple =>
{
var mapi = tuple.Key;
var g = tuple.ToList();
var mapName = tuple.Key;
var partialFileMetaData = tuple.ToList();
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log(
$"Processing map {mapi}, total dump data to process: {g.Count}",
$"Processing map {mapName}, total dump data to process: {partialFileMetaData.Count}",
LogLevel.Info
);
dumpProcessData.MapCounts[mapi] = g.Count;
dumpProcessData.MapCounts[mapName] = partialFileMetaData.Count;
var lockObjectContainerCounts = new object();
var lockObjectCounts = new object();
var counts = new LooseLootCounts();
var looseLootCounts = new LooseLootCounts();
var lockObjectDictionaryCounts = new object();
var dictionaryCounts = new FlatKeyableDictionary<string, int>();
counts.Counts = dictionaryCounts.GetKey();
looseLootCounts.Counts = dictionaryCounts.GetKey();
/*
var dictionaryItemCounts = new FlatKeyableDictionary<string, List<string>>();
@ -225,22 +230,21 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
var dictionaryItemProperties = new FlatKeyableDictionary<string, FlatKeyableList<Template>>();
var actualDictionaryItemProperties = new FlatKeyableDictionary<string, IKey>();
counts.ItemProperties = actualDictionaryItemProperties.GetKey();
looseLootCounts.ItemProperties = actualDictionaryItemProperties.GetKey();
dumpProcessData.LooseLootCounts.Add(mapi, counts.GetKey());
dumpProcessData.LooseLootCounts.Add(mapName, looseLootCounts.GetKey());
// add the items to the queue
foreach (var gi in g)
foreach (var partialData in partialFileMetaData)
{
_partialDataToProcess.Add(gi);
_partialDataToProcess.Add(partialData);
}
// Call GC before running threads
g = null;
partialFileMetaData = null;
tuple = null;
GCHandler.Collect();
// The data storage factory has a lock, we dont want the locks to occur when multithreading
for (int i = 0; i < LootDumpProcessorContext.GetConfig().Threads; i++)
{
Runners.Add(
@ -253,26 +257,35 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
try
{
var dumpData = _dataStorage.GetItem<ParsedDump>(partialData.ParsedDumpKey);
// Static containers
lock (lockObjectContainerCounts)
{
dumpProcessData.ContainerCounts.AddRange(dumpData.Containers);
if (!dumpProcessData.ContainerCounts.ContainsKey(mapName))
{
dumpProcessData.ContainerCounts.Add(mapName, dumpData.Containers);
}
else
{
dumpProcessData.ContainerCounts[mapName].AddRange(dumpData.Containers);
}
}
// loose loot into ids on files
// Loose loot into ids on files
var loadedDictionary =
_dataStorage
.GetItem<SubdivisionedKeyableDictionary<string, List<Template>>>(
dumpData.LooseLoot.ItemProperties
);
foreach (var (k, v) in loadedDictionary)
foreach (var (uniqueKey, containerTemplate) in loadedDictionary)
{
var count = dumpData.LooseLoot.Counts[k];
var count = dumpData.LooseLoot.Counts[uniqueKey];
lock (lockObjectDictionaryCounts)
{
if (dictionaryCounts.ContainsKey(k))
dictionaryCounts[k] += count;
if (dictionaryCounts.ContainsKey(uniqueKey))
dictionaryCounts[uniqueKey] += count;
else
dictionaryCounts[k] = count;
dictionaryCounts[uniqueKey] = count;
}
/*
@ -287,20 +300,20 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
lock (lockObjectDictionaryItemProperties)
{
if (!dictionaryItemProperties.TryGetValue(k, out var values))
if (!dictionaryItemProperties.TryGetValue(uniqueKey, out var values))
{
values = new FlatKeyableList<Template>();
dictionaryItemProperties.Add(k, values);
actualDictionaryItemProperties.Add(k, values.GetKey());
dictionaryItemProperties.Add(uniqueKey, values);
actualDictionaryItemProperties.Add(uniqueKey, values.GetKey());
}
values.AddRange(v);
values.AddRange(containerTemplate);
}
}
lock (lockObjectCounts)
{
counts.MapSpawnpointCount.Add(dumpData.LooseLoot.MapSpawnpointCount);
looseLootCounts.MapSpawnpointCount.Add(dumpData.LooseLoot.MapSpawnpointCount);
}
}
catch (Exception e)
@ -345,8 +358,8 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
_dataStorage.Store(actualDictionaryItemProperties);
actualDictionaryItemProperties = null;
GCHandler.Collect();
_dataStorage.Store(counts);
counts = null;
_dataStorage.Store(looseLootCounts);
looseLootCounts = null;
GCHandler.Collect();
});
return dumpProcessData;

147
Process/Processor/StaticLootProcessor.cs
View File

@ -112,62 +112,119 @@ public static class StaticLootProcessor
return ammo_distribution;
}
public static Dictionary<string, StaticItemDistribution> CreateStaticLootDistribution(
List<PreProcessedStaticLoot> container_counts,
/// <summary>
/// Dict key = map,
/// value = sub dit:
/// key = container Ids
/// value = items + counts
/// </summary>
public static Dictionary<string, Dictionary<string, StaticItemDistribution>> CreateStaticLootDistribution(
Dictionary<string, List<PreProcessedStaticLoot>> container_counts,
Dictionary<string, MapStaticLoot> staticContainers)
{
var allMapsStaticLootDisto = new Dictionary< string, Dictionary<string, StaticItemDistribution>>();
// Iterate over each map we have containers for
foreach (var map in staticContainers)
foreach (var mapContainersKvp in container_counts)
{
var mapName = map.Key;
var mapContainers = map.Value;
var mapName = mapContainersKvp.Key;
var containers = mapContainersKvp.Value;
var static_loot_distribution = new Dictionary<string, StaticItemDistribution>();
var uniqueContainerTypeIds = Enumerable.Distinct((from ci in containers
select ci.Type).ToList());
foreach (var typeId in uniqueContainerTypeIds)
{
var container_counts_selected = (from ci in containers
where ci.Type == typeId
select ci).ToList();
// Get array of all times a count of items was found in container
List<int> itemCountsInContainer = GetCountOfItemsInContainer(container_counts_selected);
// Create structure to hold item count + weight that it will be picked
// Group same counts together
static_loot_distribution[typeId] = new StaticItemDistribution();
static_loot_distribution[typeId].ItemCountDistribution = itemCountsInContainer.GroupBy(i => i)
.Select(g => new ItemCountDistribution
{
Count = g.Key,
RelativeProbability = g.Count()
}).ToList();
static_loot_distribution[typeId].ItemDistribution = CreateItemDistribution(container_counts_selected);
}
// Key = containers tpl, value = items + count weights
allMapsStaticLootDisto.TryAdd(mapName, static_loot_distribution);
}
var static_loot_distribution = new Dictionary<string, StaticItemDistribution>();
var uniqueContainerTypeIds = Enumerable.Distinct((from ci in container_counts
select ci.Type).ToList());
return allMapsStaticLootDisto;
foreach (var typeId in uniqueContainerTypeIds)
//var static_loot_distribution = new Dictionary<string, StaticItemDistribution>();
//var uniqueContainerTypeIds = Enumerable.Distinct((from ci in container_counts
// select ci.Type).ToList());
//foreach (var typeId in uniqueContainerTypeIds)
//{
// var container_counts_selected = (from ci in container_counts
// where ci.Type == typeId
// select ci).ToList();
// // Get array of all times a count of items was found in container
// List<int> itemCountsInContainer = GetCountOfItemsInContainer(container_counts_selected);
// // Create structure to hold item count + weight that it will be picked
// // Group same counts together
// static_loot_distribution[typeId] = new StaticItemDistribution();
// static_loot_distribution[typeId].ItemCountDistribution = itemCountsInContainer.GroupBy(i => i)
// .Select(g => new ItemCountDistribution
// {
// Count = g.Key,
// RelativeProbability = g.Count()
// }).ToList();
// static_loot_distribution[typeId].ItemDistribution = CreateItemDistribution(container_counts_selected);
//}
//// Key = containers tpl, value = items + count weights
//return static_loot_distribution;
}
private static List<StaticDistribution> CreateItemDistribution(List<PreProcessedStaticLoot> container_counts_selected)
{
// TODO: Change for different algo that splits items per parent once parentid = containerid, then compose
// TODO: key and finally create distribution based on composed Id instead
var itemsHitCounts = new Dictionary<string, int>();
foreach (var ci in container_counts_selected)
{
var container_counts_selected = (from ci in container_counts
where ci.Type == typeId
select ci).ToList();
var itemscounts = new List<int>();
foreach (var ci in container_counts_selected)
foreach (var cii in ci.Items.Where(cii => cii.ParentId == ci.ContainerId))
{
itemscounts.Add((from cii in ci.Items
where cii.ParentId == ci.ContainerId
select cii).ToList().Count);
if (itemsHitCounts.ContainsKey(cii.Tpl))
itemsHitCounts[cii.Tpl] += 1;
else
itemsHitCounts[cii.Tpl] = 1;
}
static_loot_distribution[typeId] = new StaticItemDistribution();
static_loot_distribution[typeId].ItemCountDistribution = itemscounts.GroupBy(i => i)
.Select(g => new ItemCountDistribution
{
Count = g.Key,
RelativeProbability = g.Count()
}).ToList();
// TODO: Change for different algo that splits items per parent once parentid = containerid, then compose
// TODO: key and finally create distribution based on composed Id instead
var itemsHitCounts = new Dictionary<string, int>();
foreach (var ci in container_counts_selected)
{
foreach (var cii in ci.Items.Where(cii => cii.ParentId == ci.ContainerId))
{
if (itemsHitCounts.ContainsKey(cii.Tpl))
itemsHitCounts[cii.Tpl] += 1;
else
itemsHitCounts[cii.Tpl] = 1;
}
}
static_loot_distribution[typeId].ItemDistribution = itemsHitCounts.Select(v => new StaticDistribution
{
Tpl = v.Key,
RelativeProbability = v.Value
}).ToList();
}
return static_loot_distribution;
// WIll create array of objects that have a tpl + relative probability weight value
return itemsHitCounts.Select(v => new StaticDistribution
{
Tpl = v.Key,
RelativeProbability = v.Value
}).ToList();
}
private static List<int> GetCountOfItemsInContainer(List<PreProcessedStaticLoot> container_counts_selected)
{
var itemCountsInContainer = new List<int>();
foreach (var containerWithItems in container_counts_selected)
{
// Only count item if its parent is the container, only root items are counted (not mod/attachment items)
itemCountsInContainer.Add((from cii in containerWithItems.Items
where cii.ParentId == containerWithItems.ContainerId
select cii).ToList().Count);
}
return itemCountsInContainer;
}
}

19
Process/Writer/FileWriter.cs
View File

@ -1,7 +1,9 @@
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Serializers.Json;
using System.Collections.Generic;
namespace LootDumpProcessor.Process.Writer;
@ -59,16 +61,27 @@ public class FileWriter : IWriter
var staticContainer = (Dictionary<string, MapStaticLoot>)data;
File.WriteAllText($@"{_outputPath}\loot\staticContainers.json",
_jsonSerializer.Serialize(staticContainer));
break;
case OutputFileType.StaticLoot:
var staticLoot = (Dictionary<string, StaticItemDistribution>)data;
File.WriteAllText($@"{_outputPath}\loot\staticLoot.json",
_jsonSerializer.Serialize(staticLoot));
var staticLootData = (Dictionary<string, Dictionary<string, StaticItemDistribution>>)data;
foreach (var (key, value) in staticLootData)
{
foreach (var s in LootDumpProcessorContext.GetDirectoryMappings()[key].Name)
{
if (!Directory.Exists($@"{_outputPath}\locations\{s}"))
Directory.CreateDirectory($@"{_outputPath}\locations\{s}");
File.WriteAllText($@"{_outputPath}\locations\{s}\staticLoot.json",
_jsonSerializer.Serialize(value));
}
}
break;
case OutputFileType.StaticAmmo:
var staticAmmo = (Dictionary<string, List<AmmoDistribution>>)data;
File.WriteAllText($@"{_outputPath}\loot\staticAmmo.json",
_jsonSerializer.Serialize(staticAmmo));
break;
default:
throw new ArgumentOutOfRangeException(nameof(type), type, null);