Merge branch 'PerMapStaticLoot'

This commit is contained in:
Refringe 2024-08-22 14:33:48 -04:00
commit f55ab3cdc0
Signed by untrusted user: Refringe
GPG Key ID: 7715B85B4A6306ED
10 changed files with 336 additions and 167 deletions

View File

@ -1,5 +1,5 @@
--- ---
Customs: bigmap:
- 5938188786f77474f723e87f # Case 0031 - 5938188786f77474f723e87f # Case 0031
- 5c12301c86f77419522ba7e4 # Flash drive with fake info - 5c12301c86f77419522ba7e4 # Flash drive with fake info
- 593965cf86f774087a77e1b6 # Case 0048 - 593965cf86f774087a77e1b6 # Case 0048
@ -13,9 +13,10 @@ Customs:
- 5939a00786f7742fe8132936 # Golden Zibbo lighter - 5939a00786f7742fe8132936 # Golden Zibbo lighter
- 5939e5a786f77461f11c0098 # Secure Folder 0013 - 5939e5a786f77461f11c0098 # Secure Folder 0013
- 64e74a3d4d49d23b2c39d319 # item_quest_clock_07 (Out of Time) - 64e74a3d4d49d23b2c39d319 # item_quest_clock_07 (Out of Time)
- 6614230055afee107f05e998 # The Unheard's phone
#- 64bd1abff3a668f08805ce4f # Secure Flash drive V4 REMOVED BY BSG #- 64bd1abff3a668f08805ce4f # Secure Flash drive V4 REMOVED BY BSG
Woods: woods:
- 5938878586f7741b797c562f # Case 0052 - 5938878586f7741b797c562f # Case 0052
- 5d3ec50586f774183a607442 # Jaeger's message Underneath the wooden lookout post. - 5d3ec50586f774183a607442 # Jaeger's message Underneath the wooden lookout post.
- 5af04e0a86f7743a532b79e2 # Single-axis Fiber Optic Gyroscope: item_barter_electr_gyroscope - 5af04e0a86f7743a532b79e2 # Single-axis Fiber Optic Gyroscope: item_barter_electr_gyroscope
@ -24,7 +25,7 @@ Woods:
#- 64bde2248f3a947a990aa4a5 # Sliderkey Secure Flash drive #1 REMOVED BY BSG #- 64bde2248f3a947a990aa4a5 # Sliderkey Secure Flash drive #1 REMOVED BY BSG
#- 64bde265807321a9b905f076 # Sliderkey Secure Flash drive #2 REMOVED BY BSG #- 64bde265807321a9b905f076 # Sliderkey Secure Flash drive #2 REMOVED BY BSG
Shoreline: shoreline:
- 5a294d7c86f7740651337cf9 # Drone 1 SAS disk - 5a294d7c86f7740651337cf9 # Drone 1 SAS disk
- 5a294d8486f774068638cd93 # Drone 2 SAS disk: ambiguous with itemTpl 5a294f1686f774340c7b7e4a - 5a294d8486f774068638cd93 # Drone 2 SAS disk: ambiguous with itemTpl 5a294f1686f774340c7b7e4a
- 5efdafc1e70b5e33f86de058 # Sanitar's Surgery kit marked with a blue symbol - 5efdafc1e70b5e33f86de058 # Sanitar's Surgery kit marked with a blue symbol
@ -41,8 +42,12 @@ Shoreline:
- 5b43237186f7742f3a4ab252 # Chemical container: item_quest_chem_container - 5b43237186f7742f3a4ab252 # Chemical container: item_quest_chem_container
- 5a29284f86f77463ef3db363 # Toughbook reinforced laptop - 5a29284f86f77463ef3db363 # Toughbook reinforced laptop
- 64e74a534d49d23b2c39d31b # item_quest_clock_10 (Out of Time) - 64e74a534d49d23b2c39d31b # item_quest_clock_10 (Out of Time)
- 6614238e0d240a5f5d0f679d # Skier and Peacekeeper correspondence
- 661421c7c1f2f548c50ee649 # The Unheard's laptop
- 6614217b6d9d5abcad0ff098 # The Unheard's phone
- 661423200d240a5f5d0f679b # The Unheard's laptop
Interchange: interchange:
- 5ae9a18586f7746e381e16a3 # OLI cargo manifests - 5ae9a18586f7746e381e16a3 # OLI cargo manifests
- 5ae9a0dd86f7742e5f454a05 # Goshan cargo manifests - 5ae9a0dd86f7742e5f454a05 # Goshan cargo manifests
- 5ae9a1b886f77404c8537c62 # Idea cargo manifests - 5ae9a1b886f77404c8537c62 # Idea cargo manifests
@ -53,11 +58,15 @@ Interchange:
- 5b4c81bd86f77418a75ae159 # Chemical container item_quest_chem_container3 - 5b4c81bd86f77418a75ae159 # Chemical container item_quest_chem_container3
- 64e74a5ac2b4f829615ec336 # item_quest_clock_11 (Out of Time) - 64e74a5ac2b4f829615ec336 # item_quest_clock_11 (Out of Time)
Factory: factory4_day:
- 591093bb86f7747caa7bb2ee # On the neck of the dead scav in the bunker (Postman Pat Part 2) - 591093bb86f7747caa7bb2ee # On the neck of the dead scav in the bunker (Postman Pat Part 2)
- 593a87af86f774122f54a951 # Syringe with a chemical - 593a87af86f774122f54a951 # Syringe with a chemical
Lighthouse: factory4_night:
- 591093bb86f7747caa7bb2ee # On the neck of the dead scav in the bunker (Postman Pat Part 2)
- 593a87af86f774122f54a951 # Syringe with a chemical
lighthouse:
- 61904c9df62c89219a56e034 # The message is tucked under the bottom of the door to the cabin. - 61904c9df62c89219a56e034 # The message is tucked under the bottom of the door to the cabin.
- 619268ad78f4fa33f173dbe5 # Water pump operation data On the desk between other documents in the upper office. - 619268ad78f4fa33f173dbe5 # Water pump operation data On the desk between other documents in the upper office.
- 619268de2be33f2604340159 # Pumping Station Operation Data In the upper floor office on the shelf. - 619268de2be33f2604340159 # Pumping Station Operation Data In the upper floor office on the shelf.
@ -69,9 +78,10 @@ Lighthouse:
- 6399f54b0a36db13c823ad21 # Radio transmitter body (Key to the Tower) - 6399f54b0a36db13c823ad21 # Radio transmitter body (Key to the Tower)
- 64e74a64aac4cd0a7264ecdf # item_quest_clock_12 (Out of Time) - 64e74a64aac4cd0a7264ecdf # item_quest_clock_12 (Out of Time)
- 578f87a3245977356274f2cb # - 578f87a3245977356274f2cb #
- 661666458c2aa9cb1602503b # Hard drive
# - 64b91627dd13d43b9d01d6d1 # Toughbook reinforced laptop (Event quest) REMOVED BY BSG # - 64b91627dd13d43b9d01d6d1 # Toughbook reinforced laptop (Event quest) REMOVED BY BSG
ReserveBase: rezervbase:
- 60915994c49cf53e4772cc38 # Military documents 1 on the table inside bunker control room (Documents) - 60915994c49cf53e4772cc38 # Military documents 1 on the table inside bunker control room (Documents)
- 60a3b6359c427533db36cf84 # Military documents 2 On the bottom shelf of the cupboard near the corner. - 60a3b6359c427533db36cf84 # Military documents 2 On the bottom shelf of the cupboard near the corner.
- 60a3b65c27adf161da7b6e14 # Military documents 3 Inside the cupboard next to the 4x4 Weapon Box. - 60a3b65c27adf161da7b6e14 # Military documents 3 Inside the cupboard next to the 4x4 Weapon Box.
@ -82,15 +92,16 @@ ReserveBase:
- 6398a072e301557ae24cec92 # Original Lightkeeper Intelligence (Snatch) - 6398a072e301557ae24cec92 # Original Lightkeeper Intelligence (Snatch)
- 64e74a4baac4cd0a7264ecdd # item_quest_clock_09 (Out of Time) - 64e74a4baac4cd0a7264ecdd # item_quest_clock_09 (Out of Time)
Laboratory: laboratory:
- 5eff135be0d3331e9d282b7b # Flash drive marked with blue tape (TerraGroup employee) - 5eff135be0d3331e9d282b7b # Flash drive marked with blue tape (TerraGroup employee)
- 6398a4cfb5992f573c6562b3 # Secured tape - 6398a4cfb5992f573c6562b3 # Secured tape
- 64e74a44c2b4f829615ec334 # Picture 8
#- 64e74a44c2b4f829615ec334 # item_quest_clock_08 (Out of Time) #1 REMOVED BY BSG #- 64e74a44c2b4f829615ec334 # item_quest_clock_08 (Out of Time) #1 REMOVED BY BSG
#- 64bdcfed8f3a947a990aa49a # Hermetic container for storing various chemicals #1 REMOVED BY BSG #- 64bdcfed8f3a947a990aa49a # Hermetic container for storing various chemicals #1 REMOVED BY BSG
#- 64bdd008b0bf3baa6702f35f # Hermetic container for storing various chemicals #2 REMOVED BY BSG #- 64bdd008b0bf3baa6702f35f # Hermetic container for storing various chemicals #2 REMOVED BY BSG
#- 64bdd014f3a668f08805ce64 # Hermetic container for storing various chemicals #3 REMOVED BY BSG #- 64bdd014f3a668f08805ce64 # Hermetic container for storing various chemicals #3 REMOVED BY BSG
Streets of Tarkov: tarkovstreets:
- 63a943cead5cc12f22161ff7 # Accountant's notes (Audit) - 63a943cead5cc12f22161ff7 # Accountant's notes (Audit)
- 638cbc68a63f1b49be6a3010 # Registered letter (Youve Got Mail) - 638cbc68a63f1b49be6a3010 # Registered letter (Youve Got Mail)
- 638df4cc7b560b03794a18d2 # AG guitar pick (Audiophile) - 638df4cc7b560b03794a18d2 # AG guitar pick (Audiophile)
@ -114,6 +125,10 @@ Streets of Tarkov:
- 64f5b4f71a5f313cb144c06c # Secret component (Beyond the Red Meat - Part 2) - 64f5b4f71a5f313cb144c06c # Secret component (Beyond the Red Meat - Part 2)
- 657acb2ac900be5902191ac9 # Cadastral registry records - 657acb2ac900be5902191ac9 # Cadastral registry records
Sandbox: sandbox:
- 6582bd252b50c61c565828e2 # Bottle of Le Jean wine
- 6575a6ca8778e96ded05a802 # TerraGroup scientist's hard drive
sandbox_high:
- 6582bd252b50c61c565828e2 # Bottle of Le Jean wine - 6582bd252b50c61c565828e2 # Bottle of Le Jean wine
- 6575a6ca8778e96ded05a802 # TerraGroup scientist's hard drive - 6575a6ca8778e96ded05a802 # TerraGroup scientist's hard drive

View File

@ -5,17 +5,17 @@ static_weapon_ids:
- 5cdeb229d7f00c000e7ce174 - 5cdeb229d7f00c000e7ce174
forced_items: forced_items:
Customs: bigmap:
# unknown key # unknown key
- containerId: custom_multiScene_00058 - containerId: custom_multiScene_00058
itemTpl: 593962ca86f774068014d9af itemTpl: 593962ca86f774068014d9af
Streets of Tarkov: tarkovstreets:
# Backup hideout key # Backup hideout key
- containerId: container_City_SE_02_DesignStuff_00025 - containerId: container_City_SE_02_Primorskiy_51_indoor_00001
itemTpl: 6398fd8ad3de3849057f5128 itemTpl: 6398fd8ad3de3849057f5128
Sandbox: sandbox:
- containerId: container_Test_for_export_00002 # Lab technician body - containerId: container_Test_for_export_00002 # Lab technician body
itemTpl: 658199aa38c79576a2569e13 # TerraGroup science office key itemTpl: 658199aa38c79576a2569e13 # TerraGroup science office key
- containerId: container_custom_DesignStuff_00029 # dead body of scav near exit - containerId: container_custom_DesignStuff_00029 # dead body of scav near exit

View File

@ -30,3 +30,6 @@ Streets of Tarkov:
Sandbox: Sandbox:
name: name:
- Sandbox - Sandbox
SandboxHigh:
name:
- Sandbox_high

View File

@ -5,6 +5,6 @@ namespace LootDumpProcessor.Model.Processing;
public class DumpProcessData public class DumpProcessData
{ {
public Dictionary<string, IKey> LooseLootCounts { get; set; } = new(); public Dictionary<string, IKey> LooseLootCounts { get; set; } = new();
public List<PreProcessedStaticLoot> ContainerCounts { get; set; } = new(); public Dictionary<string, List<PreProcessedStaticLoot>> ContainerCounts { get; set; } = new();
public Dictionary<string, int> MapCounts { get; set; } = new(); public Dictionary<string, int> MapCounts { get; set; } = new();
} }

View File

@ -1,5 +1,4 @@
using System.Text.Json.Serialization; using System.Text.Json.Serialization;
using LootDumpProcessor.Process.Processor;
using LootDumpProcessor.Utils; using LootDumpProcessor.Utils;
using Newtonsoft.Json; using Newtonsoft.Json;
@ -9,7 +8,7 @@ namespace LootDumpProcessor.Model
{ {
[JsonProperty("StackObjectsCount", NullValueHandling = NullValueHandling.Ignore)] [JsonProperty("StackObjectsCount", NullValueHandling = NullValueHandling.Ignore)]
[JsonPropertyName("StackObjectsCount")] [JsonPropertyName("StackObjectsCount")]
public int? StackObjectsCount { get; set; } public object? StackObjectsCount { get; set; }
[JsonProperty("FireMode", NullValueHandling = NullValueHandling.Ignore)] [JsonProperty("FireMode", NullValueHandling = NullValueHandling.Ignore)]
[JsonPropertyName("FireMode")] [JsonPropertyName("FireMode")]

View File

@ -52,22 +52,23 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
Runners.Add( Runners.Add(
Task.Factory.StartNew(() => Task.Factory.StartNew(() =>
{ {
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug)) if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", LogLevel.Debug); LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", LogLevel.Debug);
var data = _jsonSerializer.Deserialize<RootData>(File.ReadAllText(dumped.BasicInfo.FileName)); var dataDump = _jsonSerializer.Deserialize<RootData>(File.ReadAllText(dumped.BasicInfo.FileName));
var mapName = data.Data.Name; //var mapName = dataDump.Data.Name;
var mapId = dataDump.Data.Id.ToLower();
// the if statement below takes care of processing "forced" or real static data for each map, only need // the if statement below takes care of processing "forced" or real static data for each map, only need
// to do this once per map, we dont care about doing it again // to do this once per map, we dont care about doing it again
lock (staticContainersLock) lock (staticContainersLock)
{ {
if (!staticContainers.ContainsKey(mapName)) if (!staticContainers.ContainsKey(mapId))
{ {
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log($"Doing first time process for map {mapName} of real static data", LogLevel.Info); LoggerFactory.GetInstance().Log($"Doing first time process for map {mapId} of real static data", LogLevel.Info);
var mapStaticContainers = StaticLootProcessor.CreateStaticWeaponsAndStaticForcedContainers(data); var mapStaticContainers = StaticLootProcessor.CreateStaticWeaponsAndStaticForcedContainers(dataDump);
// .Item1 = map name // .Item1 = map name
// .Item2 = force/weapon static arrays
staticContainers[mapStaticContainers.Item1] = mapStaticContainers.Item2; staticContainers[mapStaticContainers.Item1] = mapStaticContainers.Item2;
} }
} }
@ -77,36 +78,40 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
lock (mapStaticContainersAggregatedLock) lock (mapStaticContainersAggregatedLock)
{ {
// Init dict if map key doesnt exist // Init dict if map key doesnt exist
if (!mapStaticContainersAggregated.TryGetValue(mapName, out mapAggregatedDataDict)) if (!mapStaticContainersAggregated.TryGetValue(mapId, out mapAggregatedDataDict))
{ {
mapAggregatedDataDict = new Dictionary<Template, int>(); mapAggregatedDataDict = new Dictionary<Template, int>();
mapStaticContainersAggregated.Add(mapName, mapAggregatedDataDict); mapStaticContainersAggregated.Add(mapId, mapAggregatedDataDict);
} }
} }
// Only process the dump file if the date is higher (after) the configuration date // Only process the dump file if the date is higher (after) the configuration date
if (DumpWasMadeAfterConfigThresholdDate(dumped)) if (!DumpWasMadeAfterConfigThresholdDate(dumped))
{ {
// Keep track of how many dumps we have for each map return;
lock (mapDumpCounterLock) }
{
IncrementMapCounterDictionaryValue(mapDumpCounter, mapName);
}
var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(data.Data.Id.ToLower(), out string[]? ignoreListForMap); // Keep track of how many dumps we have for each map
foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(data)) lock (mapDumpCounterLock)
{
IncrementMapCounterDictionaryValue(mapDumpCounter, mapId);
}
var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(mapId, out string[]? ignoreListForMap);
foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(dataDump))
{
lock (mapStaticContainersAggregatedLock)
{ {
lock (mapStaticContainersAggregatedLock) if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id))
{ {
// Skip adding containers to aggredated data if container id is in ignore list // Skip adding containers to aggregated data if container id is in ignore list
if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id)) continue;
{ }
continue;
}
// Increment count by 1 // Increment times container seen in dump by 1
if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1)) if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1))
mapAggregatedDataDict[dynamicStaticContainer] += 1; {
mapAggregatedDataDict[dynamicStaticContainer] += 1;
} }
} }
} }
@ -126,10 +131,10 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
td => new StaticDataPoint td => new StaticDataPoint
{ {
Template = td.Key, Template = td.Key,
Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) // kv.Key = map name
} }
).ToList() ).ToList()
).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); ).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); // Hydrate staticContainers.StaticContainers
// Static containers // Static containers
output.Add(OutputFileType.StaticContainer, staticContainers); output.Add(OutputFileType.StaticContainer, staticContainers);
@ -146,7 +151,7 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
// Static loot distribution // Static loot distribution
output.Add( output.Add(
OutputFileType.StaticLoot, OutputFileType.StaticLoot,
StaticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts) StaticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts, staticContainers)
); );
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
@ -199,23 +204,23 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
.ToList() .ToList()
.ForEach(tuple => .ForEach(tuple =>
{ {
var mapi = tuple.Key; var mapName = tuple.Key;
var g = tuple.ToList(); var partialFileMetaData = tuple.ToList();
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log( LoggerFactory.GetInstance().Log(
$"Processing map {mapi}, total dump data to process: {g.Count}", $"Processing map {mapName}, total dump data to process: {partialFileMetaData.Count}",
LogLevel.Info LogLevel.Info
); );
dumpProcessData.MapCounts[mapi] = g.Count; dumpProcessData.MapCounts[mapName] = partialFileMetaData.Count;
var lockObjectContainerCounts = new object(); var lockObjectContainerCounts = new object();
var lockObjectCounts = new object(); var lockObjectCounts = new object();
var counts = new LooseLootCounts(); var looseLootCounts = new LooseLootCounts();
var lockObjectDictionaryCounts = new object(); var lockObjectDictionaryCounts = new object();
var dictionaryCounts = new FlatKeyableDictionary<string, int>(); var dictionaryCounts = new FlatKeyableDictionary<string, int>();
counts.Counts = dictionaryCounts.GetKey(); looseLootCounts.Counts = dictionaryCounts.GetKey();
/* /*
var dictionaryItemCounts = new FlatKeyableDictionary<string, List<string>>(); var dictionaryItemCounts = new FlatKeyableDictionary<string, List<string>>();
@ -226,22 +231,21 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
var dictionaryItemProperties = new FlatKeyableDictionary<string, FlatKeyableList<Template>>(); var dictionaryItemProperties = new FlatKeyableDictionary<string, FlatKeyableList<Template>>();
var actualDictionaryItemProperties = new FlatKeyableDictionary<string, IKey>(); var actualDictionaryItemProperties = new FlatKeyableDictionary<string, IKey>();
counts.ItemProperties = actualDictionaryItemProperties.GetKey(); looseLootCounts.ItemProperties = actualDictionaryItemProperties.GetKey();
dumpProcessData.LooseLootCounts.Add(mapi, counts.GetKey()); dumpProcessData.LooseLootCounts.Add(mapName, looseLootCounts.GetKey());
// add the items to the queue // add the items to the queue
foreach (var gi in g) foreach (var partialData in partialFileMetaData)
{ {
_partialDataToProcess.Add(gi); _partialDataToProcess.Add(partialData);
} }
// Call GC before running threads // Call GC before running threads
g = null; partialFileMetaData = null;
tuple = null; tuple = null;
GCHandler.Collect(); GCHandler.Collect();
// The data storage factory has a lock, we dont want the locks to occur when multithreading // The data storage factory has a lock, we dont want the locks to occur when multithreading
for (int i = 0; i < LootDumpProcessorContext.GetConfig().Threads; i++) for (int i = 0; i < LootDumpProcessorContext.GetConfig().Threads; i++)
{ {
Runners.Add( Runners.Add(
@ -254,26 +258,35 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
try try
{ {
var dumpData = _dataStorage.GetItem<ParsedDump>(partialData.ParsedDumpKey); var dumpData = _dataStorage.GetItem<ParsedDump>(partialData.ParsedDumpKey);
// Static containers
lock (lockObjectContainerCounts) lock (lockObjectContainerCounts)
{ {
dumpProcessData.ContainerCounts.AddRange(dumpData.Containers); if (!dumpProcessData.ContainerCounts.ContainsKey(mapName))
{
dumpProcessData.ContainerCounts.Add(mapName, dumpData.Containers);
}
else
{
dumpProcessData.ContainerCounts[mapName].AddRange(dumpData.Containers);
}
} }
// loose loot into ids on files // Loose loot into ids on files
var loadedDictionary = var loadedDictionary =
_dataStorage _dataStorage
.GetItem<SubdivisionedKeyableDictionary<string, List<Template>>>( .GetItem<SubdivisionedKeyableDictionary<string, List<Template>>>(
dumpData.LooseLoot.ItemProperties dumpData.LooseLoot.ItemProperties
); );
foreach (var (k, v) in loadedDictionary) foreach (var (uniqueKey, containerTemplate) in loadedDictionary)
{ {
var count = dumpData.LooseLoot.Counts[k]; var count = dumpData.LooseLoot.Counts[uniqueKey];
lock (lockObjectDictionaryCounts) lock (lockObjectDictionaryCounts)
{ {
if (dictionaryCounts.ContainsKey(k)) if (dictionaryCounts.ContainsKey(uniqueKey))
dictionaryCounts[k] += count; dictionaryCounts[uniqueKey] += count;
else else
dictionaryCounts[k] = count; dictionaryCounts[uniqueKey] = count;
} }
/* /*
@ -288,20 +301,20 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
lock (lockObjectDictionaryItemProperties) lock (lockObjectDictionaryItemProperties)
{ {
if (!dictionaryItemProperties.TryGetValue(k, out var values)) if (!dictionaryItemProperties.TryGetValue(uniqueKey, out var values))
{ {
values = new FlatKeyableList<Template>(); values = new FlatKeyableList<Template>();
dictionaryItemProperties.Add(k, values); dictionaryItemProperties.Add(uniqueKey, values);
actualDictionaryItemProperties.Add(k, values.GetKey()); actualDictionaryItemProperties.Add(uniqueKey, values.GetKey());
} }
values.AddRange(v); values.AddRange(containerTemplate);
} }
} }
lock (lockObjectCounts) lock (lockObjectCounts)
{ {
counts.MapSpawnpointCount.Add(dumpData.LooseLoot.MapSpawnpointCount); looseLootCounts.MapSpawnpointCount.Add(dumpData.LooseLoot.MapSpawnpointCount);
} }
} }
catch (Exception e) catch (Exception e)
@ -346,8 +359,8 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
_dataStorage.Store(actualDictionaryItemProperties); _dataStorage.Store(actualDictionaryItemProperties);
actualDictionaryItemProperties = null; actualDictionaryItemProperties = null;
GCHandler.Collect(); GCHandler.Collect();
_dataStorage.Store(counts); _dataStorage.Store(looseLootCounts);
counts = null; looseLootCounts = null;
GCHandler.Collect(); GCHandler.Collect();
}); });
return dumpProcessData; return dumpProcessData;

View File

@ -85,7 +85,8 @@ public static class LooseLootProcessor
// we want to cleanup the data, so we calculate the mean for the values we get raw // we want to cleanup the data, so we calculate the mean for the values we get raw
// For whatever reason, we sometimes get dumps that have A LOT more loose loot point than // For whatever reason, we sometimes get dumps that have A LOT more loose loot point than
// the average // the average
var initialMean = np.mean(np.array(looseLootCounts.MapSpawnpointCount)).ToArray<double>().First(); var values = looseLootCounts.MapSpawnpointCount.Select(Convert.ToDouble);
var initialMean = np.mean(np.array(values)).ToArray<double>().First();
var looseLootCountTolerancePercentage = LootDumpProcessorContext.GetConfig().ProcessorConfig.LooseLootCountTolerancePercentage / 100; var looseLootCountTolerancePercentage = LootDumpProcessorContext.GetConfig().ProcessorConfig.LooseLootCountTolerancePercentage / 100;
// We calculate here a high point to check, anything above this value will be ignored // We calculate here a high point to check, anything above this value will be ignored
// The data that was inside those loose loot points still counts for them though! // The data that was inside those loose loot points still counts for them though!

View File

@ -4,6 +4,7 @@ using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.StaticContainer; using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing; using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Utils; using LootDumpProcessor.Utils;
using System.Collections.Generic;
namespace LootDumpProcessor.Process.Processor; namespace LootDumpProcessor.Process.Processor;
@ -33,6 +34,7 @@ public static class StaticLootProcessor
public static Tuple<string, MapStaticLoot> CreateStaticWeaponsAndStaticForcedContainers(RootData rawMapDump) public static Tuple<string, MapStaticLoot> CreateStaticWeaponsAndStaticForcedContainers(RootData rawMapDump)
{ {
var mapName = rawMapDump.Data.Name; var mapName = rawMapDump.Data.Name;
var mapId = rawMapDump.Data.Id.ToLower();
var staticLootPositions = (from li in rawMapDump.Data.Loot var staticLootPositions = (from li in rawMapDump.Data.Loot
where li.IsContainer ?? false where li.IsContainer ?? false
select li).ToList(); select li).ToList();
@ -46,8 +48,8 @@ public static class StaticLootProcessor
} }
} }
var forcedStaticItems = LootDumpProcessorContext.GetForcedItems().ContainsKey(mapName) var forcedStaticItems = LootDumpProcessorContext.GetForcedItems().ContainsKey(mapId)
? LootDumpProcessorContext.GetForcedItems()[mapName] ? LootDumpProcessorContext.GetForcedItems()[mapId]
: new List<StaticForced>(); : new List<StaticForced>();
var mapStaticData = new MapStaticLoot var mapStaticData = new MapStaticLoot
@ -55,7 +57,7 @@ public static class StaticLootProcessor
StaticWeapons = staticWeapons, StaticWeapons = staticWeapons,
StaticForced = forcedStaticItems StaticForced = forcedStaticItems
}; };
return Tuple.Create(mapName, mapStaticData); return Tuple.Create(mapId, mapStaticData);
} }
public static List<Template> CreateDynamicStaticContainers(RootData rawMapDump) public static List<Template> CreateDynamicStaticContainers(RootData rawMapDump)
@ -73,94 +75,207 @@ public static class StaticLootProcessor
return data; return data;
} }
public static Dictionary<string, List<AmmoDistribution>> CreateAmmoDistribution( /// <summary>
List<PreProcessedStaticLoot> container_counts ///
/// </summary>
/// <param name="container_counts"></param>
/// <returns>key = mapid / </returns>
public static Dictionary<string, Dictionary<string, List<AmmoDistribution>>> CreateAmmoDistribution(
Dictionary<string, List<PreProcessedStaticLoot>> container_counts
) )
{ {
var ammo = new List<string>(); var allMapsAmmoDistro = new Dictionary<string, Dictionary<string, List<AmmoDistribution>>>();
foreach (var ci in container_counts) foreach (var mapAndContainers in container_counts)
{ {
ammo.AddRange(from item in ci.Items var mapid = mapAndContainers.Key;
where LootDumpProcessorContext.GetTarkovItems().IsBaseClass(item.Tpl, BaseClasses.Ammo) var containers = mapAndContainers.Value;
select item.Tpl);
var ammo = new List<string>();
foreach (var ci in containers)
{
ammo.AddRange(from item in ci.Items
where LootDumpProcessorContext.GetTarkovItems().IsBaseClass(item.Tpl, BaseClasses.Ammo)
select item.Tpl);
}
var ammo_counts = new List<CaliberTemplateCount>();
ammo_counts.AddRange(
ammo.GroupBy(a => a)
.Select(g => new CaliberTemplateCount
{
Caliber = LootDumpProcessorContext.GetTarkovItems().AmmoCaliber(g.Key),
Template = g.Key,
Count = g.Count()
})
);
ammo_counts = ammo_counts.OrderBy(x => x.Caliber).ToList();
var ammo_distribution = new Dictionary<string, List<AmmoDistribution>>();
foreach (var _tup_3 in ammo_counts.GroupBy(x => x.Caliber))
{
var k = _tup_3.Key;
var g = _tup_3.ToList();
ammo_distribution[k] = (from gi in g
select new AmmoDistribution
{
Tpl = gi.Template,
RelativeProbability = gi.Count
}).ToList();
}
allMapsAmmoDistro.TryAdd(mapid, ammo_distribution);
} }
var ammo_counts = new List<CaliberTemplateCount>(); return allMapsAmmoDistro;
ammo_counts.AddRange(
ammo.GroupBy(a => a)
.Select(g => new CaliberTemplateCount
{
Caliber = LootDumpProcessorContext.GetTarkovItems().AmmoCaliber(g.Key),
Template = g.Key,
Count = g.Count()
})
);
ammo_counts = ammo_counts.OrderBy(x => x.Caliber).ToList();
var ammo_distribution = new Dictionary<string, List<AmmoDistribution>>();
foreach (var _tup_3 in ammo_counts.GroupBy(x => x.Caliber))
{
var k = _tup_3.Key;
var g = _tup_3.ToList();
ammo_distribution[k] = (from gi in g
select new AmmoDistribution
{
Tpl = gi.Template,
RelativeProbability = gi.Count
}).ToList();
}
return ammo_distribution; //var ammo = new List<string>();
//foreach (var ci in container_counts)
//{
// ammo.AddRange(from item in ci.Items
// where LootDumpProcessorContext.GetTarkovItems().IsBaseClass(item.Tpl, BaseClasses.Ammo)
// select item.Tpl);
//}
//var ammo_counts = new List<CaliberTemplateCount>();
//ammo_counts.AddRange(
// ammo.GroupBy(a => a)
// .Select(g => new CaliberTemplateCount
// {
// Caliber = LootDumpProcessorContext.GetTarkovItems().AmmoCaliber(g.Key),
// Template = g.Key,
// Count = g.Count()
// })
//);
//ammo_counts = ammo_counts.OrderBy(x => x.Caliber).ToList();
//var ammo_distribution = new Dictionary<string, List<AmmoDistribution>>();
//foreach (var _tup_3 in ammo_counts.GroupBy(x => x.Caliber))
//{
// var k = _tup_3.Key;
// var g = _tup_3.ToList();
// ammo_distribution[k] = (from gi in g
// select new AmmoDistribution
// {
// Tpl = gi.Template,
// RelativeProbability = gi.Count
// }).ToList();
//}
//return ammo_distribution;
} }
public static Dictionary<string, StaticItemDistribution> CreateStaticLootDistribution( /// <summary>
List<PreProcessedStaticLoot> container_counts /// Dict key = map,
) /// value = sub dit:
/// key = container Ids
/// value = items + counts
/// </summary>
public static Dictionary<string, Dictionary<string, StaticItemDistribution>> CreateStaticLootDistribution(
Dictionary<string, List<PreProcessedStaticLoot>> container_counts,
Dictionary<string, MapStaticLoot> staticContainers)
{ {
var static_loot_distribution = new Dictionary<string, StaticItemDistribution>(); var allMapsStaticLootDisto = new Dictionary< string, Dictionary<string, StaticItemDistribution>>();
var types = Enumerable.Distinct((from ci in container_counts // Iterate over each map we have containers for
select ci.Type).ToList()); foreach (var mapContainersKvp in container_counts)
foreach (var typei in types)
{ {
var container_counts_selected = (from ci in container_counts var mapName = mapContainersKvp.Key;
where ci.Type == typei var containers = mapContainersKvp.Value;
select ci).ToList();
var itemscounts = new List<int>();
foreach (var ci in container_counts_selected)
{
itemscounts.Add((from cii in ci.Items
where cii.ParentId == ci.ContainerId
select cii).ToList().Count);
}
static_loot_distribution[typei] = new StaticItemDistribution(); var static_loot_distribution = new Dictionary<string, StaticItemDistribution>();
static_loot_distribution[typei].ItemCountDistribution = itemscounts.GroupBy(i => i) var uniqueContainerTypeIds = Enumerable.Distinct((from ci in containers
.Select(g => new ItemCountDistribution select ci.Type).ToList());
{
Count = g.Key,
RelativeProbability = g.Count()
}).ToList();
// TODO: Change for different algo that splits items per parent once parentid = containerid, then compose
// TODO: key and finally create distribution based on composed Id instead
var itemsHitCounts = new Dictionary<string, int>();
foreach (var ci in container_counts_selected)
{
foreach (var cii in ci.Items.Where(cii => cii.ParentId == ci.ContainerId))
{
if (itemsHitCounts.ContainsKey(cii.Tpl))
itemsHitCounts[cii.Tpl] += 1;
else
itemsHitCounts[cii.Tpl] = 1;
}
}
static_loot_distribution[typei].ItemDistribution = itemsHitCounts.Select(v => new StaticDistribution foreach (var typeId in uniqueContainerTypeIds)
{ {
Tpl = v.Key, var container_counts_selected = (from ci in containers
RelativeProbability = v.Value where ci.Type == typeId
}).ToList(); select ci).ToList();
// Get array of all times a count of items was found in container
List<int> itemCountsInContainer = GetCountOfItemsInContainer(container_counts_selected);
// Create structure to hold item count + weight that it will be picked
// Group same counts together
static_loot_distribution[typeId] = new StaticItemDistribution();
static_loot_distribution[typeId].ItemCountDistribution = itemCountsInContainer.GroupBy(i => i)
.Select(g => new ItemCountDistribution
{
Count = g.Key,
RelativeProbability = g.Count()
}).ToList();
static_loot_distribution[typeId].ItemDistribution = CreateItemDistribution(container_counts_selected);
}
// Key = containers tpl, value = items + count weights
allMapsStaticLootDisto.TryAdd(mapName, static_loot_distribution);
} }
return static_loot_distribution; return allMapsStaticLootDisto;
//var static_loot_distribution = new Dictionary<string, StaticItemDistribution>();
//var uniqueContainerTypeIds = Enumerable.Distinct((from ci in container_counts
// select ci.Type).ToList());
//foreach (var typeId in uniqueContainerTypeIds)
//{
// var container_counts_selected = (from ci in container_counts
// where ci.Type == typeId
// select ci).ToList();
// // Get array of all times a count of items was found in container
// List<int> itemCountsInContainer = GetCountOfItemsInContainer(container_counts_selected);
// // Create structure to hold item count + weight that it will be picked
// // Group same counts together
// static_loot_distribution[typeId] = new StaticItemDistribution();
// static_loot_distribution[typeId].ItemCountDistribution = itemCountsInContainer.GroupBy(i => i)
// .Select(g => new ItemCountDistribution
// {
// Count = g.Key,
// RelativeProbability = g.Count()
// }).ToList();
// static_loot_distribution[typeId].ItemDistribution = CreateItemDistribution(container_counts_selected);
//}
//// Key = containers tpl, value = items + count weights
//return static_loot_distribution;
}
private static List<StaticDistribution> CreateItemDistribution(List<PreProcessedStaticLoot> container_counts_selected)
{
// TODO: Change for different algo that splits items per parent once parentid = containerid, then compose
// TODO: key and finally create distribution based on composed Id instead
var itemsHitCounts = new Dictionary<string, int>();
foreach (var ci in container_counts_selected)
{
foreach (var cii in ci.Items.Where(cii => cii.ParentId == ci.ContainerId))
{
if (itemsHitCounts.ContainsKey(cii.Tpl))
itemsHitCounts[cii.Tpl] += 1;
else
itemsHitCounts[cii.Tpl] = 1;
}
}
// WIll create array of objects that have a tpl + relative probability weight value
return itemsHitCounts.Select(v => new StaticDistribution
{
Tpl = v.Key,
RelativeProbability = v.Value
}).ToList();
}
private static List<int> GetCountOfItemsInContainer(List<PreProcessedStaticLoot> container_counts_selected)
{
var itemCountsInContainer = new List<int>();
foreach (var containerWithItems in container_counts_selected)
{
// Only count item if its parent is the container, only root items are counted (not mod/attachment items)
itemCountsInContainer.Add((from cii in containerWithItems.Items
where cii.ParentId == containerWithItems.ContainerId
select cii).ToList().Count);
}
return itemCountsInContainer;
} }
} }

View File

@ -39,7 +39,7 @@ public class JsonFileIntakeReader : IIntakeReader
{ {
basicInfo = new BasicInfo basicInfo = new BasicInfo
{ {
Map = fi.Data.Name, Map = fi.Data.Id.ToLower(),
FileHash = ProcessorUtil.HashFile(fileData), FileHash = ProcessorUtil.HashFile(fileData),
Data = fi, Data = fi,
Date = date.Value, Date = date.Value,
@ -48,10 +48,16 @@ public class JsonFileIntakeReader : IIntakeReader
_totalMapDumpsCounter[fi.Data.Name] += 1; _totalMapDumpsCounter[fi.Data.Name] += 1;
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug)) if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"File {file} fully read, returning data", LogLevel.Debug); LoggerFactory.GetInstance().Log($"File {file} fully read, returning data", LogLevel.Debug);
return true; return true;
} }
// Map dump limit reached, exit
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug)) if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"Ignoring file {file} as the file cap for map {fi.Data.Name} has been reached", LogLevel.Debug); LoggerFactory.GetInstance().Log($"Ignoring file {file} as the file cap for map {fi.Data.Id} has been reached", LogLevel.Debug);
basicInfo = null;
return false;
} }
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning)) if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))

View File

@ -1,7 +1,9 @@
using LootDumpProcessor.Model.Output; using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot; using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Output.StaticContainer; using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Serializers.Json; using LootDumpProcessor.Serializers.Json;
using System.Collections.Generic;
namespace LootDumpProcessor.Process.Writer; namespace LootDumpProcessor.Process.Writer;
@ -45,30 +47,45 @@ public class FileWriter : IWriter
var looseLootData = (Dictionary<string, LooseLootRoot>)data; var looseLootData = (Dictionary<string, LooseLootRoot>)data;
foreach (var (key, value) in looseLootData) foreach (var (key, value) in looseLootData)
{ {
foreach (var s in LootDumpProcessorContext.GetDirectoryMappings()[key].Name) if (!Directory.Exists($@"{_outputPath}\locations\{key}"))
{ Directory.CreateDirectory($@"{_outputPath}\locations\{key}");
if (!Directory.Exists($@"{_outputPath}\locations\{s}")) File.WriteAllText($@"{_outputPath}\locations\{key}\looseLoot.json",
Directory.CreateDirectory($@"{_outputPath}\locations\{s}"); _jsonSerializer.Serialize(value));
File.WriteAllText($@"{_outputPath}\locations\{s}\looseLoot.json",
_jsonSerializer.Serialize(value));
}
} }
break; break;
case OutputFileType.StaticContainer: case OutputFileType.StaticContainer:
var staticContainer = (Dictionary<string, MapStaticLoot>)data; var staticContainer = (Dictionary<string, MapStaticLoot>)data;
File.WriteAllText($@"{_outputPath}\loot\staticContainers.json", foreach (var (key, value) in staticContainer)
_jsonSerializer.Serialize(staticContainer)); {
if (!Directory.Exists($@"{_outputPath}\locations\{key}"))
Directory.CreateDirectory($@"{_outputPath}\locations\{key}");
File.WriteAllText($@"{_outputPath}\locations\{key}\staticContainers.json",
_jsonSerializer.Serialize(value));
}
break; break;
case OutputFileType.StaticLoot: case OutputFileType.StaticLoot:
var staticLoot = (Dictionary<string, StaticItemDistribution>)data; var staticLootData = (Dictionary<string, Dictionary<string, StaticItemDistribution>>)data;
File.WriteAllText($@"{_outputPath}\loot\staticLoot.json", foreach (var (key, value) in staticLootData)
_jsonSerializer.Serialize(staticLoot)); {
if (!Directory.Exists($@"{_outputPath}\locations\{key}"))
Directory.CreateDirectory($@"{_outputPath}\locations\{key}");
File.WriteAllText($@"{_outputPath}\locations\{key}\staticLoot.json",
_jsonSerializer.Serialize(value));
}
break; break;
case OutputFileType.StaticAmmo: case OutputFileType.StaticAmmo:
var staticAmmo = (Dictionary<string, List<AmmoDistribution>>)data; var staticAmmo = (Dictionary<string, Dictionary<string, List<AmmoDistribution>>>)data;
File.WriteAllText($@"{_outputPath}\loot\staticAmmo.json", foreach (var (key, value) in staticAmmo)
_jsonSerializer.Serialize(staticAmmo)); {
if (!Directory.Exists($@"{_outputPath}\locations\{key}"))
Directory.CreateDirectory($@"{_outputPath}\locations\{key}");
File.WriteAllText($@"{_outputPath}\locations\{key}\staticAmmo.json",
_jsonSerializer.Serialize(value));
}
break; break;
default: default:
throw new ArgumentOutOfRangeException(nameof(type), type, null); throw new ArgumentOutOfRangeException(nameof(type), type, null);