From 9ddbc9197728dfca2e22af0c886eef4620442001 Mon Sep 17 00:00:00 2001 From: DrakiaXYZ <565558+TheDgtl@users.noreply.github.com> Date: Sun, 3 Nov 2024 23:29:50 -0800 Subject: [PATCH] Drastically reduce memory footprint at the cost of removing threading - Drastically improve performance, where non-threaded perf is now better than previous threaded perf - Process all data in-line, as opposed to building huge lists of bot data that is then processed in groups later - Memory footprint should now be relatively static no matter the amount of dumps --- Common.Models/Output/Output.cs | 26 +- Generator/BaseBotGenerator.cs | 139 +++----- Generator/BotChancesGenerator.cs | 48 +-- Generator/BotGearGenerator.cs | 50 +-- Generator/BotLootGenerator.cs | 131 +++----- {Common/Bot => Generator}/BotParser.cs | 115 +++++++ Generator/Helpers/Gear/GearChanceHelpers.cs | 346 +++++++++++--------- Generator/Helpers/Gear/GearHelpers.cs | 4 +- Generator/Program.cs | 36 +- Generator/Weighting/WeightingService.cs | 7 +- 10 files changed, 440 insertions(+), 462 deletions(-) rename {Common/Bot => Generator}/BotParser.cs (51%) diff --git a/Common.Models/Output/Output.cs b/Common.Models/Output/Output.cs index 2a1f4dd..71aa21c 100644 --- a/Common.Models/Output/Output.cs +++ b/Common.Models/Output/Output.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Text.Json.Serialization; namespace Common.Models.Output; @@ -18,6 +19,7 @@ public class Bot public Bot(BotType botType) { this.botType = botType; + botCount = 0; appearance = new Appearance(); experience = new Experience(); health = new Health(); @@ -32,6 +34,8 @@ public class Bot [JsonIgnore] public BotType botType { get; set; } + [JsonIgnore] + public int botCount { get; set; } public Appearance appearance { get; set; } public Experience experience { get; set; } public Health health { get; set; } @@ -79,7 +83,7 @@ public class Experience public bool useSimpleAnimator { get; set; } } -public class BodyParts +public class BodyParts : IEquatable { public BodyParts() { @@ -92,6 +96,17 @@ public class BodyParts RightLeg = new MinMax(65, 65); } + public bool Equals(BodyParts other) + { + return this.Head.Equals(other.Head) && + this.Chest.Equals(other.Chest) && + this.Stomach.Equals(other.Stomach) && + this.LeftArm.Equals(other.LeftArm) && + this.RightArm.Equals(other.RightArm) && + this.LeftLeg.Equals(other.LeftLeg) && + this.RightLeg.Equals(other.RightLeg); + } + public MinMax Head { get; set; } public MinMax Chest { get; set; } public MinMax Stomach { get; set; } @@ -256,7 +271,7 @@ public class ItemChances public GenerationWeightData grenades { get; set; } } -public class MinMax +public class MinMax : IEquatable { public MinMax(int min, int max) { @@ -264,6 +279,11 @@ public class MinMax this.max = max; } + public bool Equals(MinMax other) + { + return this.min == other.min && this.max == other.max; + } + public int min { get; set; } public int max { get; set; } } diff --git a/Generator/BaseBotGenerator.cs b/Generator/BaseBotGenerator.cs index 36e8be6..9ede243 100644 --- a/Generator/BaseBotGenerator.cs +++ b/Generator/BaseBotGenerator.cs @@ -10,104 +10,56 @@ namespace Generator { public static class BaseBotGenerator { - //TODO: pass in bot types and use those to create the classes in rawBots list - public static IEnumerable GenerateBaseDetails(IEnumerable rawBots, string workingPath, IEnumerable botTypes) + public static void UpdateBaseDetails(Bot botData, Datum rawBotData) { - var stopwatch = Stopwatch.StartNew(); - LoggingHelpers.LogToConsole("Started processing bot base"); + UpdateBodyPartHealth(botData, rawBotData); + AddExperience(botData, rawBotData); + AddStandingForKill(botData, rawBotData); + AddSkills(botData, rawBotData); + botData.experience.useSimpleAnimator = rawBotData.Info.Settings.UseSimpleAnimator; - // Create a list of bot objects ready to be hydrated - var baseBots = new List(); - foreach (var botType in botTypes) - { - var typeToAdd = (BotType)Enum.Parse(typeof(BotType), botType); - baseBots.Add(new Bot(typeToAdd)); - } - - // Iterate over each bot type we just made and put some data into them - foreach (var botToUpdate in baseBots) - { - var rawBotType = botToUpdate.botType.ToString(); - var rawBotsOfSameType = rawBots.Where(x => string.Equals(x.Info.Settings.Role, rawBotType, StringComparison.OrdinalIgnoreCase)) - .ToList(); - var rawBotsOfSameTypeCount = rawBotsOfSameType.Count.ToString(); - - if (rawBotsOfSameType.Count == 0) - { - LoggingHelpers.LogToConsole($"no bots of type {rawBotType}, skipping", ConsoleColor.DarkRed); - continue; - } - - LoggingHelpers.LogToConsole($"Found {rawBotsOfSameTypeCount} bots of type: {rawBotType}"); - - UpdateBodyPartHealth(botToUpdate, rawBotsOfSameType); - AddDifficulties(botToUpdate, workingPath); - AddExperience(botToUpdate, rawBotsOfSameType); - AddStandingForKill(botToUpdate, rawBotsOfSameType); - AddSkills(botToUpdate, rawBotsOfSameType); - botToUpdate.experience.useSimpleAnimator = rawBotsOfSameType.First().Info.Settings.UseSimpleAnimator; - - foreach (var rawParsedBot in rawBotsOfSameType) - { - AddVisualAppearanceItems(botToUpdate, rawParsedBot); - AddName(botToUpdate, rawParsedBot); - AddVoice(botToUpdate, rawParsedBot); - } - } - - stopwatch.Stop(); - LoggingHelpers.LogToConsole($"Finished processing bot base. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds"); - - return baseBots; + AddVisualAppearanceItems(botData, rawBotData); + AddName(botData, rawBotData); + AddVoice(botData, rawBotData); } - private static void AddSkills(Bot botToUpdate, IEnumerable rawBotsOfSameType) + private static void AddSkills(Bot botToUpdate, Datum rawBotData) { - var firstBotOfDesiredType = rawBotsOfSameType.Last(); - // Find the smallest and biggest value for each skill - foreach (var skill in firstBotOfDesiredType.Skills.Common) + foreach (var skill in rawBotData.Skills.Common) { - var skills = new List(); - foreach (var bot in rawBotsOfSameType) + if (botToUpdate.skills.Common.TryGetValue(skill.Id, out var existingSkill)) { - skills.Add(bot.Skills.Common.Find(x => x.Id == skill.Id)); + existingSkill.min = Math.Min(existingSkill.min, skill.Progress); + existingSkill.max = Math.Max(existingSkill.max, skill.Progress); } - - var min = skills.Min(x => x?.Progress); - var max = skills.Max(x => x?.Progress); - - if (min.HasValue && max.HasValue) + else { - botToUpdate.skills.Common.Add(skill.Id, new MinMax(min.Value, max.Value)); + botToUpdate.skills.Common.Add(skill.Id, new MinMax(skill.Progress, skill.Progress)); } } } - private static void AddStandingForKill(Bot botToUpdate, IEnumerable rawBotsOfSameType) + private static void AddStandingForKill(Bot botToUpdate, Datum rawBotData) { - var firstBotOfDesiredType = rawBotsOfSameType.Last(); - - botToUpdate.experience.standingForKill = firstBotOfDesiredType.Info.Settings.StandingForKill; - botToUpdate.experience.aggressorBonus = firstBotOfDesiredType.Info.Settings.AggressorBonus; + botToUpdate.experience.standingForKill = rawBotData.Info.Settings.StandingForKill; + botToUpdate.experience.aggressorBonus = rawBotData.Info.Settings.AggressorBonus; } - private static void AddExperience(Bot botToUpdate, IEnumerable rawBotsOfSameType) + private static void AddExperience(Bot botToUpdate, Datum rawBotData) { - var firstBotOfDesiredType = rawBotsOfSameType.Last(); - - botToUpdate.experience.reward.min = firstBotOfDesiredType.Info.Settings.Experience; - botToUpdate.experience.reward.max = firstBotOfDesiredType.Info.Settings.Experience; + botToUpdate.experience.reward.min = rawBotData.Info.Settings.Experience; + botToUpdate.experience.reward.max = rawBotData.Info.Settings.Experience; } private static void AddVoice(Bot bot, Datum rawBot) { GearHelpers.IncrementDictionaryValue(bot.appearance.voice, rawBot.Info.Voice); - GearHelpers.ReduceWeightValues(bot.appearance.voice); } - private static void AddDifficulties(Bot bot, string workingPath) + public static void AddDifficulties(Bot bot) { + string workingPath = Directory.GetCurrentDirectory(); string botType = bot.botType.ToString(); var botDifficultyFiles = Directory .GetFiles($"{workingPath}//Assets", "*.txt", SearchOption.TopDirectoryOnly) @@ -117,51 +69,34 @@ namespace Generator DifficultyHelper.AddDifficultySettings(bot, botDifficultyFiles); } - private static void UpdateBodyPartHealth(Bot botToUpdate, List rawBots) + private static void UpdateBodyPartHealth(Bot botToUpdate, Datum rawBot) { - var uniqueHealthSetups = new Dictionary(); - foreach (var bot in rawBots) + var bodyPartHpToAdd = new Common.Models.Output.BodyParts() { - var healthTotal = bot.Health.BodyParts.GetHpMaxTotal(); - var alreadyExists = uniqueHealthSetups.ContainsKey(healthTotal); + Head = new MinMax(rawBot.Health.BodyParts.Head.Health.Current, rawBot.Health.BodyParts.Head.Health.Maximum), + Chest = new MinMax(rawBot.Health.BodyParts.Chest.Health.Current, rawBot.Health.BodyParts.Chest.Health.Maximum), + Stomach = new MinMax(rawBot.Health.BodyParts.Stomach.Health.Current, rawBot.Health.BodyParts.Stomach.Health.Maximum), + LeftArm = new MinMax(rawBot.Health.BodyParts.LeftArm.Health.Current, rawBot.Health.BodyParts.LeftArm.Health.Maximum), + RightArm = new MinMax(rawBot.Health.BodyParts.RightArm.Health.Current, rawBot.Health.BodyParts.RightArm.Health.Maximum), + LeftLeg = new MinMax(rawBot.Health.BodyParts.LeftLeg.Health.Current, rawBot.Health.BodyParts.LeftLeg.Health.Maximum), + RightLeg = new MinMax(rawBot.Health.BodyParts.RightLeg.Health.Current, rawBot.Health.BodyParts.RightLeg.Health.Maximum) + }; - if (!alreadyExists) - { - var bodyPartHpToAdd = new Common.Models.Output.BodyParts() - { - Head = new MinMax(bot.Health.BodyParts.Head.Health.Current, bot.Health.BodyParts.Head.Health.Maximum), - Chest = new MinMax(bot.Health.BodyParts.Chest.Health.Current, bot.Health.BodyParts.Chest.Health.Maximum), - Stomach = new MinMax(bot.Health.BodyParts.Stomach.Health.Current, bot.Health.BodyParts.Stomach.Health.Maximum), - LeftArm = new MinMax(bot.Health.BodyParts.LeftArm.Health.Current, bot.Health.BodyParts.LeftArm.Health.Maximum), - RightArm = new MinMax(bot.Health.BodyParts.RightArm.Health.Current, bot.Health.BodyParts.RightArm.Health.Maximum), - }; - - bodyPartHpToAdd.LeftLeg.min = bot.Health.BodyParts.LeftLeg.Health.Current; - bodyPartHpToAdd.LeftLeg.max = bot.Health.BodyParts.LeftLeg.Health.Maximum; - - bodyPartHpToAdd.RightLeg.min = bot.Health.BodyParts.RightLeg.Health.Current; - bodyPartHpToAdd.RightLeg.max = bot.Health.BodyParts.RightLeg.Health.Maximum; - - uniqueHealthSetups.Add(healthTotal, bodyPartHpToAdd); - } + if (!botToUpdate.health.BodyParts.Contains(bodyPartHpToAdd)) + { + botToUpdate.health.BodyParts.Add(bodyPartHpToAdd); } - - botToUpdate.health.BodyParts = uniqueHealthSetups.Values.ToList(); } private static void AddVisualAppearanceItems(Bot botToUpdate, Datum rawBot) { GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.feet, rawBot.Customization.Feet); - GearHelpers.ReduceWeightValues(botToUpdate.appearance.feet); GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.body, rawBot.Customization.Body); - GearHelpers.ReduceWeightValues(botToUpdate.appearance.body); GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.head, rawBot.Customization.Head); - GearHelpers.ReduceWeightValues(botToUpdate.appearance.head); GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.hands, rawBot.Customization.Hands); - GearHelpers.ReduceWeightValues(botToUpdate.appearance.hands); } private static void AddName(Bot botToUpdate, Datum rawBot) diff --git a/Generator/BotChancesGenerator.cs b/Generator/BotChancesGenerator.cs index cfeb00b..2847138 100644 --- a/Generator/BotChancesGenerator.cs +++ b/Generator/BotChancesGenerator.cs @@ -8,51 +8,15 @@ namespace Generator { public static class BotChancesGenerator { - public static IEnumerable AddChances(this IEnumerable botsToUpdate, Dictionary> rawBots) + public static void AddChances(Bot botToUpdate, Datum rawBot) { - var stopwatch = Stopwatch.StartNew(); - LoggingHelpers.LogToConsole("Started processing bot gear"); - - // use lock for lock safety - var dictionaryLock = new object(); var weightHelper = new WeightingService(); - // multithread - var tasks = new List(); - foreach (var botToUpdate in botsToUpdate) - { - tasks.Add(Task.Factory.StartNew(() => - { - var botType = botToUpdate.botType.ToString().ToLower(); - List rawBotsOfSameType; - lock (dictionaryLock) - { - if (!rawBots.TryGetValue(botType, out rawBotsOfSameType)) - { - Console.WriteLine($"(chances) Unable to find {botType} on rawBots data"); - return; - } - } - if (rawBotsOfSameType.Count == 0) - { - return; - } - - // TODO: Add check to make sure incoming bot list has gear - GearChanceHelpers.CalculateEquipmentChances(botToUpdate, rawBotsOfSameType); - GearChanceHelpers.AddGenerationChances(botToUpdate, weightHelper); - GearChanceHelpers.CalculateModChances(botToUpdate, rawBotsOfSameType); - GearChanceHelpers.CalculateEquipmentModChances(botToUpdate, rawBotsOfSameType); - GearChanceHelpers.ApplyModChanceOverrides(botToUpdate); - GearChanceHelpers.ApplyEquipmentChanceOverrides(botToUpdate); - })); - } - - Task.WaitAll(tasks.ToArray()); - stopwatch.Stop(); - LoggingHelpers.LogToConsole($"Finished processing bot chances. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds"); - - return botsToUpdate; + // TODO: Add check to make sure incoming bot list has gear + GearChanceHelpers.AddEquipmentChances(botToUpdate, rawBot); + GearChanceHelpers.AddGenerationChances(botToUpdate, weightHelper); + GearChanceHelpers.AddModChances(botToUpdate, rawBot); + GearChanceHelpers.AddEquipmentModChances(botToUpdate, rawBot); } } } diff --git a/Generator/BotGearGenerator.cs b/Generator/BotGearGenerator.cs index 3c8febe..11286d4 100644 --- a/Generator/BotGearGenerator.cs +++ b/Generator/BotGearGenerator.cs @@ -7,52 +7,12 @@ namespace Generator { public static class BotGearGenerator { - public static IEnumerable AddGear(this IEnumerable baseBots, Dictionary> rawBots) + public static void AddGear(Bot botToUpdate, Datum rawBotData) { - var stopwatch = Stopwatch.StartNew(); - LoggingHelpers.LogToConsole("Started processing bot gear"); - - var dictionaryLock = new object(); - var tasks = new List(); - - foreach (var botToUpdate in baseBots) - { - tasks.Add(Task.Factory.StartNew(() => - { - var botType = botToUpdate.botType.ToString().ToLower(); - List rawBotsOfSameType; - lock (dictionaryLock) - { - if (!rawBots.TryGetValue(botType, out rawBotsOfSameType)) - { - Console.WriteLine($"(gear) Unable to find {botType} on rawBots data"); - return; - } - } - - if (rawBotsOfSameType.Count == 0) - { - return; - } - - foreach (var rawParsedBot in rawBotsOfSameType) - { - GearHelpers.AddEquippedGear(botToUpdate, rawParsedBot); - GearHelpers.AddAmmo(botToUpdate, rawParsedBot); - GearHelpers.AddEquippedMods(botToUpdate, rawParsedBot); - //GearHelpers.AddCartridges(botToUpdate, rawParsedBot); - } - - GearHelpers.ReduceAmmoWeightValues(botToUpdate); - GearHelpers.ReduceEquipmentWeightValues(botToUpdate.inventory.equipment); - })); - } - - Task.WaitAll(tasks.ToArray()); - stopwatch.Stop(); - LoggingHelpers.LogToConsole($"Finished processing bot gear. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds"); - - return baseBots; + GearHelpers.AddEquippedGear(botToUpdate, rawBotData); + GearHelpers.AddAmmo(botToUpdate, rawBotData); + GearHelpers.AddEquippedMods(botToUpdate, rawBotData); + //GearHelpers.AddCartridges(botToUpdate, rawBotData); } } } diff --git a/Generator/BotLootGenerator.cs b/Generator/BotLootGenerator.cs index 794894e..65c860c 100644 --- a/Generator/BotLootGenerator.cs +++ b/Generator/BotLootGenerator.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Linq; using Common.Extensions; +using Common.Models; using Common.Models.Input; using Common.Models.Output; using Generator.Helpers.Gear; @@ -9,107 +10,54 @@ namespace Generator { public static class BotLootGenerator { - internal static IEnumerable AddLoot(this IEnumerable botsWithGear, Dictionary> rawBots) + internal static void AddLoot(Bot botToUpdate, Datum rawBotData) { - var stopwatch = Stopwatch.StartNew(); - LoggingHelpers.LogToConsole("Started processing bot loot"); - - var dictionaryLock = new object(); - - var tasks = new List(50); - foreach (var botToUpdate in botsWithGear) - { - tasks.Add(Task.Factory.StartNew(() => - { - var botType = botToUpdate.botType.ToString().ToLower(); - List rawBotsOfSameType; - lock (dictionaryLock) - { - if (!rawBots.TryGetValue(botType, out rawBotsOfSameType)) - { - Console.WriteLine($"(loot) Unable to find {botType} on rawBots data"); - return; - } - } - - if (rawBotsOfSameType.Count == 0) - { - return; - } - - AddLootToContainers(botType, botToUpdate, rawBotsOfSameType); - GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.Backpack); - GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.Pockets); - GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.TacticalVest); - GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.SecuredContainer); - - //foreach (var rawParsedBot in rawBotsOfSameType) - //{ - // AddPocketLoot(botToUpdate, rawParsedBot); - //} - - //AddTacticalVestLoot(botToUpdate, rawBotsOfSameType); - //AddBackpackLoot(botToUpdate, rawBotsOfSameType); - //AddSecureContainerLoot(botToUpdate, rawBotsOfSameType); - //AddSpecialLoot(botToUpdate); - })); - } - - Task.WaitAll(tasks.ToArray()); - - stopwatch.Stop(); - LoggingHelpers.LogToConsole($"Finished processing bot loot. Took: {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds"); - - return botsWithGear; + AddLootToContainers(botToUpdate, rawBotData); } - private static void AddLootToContainers(string botType, Bot botToUpdate, List rawBotsOfSameType) + private static void AddLootToContainers(Bot botToUpdate, Datum rawBot) { - // Process each bot - foreach (var rawBot in rawBotsOfSameType) - { - // Filter out base inventory items and equipment mod items - var rawBotItems = rawBot.Inventory.items.Where(item => item.location != null).ToList(); + // Filter out base inventory items and equipment mod items + var rawBotItems = rawBot.Inventory.items.Where(item => item.location != null); - var botBackpack = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "Backpack"); - if (botBackpack != null) - { - AddLootItemsToContainerDictionary(rawBotItems, botBackpack._id, botToUpdate.inventory.items.Backpack); - } + var botBackpack = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "Backpack"); + if (botBackpack != null) + { + AddLootItemsToContainerDictionary(rawBotItems, botBackpack._id, botToUpdate.inventory.items.Backpack, "backpack", botToUpdate.botType); + } - var botPockets = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "Pockets"); - if (botPockets != null) - { - AddLootItemsToContainerDictionary(rawBotItems, botPockets._id, botToUpdate.inventory.items.Pockets); - } + var botPockets = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "Pockets"); + if (botPockets != null) + { + AddLootItemsToContainerDictionary(rawBotItems, botPockets._id, botToUpdate.inventory.items.Pockets); + } - var botVest = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "TacticalVest"); - if (botVest != null) - { - AddLootItemsToContainerDictionary(rawBotItems, botVest._id, botToUpdate.inventory.items.TacticalVest); - } + var botVest = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "TacticalVest"); + if (botVest != null) + { + AddLootItemsToContainerDictionary(rawBotItems, botVest._id, botToUpdate.inventory.items.TacticalVest); + } - var botSecure = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "SecuredContainer"); - if (botSecure != null) - { - AddLootItemsToContainerDictionary(rawBotItems, botSecure._id, botToUpdate.inventory.items.SecuredContainer); - } + var botSecure = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "SecuredContainer"); + if (botSecure != null) + { + AddLootItemsToContainerDictionary(rawBotItems, botSecure._id, botToUpdate.inventory.items.SecuredContainer); + } - // Add generic keys to bosses - if (botToUpdate.botType.IsBoss()) + // Add generic keys to bosses + if (botToUpdate.botType.IsBoss()) + { + var keys = SpecialLootHelper.GetGenericBossKeysDictionary(); + foreach (var bosskey in keys) { - var keys = SpecialLootHelper.GetGenericBossKeysDictionary(); - foreach (var bosskey in keys) + if (!botToUpdate.inventory.items.Backpack.ContainsKey(bosskey.Key)) { - if (!botToUpdate.inventory.items.Backpack.ContainsKey(bosskey.Key)) - { - botToUpdate.inventory.items.Backpack.Add(bosskey.Key, bosskey.Value); - } + botToUpdate.inventory.items.Backpack.Add(bosskey.Key, bosskey.Value); } } - - AddSpecialLoot(botToUpdate); } + + AddSpecialLoot(botToUpdate); } /// @@ -119,16 +67,17 @@ namespace Generator /// Bots inventory items /// /// - private static void AddLootItemsToContainerDictionary(List itemsToFilter, string containerId, Dictionary dictToAddTo) + private static void AddLootItemsToContainerDictionary(IEnumerable itemsToFilter, string containerId, Dictionary dictToAddTo, string container = "", BotType type = BotType.arenafighterevent) { - var lootItems = itemsToFilter.Where(item => item.parentId == containerId); - foreach (var itemToAdd in lootItems) + foreach (var itemToAdd in itemsToFilter) { + if (itemToAdd.parentId != containerId) continue; + if (!dictToAddTo.ContainsKey(itemToAdd._tpl)) { dictToAddTo[itemToAdd._tpl] = 1; - return; + continue; } dictToAddTo[itemToAdd._tpl]++; diff --git a/Common/Bot/BotParser.cs b/Generator/BotParser.cs similarity index 51% rename from Common/Bot/BotParser.cs rename to Generator/BotParser.cs index c06f233..3843470 100644 --- a/Common/Bot/BotParser.cs +++ b/Generator/BotParser.cs @@ -8,6 +8,9 @@ using System.Linq; using System.Text.Json; using System.Threading.Tasks; using Common.Models.Output; +using Common.Models; +using Generator; +using Generator.Helpers.Gear; namespace Common.Bots; @@ -15,6 +18,118 @@ public static class BotParser { private static readonly JsonSerializerOptions serialiserOptions = new() { }; + public static List Parse(string dumpPath, HashSet botTypes) + { + var stopwatch = Stopwatch.StartNew(); + + // Build the list of base bot data + var baseBots = new HashSet(); + foreach (var botType in botTypes) + { + var typeToAdd = (BotType)Enum.Parse(typeof(BotType), botType); + baseBots.Add(new Bot(typeToAdd)); + } + + DiskHelpers.CreateDirIfDoesntExist(dumpPath); + + var botFiles = Directory.GetFiles(dumpPath, "*.json", SearchOption.TopDirectoryOnly); + LoggingHelpers.LogToConsole($"{botFiles.Length} bot dump files found"); + + // Store a list of parsed bots so we don't parse the same bot twice + int totalDupeCount = 0; + var parsedBotIds = new HashSet(); + int i = 0; + foreach (var filePath in botFiles) + { + i++; + if (i % 100 == 0) Console.WriteLine($"Processing file {i}"); + ProcessBotFileSync(baseBots, filePath, parsedBotIds, totalDupeCount); + } + + // Handle things we can only do once all data has been processed, or only needs to run once per bot type + foreach (var bot in baseBots) + { + // Skip any bots we didn't handle + if (bot.botCount == 0) continue; + + BaseBotGenerator.AddDifficulties(bot); + GearChanceHelpers.CalculateModChances(bot); + GearChanceHelpers.CalculateEquipmentModChances(bot); + GearChanceHelpers.CalculateEquipmentChances(bot); + GearChanceHelpers.ApplyModChanceOverrides(bot); + GearChanceHelpers.ApplyEquipmentChanceOverrides(bot); + + GearHelpers.ReduceAmmoWeightValues(bot); + GearHelpers.ReduceEquipmentWeightValues(bot.inventory.equipment); + GearHelpers.ReduceWeightValues(bot.appearance.voice); + GearHelpers.ReduceWeightValues(bot.appearance.feet); + GearHelpers.ReduceWeightValues(bot.appearance.body); + GearHelpers.ReduceWeightValues(bot.appearance.head); + GearHelpers.ReduceWeightValues(bot.appearance.hands); + } + + stopwatch.Stop(); + LoggingHelpers.LogToConsole($"{totalDupeCount} dupes were ignored. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds"); + + return baseBots.ToList(); + } + + private static void ProcessBotFileSync( + HashSet baseBots, + string filePath, + HashSet parsedBotIds, + int totalDupeCount) + { + var splitFilePath = filePath.Split("\\"); + + int dupeCount = 0; + + List bots = []; + try + { + // Parse the bots inside the json file + using (var reader = new StreamReader(filePath)) + { + var deSerialisedObject = JsonSerializer.Deserialize(reader.ReadToEnd(), serialiserOptions); + + foreach (var botData in deSerialisedObject.data) + { + // Bot fucks up something, never allow it in + if (botData._id == "6483938c53cc9087c70eae86") + { + Console.WriteLine("oh no"); + continue; + } + + var baseBot = baseBots.SingleOrDefault(bot => bot.botType.ToString().Equals(botData.Info.Settings.Role, StringComparison.OrdinalIgnoreCase)); + if (baseBot == null) + { + //Console.WriteLine($"Skipping {botData._id} due to unknown role {botData.Info.Settings.Role}"); + continue; + } + + // Add bot if not already added + if (!parsedBotIds.Add(botData._id)) + { + dupeCount++; + } + baseBot.botCount += 1; + BaseBotGenerator.UpdateBaseDetails(baseBot, botData); + BotGearGenerator.AddGear(baseBot, botData); + BotLootGenerator.AddLoot(baseBot, botData); + BotChancesGenerator.AddChances(baseBot, botData); + } + } + } + catch (Exception ex) + { + Console.WriteLine($"File parse fucked up: {filePath}"); + throw; + } + + totalDupeCount += dupeCount; + } + public static async Task> ParseAsync(string dumpPath, HashSet botTypes) { var stopwatch = Stopwatch.StartNew(); diff --git a/Generator/Helpers/Gear/GearChanceHelpers.cs b/Generator/Helpers/Gear/GearChanceHelpers.cs index 5f8414e..936cc90 100644 --- a/Generator/Helpers/Gear/GearChanceHelpers.cs +++ b/Generator/Helpers/Gear/GearChanceHelpers.cs @@ -7,75 +7,102 @@ namespace Generator.Helpers.Gear { public static class GearChanceHelpers { - public static void CalculateModChances(Bot bot, List baseBots) + private static Dictionary> weaponModCount = new Dictionary>(); + private static Dictionary> weaponSlotCount = new Dictionary>(); + private static Dictionary> equipmentModCount = new Dictionary>(); + private static Dictionary> equipmentSlotCount = new Dictionary>(); + + + public static void AddModChances(Bot bot, Datum baseBot) { // TODO: Further split these counts by equipment slot? (ex. "FirstPrimaryWeapon", "Holster", etc.) var validSlots = new List { "FirstPrimaryWeapon", "SecondPrimaryWeapon", "Holster" }; - var modCounts = new Dictionary(); - var slotCounts = new Dictionary(); - - foreach (var baseBot in baseBots) + if (!weaponModCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var modCounts)) { - var validParents = new List(); - foreach (var inventoryItem in baseBot.Inventory.items) + modCounts = new Dictionary(); + weaponModCount.Add(baseBot.Info.Settings.Role.ToLower(), modCounts); + } + + if (!weaponSlotCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var slotCounts)) + { + slotCounts = new Dictionary(); + weaponSlotCount.Add(baseBot.Info.Settings.Role.ToLower(), slotCounts); + } + + var validParents = new List(); + foreach (var inventoryItem in baseBot.Inventory.items) + { + if (validSlots.Contains(inventoryItem.slotId)) { - if (validSlots.Contains(inventoryItem.slotId)) + validParents.Add(inventoryItem._id); + } + else if (validParents.Contains(inventoryItem.parentId)) + { + validParents.Add(inventoryItem._id); + } + else + { + continue; + } + + var template = ItemTemplateHelper.GetTemplateById(inventoryItem._tpl); + var parentTemplate = ItemTemplateHelper.GetTemplateById(baseBot.Inventory.items.Single(i => i._id == inventoryItem.parentId)._tpl); + + if (!(parentTemplate?._props?.Slots?.FirstOrDefault(slot => slot._name == inventoryItem.slotId)?._required ?? false)) + { + if (modCounts.ContainsKey(inventoryItem.slotId.ToLower())) { - validParents.Add(inventoryItem._id); - } - else if (validParents.Contains(inventoryItem.parentId)) - { - validParents.Add(inventoryItem._id); + modCounts[inventoryItem.slotId.ToLower()]++; } else { - continue; - } - - var template = ItemTemplateHelper.GetTemplateById(inventoryItem._tpl); - var parentTemplate = ItemTemplateHelper.GetTemplateById(baseBot.Inventory.items.Single(i => i._id == inventoryItem.parentId)._tpl); - - if (!(parentTemplate?._props?.Slots?.FirstOrDefault(slot => slot._name == inventoryItem.slotId)?._required ?? false)) - { - if (modCounts.ContainsKey(inventoryItem.slotId.ToLower())) - { - modCounts[inventoryItem.slotId.ToLower()]++; - } - else - { - modCounts.Add(inventoryItem.slotId.ToLower(), 1); - } - } - - if ((template?._props?.Slots?.Count ?? 0) < 1) - { - // Item has no slots, nothing to count here - continue; - } - - foreach (var slot in template._props.Slots) - { - if (slot._required) - { - continue; - } - - if (slot._name.StartsWith("camora")) - { - continue; - } - - if (slotCounts.ContainsKey(slot._name.ToLower())) - { - slotCounts[slot._name.ToLower()]++; - } - else - { - slotCounts.Add(slot._name.ToLower(), 1); - } + modCounts.Add(inventoryItem.slotId.ToLower(), 1); } } + + if ((template?._props?.Slots?.Count ?? 0) < 1) + { + // Item has no slots, nothing to count here + continue; + } + + foreach (var slot in template._props.Slots) + { + if (slot._required) + { + continue; + } + + if (slot._name.StartsWith("camora")) + { + continue; + } + + if (slotCounts.ContainsKey(slot._name.ToLower())) + { + slotCounts[slot._name.ToLower()]++; + } + else + { + slotCounts.Add(slot._name.ToLower(), 1); + } + } + } + } + + public static void CalculateModChances(Bot bot) + { + if (!weaponModCount.TryGetValue(bot.botType.ToString(), out var modCounts)) + { + modCounts = new Dictionary(); + weaponModCount.Add(bot.botType.ToString(), modCounts); + } + + if (!weaponSlotCount.TryGetValue(bot.botType.ToString(), out var slotCounts)) + { + slotCounts = new Dictionary(); + weaponSlotCount.Add(bot.botType.ToString(), slotCounts); } bot.chances.weaponMods = slotCounts.ToDictionary( @@ -83,76 +110,97 @@ namespace Generator.Helpers.Gear kvp => GetPercent(kvp.Value, modCounts.GetValueOrDefault(kvp.Key))); } - public static void CalculateEquipmentModChances(Bot bot, List baseBots) + public static void AddEquipmentModChances(Bot bot, Datum baseBot) { + if (!equipmentModCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var modCounts)) + { + modCounts = new Dictionary(); + equipmentModCount.Add(baseBot.Info.Settings.Role.ToLower(), modCounts); + } + + if (!equipmentSlotCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var slotCounts)) + { + slotCounts = new Dictionary(); + equipmentSlotCount.Add(baseBot.Info.Settings.Role.ToLower(), slotCounts); + } + // TODO: Further split these counts by equipment slot? (ex. "FirstPrimaryWeapon", "Holster", etc.) var validSlots = new List { "Headwear", "ArmorVest", "TacticalVest" }; - var modCounts = new Dictionary(); - var slotCounts = new Dictionary(); + var validParents = new List(); - foreach (var baseBot in baseBots) + foreach (var inventoryItem in baseBot.Inventory.items) { - var validParents = new List(); - - foreach (var inventoryItem in baseBot.Inventory.items) + if (validSlots.Contains(inventoryItem.slotId)) { - if (validSlots.Contains(inventoryItem.slotId)) + validParents.Add(inventoryItem._id); + } + else if (validParents.Contains(inventoryItem.parentId)) + { + validParents.Add(inventoryItem._id); + } + else + { + continue; + } + + var template = ItemTemplateHelper.GetTemplateById(inventoryItem._tpl); + var parentTemplate = ItemTemplateHelper.GetTemplateById(baseBot.Inventory.items.Single(i => i._id == inventoryItem.parentId)._tpl); + + if (!(parentTemplate?._props?.Slots?.FirstOrDefault(slot => slot._name == inventoryItem.slotId)?._required ?? false)) + { + if (modCounts.ContainsKey(inventoryItem.slotId.ToLower())) { - validParents.Add(inventoryItem._id); - } - else if (validParents.Contains(inventoryItem.parentId)) - { - validParents.Add(inventoryItem._id); + modCounts[inventoryItem.slotId.ToLower()]++; } else { - continue; - } - - var template = ItemTemplateHelper.GetTemplateById(inventoryItem._tpl); - var parentTemplate = ItemTemplateHelper.GetTemplateById(baseBot.Inventory.items.Single(i => i._id == inventoryItem.parentId)._tpl); - - if (!(parentTemplate?._props?.Slots?.FirstOrDefault(slot => slot._name == inventoryItem.slotId)?._required ?? false)) - { - if (modCounts.ContainsKey(inventoryItem.slotId.ToLower())) - { - modCounts[inventoryItem.slotId.ToLower()]++; - } - else - { - modCounts.Add(inventoryItem.slotId.ToLower(), 1); - } - } - - if ((template?._props?.Slots?.Count ?? 0) < 1) - { - // Item has no slots, nothing to count here - continue; - } - - foreach (var slot in template._props.Slots) - { - if (slot._required) - { - continue; - } - - if (slot._name.StartsWith("camora")) - { - continue; - } - - if (slotCounts.ContainsKey(slot._name.ToLower())) - { - slotCounts[slot._name.ToLower()]++; - } - else - { - slotCounts.Add(slot._name.ToLower(), 1); - } + modCounts.Add(inventoryItem.slotId.ToLower(), 1); } } + + if ((template?._props?.Slots?.Count ?? 0) < 1) + { + // Item has no slots, nothing to count here + continue; + } + + foreach (var slot in template._props.Slots) + { + if (slot._required) + { + continue; + } + + if (slot._name.StartsWith("camora")) + { + continue; + } + + if (slotCounts.ContainsKey(slot._name.ToLower())) + { + slotCounts[slot._name.ToLower()]++; + } + else + { + slotCounts.Add(slot._name.ToLower(), 1); + } + } + } + } + + public static void CalculateEquipmentModChances(Bot bot) + { + if (!equipmentModCount.TryGetValue(bot.botType.ToString(), out var modCounts)) + { + modCounts = new Dictionary(); + equipmentModCount.Add(bot.botType.ToString(), modCounts); + } + + if (!equipmentSlotCount.TryGetValue(bot.botType.ToString(), out var slotCounts)) + { + slotCounts = new Dictionary(); + equipmentSlotCount.Add(bot.botType.ToString(), slotCounts); } bot.chances.equipmentMods = slotCounts.ToDictionary( @@ -245,47 +293,47 @@ namespace Generator.Helpers.Gear weightsData["grenades"]); } - public static void CalculateEquipmentChances(Bot bot, List baseBots) + public static void AddEquipmentChances(Bot bot, Datum baseBot) { - // TODO: Convert to dynamic? - var totalBotsCount = baseBots.Count; - int headwearCount = 0, earCount = 0, faceCoverCount = 0, armorVestCount = 0, eyeWearCount = 0, armBandCount = 0, - tacticalVestCount = 0, backpackCount = 0, firstPrimaryCount = 0, secondPrimaryCount = 0, holsterCount = 0, - scabbardCount = 0, pocketsCount = 0, securedContainerCount = 0; + bot.chances.equipment.Headwear += baseBot.Inventory.items.Count(x => x.slotId == "Headwear"); + bot.chances.equipment.Earpiece += baseBot.Inventory.items.Count(x => x.slotId == "Earpiece"); + bot.chances.equipment.FaceCover += baseBot.Inventory.items.Count(x => x.slotId == "FaceCover"); + bot.chances.equipment.ArmorVest += baseBot.Inventory.items.Count(x => x.slotId == "ArmorVest"); + bot.chances.equipment.Eyewear += baseBot.Inventory.items.Count(x => x.slotId == "Eyewear"); + bot.chances.equipment.ArmBand += baseBot.Inventory.items.Count(x => x.slotId == "ArmBand"); + bot.chances.equipment.TacticalVest += baseBot.Inventory.items.Count(x => x.slotId == "TacticalVest"); + bot.chances.equipment.Backpack += baseBot.Inventory.items.Count(x => x.slotId == "Backpack"); + bot.chances.equipment.FirstPrimaryWeapon += baseBot.Inventory.items.Count(x => x.slotId == "FirstPrimaryWeapon"); + bot.chances.equipment.SecondPrimaryWeapon += baseBot.Inventory.items.Count(x => x.slotId == "SecondPrimaryWeapon"); + bot.chances.equipment.Holster += baseBot.Inventory.items.Count(x => x.slotId == "Holster"); + bot.chances.equipment.Scabbard += baseBot.Inventory.items.Count(x => x.slotId == "Scabbard"); + bot.chances.equipment.Pockets += baseBot.Inventory.items.Count(x => x.slotId == "Pockets"); + bot.chances.equipment.SecuredContainer += baseBot.Inventory.items.Count(x => x.slotId == "SecuredContainer"); + } - foreach (var baseBot in baseBots) + public static void CalculateEquipmentChances(Bot bot) + { + if (bot.botCount == 0) { - headwearCount += baseBot.Inventory.items.Count(x => x.slotId == "Headwear"); - earCount += baseBot.Inventory.items.Count(x => x.slotId == "Earpiece"); - faceCoverCount += baseBot.Inventory.items.Count(x => x.slotId == "FaceCover"); - armorVestCount += baseBot.Inventory.items.Count(x => x.slotId == "ArmorVest"); - eyeWearCount += baseBot.Inventory.items.Count(x => x.slotId == "Eyewear"); - armBandCount += baseBot.Inventory.items.Count(x => x.slotId == "ArmBand"); - tacticalVestCount += baseBot.Inventory.items.Count(x => x.slotId == "TacticalVest"); - backpackCount += baseBot.Inventory.items.Count(x => x.slotId == "Backpack"); - firstPrimaryCount += baseBot.Inventory.items.Count(x => x.slotId == "FirstPrimaryWeapon"); - secondPrimaryCount += baseBot.Inventory.items.Count(x => x.slotId == "SecondPrimaryWeapon"); - holsterCount += baseBot.Inventory.items.Count(x => x.slotId == "Holster"); - scabbardCount += baseBot.Inventory.items.Count(x => x.slotId == "Scabbard"); - pocketsCount += baseBot.Inventory.items.Count(x => x.slotId == "Pockets"); - securedContainerCount += baseBot.Inventory.items.Count(x => x.slotId == "SecuredContainer"); + // No bots, don't do anything + return; } bot.chances.equipment = new EquipmentChances( - GetPercent(totalBotsCount, headwearCount), - GetPercent(totalBotsCount, earCount), - GetPercent(totalBotsCount, faceCoverCount), - GetPercent(totalBotsCount, armorVestCount), - GetPercent(totalBotsCount, eyeWearCount), - GetPercent(totalBotsCount, armBandCount), - GetPercent(totalBotsCount, tacticalVestCount), - GetPercent(totalBotsCount, backpackCount), - GetPercent(totalBotsCount, firstPrimaryCount), - GetPercent(totalBotsCount, secondPrimaryCount), - GetPercent(totalBotsCount, holsterCount), - GetPercent(totalBotsCount, scabbardCount), - GetPercent(totalBotsCount, pocketsCount), - GetPercent(totalBotsCount, securedContainerCount)); + GetPercent(bot.botCount, bot.chances.equipment.Headwear), + GetPercent(bot.botCount, bot.chances.equipment.Earpiece), + GetPercent(bot.botCount, bot.chances.equipment.FaceCover), + GetPercent(bot.botCount, bot.chances.equipment.ArmorVest), + GetPercent(bot.botCount, bot.chances.equipment.Eyewear), + GetPercent(bot.botCount, bot.chances.equipment.ArmBand), + GetPercent(bot.botCount, bot.chances.equipment.TacticalVest), + GetPercent(bot.botCount, bot.chances.equipment.Backpack), + GetPercent(bot.botCount, bot.chances.equipment.FirstPrimaryWeapon), + GetPercent(bot.botCount, bot.chances.equipment.SecondPrimaryWeapon), + GetPercent(bot.botCount, bot.chances.equipment.Holster), + GetPercent(bot.botCount, bot.chances.equipment.Scabbard), + GetPercent(bot.botCount, bot.chances.equipment.Pockets), + GetPercent(bot.botCount, bot.chances.equipment.SecuredContainer)); } private static int GetPercent(int total, int count) diff --git a/Generator/Helpers/Gear/GearHelpers.cs b/Generator/Helpers/Gear/GearHelpers.cs index 8e22cbd..0b8d7cf 100644 --- a/Generator/Helpers/Gear/GearHelpers.cs +++ b/Generator/Helpers/Gear/GearHelpers.cs @@ -65,7 +65,6 @@ namespace Generator.Helpers.Gear internal static void AddAmmo(Bot botToUpdate, Datum bot) { - //var weightService = new WeightingService(); foreach (var ammo in bot.Inventory.items.Where( x => x.slotId != null && (x.slotId == "patron_in_weapon" @@ -117,7 +116,6 @@ namespace Generator.Helpers.Gear public static void AddEquippedGear(Bot botToUpdate, Datum bot) { // add equipped gear - var weightService = new WeightingService(); foreach (var inventoryItem in bot.Inventory.items.Where(x=>x.slotId != null)) { switch (inventoryItem.slotId?.ToLower()) @@ -306,7 +304,7 @@ namespace Generator.Helpers.Gear botToUpdate.inventory.Ammo[cartridge][cartridgeWeightKvP.Key] /= commonAmmoDivisor; } } - } + } } public static void ReduceEquipmentWeightValues(Equipment equipment) diff --git a/Generator/Program.cs b/Generator/Program.cs index d4a21e4..f933627 100644 --- a/Generator/Program.cs +++ b/Generator/Program.cs @@ -1,5 +1,7 @@ using Common.Models.Input; +using Common.Models.Output; using Generator.Helpers; +using System.Diagnostics; namespace Generator; @@ -7,6 +9,9 @@ internal static class Program { internal static async Task Main(string[] args) { + var stopwatch = Stopwatch.StartNew(); + LoggingHelpers.LogToConsole("Started processing bots"); + // Create list of bots we want to process string[] botTypes = { "assault", @@ -72,38 +77,19 @@ internal static class Program // Read raw bot dumps and turn into c# objects var workingPath = Directory.GetCurrentDirectory(); var dumpPath = $"{workingPath}//dumps"; - var parsedBots = await BotParser.ParseAsync(dumpPath, botTypes.ToHashSet()); + List bots = BotParser.Parse(dumpPath, botTypes.ToHashSet()); - // Put in dictionary for better use later on - var rawBotsCache = new Dictionary>(45); - foreach (var rawBot in parsedBots) - { - if (rawBotsCache.TryGetValue(rawBot.Info.Settings.Role.ToLower(), out var botList)) - { - botList.Add(rawBot); - - continue; - } - - // Doesnt exist, add key and bot - rawBotsCache.Add(rawBot.Info.Settings.Role.ToLower(), new List { rawBot }); - } - - if (parsedBots.Count == 0) + if (bots.Count == 0) { LoggingHelpers.LogToConsole("No bots found, unable to continue"); LoggingHelpers.LogToConsole("Check your dumps are in 'Generator\\bin\\Debug\\net6.0\\dumps' and start with 'resp.' NOT 'req.'"); return; } - // Generate the base bot class with basic details (health/body part hp etc) and then append everything else - var bots = BaseBotGenerator.GenerateBaseDetails(parsedBots, workingPath, botTypes) - .AddGear(rawBotsCache) // Add weapons/armor - .AddLoot(rawBotsCache) - .AddChances(rawBotsCache); // Add mod/equipment chances - - // Output bot to json file var jsonWriter = new JsonWriter(workingPath, "output"); - jsonWriter.WriteJson(bots.ToList()); + jsonWriter.WriteJson(bots); + + stopwatch.Stop(); + LoggingHelpers.LogToConsole($"Finished processing bots. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds"); } } diff --git a/Generator/Weighting/WeightingService.cs b/Generator/Weighting/WeightingService.cs index 115c498..cf33a55 100644 --- a/Generator/Weighting/WeightingService.cs +++ b/Generator/Weighting/WeightingService.cs @@ -14,11 +14,14 @@ namespace Generator.Weighting public class WeightingService { - private readonly Dictionary _weights; - private readonly Dictionary> _generationWeights; + private static Dictionary _weights = null; + private static Dictionary> _generationWeights = null; public WeightingService() { + // Cache the loaded data + if (_weights != null && _generationWeights != null) return; + var assetsPath = $"{Directory.GetCurrentDirectory()}\\Assets"; var weightsFilePath = $"{assetsPath}\\weights.json"; if (!File.Exists(weightsFilePath)) -- 2.47.1