Merge pull request 'Drastically reduce memory footprint at the cost of removing threading' (#4) from DrakiaXYZ/BotGenerator:feat-memory-improvements into master

Reviewed-on: #4

Common.Models/Output/Output.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Text.Json.Serialization;
 
 namespace Common.Models.Output;
@@ -18,6 +19,7 @@ public class Bot
     public Bot(BotType botType)
     {
         this.botType = botType;
+        botCount = 0;
         appearance = new Appearance();
         experience = new Experience();
         health = new Health();
@@ -32,6 +34,8 @@ public class Bot
 
     [JsonIgnore]
     public BotType botType { get; set; }
+    [JsonIgnore]
+    public int botCount { get; set; }
     public Appearance appearance { get; set; }
     public Experience experience { get; set; }
     public Health health { get; set; }
@@ -79,7 +83,7 @@ public class Experience
     public bool useSimpleAnimator { get; set; }
 }
 
-public class BodyParts
+public class BodyParts : IEquatable<BodyParts>
 {
     public BodyParts()
     {
@@ -92,6 +96,17 @@ public class BodyParts
         RightLeg = new MinMax(65, 65);
     }
 
+    public bool Equals(BodyParts other)
+    {
+        return this.Head.Equals(other.Head) &&
+            this.Chest.Equals(other.Chest) &&
+            this.Stomach.Equals(other.Stomach) &&
+            this.LeftArm.Equals(other.LeftArm) &&
+            this.RightArm.Equals(other.RightArm) &&
+            this.LeftLeg.Equals(other.LeftLeg) &&
+            this.RightLeg.Equals(other.RightLeg);
+    }
+
     public MinMax Head { get; set; }
     public MinMax Chest { get; set; }
     public MinMax Stomach { get; set; }
@@ -256,7 +271,7 @@ public class ItemChances
     public GenerationWeightData grenades { get; set; }
 }
 
-public class MinMax
+public class MinMax : IEquatable<MinMax>
 {
     public MinMax(int min, int max)
     {
@@ -264,6 +279,11 @@ public class MinMax
         this.max = max;
     }
 
+    public bool Equals(MinMax other)
+    {
+        return this.min == other.min && this.max == other.max;
+    }
+
     public int min { get; set; }
     public int max { get; set; }
 }

Generator/BaseBotGenerator.cs

@@ -10,104 +10,56 @@ namespace Generator
 {
     public static class BaseBotGenerator
     {
-        //TODO: pass in bot types and use those to create the classes in rawBots list
+        public static void UpdateBaseDetails(Bot botData, Datum rawBotData)
-        public static IEnumerable<Bot> GenerateBaseDetails(IEnumerable<Datum> rawBots, string workingPath, IEnumerable<string> botTypes)
         {
-            var stopwatch = Stopwatch.StartNew();
+            UpdateBodyPartHealth(botData, rawBotData);
-            LoggingHelpers.LogToConsole("Started processing bot base");
+            AddExperience(botData, rawBotData);
+            AddStandingForKill(botData, rawBotData);
+            AddSkills(botData, rawBotData);
+            botData.experience.useSimpleAnimator = rawBotData.Info.Settings.UseSimpleAnimator;
 
-            // Create a list of bot objects ready to be hydrated
+            AddVisualAppearanceItems(botData, rawBotData);
-            var baseBots = new List<Bot>();
+            AddName(botData, rawBotData);
-            foreach (var botType in botTypes)
+            AddVoice(botData, rawBotData);
-            {
-                var typeToAdd = (BotType)Enum.Parse(typeof(BotType), botType);
-                baseBots.Add(new Bot(typeToAdd));
         }
 
-            // Iterate over each bot type we just made and put some data into them
+        private static void AddSkills(Bot botToUpdate, Datum rawBotData)
-            foreach (var botToUpdate in baseBots)
         {
-                var rawBotType = botToUpdate.botType.ToString();
-                var rawBotsOfSameType = rawBots.Where(x => string.Equals(x.Info.Settings.Role, rawBotType, StringComparison.OrdinalIgnoreCase))
-                                                .ToList();
-                var rawBotsOfSameTypeCount = rawBotsOfSameType.Count.ToString();
-
-                if (rawBotsOfSameType.Count == 0)
-                {
-                    LoggingHelpers.LogToConsole($"no bots of type {rawBotType}, skipping", ConsoleColor.DarkRed);
-                    continue;
-                }
-
-                LoggingHelpers.LogToConsole($"Found {rawBotsOfSameTypeCount} bots of type: {rawBotType}");
-
-                UpdateBodyPartHealth(botToUpdate, rawBotsOfSameType);
-                AddDifficulties(botToUpdate, workingPath);
-                AddExperience(botToUpdate, rawBotsOfSameType);
-                AddStandingForKill(botToUpdate, rawBotsOfSameType);
-                AddSkills(botToUpdate, rawBotsOfSameType);
-                botToUpdate.experience.useSimpleAnimator = rawBotsOfSameType.First().Info.Settings.UseSimpleAnimator;
-
-                foreach (var rawParsedBot in rawBotsOfSameType)
-                {
-                    AddVisualAppearanceItems(botToUpdate, rawParsedBot);
-                    AddName(botToUpdate, rawParsedBot);
-                    AddVoice(botToUpdate, rawParsedBot);
-                }
-            }
-
-            stopwatch.Stop();
-            LoggingHelpers.LogToConsole($"Finished processing bot base. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
-
-            return baseBots;
-        }
-
-        private static void AddSkills(Bot botToUpdate, IEnumerable<Datum> rawBotsOfSameType)
-        {
-            var firstBotOfDesiredType = rawBotsOfSameType.Last();
-
             // Find the smallest and biggest value for each skill
-            foreach (var skill in firstBotOfDesiredType.Skills.Common)
+            foreach (var skill in rawBotData.Skills.Common)
             {
-                var skills = new List<Common.Models.Input.Common>();
+                if (botToUpdate.skills.Common.TryGetValue(skill.Id, out var existingSkill))
-                foreach (var bot in rawBotsOfSameType)
                 {
-                    skills.Add(bot.Skills.Common.Find(x => x.Id == skill.Id));
+                    existingSkill.min = Math.Min(existingSkill.min, skill.Progress);
+                    existingSkill.max = Math.Max(existingSkill.max, skill.Progress);
                 }
-
+                else
-                var min = skills.Min(x => x?.Progress);
-                var max = skills.Max(x => x?.Progress);
-
-                if (min.HasValue && max.HasValue)
                 {
-                    botToUpdate.skills.Common.Add(skill.Id, new MinMax(min.Value, max.Value));
+                    botToUpdate.skills.Common.Add(skill.Id, new MinMax(skill.Progress, skill.Progress));
                 }
             }
         }
 
-        private static void AddStandingForKill(Bot botToUpdate, IEnumerable<Datum> rawBotsOfSameType)
+        private static void AddStandingForKill(Bot botToUpdate, Datum rawBotData)
         {
-            var firstBotOfDesiredType = rawBotsOfSameType.Last();
+            botToUpdate.experience.standingForKill = rawBotData.Info.Settings.StandingForKill;
-
+            botToUpdate.experience.aggressorBonus = rawBotData.Info.Settings.AggressorBonus;
-            botToUpdate.experience.standingForKill = firstBotOfDesiredType.Info.Settings.StandingForKill;
-            botToUpdate.experience.aggressorBonus = firstBotOfDesiredType.Info.Settings.AggressorBonus;
         }
 
-        private static void AddExperience(Bot botToUpdate, IEnumerable<Datum> rawBotsOfSameType)
+        private static void AddExperience(Bot botToUpdate, Datum rawBotData)
         {
-            var firstBotOfDesiredType = rawBotsOfSameType.Last();
+            botToUpdate.experience.reward.min = rawBotData.Info.Settings.Experience;
-
+            botToUpdate.experience.reward.max = rawBotData.Info.Settings.Experience;
-            botToUpdate.experience.reward.min = firstBotOfDesiredType.Info.Settings.Experience;
-            botToUpdate.experience.reward.max = firstBotOfDesiredType.Info.Settings.Experience;
         }
 
         private static void AddVoice(Bot bot, Datum rawBot)
         {
             GearHelpers.IncrementDictionaryValue(bot.appearance.voice, rawBot.Info.Voice);
-            GearHelpers.ReduceWeightValues(bot.appearance.voice);
         }
 
-        private static void AddDifficulties(Bot bot, string workingPath)
+        public static void AddDifficulties(Bot bot)
         {
+            string workingPath = Directory.GetCurrentDirectory();
             string botType = bot.botType.ToString();
             var botDifficultyFiles = Directory
                 .GetFiles($"{workingPath}//Assets", "*.txt", SearchOption.TopDirectoryOnly)
@@ -117,51 +69,34 @@ namespace Generator
             DifficultyHelper.AddDifficultySettings(bot, botDifficultyFiles);
         }
 
-        private static void UpdateBodyPartHealth(Bot botToUpdate, List<Datum> rawBots)
+        private static void UpdateBodyPartHealth(Bot botToUpdate, Datum rawBot)
-        {
-            var uniqueHealthSetups = new Dictionary<int, Common.Models.Output.BodyParts>();
-            foreach (var bot in rawBots)
-            {
-                var healthTotal = bot.Health.BodyParts.GetHpMaxTotal();
-                var alreadyExists = uniqueHealthSetups.ContainsKey(healthTotal);
-
-                if (!alreadyExists)
         {
             var bodyPartHpToAdd = new Common.Models.Output.BodyParts()
             {
-                        Head = new MinMax(bot.Health.BodyParts.Head.Health.Current, bot.Health.BodyParts.Head.Health.Maximum),
+                Head = new MinMax(rawBot.Health.BodyParts.Head.Health.Current, rawBot.Health.BodyParts.Head.Health.Maximum),
-                        Chest = new MinMax(bot.Health.BodyParts.Chest.Health.Current, bot.Health.BodyParts.Chest.Health.Maximum),
+                Chest = new MinMax(rawBot.Health.BodyParts.Chest.Health.Current, rawBot.Health.BodyParts.Chest.Health.Maximum),
-                        Stomach = new MinMax(bot.Health.BodyParts.Stomach.Health.Current, bot.Health.BodyParts.Stomach.Health.Maximum),
+                Stomach = new MinMax(rawBot.Health.BodyParts.Stomach.Health.Current, rawBot.Health.BodyParts.Stomach.Health.Maximum),
-                        LeftArm = new MinMax(bot.Health.BodyParts.LeftArm.Health.Current, bot.Health.BodyParts.LeftArm.Health.Maximum),
+                LeftArm = new MinMax(rawBot.Health.BodyParts.LeftArm.Health.Current, rawBot.Health.BodyParts.LeftArm.Health.Maximum),
-                        RightArm = new MinMax(bot.Health.BodyParts.RightArm.Health.Current, bot.Health.BodyParts.RightArm.Health.Maximum),
+                RightArm = new MinMax(rawBot.Health.BodyParts.RightArm.Health.Current, rawBot.Health.BodyParts.RightArm.Health.Maximum),
+                LeftLeg = new MinMax(rawBot.Health.BodyParts.LeftLeg.Health.Current, rawBot.Health.BodyParts.LeftLeg.Health.Maximum),
+                RightLeg = new MinMax(rawBot.Health.BodyParts.RightLeg.Health.Current, rawBot.Health.BodyParts.RightLeg.Health.Maximum)
             };
 
-                    bodyPartHpToAdd.LeftLeg.min = bot.Health.BodyParts.LeftLeg.Health.Current;
+            if (!botToUpdate.health.BodyParts.Contains(bodyPartHpToAdd))
-                    bodyPartHpToAdd.LeftLeg.max = bot.Health.BodyParts.LeftLeg.Health.Maximum;
+            {
-
+                botToUpdate.health.BodyParts.Add(bodyPartHpToAdd);
-                    bodyPartHpToAdd.RightLeg.min = bot.Health.BodyParts.RightLeg.Health.Current;
-                    bodyPartHpToAdd.RightLeg.max = bot.Health.BodyParts.RightLeg.Health.Maximum;
-
-                    uniqueHealthSetups.Add(healthTotal, bodyPartHpToAdd);
             }
         }
 
-            botToUpdate.health.BodyParts = uniqueHealthSetups.Values.ToList();
-        }
-
         private static void AddVisualAppearanceItems(Bot botToUpdate, Datum rawBot)
         {
             GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.feet, rawBot.Customization.Feet);
-            GearHelpers.ReduceWeightValues(botToUpdate.appearance.feet);
 
             GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.body, rawBot.Customization.Body);
-            GearHelpers.ReduceWeightValues(botToUpdate.appearance.body);
 
             GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.head, rawBot.Customization.Head);
-            GearHelpers.ReduceWeightValues(botToUpdate.appearance.head);
 
             GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.hands, rawBot.Customization.Hands);
-            GearHelpers.ReduceWeightValues(botToUpdate.appearance.hands);
         }
 
         private static void AddName(Bot botToUpdate, Datum rawBot)

Generator/BotChancesGenerator.cs

@@ -8,51 +8,15 @@ namespace Generator
 {
     public static class BotChancesGenerator
     {
-        public static IEnumerable<Bot> AddChances(this IEnumerable<Bot> botsToUpdate, Dictionary<string, List<Datum>> rawBots)
+        public static void AddChances(Bot botToUpdate, Datum rawBot)
         {
-            var stopwatch = Stopwatch.StartNew();
-            LoggingHelpers.LogToConsole("Started processing bot gear");
-
-            // use lock for lock safety
-            var dictionaryLock = new object();
             var weightHelper = new WeightingService();
-            // multithread
-            var tasks = new List<Task>();
-            foreach (var botToUpdate in botsToUpdate)
-            {
-                tasks.Add(Task.Factory.StartNew(() =>
-                {
-                    var botType = botToUpdate.botType.ToString().ToLower();
-                    List<Datum> rawBotsOfSameType;
-                    lock (dictionaryLock)
-                    {
-                        if (!rawBots.TryGetValue(botType, out rawBotsOfSameType))
-                        {
-                            Console.WriteLine($"(chances) Unable to find {botType} on rawBots data");
-                            return;
-                        }
-                    }
-
-                    if (rawBotsOfSameType.Count == 0)
-                    {
-                        return;
-                    }
 
             // TODO: Add check to make sure incoming bot list has gear
-                    GearChanceHelpers.CalculateEquipmentChances(botToUpdate, rawBotsOfSameType);
+            GearChanceHelpers.AddEquipmentChances(botToUpdate, rawBot);
             GearChanceHelpers.AddGenerationChances(botToUpdate, weightHelper);
-                    GearChanceHelpers.CalculateModChances(botToUpdate, rawBotsOfSameType);
+            GearChanceHelpers.AddModChances(botToUpdate, rawBot);
-                    GearChanceHelpers.CalculateEquipmentModChances(botToUpdate, rawBotsOfSameType);
+            GearChanceHelpers.AddEquipmentModChances(botToUpdate, rawBot);
-                    GearChanceHelpers.ApplyModChanceOverrides(botToUpdate);
-                    GearChanceHelpers.ApplyEquipmentChanceOverrides(botToUpdate);
-                }));
-            }
-
-            Task.WaitAll(tasks.ToArray());
-            stopwatch.Stop();
-            LoggingHelpers.LogToConsole($"Finished processing bot chances. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
-
-            return botsToUpdate;
         }
     }
 }

Generator/BotGearGenerator.cs

@@ -7,52 +7,12 @@ namespace Generator
 {
     public static class BotGearGenerator
     {
-        public static IEnumerable<Bot> AddGear(this IEnumerable<Bot> baseBots, Dictionary<string, List<Datum>> rawBots)
+        public static void AddGear(Bot botToUpdate, Datum rawBotData)
         {
-            var stopwatch = Stopwatch.StartNew();
+            GearHelpers.AddEquippedGear(botToUpdate, rawBotData);
-            LoggingHelpers.LogToConsole("Started processing bot gear");
+            GearHelpers.AddAmmo(botToUpdate, rawBotData);
-
+            GearHelpers.AddEquippedMods(botToUpdate, rawBotData);
-            var dictionaryLock = new object();
+            //GearHelpers.AddCartridges(botToUpdate, rawBotData);
-            var tasks = new List<Task>();
-
-            foreach (var botToUpdate in baseBots)
-            {
-                tasks.Add(Task.Factory.StartNew(() =>
-                {
-                    var botType = botToUpdate.botType.ToString().ToLower();
-                    List<Datum> rawBotsOfSameType;
-                    lock (dictionaryLock)
-                    {
-                        if (!rawBots.TryGetValue(botType, out rawBotsOfSameType))
-                        {
-                            Console.WriteLine($"(gear) Unable to find {botType} on rawBots data");
-                            return;
-                        }  
-                    }
-
-                    if (rawBotsOfSameType.Count == 0)
-                    {
-                        return;
-                    }
-
-                    foreach (var rawParsedBot in rawBotsOfSameType)
-                    {
-                        GearHelpers.AddEquippedGear(botToUpdate, rawParsedBot);
-                        GearHelpers.AddAmmo(botToUpdate, rawParsedBot);
-                        GearHelpers.AddEquippedMods(botToUpdate, rawParsedBot);
-                        //GearHelpers.AddCartridges(botToUpdate, rawParsedBot);
-                    }
-
-                    GearHelpers.ReduceAmmoWeightValues(botToUpdate);
-                    GearHelpers.ReduceEquipmentWeightValues(botToUpdate.inventory.equipment);
-                }));
-            }
-
-            Task.WaitAll(tasks.ToArray());
-            stopwatch.Stop();
-            LoggingHelpers.LogToConsole($"Finished processing bot gear. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
-
-            return baseBots;
         }
     }
 }

Generator/BotLootGenerator.cs

@@ -1,6 +1,7 @@
 using System.Diagnostics;
 using System.Linq;
 using Common.Extensions;
+using Common.Models;
 using Common.Models.Input;
 using Common.Models.Output;
 using Generator.Helpers.Gear;
@@ -9,72 +10,20 @@ namespace Generator
 {
     public static class BotLootGenerator
     {
-        internal static IEnumerable<Bot> AddLoot(this IEnumerable<Bot> botsWithGear, Dictionary<string, List<Datum>> rawBots)
+        internal static void AddLoot(Bot botToUpdate, Datum rawBotData)
         {
-            var stopwatch = Stopwatch.StartNew();
+            AddLootToContainers(botToUpdate, rawBotData);
-            LoggingHelpers.LogToConsole("Started processing bot loot");
-
-            var dictionaryLock = new object();
-
-            var tasks = new List<Task>(50);
-            foreach (var botToUpdate in botsWithGear)
-            {
-                tasks.Add(Task.Factory.StartNew(() =>
-                {
-                    var botType = botToUpdate.botType.ToString().ToLower();
-                    List<Datum> rawBotsOfSameType;
-                    lock (dictionaryLock)
-                    {
-                        if (!rawBots.TryGetValue(botType, out rawBotsOfSameType))
-                        {
-                            Console.WriteLine($"(loot) Unable to find {botType} on rawBots data");
-                            return;
-                        }
         }
 
-                    if (rawBotsOfSameType.Count == 0)
+        private static void AddLootToContainers(Bot botToUpdate, Datum rawBot)
-                    {
-                        return;
-                    }
-
-                    AddLootToContainers(botType, botToUpdate, rawBotsOfSameType);
-                    GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.Backpack);
-                    GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.Pockets);
-                    GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.TacticalVest);
-                    GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.SecuredContainer);
-
-                    //foreach (var rawParsedBot in rawBotsOfSameType)
-                    //{
-                    //    AddPocketLoot(botToUpdate, rawParsedBot);
-                    //}
-
-                    //AddTacticalVestLoot(botToUpdate, rawBotsOfSameType);
-                    //AddBackpackLoot(botToUpdate, rawBotsOfSameType);
-                    //AddSecureContainerLoot(botToUpdate, rawBotsOfSameType);
-                    //AddSpecialLoot(botToUpdate);
-                }));
-            }
-
-            Task.WaitAll(tasks.ToArray());
-
-            stopwatch.Stop();
-            LoggingHelpers.LogToConsole($"Finished processing bot loot. Took: {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
-
-            return botsWithGear;
-        }
-
-        private static void AddLootToContainers(string botType, Bot botToUpdate, List<Datum> rawBotsOfSameType)
-        {
-            // Process each bot
-            foreach (var rawBot in rawBotsOfSameType)
         {
             // Filter out base inventory items and equipment mod items
-                var rawBotItems = rawBot.Inventory.items.Where(item => item.location != null).ToList();
+            var rawBotItems = rawBot.Inventory.items.Where(item => item.location != null);
                 
             var botBackpack = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "Backpack");
             if (botBackpack != null)
             {
-                    AddLootItemsToContainerDictionary(rawBotItems, botBackpack._id, botToUpdate.inventory.items.Backpack);
+                AddLootItemsToContainerDictionary(rawBotItems, botBackpack._id, botToUpdate.inventory.items.Backpack, "backpack", botToUpdate.botType);
             }
 
             var botPockets = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "Pockets");
@@ -110,7 +59,6 @@ namespace Generator
 
             AddSpecialLoot(botToUpdate);
         }
-        }
 
         /// <summary>
         /// Look for items inside itemsToFilter that have the parentid of `containerId` and add them to dictToAddTo
@@ -119,16 +67,17 @@ namespace Generator
         /// <param name="itemsToFilter">Bots inventory items</param>
         /// <param name="containerId"></param>
         /// <param name="dictToAddTo"></param>
-        private static void AddLootItemsToContainerDictionary(List<Item> itemsToFilter, string containerId, Dictionary<string, int> dictToAddTo)
+        private static void AddLootItemsToContainerDictionary(IEnumerable<Common.Models.Input.Item> itemsToFilter, string containerId, Dictionary<string, int> dictToAddTo, string container = "", BotType type = BotType.arenafighterevent)
         {
-            var lootItems = itemsToFilter.Where(item => item.parentId == containerId);
+            foreach (var itemToAdd in itemsToFilter)
-            foreach (var itemToAdd in lootItems)
             {
+                if (itemToAdd.parentId != containerId) continue;
+
                 if (!dictToAddTo.ContainsKey(itemToAdd._tpl))
                 {
                     dictToAddTo[itemToAdd._tpl] = 1;
 
-                    return;
+                    continue;
                 }
 
                 dictToAddTo[itemToAdd._tpl]++;

Generator/BotParser.cs

@@ -8,6 +8,9 @@ using System.Linq;
 using System.Text.Json;
 using System.Threading.Tasks;
 using Common.Models.Output;
+using Common.Models;
+using Generator;
+using Generator.Helpers.Gear;
 
 namespace Common.Bots;
 
@@ -15,6 +18,118 @@ public static class BotParser
 {
     private static readonly JsonSerializerOptions serialiserOptions = new() { };
 
+    public static List<Bot> Parse(string dumpPath, HashSet<string> botTypes)
+    {
+        var stopwatch = Stopwatch.StartNew();
+
+        // Build the list of base bot data
+        var baseBots = new HashSet<Bot>();
+        foreach (var botType in botTypes)
+        {
+            var typeToAdd = (BotType)Enum.Parse(typeof(BotType), botType);
+            baseBots.Add(new Bot(typeToAdd));
+        }
+
+        DiskHelpers.CreateDirIfDoesntExist(dumpPath);
+
+        var botFiles = Directory.GetFiles(dumpPath, "*.json", SearchOption.TopDirectoryOnly);
+        LoggingHelpers.LogToConsole($"{botFiles.Length} bot dump files found");
+
+        // Store a list of parsed bots so we don't parse the same bot twice
+        int totalDupeCount = 0;
+        var parsedBotIds = new HashSet<string>();
+        int i = 0;
+        foreach (var filePath in botFiles)
+        {
+            i++;
+            if (i % 100 == 0) Console.WriteLine($"Processing file {i}");
+            ProcessBotFileSync(baseBots, filePath, parsedBotIds, totalDupeCount);
+        }
+
+        // Handle things we can only do once all data has been processed, or only needs to run once per bot type
+        foreach (var bot in baseBots)
+        {
+            // Skip any bots we didn't handle
+            if (bot.botCount == 0) continue;
+
+            BaseBotGenerator.AddDifficulties(bot);
+            GearChanceHelpers.CalculateModChances(bot);
+            GearChanceHelpers.CalculateEquipmentModChances(bot);
+            GearChanceHelpers.CalculateEquipmentChances(bot);
+            GearChanceHelpers.ApplyModChanceOverrides(bot);
+            GearChanceHelpers.ApplyEquipmentChanceOverrides(bot);
+
+            GearHelpers.ReduceAmmoWeightValues(bot);
+            GearHelpers.ReduceEquipmentWeightValues(bot.inventory.equipment);
+            GearHelpers.ReduceWeightValues(bot.appearance.voice);
+            GearHelpers.ReduceWeightValues(bot.appearance.feet);
+            GearHelpers.ReduceWeightValues(bot.appearance.body);
+            GearHelpers.ReduceWeightValues(bot.appearance.head);
+            GearHelpers.ReduceWeightValues(bot.appearance.hands);
+        }
+
+        stopwatch.Stop();
+        LoggingHelpers.LogToConsole($"{totalDupeCount} dupes were ignored. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
+
+        return baseBots.ToList();
+    }
+
+    private static void ProcessBotFileSync(
+        HashSet<Bot> baseBots,
+        string filePath,
+        HashSet<string> parsedBotIds,
+        int totalDupeCount)
+    {
+        var splitFilePath = filePath.Split("\\");
+
+        int dupeCount = 0;
+
+        List<Datum> bots = [];
+        try
+        {
+            // Parse the bots inside the json file
+            using (var reader = new StreamReader(filePath))
+            {
+                var deSerialisedObject = JsonSerializer.Deserialize<Root>(reader.ReadToEnd(), serialiserOptions);
+
+                foreach (var botData in deSerialisedObject.data)
+                {
+                    // Bot fucks up something, never allow it in
+                    if (botData._id == "6483938c53cc9087c70eae86")
+                    {
+                        Console.WriteLine("oh no");
+                        continue;
+                    }
+
+                    var baseBot = baseBots.SingleOrDefault(bot => bot.botType.ToString().Equals(botData.Info.Settings.Role, StringComparison.OrdinalIgnoreCase));
+                    if (baseBot == null)
+                    {
+                        //Console.WriteLine($"Skipping {botData._id} due to unknown role {botData.Info.Settings.Role}");
+                        continue;
+                    }
+
+                    // Add bot if not already added
+                    if (!parsedBotIds.Add(botData._id))
+                    {
+                        dupeCount++;
+                    }
+                    baseBot.botCount += 1;
+                    BaseBotGenerator.UpdateBaseDetails(baseBot, botData);
+                    BotGearGenerator.AddGear(baseBot, botData);
+                    BotLootGenerator.AddLoot(baseBot, botData);
+                    BotChancesGenerator.AddChances(baseBot, botData);
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"File parse fucked up: {filePath}");
+            throw;
+        }
+
+        totalDupeCount += dupeCount;
+    }
+
     public static async Task<List<Datum>> ParseAsync(string dumpPath, HashSet<string> botTypes)
     {
         var stopwatch = Stopwatch.StartNew();

Generator/Helpers/Gear/GearChanceHelpers.cs

@@ -7,16 +7,29 @@ namespace Generator.Helpers.Gear
 {
     public static class GearChanceHelpers
     {
-        public static void CalculateModChances(Bot bot, List<Datum> baseBots)
+        private static Dictionary<string, Dictionary<string, int>> weaponModCount = new Dictionary<string, Dictionary<string, int>>();
+        private static Dictionary<string, Dictionary<string, int>> weaponSlotCount = new Dictionary<string, Dictionary<string, int>>();
+        private static Dictionary<string, Dictionary<string, int>> equipmentModCount = new Dictionary<string, Dictionary<string, int>>();
+        private static Dictionary<string, Dictionary<string, int>> equipmentSlotCount = new Dictionary<string, Dictionary<string, int>>();
+
+
+        public static void AddModChances(Bot bot, Datum baseBot)
         {
             // TODO: Further split these counts by equipment slot? (ex. "FirstPrimaryWeapon", "Holster", etc.)
             var validSlots = new List<string> { "FirstPrimaryWeapon", "SecondPrimaryWeapon", "Holster" };
 
-            var modCounts = new Dictionary<string, int>();
+            if (!weaponModCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var modCounts))
-            var slotCounts = new Dictionary<string, int>();
-
-            foreach (var baseBot in baseBots)
             {
+                modCounts = new Dictionary<string, int>();
+                weaponModCount.Add(baseBot.Info.Settings.Role.ToLower(), modCounts);
+            }
+
+            if (!weaponSlotCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var slotCounts))
+            {
+                slotCounts = new Dictionary<string, int>();
+                weaponSlotCount.Add(baseBot.Info.Settings.Role.ToLower(), slotCounts);
+            }
+
             var validParents = new List<string>();
             foreach (var inventoryItem in baseBot.Inventory.items)
             {
@@ -78,21 +91,42 @@ namespace Generator.Helpers.Gear
             }
         }
 
+        public static void CalculateModChances(Bot bot)
+        {
+            if (!weaponModCount.TryGetValue(bot.botType.ToString(), out var modCounts))
+            {
+                modCounts = new Dictionary<string, int>();
+                weaponModCount.Add(bot.botType.ToString(), modCounts);
+            }
+
+            if (!weaponSlotCount.TryGetValue(bot.botType.ToString(), out var slotCounts))
+            {
+                slotCounts = new Dictionary<string, int>();
+                weaponSlotCount.Add(bot.botType.ToString(), slotCounts);
+            }
+
             bot.chances.weaponMods = slotCounts.ToDictionary(
                 kvp => kvp.Key,
                 kvp => GetPercent(kvp.Value, modCounts.GetValueOrDefault(kvp.Key)));
         }
 
-        public static void CalculateEquipmentModChances(Bot bot, List<Datum> baseBots)
+        public static void AddEquipmentModChances(Bot bot, Datum baseBot)
         {
+            if (!equipmentModCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var modCounts))
+            {
+                modCounts = new Dictionary<string, int>();
+                equipmentModCount.Add(baseBot.Info.Settings.Role.ToLower(), modCounts);
+            }
+
+            if (!equipmentSlotCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var slotCounts))
+            {
+                slotCounts = new Dictionary<string, int>();
+                equipmentSlotCount.Add(baseBot.Info.Settings.Role.ToLower(), slotCounts);
+            }
+
             // TODO: Further split these counts by equipment slot? (ex. "FirstPrimaryWeapon", "Holster", etc.)
             var validSlots = new List<string> { "Headwear", "ArmorVest", "TacticalVest" };
 
-            var modCounts = new Dictionary<string, int>();
-            var slotCounts = new Dictionary<string, int>();
-
-            foreach (var baseBot in baseBots)
-            {
             var validParents = new List<string>();
 
             foreach (var inventoryItem in baseBot.Inventory.items)
@@ -155,6 +189,20 @@ namespace Generator.Helpers.Gear
             }
         }
 
+        public static void CalculateEquipmentModChances(Bot bot)
+        {
+            if (!equipmentModCount.TryGetValue(bot.botType.ToString(), out var modCounts))
+            {
+                modCounts = new Dictionary<string, int>();
+                equipmentModCount.Add(bot.botType.ToString(), modCounts);
+            }
+
+            if (!equipmentSlotCount.TryGetValue(bot.botType.ToString(), out var slotCounts))
+            {
+                slotCounts = new Dictionary<string, int>();
+                equipmentSlotCount.Add(bot.botType.ToString(), slotCounts);
+            }
+
             bot.chances.equipmentMods = slotCounts.ToDictionary(
                 kvp => kvp.Key,
                 kvp => GetPercent(kvp.Value, modCounts.GetValueOrDefault(kvp.Key)));
@@ -245,47 +293,47 @@ namespace Generator.Helpers.Gear
                 weightsData["grenades"]);
         }
 
-        public static void CalculateEquipmentChances(Bot bot, List<Datum> baseBots)
+        public static void AddEquipmentChances(Bot bot, Datum baseBot)
         {
-            // TODO: Convert to dynamic?
+            bot.chances.equipment.Headwear += baseBot.Inventory.items.Count(x => x.slotId == "Headwear");
-            var totalBotsCount = baseBots.Count;
+            bot.chances.equipment.Earpiece += baseBot.Inventory.items.Count(x => x.slotId == "Earpiece");
-            int headwearCount = 0, earCount = 0, faceCoverCount = 0, armorVestCount = 0, eyeWearCount = 0, armBandCount = 0,
+            bot.chances.equipment.FaceCover += baseBot.Inventory.items.Count(x => x.slotId == "FaceCover");
-                tacticalVestCount = 0, backpackCount = 0, firstPrimaryCount = 0, secondPrimaryCount = 0, holsterCount = 0,
+            bot.chances.equipment.ArmorVest += baseBot.Inventory.items.Count(x => x.slotId == "ArmorVest");
-                scabbardCount = 0, pocketsCount = 0, securedContainerCount = 0;
+            bot.chances.equipment.Eyewear += baseBot.Inventory.items.Count(x => x.slotId == "Eyewear");
+            bot.chances.equipment.ArmBand += baseBot.Inventory.items.Count(x => x.slotId == "ArmBand");
+            bot.chances.equipment.TacticalVest += baseBot.Inventory.items.Count(x => x.slotId == "TacticalVest");
+            bot.chances.equipment.Backpack += baseBot.Inventory.items.Count(x => x.slotId == "Backpack");
+            bot.chances.equipment.FirstPrimaryWeapon += baseBot.Inventory.items.Count(x => x.slotId == "FirstPrimaryWeapon");
+            bot.chances.equipment.SecondPrimaryWeapon += baseBot.Inventory.items.Count(x => x.slotId == "SecondPrimaryWeapon");
+            bot.chances.equipment.Holster += baseBot.Inventory.items.Count(x => x.slotId == "Holster");
+            bot.chances.equipment.Scabbard += baseBot.Inventory.items.Count(x => x.slotId == "Scabbard");
+            bot.chances.equipment.Pockets += baseBot.Inventory.items.Count(x => x.slotId == "Pockets");
+            bot.chances.equipment.SecuredContainer += baseBot.Inventory.items.Count(x => x.slotId == "SecuredContainer");
+        }
 
-            foreach (var baseBot in baseBots)
+        public static void CalculateEquipmentChances(Bot bot)
         {
-                headwearCount += baseBot.Inventory.items.Count(x => x.slotId == "Headwear");
+            if (bot.botCount == 0)
-                earCount += baseBot.Inventory.items.Count(x => x.slotId == "Earpiece");
+            {
-                faceCoverCount += baseBot.Inventory.items.Count(x => x.slotId == "FaceCover");
+                // No bots, don't do anything
-                armorVestCount += baseBot.Inventory.items.Count(x => x.slotId == "ArmorVest");
+                return;
-                eyeWearCount += baseBot.Inventory.items.Count(x => x.slotId == "Eyewear");
-                armBandCount += baseBot.Inventory.items.Count(x => x.slotId == "ArmBand");
-                tacticalVestCount += baseBot.Inventory.items.Count(x => x.slotId == "TacticalVest");
-                backpackCount += baseBot.Inventory.items.Count(x => x.slotId == "Backpack");
-                firstPrimaryCount += baseBot.Inventory.items.Count(x => x.slotId == "FirstPrimaryWeapon");
-                secondPrimaryCount += baseBot.Inventory.items.Count(x => x.slotId == "SecondPrimaryWeapon");
-                holsterCount += baseBot.Inventory.items.Count(x => x.slotId == "Holster");
-                scabbardCount += baseBot.Inventory.items.Count(x => x.slotId == "Scabbard");
-                pocketsCount += baseBot.Inventory.items.Count(x => x.slotId == "Pockets");
-                securedContainerCount += baseBot.Inventory.items.Count(x => x.slotId == "SecuredContainer");
             }
 
             bot.chances.equipment = new EquipmentChances(
-                  GetPercent(totalBotsCount, headwearCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.Headwear),
-                  GetPercent(totalBotsCount, earCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.Earpiece),
-                  GetPercent(totalBotsCount, faceCoverCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.FaceCover),
-                  GetPercent(totalBotsCount, armorVestCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.ArmorVest),
-                  GetPercent(totalBotsCount, eyeWearCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.Eyewear),
-                  GetPercent(totalBotsCount, armBandCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.ArmBand),
-                  GetPercent(totalBotsCount, tacticalVestCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.TacticalVest),
-                  GetPercent(totalBotsCount, backpackCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.Backpack),
-                  GetPercent(totalBotsCount, firstPrimaryCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.FirstPrimaryWeapon),
-                  GetPercent(totalBotsCount, secondPrimaryCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.SecondPrimaryWeapon),
-                  GetPercent(totalBotsCount, holsterCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.Holster),
-                  GetPercent(totalBotsCount, scabbardCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.Scabbard),
-                  GetPercent(totalBotsCount, pocketsCount),
+                  GetPercent(bot.botCount, bot.chances.equipment.Pockets),
-                  GetPercent(totalBotsCount, securedContainerCount));
+                  GetPercent(bot.botCount, bot.chances.equipment.SecuredContainer));
         }
 
         private static int GetPercent(int total, int count)

Generator/Helpers/Gear/GearHelpers.cs

@@ -65,7 +65,6 @@ namespace Generator.Helpers.Gear
 
         internal static void AddAmmo(Bot botToUpdate, Datum bot)
         {
-            //var weightService = new WeightingService();
             foreach (var ammo in bot.Inventory.items.Where(
                 x => x.slotId != null 
             && (x.slotId == "patron_in_weapon"
@@ -117,7 +116,6 @@ namespace Generator.Helpers.Gear
         public static void AddEquippedGear(Bot botToUpdate, Datum bot)
         {
             // add equipped gear
-            var weightService = new WeightingService();
             foreach (var inventoryItem in bot.Inventory.items.Where(x=>x.slotId != null))
             {
                 switch (inventoryItem.slotId?.ToLower())

Generator/Program.cs

@@ -1,5 +1,7 @@
 using Common.Models.Input;
+using Common.Models.Output;
 using Generator.Helpers;
+using System.Diagnostics;
 
 namespace Generator;
 
@@ -7,6 +9,9 @@ internal static class Program
 {
     internal static async Task Main(string[] args)
     {
+        var stopwatch = Stopwatch.StartNew();
+        LoggingHelpers.LogToConsole("Started processing bots");
+
         // Create list of bots we want to process
         string[] botTypes = {
         "assault",
@@ -72,38 +77,19 @@ internal static class Program
         // Read raw bot dumps and turn into c# objects
         var workingPath = Directory.GetCurrentDirectory();
         var dumpPath = $"{workingPath}//dumps";
-        var parsedBots = await BotParser.ParseAsync(dumpPath, botTypes.ToHashSet());
+        List<Bot> bots = BotParser.Parse(dumpPath, botTypes.ToHashSet());
 
-        // Put in dictionary for better use later on
+        if (bots.Count == 0)
-        var rawBotsCache = new Dictionary<string, List<Datum>>(45);
-        foreach (var rawBot in parsedBots)
-        {
-            if (rawBotsCache.TryGetValue(rawBot.Info.Settings.Role.ToLower(), out var botList))
-            {
-                botList.Add(rawBot);
-
-                continue;
-            }
-
-            // Doesnt exist, add key and bot
-            rawBotsCache.Add(rawBot.Info.Settings.Role.ToLower(), new List<Datum> { rawBot });
-        }
-
-        if (parsedBots.Count == 0)
         {
             LoggingHelpers.LogToConsole("No bots found, unable to continue");
             LoggingHelpers.LogToConsole("Check your dumps are in 'Generator\\bin\\Debug\\net6.0\\dumps' and start with 'resp.' NOT 'req.'");
             return;
         }
 
-        // Generate the base bot class with basic details (health/body part hp etc) and then append everything else
-        var bots = BaseBotGenerator.GenerateBaseDetails(parsedBots, workingPath, botTypes)
-                                .AddGear(rawBotsCache) // Add weapons/armor
-                                .AddLoot(rawBotsCache)
-                                .AddChances(rawBotsCache); // Add mod/equipment chances
-
-        // Output bot to json file
         var jsonWriter = new JsonWriter(workingPath, "output");
-        jsonWriter.WriteJson(bots.ToList());
+        jsonWriter.WriteJson(bots);
+
+        stopwatch.Stop();
+        LoggingHelpers.LogToConsole($"Finished processing bots. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
     }
 }

Generator/Weighting/WeightingService.cs

@@ -14,11 +14,14 @@ namespace Generator.Weighting
     
     public class WeightingService
     {
-        private readonly Dictionary<BotType, Weightings> _weights;
+        private static Dictionary<BotType, Weightings> _weights = null;
-        private readonly Dictionary<string, Dictionary<string, GenerationWeightData>> _generationWeights;
+        private static Dictionary<string, Dictionary<string, GenerationWeightData>> _generationWeights = null;
 
         public WeightingService()
         {
+            // Cache the loaded  data
+            if (_weights != null && _generationWeights != null) return;
+
             var assetsPath = $"{Directory.GetCurrentDirectory()}\\Assets";
             var weightsFilePath = $"{assetsPath}\\weights.json";
             if (!File.Exists(weightsFilePath))