Drastically reduce memory footprint at the cost of removing threading

- Drastically improve performance, where non-threaded perf is now better than previous threaded perf
- Process all data in-line, as opposed to building huge lists of bot data that is then processed in groups later
- Memory footprint should now be relatively static no matter the amount of dumps
This commit is contained in:
DrakiaXYZ 2024-11-03 23:29:50 -08:00
parent e0f9cfbfaa
commit 9ddbc91977
10 changed files with 440 additions and 462 deletions

View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace Common.Models.Output;
@ -18,6 +19,7 @@ public class Bot
public Bot(BotType botType)
{
this.botType = botType;
botCount = 0;
appearance = new Appearance();
experience = new Experience();
health = new Health();
@ -32,6 +34,8 @@ public class Bot
[JsonIgnore]
public BotType botType { get; set; }
[JsonIgnore]
public int botCount { get; set; }
public Appearance appearance { get; set; }
public Experience experience { get; set; }
public Health health { get; set; }
@ -79,7 +83,7 @@ public class Experience
public bool useSimpleAnimator { get; set; }
}
public class BodyParts
public class BodyParts : IEquatable<BodyParts>
{
public BodyParts()
{
@ -92,6 +96,17 @@ public class BodyParts
RightLeg = new MinMax(65, 65);
}
public bool Equals(BodyParts other)
{
return this.Head.Equals(other.Head) &&
this.Chest.Equals(other.Chest) &&
this.Stomach.Equals(other.Stomach) &&
this.LeftArm.Equals(other.LeftArm) &&
this.RightArm.Equals(other.RightArm) &&
this.LeftLeg.Equals(other.LeftLeg) &&
this.RightLeg.Equals(other.RightLeg);
}
public MinMax Head { get; set; }
public MinMax Chest { get; set; }
public MinMax Stomach { get; set; }
@ -256,7 +271,7 @@ public class ItemChances
public GenerationWeightData grenades { get; set; }
}
public class MinMax
public class MinMax : IEquatable<MinMax>
{
public MinMax(int min, int max)
{
@ -264,6 +279,11 @@ public class MinMax
this.max = max;
}
public bool Equals(MinMax other)
{
return this.min == other.min && this.max == other.max;
}
public int min { get; set; }
public int max { get; set; }
}

View File

@ -10,104 +10,56 @@ namespace Generator
{
public static class BaseBotGenerator
{
//TODO: pass in bot types and use those to create the classes in rawBots list
public static IEnumerable<Bot> GenerateBaseDetails(IEnumerable<Datum> rawBots, string workingPath, IEnumerable<string> botTypes)
public static void UpdateBaseDetails(Bot botData, Datum rawBotData)
{
var stopwatch = Stopwatch.StartNew();
LoggingHelpers.LogToConsole("Started processing bot base");
UpdateBodyPartHealth(botData, rawBotData);
AddExperience(botData, rawBotData);
AddStandingForKill(botData, rawBotData);
AddSkills(botData, rawBotData);
botData.experience.useSimpleAnimator = rawBotData.Info.Settings.UseSimpleAnimator;
// Create a list of bot objects ready to be hydrated
var baseBots = new List<Bot>();
foreach (var botType in botTypes)
{
var typeToAdd = (BotType)Enum.Parse(typeof(BotType), botType);
baseBots.Add(new Bot(typeToAdd));
}
// Iterate over each bot type we just made and put some data into them
foreach (var botToUpdate in baseBots)
{
var rawBotType = botToUpdate.botType.ToString();
var rawBotsOfSameType = rawBots.Where(x => string.Equals(x.Info.Settings.Role, rawBotType, StringComparison.OrdinalIgnoreCase))
.ToList();
var rawBotsOfSameTypeCount = rawBotsOfSameType.Count.ToString();
if (rawBotsOfSameType.Count == 0)
{
LoggingHelpers.LogToConsole($"no bots of type {rawBotType}, skipping", ConsoleColor.DarkRed);
continue;
}
LoggingHelpers.LogToConsole($"Found {rawBotsOfSameTypeCount} bots of type: {rawBotType}");
UpdateBodyPartHealth(botToUpdate, rawBotsOfSameType);
AddDifficulties(botToUpdate, workingPath);
AddExperience(botToUpdate, rawBotsOfSameType);
AddStandingForKill(botToUpdate, rawBotsOfSameType);
AddSkills(botToUpdate, rawBotsOfSameType);
botToUpdate.experience.useSimpleAnimator = rawBotsOfSameType.First().Info.Settings.UseSimpleAnimator;
foreach (var rawParsedBot in rawBotsOfSameType)
{
AddVisualAppearanceItems(botToUpdate, rawParsedBot);
AddName(botToUpdate, rawParsedBot);
AddVoice(botToUpdate, rawParsedBot);
}
}
stopwatch.Stop();
LoggingHelpers.LogToConsole($"Finished processing bot base. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
return baseBots;
AddVisualAppearanceItems(botData, rawBotData);
AddName(botData, rawBotData);
AddVoice(botData, rawBotData);
}
private static void AddSkills(Bot botToUpdate, IEnumerable<Datum> rawBotsOfSameType)
private static void AddSkills(Bot botToUpdate, Datum rawBotData)
{
var firstBotOfDesiredType = rawBotsOfSameType.Last();
// Find the smallest and biggest value for each skill
foreach (var skill in firstBotOfDesiredType.Skills.Common)
foreach (var skill in rawBotData.Skills.Common)
{
var skills = new List<Common.Models.Input.Common>();
foreach (var bot in rawBotsOfSameType)
if (botToUpdate.skills.Common.TryGetValue(skill.Id, out var existingSkill))
{
skills.Add(bot.Skills.Common.Find(x => x.Id == skill.Id));
existingSkill.min = Math.Min(existingSkill.min, skill.Progress);
existingSkill.max = Math.Max(existingSkill.max, skill.Progress);
}
var min = skills.Min(x => x?.Progress);
var max = skills.Max(x => x?.Progress);
if (min.HasValue && max.HasValue)
else
{
botToUpdate.skills.Common.Add(skill.Id, new MinMax(min.Value, max.Value));
botToUpdate.skills.Common.Add(skill.Id, new MinMax(skill.Progress, skill.Progress));
}
}
}
private static void AddStandingForKill(Bot botToUpdate, IEnumerable<Datum> rawBotsOfSameType)
private static void AddStandingForKill(Bot botToUpdate, Datum rawBotData)
{
var firstBotOfDesiredType = rawBotsOfSameType.Last();
botToUpdate.experience.standingForKill = firstBotOfDesiredType.Info.Settings.StandingForKill;
botToUpdate.experience.aggressorBonus = firstBotOfDesiredType.Info.Settings.AggressorBonus;
botToUpdate.experience.standingForKill = rawBotData.Info.Settings.StandingForKill;
botToUpdate.experience.aggressorBonus = rawBotData.Info.Settings.AggressorBonus;
}
private static void AddExperience(Bot botToUpdate, IEnumerable<Datum> rawBotsOfSameType)
private static void AddExperience(Bot botToUpdate, Datum rawBotData)
{
var firstBotOfDesiredType = rawBotsOfSameType.Last();
botToUpdate.experience.reward.min = firstBotOfDesiredType.Info.Settings.Experience;
botToUpdate.experience.reward.max = firstBotOfDesiredType.Info.Settings.Experience;
botToUpdate.experience.reward.min = rawBotData.Info.Settings.Experience;
botToUpdate.experience.reward.max = rawBotData.Info.Settings.Experience;
}
private static void AddVoice(Bot bot, Datum rawBot)
{
GearHelpers.IncrementDictionaryValue(bot.appearance.voice, rawBot.Info.Voice);
GearHelpers.ReduceWeightValues(bot.appearance.voice);
}
private static void AddDifficulties(Bot bot, string workingPath)
public static void AddDifficulties(Bot bot)
{
string workingPath = Directory.GetCurrentDirectory();
string botType = bot.botType.ToString();
var botDifficultyFiles = Directory
.GetFiles($"{workingPath}//Assets", "*.txt", SearchOption.TopDirectoryOnly)
@ -117,51 +69,34 @@ namespace Generator
DifficultyHelper.AddDifficultySettings(bot, botDifficultyFiles);
}
private static void UpdateBodyPartHealth(Bot botToUpdate, List<Datum> rawBots)
private static void UpdateBodyPartHealth(Bot botToUpdate, Datum rawBot)
{
var uniqueHealthSetups = new Dictionary<int, Common.Models.Output.BodyParts>();
foreach (var bot in rawBots)
var bodyPartHpToAdd = new Common.Models.Output.BodyParts()
{
var healthTotal = bot.Health.BodyParts.GetHpMaxTotal();
var alreadyExists = uniqueHealthSetups.ContainsKey(healthTotal);
Head = new MinMax(rawBot.Health.BodyParts.Head.Health.Current, rawBot.Health.BodyParts.Head.Health.Maximum),
Chest = new MinMax(rawBot.Health.BodyParts.Chest.Health.Current, rawBot.Health.BodyParts.Chest.Health.Maximum),
Stomach = new MinMax(rawBot.Health.BodyParts.Stomach.Health.Current, rawBot.Health.BodyParts.Stomach.Health.Maximum),
LeftArm = new MinMax(rawBot.Health.BodyParts.LeftArm.Health.Current, rawBot.Health.BodyParts.LeftArm.Health.Maximum),
RightArm = new MinMax(rawBot.Health.BodyParts.RightArm.Health.Current, rawBot.Health.BodyParts.RightArm.Health.Maximum),
LeftLeg = new MinMax(rawBot.Health.BodyParts.LeftLeg.Health.Current, rawBot.Health.BodyParts.LeftLeg.Health.Maximum),
RightLeg = new MinMax(rawBot.Health.BodyParts.RightLeg.Health.Current, rawBot.Health.BodyParts.RightLeg.Health.Maximum)
};
if (!alreadyExists)
{
var bodyPartHpToAdd = new Common.Models.Output.BodyParts()
{
Head = new MinMax(bot.Health.BodyParts.Head.Health.Current, bot.Health.BodyParts.Head.Health.Maximum),
Chest = new MinMax(bot.Health.BodyParts.Chest.Health.Current, bot.Health.BodyParts.Chest.Health.Maximum),
Stomach = new MinMax(bot.Health.BodyParts.Stomach.Health.Current, bot.Health.BodyParts.Stomach.Health.Maximum),
LeftArm = new MinMax(bot.Health.BodyParts.LeftArm.Health.Current, bot.Health.BodyParts.LeftArm.Health.Maximum),
RightArm = new MinMax(bot.Health.BodyParts.RightArm.Health.Current, bot.Health.BodyParts.RightArm.Health.Maximum),
};
bodyPartHpToAdd.LeftLeg.min = bot.Health.BodyParts.LeftLeg.Health.Current;
bodyPartHpToAdd.LeftLeg.max = bot.Health.BodyParts.LeftLeg.Health.Maximum;
bodyPartHpToAdd.RightLeg.min = bot.Health.BodyParts.RightLeg.Health.Current;
bodyPartHpToAdd.RightLeg.max = bot.Health.BodyParts.RightLeg.Health.Maximum;
uniqueHealthSetups.Add(healthTotal, bodyPartHpToAdd);
}
if (!botToUpdate.health.BodyParts.Contains(bodyPartHpToAdd))
{
botToUpdate.health.BodyParts.Add(bodyPartHpToAdd);
}
botToUpdate.health.BodyParts = uniqueHealthSetups.Values.ToList();
}
private static void AddVisualAppearanceItems(Bot botToUpdate, Datum rawBot)
{
GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.feet, rawBot.Customization.Feet);
GearHelpers.ReduceWeightValues(botToUpdate.appearance.feet);
GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.body, rawBot.Customization.Body);
GearHelpers.ReduceWeightValues(botToUpdate.appearance.body);
GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.head, rawBot.Customization.Head);
GearHelpers.ReduceWeightValues(botToUpdate.appearance.head);
GearHelpers.IncrementDictionaryValue(botToUpdate.appearance.hands, rawBot.Customization.Hands);
GearHelpers.ReduceWeightValues(botToUpdate.appearance.hands);
}
private static void AddName(Bot botToUpdate, Datum rawBot)

View File

@ -8,51 +8,15 @@ namespace Generator
{
public static class BotChancesGenerator
{
public static IEnumerable<Bot> AddChances(this IEnumerable<Bot> botsToUpdate, Dictionary<string, List<Datum>> rawBots)
public static void AddChances(Bot botToUpdate, Datum rawBot)
{
var stopwatch = Stopwatch.StartNew();
LoggingHelpers.LogToConsole("Started processing bot gear");
// use lock for lock safety
var dictionaryLock = new object();
var weightHelper = new WeightingService();
// multithread
var tasks = new List<Task>();
foreach (var botToUpdate in botsToUpdate)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var botType = botToUpdate.botType.ToString().ToLower();
List<Datum> rawBotsOfSameType;
lock (dictionaryLock)
{
if (!rawBots.TryGetValue(botType, out rawBotsOfSameType))
{
Console.WriteLine($"(chances) Unable to find {botType} on rawBots data");
return;
}
}
if (rawBotsOfSameType.Count == 0)
{
return;
}
// TODO: Add check to make sure incoming bot list has gear
GearChanceHelpers.CalculateEquipmentChances(botToUpdate, rawBotsOfSameType);
GearChanceHelpers.AddGenerationChances(botToUpdate, weightHelper);
GearChanceHelpers.CalculateModChances(botToUpdate, rawBotsOfSameType);
GearChanceHelpers.CalculateEquipmentModChances(botToUpdate, rawBotsOfSameType);
GearChanceHelpers.ApplyModChanceOverrides(botToUpdate);
GearChanceHelpers.ApplyEquipmentChanceOverrides(botToUpdate);
}));
}
Task.WaitAll(tasks.ToArray());
stopwatch.Stop();
LoggingHelpers.LogToConsole($"Finished processing bot chances. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
return botsToUpdate;
// TODO: Add check to make sure incoming bot list has gear
GearChanceHelpers.AddEquipmentChances(botToUpdate, rawBot);
GearChanceHelpers.AddGenerationChances(botToUpdate, weightHelper);
GearChanceHelpers.AddModChances(botToUpdate, rawBot);
GearChanceHelpers.AddEquipmentModChances(botToUpdate, rawBot);
}
}
}

View File

@ -7,52 +7,12 @@ namespace Generator
{
public static class BotGearGenerator
{
public static IEnumerable<Bot> AddGear(this IEnumerable<Bot> baseBots, Dictionary<string, List<Datum>> rawBots)
public static void AddGear(Bot botToUpdate, Datum rawBotData)
{
var stopwatch = Stopwatch.StartNew();
LoggingHelpers.LogToConsole("Started processing bot gear");
var dictionaryLock = new object();
var tasks = new List<Task>();
foreach (var botToUpdate in baseBots)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var botType = botToUpdate.botType.ToString().ToLower();
List<Datum> rawBotsOfSameType;
lock (dictionaryLock)
{
if (!rawBots.TryGetValue(botType, out rawBotsOfSameType))
{
Console.WriteLine($"(gear) Unable to find {botType} on rawBots data");
return;
}
}
if (rawBotsOfSameType.Count == 0)
{
return;
}
foreach (var rawParsedBot in rawBotsOfSameType)
{
GearHelpers.AddEquippedGear(botToUpdate, rawParsedBot);
GearHelpers.AddAmmo(botToUpdate, rawParsedBot);
GearHelpers.AddEquippedMods(botToUpdate, rawParsedBot);
//GearHelpers.AddCartridges(botToUpdate, rawParsedBot);
}
GearHelpers.ReduceAmmoWeightValues(botToUpdate);
GearHelpers.ReduceEquipmentWeightValues(botToUpdate.inventory.equipment);
}));
}
Task.WaitAll(tasks.ToArray());
stopwatch.Stop();
LoggingHelpers.LogToConsole($"Finished processing bot gear. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
return baseBots;
GearHelpers.AddEquippedGear(botToUpdate, rawBotData);
GearHelpers.AddAmmo(botToUpdate, rawBotData);
GearHelpers.AddEquippedMods(botToUpdate, rawBotData);
//GearHelpers.AddCartridges(botToUpdate, rawBotData);
}
}
}

View File

@ -1,6 +1,7 @@
using System.Diagnostics;
using System.Linq;
using Common.Extensions;
using Common.Models;
using Common.Models.Input;
using Common.Models.Output;
using Generator.Helpers.Gear;
@ -9,107 +10,54 @@ namespace Generator
{
public static class BotLootGenerator
{
internal static IEnumerable<Bot> AddLoot(this IEnumerable<Bot> botsWithGear, Dictionary<string, List<Datum>> rawBots)
internal static void AddLoot(Bot botToUpdate, Datum rawBotData)
{
var stopwatch = Stopwatch.StartNew();
LoggingHelpers.LogToConsole("Started processing bot loot");
var dictionaryLock = new object();
var tasks = new List<Task>(50);
foreach (var botToUpdate in botsWithGear)
{
tasks.Add(Task.Factory.StartNew(() =>
{
var botType = botToUpdate.botType.ToString().ToLower();
List<Datum> rawBotsOfSameType;
lock (dictionaryLock)
{
if (!rawBots.TryGetValue(botType, out rawBotsOfSameType))
{
Console.WriteLine($"(loot) Unable to find {botType} on rawBots data");
return;
}
}
if (rawBotsOfSameType.Count == 0)
{
return;
}
AddLootToContainers(botType, botToUpdate, rawBotsOfSameType);
GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.Backpack);
GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.Pockets);
GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.TacticalVest);
GearHelpers.ReduceWeightValues(botToUpdate.inventory.equipment.SecuredContainer);
//foreach (var rawParsedBot in rawBotsOfSameType)
//{
// AddPocketLoot(botToUpdate, rawParsedBot);
//}
//AddTacticalVestLoot(botToUpdate, rawBotsOfSameType);
//AddBackpackLoot(botToUpdate, rawBotsOfSameType);
//AddSecureContainerLoot(botToUpdate, rawBotsOfSameType);
//AddSpecialLoot(botToUpdate);
}));
}
Task.WaitAll(tasks.ToArray());
stopwatch.Stop();
LoggingHelpers.LogToConsole($"Finished processing bot loot. Took: {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
return botsWithGear;
AddLootToContainers(botToUpdate, rawBotData);
}
private static void AddLootToContainers(string botType, Bot botToUpdate, List<Datum> rawBotsOfSameType)
private static void AddLootToContainers(Bot botToUpdate, Datum rawBot)
{
// Process each bot
foreach (var rawBot in rawBotsOfSameType)
// Filter out base inventory items and equipment mod items
var rawBotItems = rawBot.Inventory.items.Where(item => item.location != null);
var botBackpack = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "Backpack");
if (botBackpack != null)
{
// Filter out base inventory items and equipment mod items
var rawBotItems = rawBot.Inventory.items.Where(item => item.location != null).ToList();
AddLootItemsToContainerDictionary(rawBotItems, botBackpack._id, botToUpdate.inventory.items.Backpack, "backpack", botToUpdate.botType);
}
var botBackpack = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "Backpack");
if (botBackpack != null)
{
AddLootItemsToContainerDictionary(rawBotItems, botBackpack._id, botToUpdate.inventory.items.Backpack);
}
var botPockets = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "Pockets");
if (botPockets != null)
{
AddLootItemsToContainerDictionary(rawBotItems, botPockets._id, botToUpdate.inventory.items.Pockets);
}
var botPockets = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "Pockets");
if (botPockets != null)
{
AddLootItemsToContainerDictionary(rawBotItems, botPockets._id, botToUpdate.inventory.items.Pockets);
}
var botVest = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "TacticalVest");
if (botVest != null)
{
AddLootItemsToContainerDictionary(rawBotItems, botVest._id, botToUpdate.inventory.items.TacticalVest);
}
var botVest = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "TacticalVest");
if (botVest != null)
{
AddLootItemsToContainerDictionary(rawBotItems, botVest._id, botToUpdate.inventory.items.TacticalVest);
}
var botSecure = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "SecuredContainer");
if (botSecure != null)
{
AddLootItemsToContainerDictionary(rawBotItems, botSecure._id, botToUpdate.inventory.items.SecuredContainer);
}
var botSecure = rawBot.Inventory.items.FirstOrDefault(item => item.slotId == "SecuredContainer");
if (botSecure != null)
// Add generic keys to bosses
if (botToUpdate.botType.IsBoss())
{
var keys = SpecialLootHelper.GetGenericBossKeysDictionary();
foreach (var bosskey in keys)
{
AddLootItemsToContainerDictionary(rawBotItems, botSecure._id, botToUpdate.inventory.items.SecuredContainer);
}
// Add generic keys to bosses
if (botToUpdate.botType.IsBoss())
{
var keys = SpecialLootHelper.GetGenericBossKeysDictionary();
foreach (var bosskey in keys)
if (!botToUpdate.inventory.items.Backpack.ContainsKey(bosskey.Key))
{
if (!botToUpdate.inventory.items.Backpack.ContainsKey(bosskey.Key))
{
botToUpdate.inventory.items.Backpack.Add(bosskey.Key, bosskey.Value);
}
botToUpdate.inventory.items.Backpack.Add(bosskey.Key, bosskey.Value);
}
}
AddSpecialLoot(botToUpdate);
}
AddSpecialLoot(botToUpdate);
}
/// <summary>
@ -119,16 +67,17 @@ namespace Generator
/// <param name="itemsToFilter">Bots inventory items</param>
/// <param name="containerId"></param>
/// <param name="dictToAddTo"></param>
private static void AddLootItemsToContainerDictionary(List<Item> itemsToFilter, string containerId, Dictionary<string, int> dictToAddTo)
private static void AddLootItemsToContainerDictionary(IEnumerable<Common.Models.Input.Item> itemsToFilter, string containerId, Dictionary<string, int> dictToAddTo, string container = "", BotType type = BotType.arenafighterevent)
{
var lootItems = itemsToFilter.Where(item => item.parentId == containerId);
foreach (var itemToAdd in lootItems)
foreach (var itemToAdd in itemsToFilter)
{
if (itemToAdd.parentId != containerId) continue;
if (!dictToAddTo.ContainsKey(itemToAdd._tpl))
{
dictToAddTo[itemToAdd._tpl] = 1;
return;
continue;
}
dictToAddTo[itemToAdd._tpl]++;

View File

@ -8,6 +8,9 @@ using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using Common.Models.Output;
using Common.Models;
using Generator;
using Generator.Helpers.Gear;
namespace Common.Bots;
@ -15,6 +18,118 @@ public static class BotParser
{
private static readonly JsonSerializerOptions serialiserOptions = new() { };
public static List<Bot> Parse(string dumpPath, HashSet<string> botTypes)
{
var stopwatch = Stopwatch.StartNew();
// Build the list of base bot data
var baseBots = new HashSet<Bot>();
foreach (var botType in botTypes)
{
var typeToAdd = (BotType)Enum.Parse(typeof(BotType), botType);
baseBots.Add(new Bot(typeToAdd));
}
DiskHelpers.CreateDirIfDoesntExist(dumpPath);
var botFiles = Directory.GetFiles(dumpPath, "*.json", SearchOption.TopDirectoryOnly);
LoggingHelpers.LogToConsole($"{botFiles.Length} bot dump files found");
// Store a list of parsed bots so we don't parse the same bot twice
int totalDupeCount = 0;
var parsedBotIds = new HashSet<string>();
int i = 0;
foreach (var filePath in botFiles)
{
i++;
if (i % 100 == 0) Console.WriteLine($"Processing file {i}");
ProcessBotFileSync(baseBots, filePath, parsedBotIds, totalDupeCount);
}
// Handle things we can only do once all data has been processed, or only needs to run once per bot type
foreach (var bot in baseBots)
{
// Skip any bots we didn't handle
if (bot.botCount == 0) continue;
BaseBotGenerator.AddDifficulties(bot);
GearChanceHelpers.CalculateModChances(bot);
GearChanceHelpers.CalculateEquipmentModChances(bot);
GearChanceHelpers.CalculateEquipmentChances(bot);
GearChanceHelpers.ApplyModChanceOverrides(bot);
GearChanceHelpers.ApplyEquipmentChanceOverrides(bot);
GearHelpers.ReduceAmmoWeightValues(bot);
GearHelpers.ReduceEquipmentWeightValues(bot.inventory.equipment);
GearHelpers.ReduceWeightValues(bot.appearance.voice);
GearHelpers.ReduceWeightValues(bot.appearance.feet);
GearHelpers.ReduceWeightValues(bot.appearance.body);
GearHelpers.ReduceWeightValues(bot.appearance.head);
GearHelpers.ReduceWeightValues(bot.appearance.hands);
}
stopwatch.Stop();
LoggingHelpers.LogToConsole($"{totalDupeCount} dupes were ignored. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
return baseBots.ToList();
}
private static void ProcessBotFileSync(
HashSet<Bot> baseBots,
string filePath,
HashSet<string> parsedBotIds,
int totalDupeCount)
{
var splitFilePath = filePath.Split("\\");
int dupeCount = 0;
List<Datum> bots = [];
try
{
// Parse the bots inside the json file
using (var reader = new StreamReader(filePath))
{
var deSerialisedObject = JsonSerializer.Deserialize<Root>(reader.ReadToEnd(), serialiserOptions);
foreach (var botData in deSerialisedObject.data)
{
// Bot fucks up something, never allow it in
if (botData._id == "6483938c53cc9087c70eae86")
{
Console.WriteLine("oh no");
continue;
}
var baseBot = baseBots.SingleOrDefault(bot => bot.botType.ToString().Equals(botData.Info.Settings.Role, StringComparison.OrdinalIgnoreCase));
if (baseBot == null)
{
//Console.WriteLine($"Skipping {botData._id} due to unknown role {botData.Info.Settings.Role}");
continue;
}
// Add bot if not already added
if (!parsedBotIds.Add(botData._id))
{
dupeCount++;
}
baseBot.botCount += 1;
BaseBotGenerator.UpdateBaseDetails(baseBot, botData);
BotGearGenerator.AddGear(baseBot, botData);
BotLootGenerator.AddLoot(baseBot, botData);
BotChancesGenerator.AddChances(baseBot, botData);
}
}
}
catch (Exception ex)
{
Console.WriteLine($"File parse fucked up: {filePath}");
throw;
}
totalDupeCount += dupeCount;
}
public static async Task<List<Datum>> ParseAsync(string dumpPath, HashSet<string> botTypes)
{
var stopwatch = Stopwatch.StartNew();

View File

@ -7,75 +7,102 @@ namespace Generator.Helpers.Gear
{
public static class GearChanceHelpers
{
public static void CalculateModChances(Bot bot, List<Datum> baseBots)
private static Dictionary<string, Dictionary<string, int>> weaponModCount = new Dictionary<string, Dictionary<string, int>>();
private static Dictionary<string, Dictionary<string, int>> weaponSlotCount = new Dictionary<string, Dictionary<string, int>>();
private static Dictionary<string, Dictionary<string, int>> equipmentModCount = new Dictionary<string, Dictionary<string, int>>();
private static Dictionary<string, Dictionary<string, int>> equipmentSlotCount = new Dictionary<string, Dictionary<string, int>>();
public static void AddModChances(Bot bot, Datum baseBot)
{
// TODO: Further split these counts by equipment slot? (ex. "FirstPrimaryWeapon", "Holster", etc.)
var validSlots = new List<string> { "FirstPrimaryWeapon", "SecondPrimaryWeapon", "Holster" };
var modCounts = new Dictionary<string, int>();
var slotCounts = new Dictionary<string, int>();
foreach (var baseBot in baseBots)
if (!weaponModCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var modCounts))
{
var validParents = new List<string>();
foreach (var inventoryItem in baseBot.Inventory.items)
modCounts = new Dictionary<string, int>();
weaponModCount.Add(baseBot.Info.Settings.Role.ToLower(), modCounts);
}
if (!weaponSlotCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var slotCounts))
{
slotCounts = new Dictionary<string, int>();
weaponSlotCount.Add(baseBot.Info.Settings.Role.ToLower(), slotCounts);
}
var validParents = new List<string>();
foreach (var inventoryItem in baseBot.Inventory.items)
{
if (validSlots.Contains(inventoryItem.slotId))
{
if (validSlots.Contains(inventoryItem.slotId))
validParents.Add(inventoryItem._id);
}
else if (validParents.Contains(inventoryItem.parentId))
{
validParents.Add(inventoryItem._id);
}
else
{
continue;
}
var template = ItemTemplateHelper.GetTemplateById(inventoryItem._tpl);
var parentTemplate = ItemTemplateHelper.GetTemplateById(baseBot.Inventory.items.Single(i => i._id == inventoryItem.parentId)._tpl);
if (!(parentTemplate?._props?.Slots?.FirstOrDefault(slot => slot._name == inventoryItem.slotId)?._required ?? false))
{
if (modCounts.ContainsKey(inventoryItem.slotId.ToLower()))
{
validParents.Add(inventoryItem._id);
}
else if (validParents.Contains(inventoryItem.parentId))
{
validParents.Add(inventoryItem._id);
modCounts[inventoryItem.slotId.ToLower()]++;
}
else
{
continue;
}
var template = ItemTemplateHelper.GetTemplateById(inventoryItem._tpl);
var parentTemplate = ItemTemplateHelper.GetTemplateById(baseBot.Inventory.items.Single(i => i._id == inventoryItem.parentId)._tpl);
if (!(parentTemplate?._props?.Slots?.FirstOrDefault(slot => slot._name == inventoryItem.slotId)?._required ?? false))
{
if (modCounts.ContainsKey(inventoryItem.slotId.ToLower()))
{
modCounts[inventoryItem.slotId.ToLower()]++;
}
else
{
modCounts.Add(inventoryItem.slotId.ToLower(), 1);
}
}
if ((template?._props?.Slots?.Count ?? 0) < 1)
{
// Item has no slots, nothing to count here
continue;
}
foreach (var slot in template._props.Slots)
{
if (slot._required)
{
continue;
}
if (slot._name.StartsWith("camora"))
{
continue;
}
if (slotCounts.ContainsKey(slot._name.ToLower()))
{
slotCounts[slot._name.ToLower()]++;
}
else
{
slotCounts.Add(slot._name.ToLower(), 1);
}
modCounts.Add(inventoryItem.slotId.ToLower(), 1);
}
}
if ((template?._props?.Slots?.Count ?? 0) < 1)
{
// Item has no slots, nothing to count here
continue;
}
foreach (var slot in template._props.Slots)
{
if (slot._required)
{
continue;
}
if (slot._name.StartsWith("camora"))
{
continue;
}
if (slotCounts.ContainsKey(slot._name.ToLower()))
{
slotCounts[slot._name.ToLower()]++;
}
else
{
slotCounts.Add(slot._name.ToLower(), 1);
}
}
}
}
public static void CalculateModChances(Bot bot)
{
if (!weaponModCount.TryGetValue(bot.botType.ToString(), out var modCounts))
{
modCounts = new Dictionary<string, int>();
weaponModCount.Add(bot.botType.ToString(), modCounts);
}
if (!weaponSlotCount.TryGetValue(bot.botType.ToString(), out var slotCounts))
{
slotCounts = new Dictionary<string, int>();
weaponSlotCount.Add(bot.botType.ToString(), slotCounts);
}
bot.chances.weaponMods = slotCounts.ToDictionary(
@ -83,76 +110,97 @@ namespace Generator.Helpers.Gear
kvp => GetPercent(kvp.Value, modCounts.GetValueOrDefault(kvp.Key)));
}
public static void CalculateEquipmentModChances(Bot bot, List<Datum> baseBots)
public static void AddEquipmentModChances(Bot bot, Datum baseBot)
{
if (!equipmentModCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var modCounts))
{
modCounts = new Dictionary<string, int>();
equipmentModCount.Add(baseBot.Info.Settings.Role.ToLower(), modCounts);
}
if (!equipmentSlotCount.TryGetValue(baseBot.Info.Settings.Role.ToLower(), out var slotCounts))
{
slotCounts = new Dictionary<string, int>();
equipmentSlotCount.Add(baseBot.Info.Settings.Role.ToLower(), slotCounts);
}
// TODO: Further split these counts by equipment slot? (ex. "FirstPrimaryWeapon", "Holster", etc.)
var validSlots = new List<string> { "Headwear", "ArmorVest", "TacticalVest" };
var modCounts = new Dictionary<string, int>();
var slotCounts = new Dictionary<string, int>();
var validParents = new List<string>();
foreach (var baseBot in baseBots)
foreach (var inventoryItem in baseBot.Inventory.items)
{
var validParents = new List<string>();
foreach (var inventoryItem in baseBot.Inventory.items)
if (validSlots.Contains(inventoryItem.slotId))
{
if (validSlots.Contains(inventoryItem.slotId))
validParents.Add(inventoryItem._id);
}
else if (validParents.Contains(inventoryItem.parentId))
{
validParents.Add(inventoryItem._id);
}
else
{
continue;
}
var template = ItemTemplateHelper.GetTemplateById(inventoryItem._tpl);
var parentTemplate = ItemTemplateHelper.GetTemplateById(baseBot.Inventory.items.Single(i => i._id == inventoryItem.parentId)._tpl);
if (!(parentTemplate?._props?.Slots?.FirstOrDefault(slot => slot._name == inventoryItem.slotId)?._required ?? false))
{
if (modCounts.ContainsKey(inventoryItem.slotId.ToLower()))
{
validParents.Add(inventoryItem._id);
}
else if (validParents.Contains(inventoryItem.parentId))
{
validParents.Add(inventoryItem._id);
modCounts[inventoryItem.slotId.ToLower()]++;
}
else
{
continue;
}
var template = ItemTemplateHelper.GetTemplateById(inventoryItem._tpl);
var parentTemplate = ItemTemplateHelper.GetTemplateById(baseBot.Inventory.items.Single(i => i._id == inventoryItem.parentId)._tpl);
if (!(parentTemplate?._props?.Slots?.FirstOrDefault(slot => slot._name == inventoryItem.slotId)?._required ?? false))
{
if (modCounts.ContainsKey(inventoryItem.slotId.ToLower()))
{
modCounts[inventoryItem.slotId.ToLower()]++;
}
else
{
modCounts.Add(inventoryItem.slotId.ToLower(), 1);
}
}
if ((template?._props?.Slots?.Count ?? 0) < 1)
{
// Item has no slots, nothing to count here
continue;
}
foreach (var slot in template._props.Slots)
{
if (slot._required)
{
continue;
}
if (slot._name.StartsWith("camora"))
{
continue;
}
if (slotCounts.ContainsKey(slot._name.ToLower()))
{
slotCounts[slot._name.ToLower()]++;
}
else
{
slotCounts.Add(slot._name.ToLower(), 1);
}
modCounts.Add(inventoryItem.slotId.ToLower(), 1);
}
}
if ((template?._props?.Slots?.Count ?? 0) < 1)
{
// Item has no slots, nothing to count here
continue;
}
foreach (var slot in template._props.Slots)
{
if (slot._required)
{
continue;
}
if (slot._name.StartsWith("camora"))
{
continue;
}
if (slotCounts.ContainsKey(slot._name.ToLower()))
{
slotCounts[slot._name.ToLower()]++;
}
else
{
slotCounts.Add(slot._name.ToLower(), 1);
}
}
}
}
public static void CalculateEquipmentModChances(Bot bot)
{
if (!equipmentModCount.TryGetValue(bot.botType.ToString(), out var modCounts))
{
modCounts = new Dictionary<string, int>();
equipmentModCount.Add(bot.botType.ToString(), modCounts);
}
if (!equipmentSlotCount.TryGetValue(bot.botType.ToString(), out var slotCounts))
{
slotCounts = new Dictionary<string, int>();
equipmentSlotCount.Add(bot.botType.ToString(), slotCounts);
}
bot.chances.equipmentMods = slotCounts.ToDictionary(
@ -245,47 +293,47 @@ namespace Generator.Helpers.Gear
weightsData["grenades"]);
}
public static void CalculateEquipmentChances(Bot bot, List<Datum> baseBots)
public static void AddEquipmentChances(Bot bot, Datum baseBot)
{
// TODO: Convert to dynamic?
var totalBotsCount = baseBots.Count;
int headwearCount = 0, earCount = 0, faceCoverCount = 0, armorVestCount = 0, eyeWearCount = 0, armBandCount = 0,
tacticalVestCount = 0, backpackCount = 0, firstPrimaryCount = 0, secondPrimaryCount = 0, holsterCount = 0,
scabbardCount = 0, pocketsCount = 0, securedContainerCount = 0;
bot.chances.equipment.Headwear += baseBot.Inventory.items.Count(x => x.slotId == "Headwear");
bot.chances.equipment.Earpiece += baseBot.Inventory.items.Count(x => x.slotId == "Earpiece");
bot.chances.equipment.FaceCover += baseBot.Inventory.items.Count(x => x.slotId == "FaceCover");
bot.chances.equipment.ArmorVest += baseBot.Inventory.items.Count(x => x.slotId == "ArmorVest");
bot.chances.equipment.Eyewear += baseBot.Inventory.items.Count(x => x.slotId == "Eyewear");
bot.chances.equipment.ArmBand += baseBot.Inventory.items.Count(x => x.slotId == "ArmBand");
bot.chances.equipment.TacticalVest += baseBot.Inventory.items.Count(x => x.slotId == "TacticalVest");
bot.chances.equipment.Backpack += baseBot.Inventory.items.Count(x => x.slotId == "Backpack");
bot.chances.equipment.FirstPrimaryWeapon += baseBot.Inventory.items.Count(x => x.slotId == "FirstPrimaryWeapon");
bot.chances.equipment.SecondPrimaryWeapon += baseBot.Inventory.items.Count(x => x.slotId == "SecondPrimaryWeapon");
bot.chances.equipment.Holster += baseBot.Inventory.items.Count(x => x.slotId == "Holster");
bot.chances.equipment.Scabbard += baseBot.Inventory.items.Count(x => x.slotId == "Scabbard");
bot.chances.equipment.Pockets += baseBot.Inventory.items.Count(x => x.slotId == "Pockets");
bot.chances.equipment.SecuredContainer += baseBot.Inventory.items.Count(x => x.slotId == "SecuredContainer");
}
foreach (var baseBot in baseBots)
public static void CalculateEquipmentChances(Bot bot)
{
if (bot.botCount == 0)
{
headwearCount += baseBot.Inventory.items.Count(x => x.slotId == "Headwear");
earCount += baseBot.Inventory.items.Count(x => x.slotId == "Earpiece");
faceCoverCount += baseBot.Inventory.items.Count(x => x.slotId == "FaceCover");
armorVestCount += baseBot.Inventory.items.Count(x => x.slotId == "ArmorVest");
eyeWearCount += baseBot.Inventory.items.Count(x => x.slotId == "Eyewear");
armBandCount += baseBot.Inventory.items.Count(x => x.slotId == "ArmBand");
tacticalVestCount += baseBot.Inventory.items.Count(x => x.slotId == "TacticalVest");
backpackCount += baseBot.Inventory.items.Count(x => x.slotId == "Backpack");
firstPrimaryCount += baseBot.Inventory.items.Count(x => x.slotId == "FirstPrimaryWeapon");
secondPrimaryCount += baseBot.Inventory.items.Count(x => x.slotId == "SecondPrimaryWeapon");
holsterCount += baseBot.Inventory.items.Count(x => x.slotId == "Holster");
scabbardCount += baseBot.Inventory.items.Count(x => x.slotId == "Scabbard");
pocketsCount += baseBot.Inventory.items.Count(x => x.slotId == "Pockets");
securedContainerCount += baseBot.Inventory.items.Count(x => x.slotId == "SecuredContainer");
// No bots, don't do anything
return;
}
bot.chances.equipment = new EquipmentChances(
GetPercent(totalBotsCount, headwearCount),
GetPercent(totalBotsCount, earCount),
GetPercent(totalBotsCount, faceCoverCount),
GetPercent(totalBotsCount, armorVestCount),
GetPercent(totalBotsCount, eyeWearCount),
GetPercent(totalBotsCount, armBandCount),
GetPercent(totalBotsCount, tacticalVestCount),
GetPercent(totalBotsCount, backpackCount),
GetPercent(totalBotsCount, firstPrimaryCount),
GetPercent(totalBotsCount, secondPrimaryCount),
GetPercent(totalBotsCount, holsterCount),
GetPercent(totalBotsCount, scabbardCount),
GetPercent(totalBotsCount, pocketsCount),
GetPercent(totalBotsCount, securedContainerCount));
GetPercent(bot.botCount, bot.chances.equipment.Headwear),
GetPercent(bot.botCount, bot.chances.equipment.Earpiece),
GetPercent(bot.botCount, bot.chances.equipment.FaceCover),
GetPercent(bot.botCount, bot.chances.equipment.ArmorVest),
GetPercent(bot.botCount, bot.chances.equipment.Eyewear),
GetPercent(bot.botCount, bot.chances.equipment.ArmBand),
GetPercent(bot.botCount, bot.chances.equipment.TacticalVest),
GetPercent(bot.botCount, bot.chances.equipment.Backpack),
GetPercent(bot.botCount, bot.chances.equipment.FirstPrimaryWeapon),
GetPercent(bot.botCount, bot.chances.equipment.SecondPrimaryWeapon),
GetPercent(bot.botCount, bot.chances.equipment.Holster),
GetPercent(bot.botCount, bot.chances.equipment.Scabbard),
GetPercent(bot.botCount, bot.chances.equipment.Pockets),
GetPercent(bot.botCount, bot.chances.equipment.SecuredContainer));
}
private static int GetPercent(int total, int count)

View File

@ -65,7 +65,6 @@ namespace Generator.Helpers.Gear
internal static void AddAmmo(Bot botToUpdate, Datum bot)
{
//var weightService = new WeightingService();
foreach (var ammo in bot.Inventory.items.Where(
x => x.slotId != null
&& (x.slotId == "patron_in_weapon"
@ -117,7 +116,6 @@ namespace Generator.Helpers.Gear
public static void AddEquippedGear(Bot botToUpdate, Datum bot)
{
// add equipped gear
var weightService = new WeightingService();
foreach (var inventoryItem in bot.Inventory.items.Where(x=>x.slotId != null))
{
switch (inventoryItem.slotId?.ToLower())

View File

@ -1,5 +1,7 @@
using Common.Models.Input;
using Common.Models.Output;
using Generator.Helpers;
using System.Diagnostics;
namespace Generator;
@ -7,6 +9,9 @@ internal static class Program
{
internal static async Task Main(string[] args)
{
var stopwatch = Stopwatch.StartNew();
LoggingHelpers.LogToConsole("Started processing bots");
// Create list of bots we want to process
string[] botTypes = {
"assault",
@ -72,38 +77,19 @@ internal static class Program
// Read raw bot dumps and turn into c# objects
var workingPath = Directory.GetCurrentDirectory();
var dumpPath = $"{workingPath}//dumps";
var parsedBots = await BotParser.ParseAsync(dumpPath, botTypes.ToHashSet());
List<Bot> bots = BotParser.Parse(dumpPath, botTypes.ToHashSet());
// Put in dictionary for better use later on
var rawBotsCache = new Dictionary<string, List<Datum>>(45);
foreach (var rawBot in parsedBots)
{
if (rawBotsCache.TryGetValue(rawBot.Info.Settings.Role.ToLower(), out var botList))
{
botList.Add(rawBot);
continue;
}
// Doesnt exist, add key and bot
rawBotsCache.Add(rawBot.Info.Settings.Role.ToLower(), new List<Datum> { rawBot });
}
if (parsedBots.Count == 0)
if (bots.Count == 0)
{
LoggingHelpers.LogToConsole("No bots found, unable to continue");
LoggingHelpers.LogToConsole("Check your dumps are in 'Generator\\bin\\Debug\\net6.0\\dumps' and start with 'resp.' NOT 'req.'");
return;
}
// Generate the base bot class with basic details (health/body part hp etc) and then append everything else
var bots = BaseBotGenerator.GenerateBaseDetails(parsedBots, workingPath, botTypes)
.AddGear(rawBotsCache) // Add weapons/armor
.AddLoot(rawBotsCache)
.AddChances(rawBotsCache); // Add mod/equipment chances
// Output bot to json file
var jsonWriter = new JsonWriter(workingPath, "output");
jsonWriter.WriteJson(bots.ToList());
jsonWriter.WriteJson(bots);
stopwatch.Stop();
LoggingHelpers.LogToConsole($"Finished processing bots. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
}
}

View File

@ -14,11 +14,14 @@ namespace Generator.Weighting
public class WeightingService
{
private readonly Dictionary<BotType, Weightings> _weights;
private readonly Dictionary<string, Dictionary<string, GenerationWeightData>> _generationWeights;
private static Dictionary<BotType, Weightings> _weights = null;
private static Dictionary<string, Dictionary<string, GenerationWeightData>> _generationWeights = null;
public WeightingService()
{
// Cache the loaded data
if (_weights != null && _generationWeights != null) return;
var assetsPath = $"{Directory.GetCurrentDirectory()}\\Assets";
var weightsFilePath = $"{assetsPath}\\weights.json";
if (!File.Exists(weightsFilePath))