From fa596241724b14cb2c12d0b77a446e934e3e7b6f Mon Sep 17 00:00:00 2001 From: Chomp Date: Fri, 26 Nov 2021 14:50:07 +0000 Subject: [PATCH] Update bot parsing code to use Parallel.ForEachAsync + make parser static + make parser async --- Common/Bot/BotParser.cs | 154 ++++++++++++++++++++----------------- Generator/Program.cs | 56 ++++++-------- UniqueTemplates/Program.cs | 122 ++++++++++++++--------------- 3 files changed, 165 insertions(+), 167 deletions(-) diff --git a/Common/Bot/BotParser.cs b/Common/Bot/BotParser.cs index b38fdca..f280c78 100644 --- a/Common/Bot/BotParser.cs +++ b/Common/Bot/BotParser.cs @@ -1,95 +1,109 @@ using Common.Models.Input; -using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; +using System.Text.Json; using System.Threading.Tasks; -namespace Common.Bots +namespace Common.Bots; + +public static class BotParser { - public class BotParser + static JsonSerializerOptions serialiserOptions = new JsonSerializerOptions { }; + + public static async Task> ParseAsync(string dumpPath) { - private readonly string _dumpPath; + var stopwatch = Stopwatch.StartNew(); - public BotParser(string dumpPath) + DiskHelpers.CreateDirIfDoesntExist(dumpPath); + + var botFiles = Directory.GetFiles(dumpPath, "*.json", SearchOption.TopDirectoryOnly).ToList(); + LoggingHelpers.LogToConsole($"{botFiles.Count} bot dump files found"); + + var parsedBotsDict = new Dictionary(10000); + int totalDupeCount = 0; + + ParallelOptions parallelOptions = new() { - _dumpPath = dumpPath; - } - - public List Parse() + MaxDegreeOfParallelism = Environment.ProcessorCount + }; + await Parallel.ForEachAsync(botFiles, parallelOptions, async(file, token) => { - var stopwatch = Stopwatch.StartNew(); + var splitFilePath = file.Split("\\"); - var failedFilesCount = 0; - DiskHelpers.CreateDirIfDoesntExist(_dumpPath); + int dupeCount = 0; + var rawInputString = await ReadFileContentsAsync(file); - var botFiles = Directory.GetFiles(_dumpPath, "*.json", SearchOption.TopDirectoryOnly).ToList(); - Console.WriteLine($"{botFiles.Count} bot dump files found"); - - var parsedBots = new List(); - Parallel.ForEach(botFiles, file => { - var splitFile = file.Split("\\"); - - var json = File.ReadAllText(file); - try - { - json = PruneMalformedBsgJson(json, splitFile.Last()); - - var bots = ParseJson(json); - - if (bots == null || bots.Count == 0) - { - Console.WriteLine($"skipping file: {splitFile.Last()}. no bots found, "); - return; - } - - Console.WriteLine($"parsing: {bots.Count} bots in file {splitFile.Last()}"); - foreach (var bot in bots) - { - parsedBots.Add(bot); - } - } - catch (JsonException jex) - { - failedFilesCount++; - Console.WriteLine($"JSON Error message: {jex.Message} || file: {splitFile.Last()}"); - } - }); - - stopwatch.Stop(); - LoggingHelpers.LogToConsole($"Cleaned and Parsed: {parsedBots.Count} bots. Failed: {failedFilesCount}. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds"); - - return parsedBots; - } - - private string PruneMalformedBsgJson(string json, string fileName) - { - // Bsg send json where an item has a location of 1 but it should be an object with x/y/z coords - var o = JObject.Parse(json); - var jItemsToReplace = o.SelectTokens("$.data[*].Inventory.items[?(@.location == 1)].location"); - //var jItemsToReplace = o.SelectTokens("$.data[*].Inventory.items[?(@.location == 1 && @.slotId == 'cartridges')].location"); - - if (jItemsToReplace != null && jItemsToReplace.Any()) + var json = rawInputString; + if (rawInputString.Contains("location\":1,")) { - LoggingHelpers.LogToConsole($"file {fileName} has {jItemsToReplace.Count()} json issues, cleaning up."); - foreach (var item in jItemsToReplace) + json = PruneMalformedBsgJson(rawInputString, splitFilePath.Last()); + } + + var bots = ParseJson(json); + if (bots == null || bots.Count == 0) + { + Console.WriteLine($"skipping file: {splitFilePath.Last()}. no bots found, "); + return; + } + + Console.WriteLine($"parsing: {bots.Count} bots in file {splitFilePath.Last()}"); + foreach (var bot in bots) + { + if (!parsedBotsDict.ContainsKey(bot._id)) { - var obj = new { x = 1, y = 0, r = 0 }; - item.Replace(JToken.FromObject(obj)); + parsedBotsDict.Add(bot._id, bot); + } + else + { + dupeCount++; } } - return o.ToString(); - } + totalDupeCount += dupeCount; + }); - private static List ParseJson(string json) + stopwatch.Stop(); + LoggingHelpers.LogToConsole($"Cleaned and Parsed: {parsedBotsDict.Count} bots. {totalDupeCount} dupes were ignored. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds"); + + return (parsedBotsDict.Select(x => x.Value)).ToList(); + } + + private static async Task ReadFileContentsAsync(string file) + { + using var reader = File.OpenText(file); + return await reader.ReadToEndAsync(); + } + + private static string PruneMalformedBsgJson(string json, string fileName) + { + // Bsg send json where an item has a location of 1 but it should be an object with x/y/z coords + var o = JObject.Parse(json); + var jItemsToReplace = o.SelectTokens("$.data[*].Inventory.items[?(@.location == 1)].location"); + //var jItemsToReplace = o.SelectTokens("$.data[*].Inventory.items[?(@.location == 1 && @.slotId == 'cartridges')].location"); + + if (jItemsToReplace != null && jItemsToReplace.Any()) { - var serialisedObject = JsonConvert.DeserializeObject(json); - - return serialisedObject.data; + LoggingHelpers.LogToConsole($"file {fileName} has {jItemsToReplace.Count()} json issues, cleaning up.", ConsoleColor.Yellow); + foreach (var item in jItemsToReplace) + { + var obj = new { x = 1, y = 0, r = 0 }; + item.Replace(JToken.FromObject(obj)); + } } + var returnString = o.ToString(); + + o = null; + jItemsToReplace = null; + + return returnString; + } + + private static List ParseJson(string json) + { + var deSerialisedObject = JsonSerializer.Deserialize(json, serialiserOptions); + return deSerialisedObject.data; } } \ No newline at end of file diff --git a/Generator/Program.cs b/Generator/Program.cs index 6f127aa..c5b6014 100644 --- a/Generator/Program.cs +++ b/Generator/Program.cs @@ -1,17 +1,11 @@ -using Common; -using Common.Bots; -using Common.Models.Output; -using System.IO; -using System.Linq; +namespace Generator; -namespace Generator +internal static class Program { - internal static class Program + internal static async Task Main(string[] args) { - internal static void Main(string[] args) - { - // Create list of bots we want to process - string[] botTypes = { + // Create list of bots we want to process + string[] botTypes = { "assault", "marksman", "pmcBot", @@ -38,28 +32,26 @@ namespace Generator "sectantwarrior", }; - // Read raw bot dumps and turn into c# objects - var workingPath = Directory.GetCurrentDirectory(); - var dumpPath = $"{workingPath}//dumps"; - var botParser = new BotParser(dumpPath); - var parsedBots = botParser.Parse(); + // Read raw bot dumps and turn into c# objects + var workingPath = Directory.GetCurrentDirectory(); + var dumpPath = $"{workingPath}//dumps"; + var parsedBots = await BotParser.ParseAsync(dumpPath); - if (parsedBots.Count == 0) - { - LoggingHelpers.LogToConsole("no bots found, unable to continue"); - LoggingHelpers.LogToConsole("Check your dumps are in 'Generator\\bin\\Debug\\netcoreapp3.1\\dumps' and start with 'resp.' NOT 'req.'"); - return; - } - - // Generate the base bot class with basic details (health/body part hp etc) and then append everything else - var bots = BaseBotGenerator.GenerateBaseDetails(parsedBots, workingPath, botTypes) - .AddGear(parsedBots) // Add weapons/armor - .AddLoot(parsedBots) - .AddChances(parsedBots); // Add mod/equipment chances - - // Output bot to json file - var jsonWriter = new JsonWriter(workingPath, "output"); - jsonWriter.WriteJson(bots.ToList()); + if (parsedBots.Count == 0) + { + LoggingHelpers.LogToConsole("no bots found, unable to continue"); + LoggingHelpers.LogToConsole("Check your dumps are in 'Generator\\bin\\Debug\\netcoreapp3.1\\dumps' and start with 'resp.' NOT 'req.'"); + return; } + + // Generate the base bot class with basic details (health/body part hp etc) and then append everything else + var bots = BaseBotGenerator.GenerateBaseDetails(parsedBots, workingPath, botTypes) + .AddGear(parsedBots) // Add weapons/armor + .AddLoot(parsedBots) + .AddChances(parsedBots); // Add mod/equipment chances + + // Output bot to json file + var jsonWriter = new JsonWriter(workingPath, "output"); + jsonWriter.WriteJson(bots.ToList()); } } diff --git a/UniqueTemplates/Program.cs b/UniqueTemplates/Program.cs index d31ad64..8a3b1c4 100644 --- a/UniqueTemplates/Program.cs +++ b/UniqueTemplates/Program.cs @@ -1,83 +1,75 @@ using Common; using Common.Bots; using Common.Models.Input; -using System; using System.Collections.Generic; using System.IO; using System.Linq; -using UniqueTemplates.Extensions; +using System.Threading.Tasks; -namespace UniqueTemplates +namespace UniqueTemplates; + +public class Program { - public class Program + static async Task Main(string[] args) { - static void Main(string[] args) + // Get the unique bot types that bsg generate from live dumps + + // Read raw bot dumps and turn into c# objects + var workingPath = Directory.GetCurrentDirectory(); + var dumpPath = $"{workingPath}//dumps"; + var parsedBots = await BotParser.ParseAsync(dumpPath); + + if (parsedBots.Count == 0) { - // Get the unique bot types that bsg generate from live dumps - - // Read raw bot dumps and turn into c# objects - var workingPath = Directory.GetCurrentDirectory(); - var dumpPath = $"{workingPath}//dumps"; - var botParser = new BotParser(dumpPath); - var parsedBots = botParser.Parse(); - - if (parsedBots.Count == 0) - { - LoggingHelpers.LogToConsole("no bots found, unable to continue"); - LoggingHelpers.LogToConsole("Check your dumps are in 'Generator\\bin\\Debug\\netcoreapp3.1\\dumps' and start with 'resp.' NOT 'req.'"); - return; - } - - //var dupeCount = 0; - //var botTemplates = new List(); - //foreach (var bot in parsedBots) - //{ - - // if (botTemplates.ContainsBot(bot)) - // { - // dupeCount++; - // continue; - // } - - // botTemplates.Add(bot); - //} - - var dictDupeCount = 0; - var uniqueBotTemplates = new Dictionary(); - foreach (var bot in parsedBots) - { - if (uniqueBotTemplates.ContainsKey(bot._id)) - { - dictDupeCount++; - continue; - } - - uniqueBotTemplates.Add(bot._id, bot); - } - - //LoggingHelpers.LogToConsole($"LIST templates found: {botTemplates.Count}, {dupeCount} bots rejected as duplicate"); - LoggingHelpers.LogToConsole($"DICTIONARY templates found: {uniqueBotTemplates.Count}, {dictDupeCount} bots rejected as duplicate"); - - var groupedBots = uniqueBotTemplates.GroupBy(x => x.Value.Info.Settings.Role).ToList(); - - foreach (var group in groupedBots) - { - var botList = new List(); - foreach (var bot in group) - { - botList.Add(bot.Value); - } - - LoggingHelpers.LogToConsole($"{botList.Count} unique {group.Key} bots found"); - - var jsonWriter = new JsonWriter(workingPath, "output"); - jsonWriter.WriteJson(botList, group.Key); - } + LoggingHelpers.LogToConsole("no bots found, unable to continue"); + LoggingHelpers.LogToConsole("Check your dumps are in 'Generator\\bin\\Debug\\netcoreapp3.1\\dumps' and start with 'resp.' NOT 'req.'"); + return; } + //var dupeCount = 0; + //var botTemplates = new List(); + //foreach (var bot in parsedBots) + //{ + // if (botTemplates.ContainsBot(bot)) + // { + // dupeCount++; + // continue; + // } + // botTemplates.Add(bot); + //} + var dictDupeCount = 0; + var uniqueBotTemplates = new Dictionary(); + foreach (var bot in parsedBots) + { + if (uniqueBotTemplates.ContainsKey(bot._id)) + { + dictDupeCount++; + continue; + } + uniqueBotTemplates.Add(bot._id, bot); + } + + //LoggingHelpers.LogToConsole($"LIST templates found: {botTemplates.Count}, {dupeCount} bots rejected as duplicate"); + LoggingHelpers.LogToConsole($"DICTIONARY templates found: {uniqueBotTemplates.Count}, {dictDupeCount} bots rejected as duplicate"); + + var groupedBots = uniqueBotTemplates.GroupBy(x => x.Value.Info.Settings.Role).ToList(); + + foreach (var group in groupedBots) + { + var botList = new List(); + foreach (var bot in group) + { + botList.Add(bot.Value); + } + + LoggingHelpers.LogToConsole($"{botList.Count} unique {group.Key} bots found"); + + var jsonWriter = new JsonWriter(workingPath, "output"); + jsonWriter.WriteJson(botList, group.Key); + } } }