DrakiaXYZ/BotGenerator
0
0
Fork 0
forked from chomp/BotGenerator
Code Pull Requests Activity

Update bot parsing code to use Parallel.ForEachAsync + make parser static + make parser async

Browse Source
This commit is contained in:
Chomp 2021-11-26 14:50:07 +00:00
parent 4489785cd3
commit fa59624172
3 changed files with 165 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
Common/Bot/BotParser.cs
View File

@ -1,95 +1,109 @@
using Common.Models.Input;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
namespace Common.Bots
namespace Common.Bots;
public static class BotParser
{
public class BotParser
static JsonSerializerOptions serialiserOptions = new JsonSerializerOptions { };
public static async Task<List<Datum>> ParseAsync(string dumpPath)
{
private readonly string _dumpPath;
var stopwatch = Stopwatch.StartNew();
public BotParser(string dumpPath)
DiskHelpers.CreateDirIfDoesntExist(dumpPath);
var botFiles = Directory.GetFiles(dumpPath, "*.json", SearchOption.TopDirectoryOnly).ToList();
LoggingHelpers.LogToConsole($"{botFiles.Count} bot dump files found");
var parsedBotsDict = new Dictionary<string, Datum>(10000);
int totalDupeCount = 0;
ParallelOptions parallelOptions = new()
{
_dumpPath = dumpPath;
}
public List<Datum> Parse()
MaxDegreeOfParallelism = Environment.ProcessorCount
};
await Parallel.ForEachAsync(botFiles, parallelOptions, async(file, token) =>
{
var stopwatch = Stopwatch.StartNew();
var splitFilePath = file.Split("\\");
var failedFilesCount = 0;
DiskHelpers.CreateDirIfDoesntExist(_dumpPath);
int dupeCount = 0;
var rawInputString = await ReadFileContentsAsync(file);
var botFiles = Directory.GetFiles(_dumpPath, "*.json", SearchOption.TopDirectoryOnly).ToList();
Console.WriteLine($"{botFiles.Count} bot dump files found");
var parsedBots = new List<Datum>();
Parallel.ForEach(botFiles, file => {
var splitFile = file.Split("\\");
var json = File.ReadAllText(file);
try
{
json = PruneMalformedBsgJson(json, splitFile.Last());
var bots = ParseJson(json);
if (bots == null || bots.Count == 0)
{
Console.WriteLine($"skipping file: {splitFile.Last()}. no bots found, ");
return;
}
Console.WriteLine($"parsing: {bots.Count} bots in file {splitFile.Last()}");
foreach (var bot in bots)
{
parsedBots.Add(bot);
}
}
catch (JsonException jex)
{
failedFilesCount++;
Console.WriteLine($"JSON Error message: {jex.Message} || file: {splitFile.Last()}");
}
});
stopwatch.Stop();
LoggingHelpers.LogToConsole($"Cleaned and Parsed: {parsedBots.Count} bots. Failed: {failedFilesCount}. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
return parsedBots;
}
private string PruneMalformedBsgJson(string json, string fileName)
{
// Bsg send json where an item has a location of 1 but it should be an object with x/y/z coords
var o = JObject.Parse(json);
var jItemsToReplace = o.SelectTokens("$.data[*].Inventory.items[?(@.location == 1)].location");
//var jItemsToReplace = o.SelectTokens("$.data[*].Inventory.items[?(@.location == 1 && @.slotId == 'cartridges')].location");
if (jItemsToReplace != null && jItemsToReplace.Any())
var json = rawInputString;
if (rawInputString.Contains("location\":1,"))
{
LoggingHelpers.LogToConsole($"file {fileName} has {jItemsToReplace.Count()} json issues, cleaning up.");
foreach (var item in jItemsToReplace)
json = PruneMalformedBsgJson(rawInputString, splitFilePath.Last());
}
var bots = ParseJson(json);
if (bots == null || bots.Count == 0)
{
Console.WriteLine($"skipping file: {splitFilePath.Last()}. no bots found, ");
return;
}
Console.WriteLine($"parsing: {bots.Count} bots in file {splitFilePath.Last()}");
foreach (var bot in bots)
{
if (!parsedBotsDict.ContainsKey(bot._id))
{
var obj = new { x = 1, y = 0, r = 0 };
item.Replace(JToken.FromObject(obj));
parsedBotsDict.Add(bot._id, bot);
}
else
{
dupeCount++;
}
}
return o.ToString();
}
totalDupeCount += dupeCount;
});
private static List<Datum> ParseJson(string json)
stopwatch.Stop();
LoggingHelpers.LogToConsole($"Cleaned and Parsed: {parsedBotsDict.Count} bots. {totalDupeCount} dupes were ignored. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
return (parsedBotsDict.Select(x => x.Value)).ToList();
}
private static async Task<string> ReadFileContentsAsync(string file)
{
using var reader = File.OpenText(file);
return await reader.ReadToEndAsync();
}
private static string PruneMalformedBsgJson(string json, string fileName)
{
// Bsg send json where an item has a location of 1 but it should be an object with x/y/z coords
var o = JObject.Parse(json);
var jItemsToReplace = o.SelectTokens("$.data[*].Inventory.items[?(@.location == 1)].location");
//var jItemsToReplace = o.SelectTokens("$.data[*].Inventory.items[?(@.location == 1 && @.slotId == 'cartridges')].location");
if (jItemsToReplace != null && jItemsToReplace.Any())
{
var serialisedObject = JsonConvert.DeserializeObject<Root>(json);
return serialisedObject.data;
LoggingHelpers.LogToConsole($"file {fileName} has {jItemsToReplace.Count()} json issues, cleaning up.", ConsoleColor.Yellow);
foreach (var item in jItemsToReplace)
{
var obj = new { x = 1, y = 0, r = 0 };
item.Replace(JToken.FromObject(obj));
}
}
var returnString = o.ToString();
o = null;
jItemsToReplace = null;
return returnString;
}
private static List<Datum> ParseJson(string json)
{
var deSerialisedObject = JsonSerializer.Deserialize<Root>(json, serialiserOptions);
return deSerialisedObject.data;
}
}

56
Generator/Program.cs
View File

@ -1,17 +1,11 @@
using Common;
using Common.Bots;
using Common.Models.Output;
using System.IO;
using System.Linq;
namespace Generator;
namespace Generator
internal static class Program
{
internal static class Program
internal static async Task Main(string[] args)
{
internal static void Main(string[] args)
{
// Create list of bots we want to process
string[] botTypes = {
// Create list of bots we want to process
string[] botTypes = {
"assault",
"marksman",
"pmcBot",
@ -38,28 +32,26 @@ namespace Generator
"sectantwarrior",
};
// Read raw bot dumps and turn into c# objects
var workingPath = Directory.GetCurrentDirectory();
var dumpPath = $"{workingPath}//dumps";
var botParser = new BotParser(dumpPath);
var parsedBots = botParser.Parse();
// Read raw bot dumps and turn into c# objects
var workingPath = Directory.GetCurrentDirectory();
var dumpPath = $"{workingPath}//dumps";
var parsedBots = await BotParser.ParseAsync(dumpPath);
if (parsedBots.Count == 0)
{
LoggingHelpers.LogToConsole("no bots found, unable to continue");
LoggingHelpers.LogToConsole("Check your dumps are in 'Generator\\bin\\Debug\\netcoreapp3.1\\dumps' and start with 'resp.' NOT 'req.'");
return;
}
// Generate the base bot class with basic details (health/body part hp etc) and then append everything else
var bots = BaseBotGenerator.GenerateBaseDetails(parsedBots, workingPath, botTypes)
.AddGear(parsedBots) // Add weapons/armor
.AddLoot(parsedBots)
.AddChances(parsedBots); // Add mod/equipment chances
// Output bot to json file
var jsonWriter = new JsonWriter(workingPath, "output");
jsonWriter.WriteJson(bots.ToList());
if (parsedBots.Count == 0)
{
LoggingHelpers.LogToConsole("no bots found, unable to continue");
LoggingHelpers.LogToConsole("Check your dumps are in 'Generator\\bin\\Debug\\netcoreapp3.1\\dumps' and start with 'resp.' NOT 'req.'");
return;
}
// Generate the base bot class with basic details (health/body part hp etc) and then append everything else
var bots = BaseBotGenerator.GenerateBaseDetails(parsedBots, workingPath, botTypes)
.AddGear(parsedBots) // Add weapons/armor
.AddLoot(parsedBots)
.AddChances(parsedBots); // Add mod/equipment chances
// Output bot to json file
var jsonWriter = new JsonWriter(workingPath, "output");
jsonWriter.WriteJson(bots.ToList());
}
}

122
UniqueTemplates/Program.cs
View File

@ -1,83 +1,75 @@
using Common;
using Common.Bots;
using Common.Models.Input;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using UniqueTemplates.Extensions;
using System.Threading.Tasks;
namespace UniqueTemplates
namespace UniqueTemplates;
public class Program
{
public class Program
static async Task Main(string[] args)
{
static void Main(string[] args)
// Get the unique bot types that bsg generate from live dumps
// Read raw bot dumps and turn into c# objects
var workingPath = Directory.GetCurrentDirectory();
var dumpPath = $"{workingPath}//dumps";
var parsedBots = await BotParser.ParseAsync(dumpPath);
if (parsedBots.Count == 0)
{
// Get the unique bot types that bsg generate from live dumps
// Read raw bot dumps and turn into c# objects
var workingPath = Directory.GetCurrentDirectory();
var dumpPath = $"{workingPath}//dumps";
var botParser = new BotParser(dumpPath);
var parsedBots = botParser.Parse();
if (parsedBots.Count == 0)
{
LoggingHelpers.LogToConsole("no bots found, unable to continue");
LoggingHelpers.LogToConsole("Check your dumps are in 'Generator\\bin\\Debug\\netcoreapp3.1\\dumps' and start with 'resp.' NOT 'req.'");
return;
}
//var dupeCount = 0;
//var botTemplates = new List<Datum>();
//foreach (var bot in parsedBots)
//{
// if (botTemplates.ContainsBot(bot))
// {
// dupeCount++;
// continue;
// }
// botTemplates.Add(bot);
//}
var dictDupeCount = 0;
var uniqueBotTemplates = new Dictionary<string, Datum>();
foreach (var bot in parsedBots)
{
if (uniqueBotTemplates.ContainsKey(bot._id))
{
dictDupeCount++;
continue;
}
uniqueBotTemplates.Add(bot._id, bot);
}
//LoggingHelpers.LogToConsole($"LIST templates found: {botTemplates.Count}, {dupeCount} bots rejected as duplicate");
LoggingHelpers.LogToConsole($"DICTIONARY templates found: {uniqueBotTemplates.Count}, {dictDupeCount} bots rejected as duplicate");
var groupedBots = uniqueBotTemplates.GroupBy(x => x.Value.Info.Settings.Role).ToList();
foreach (var group in groupedBots)
{
var botList = new List<Datum>();
foreach (var bot in group)
{
botList.Add(bot.Value);
}
LoggingHelpers.LogToConsole($"{botList.Count} unique {group.Key} bots found");
var jsonWriter = new JsonWriter(workingPath, "output");
jsonWriter.WriteJson(botList, group.Key);
}
LoggingHelpers.LogToConsole("no bots found, unable to continue");
LoggingHelpers.LogToConsole("Check your dumps are in 'Generator\\bin\\Debug\\netcoreapp3.1\\dumps' and start with 'resp.' NOT 'req.'");
return;
}
//var dupeCount = 0;
//var botTemplates = new List<Datum>();
//foreach (var bot in parsedBots)
//{
// if (botTemplates.ContainsBot(bot))
// {
// dupeCount++;
// continue;
// }
// botTemplates.Add(bot);
//}
var dictDupeCount = 0;
var uniqueBotTemplates = new Dictionary<string, Datum>();
foreach (var bot in parsedBots)
{
if (uniqueBotTemplates.ContainsKey(bot._id))
{
dictDupeCount++;
continue;
}
uniqueBotTemplates.Add(bot._id, bot);
}
//LoggingHelpers.LogToConsole($"LIST templates found: {botTemplates.Count}, {dupeCount} bots rejected as duplicate");
LoggingHelpers.LogToConsole($"DICTIONARY templates found: {uniqueBotTemplates.Count}, {dictDupeCount} bots rejected as duplicate");
var groupedBots = uniqueBotTemplates.GroupBy(x => x.Value.Info.Settings.Role).ToList();
foreach (var group in groupedBots)
{
var botList = new List<Datum>();
foreach (var bot in group)
{
botList.Add(bot.Value);
}
LoggingHelpers.LogToConsole($"{botList.Count} unique {group.Key} bots found");
var jsonWriter = new JsonWriter(workingPath, "output");
jsonWriter.WriteJson(botList, group.Key);
}
}
}