Update bot parsing code to use Parallel.ForEachAsync + make parser static + make parser async

Common/Bot/BotParser.cs

@@ -1,71 +1,83 @@
 using Common.Models.Input;
-using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
-using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Text.Json;
 using System.Threading.Tasks;
 
-namespace Common.Bots
+namespace Common.Bots;
+
+public static class BotParser
 {
-    public class BotParser
+    static JsonSerializerOptions serialiserOptions = new JsonSerializerOptions { };
-    {
-        private readonly string _dumpPath;
 
-        public BotParser(string dumpPath)
+    public static async Task<List<Datum>> ParseAsync(string dumpPath)
-        {
-            _dumpPath = dumpPath;
-        }
-
-        public List<Datum> Parse()
     {
         var stopwatch = Stopwatch.StartNew();
 
-            var failedFilesCount = 0;
+        DiskHelpers.CreateDirIfDoesntExist(dumpPath);
-            DiskHelpers.CreateDirIfDoesntExist(_dumpPath);
 
-            var botFiles = Directory.GetFiles(_dumpPath, "*.json", SearchOption.TopDirectoryOnly).ToList();
+        var botFiles = Directory.GetFiles(dumpPath, "*.json", SearchOption.TopDirectoryOnly).ToList();
-            Console.WriteLine($"{botFiles.Count} bot dump files found");
+        LoggingHelpers.LogToConsole($"{botFiles.Count} bot dump files found");
 
-            var parsedBots = new List<Datum>();
+        var parsedBotsDict = new Dictionary<string, Datum>(10000);
-            Parallel.ForEach(botFiles, file => {
+        int totalDupeCount = 0;
-                var splitFile = file.Split("\\");
 
-                var json = File.ReadAllText(file);
+        ParallelOptions parallelOptions = new()
-                try
         {
-                    json = PruneMalformedBsgJson(json, splitFile.Last());
+            MaxDegreeOfParallelism = Environment.ProcessorCount
+        };
+        await Parallel.ForEachAsync(botFiles, parallelOptions, async(file, token) =>
+        {
+            var splitFilePath = file.Split("\\");
+
+            int dupeCount = 0;
+            var rawInputString = await ReadFileContentsAsync(file);
+
+            var json = rawInputString;
+            if (rawInputString.Contains("location\":1,"))
+            {
+                json = PruneMalformedBsgJson(rawInputString, splitFilePath.Last());
+            }
 
             var bots = ParseJson(json);
-
             if (bots == null || bots.Count == 0)
             {
-                        Console.WriteLine($"skipping file: {splitFile.Last()}. no bots found, ");
+                Console.WriteLine($"skipping file: {splitFilePath.Last()}. no bots found, ");
                 return;
             }
 
-                    Console.WriteLine($"parsing: {bots.Count} bots in file {splitFile.Last()}");
+            Console.WriteLine($"parsing: {bots.Count} bots in file {splitFilePath.Last()}");
             foreach (var bot in bots)
             {
-                        parsedBots.Add(bot);
+                if (!parsedBotsDict.ContainsKey(bot._id))
-                    }
-                }
-                catch (JsonException jex)
                 {
-                    failedFilesCount++;
+                    parsedBotsDict.Add(bot._id, bot);
-                    Console.WriteLine($"JSON Error message: {jex.Message} || file: {splitFile.Last()}");
                 }
+                else
+                {
+                    dupeCount++;
+                }
+            }
+
+            totalDupeCount += dupeCount;
         });
 
         stopwatch.Stop();
-            LoggingHelpers.LogToConsole($"Cleaned and Parsed: {parsedBots.Count} bots. Failed: {failedFilesCount}. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
+        LoggingHelpers.LogToConsole($"Cleaned and Parsed: {parsedBotsDict.Count} bots. {totalDupeCount} dupes were ignored. Took {LoggingHelpers.LogTimeTaken(stopwatch.Elapsed.TotalSeconds)} seconds");
 
-            return parsedBots;
+        return (parsedBotsDict.Select(x => x.Value)).ToList();
     }
 
-        private string PruneMalformedBsgJson(string json, string fileName)
+    private static async Task<string> ReadFileContentsAsync(string file)
+    {
+        using var reader = File.OpenText(file);
+        return await reader.ReadToEndAsync();
+    }
+
+    private static string PruneMalformedBsgJson(string json, string fileName)
     {
         // Bsg send json where an item has a location of 1 but it should be an object with x/y/z coords
         var o = JObject.Parse(json);
@@ -74,22 +86,24 @@ namespace Common.Bots
 
         if (jItemsToReplace != null && jItemsToReplace.Any())
         {
-                LoggingHelpers.LogToConsole($"file {fileName} has {jItemsToReplace.Count()} json issues, cleaning up.");
+            LoggingHelpers.LogToConsole($"file {fileName} has {jItemsToReplace.Count()} json issues, cleaning up.", ConsoleColor.Yellow);
             foreach (var item in jItemsToReplace)
             {
                 var obj = new { x = 1, y = 0, r = 0 };
                 item.Replace(JToken.FromObject(obj));
             }
         }
+        var returnString = o.ToString();
 
-            return o.ToString();
+        o = null;
+        jItemsToReplace = null;
+
+        return returnString;
     }
 
     private static List<Datum> ParseJson(string json)
     {
-            var serialisedObject = JsonConvert.DeserializeObject<Root>(json);
+        var deSerialisedObject = JsonSerializer.Deserialize<Root>(json, serialiserOptions);
-
+        return deSerialisedObject.data;
-            return serialisedObject.data;
-        }
     }
 }

Generator/Program.cs

@@ -1,14 +1,8 @@
-using Common;
+namespace Generator;
-using Common.Bots;
-using Common.Models.Output;
-using System.IO;
-using System.Linq;
 
-namespace Generator
+internal static class Program
 {
-    internal static class Program
+    internal static async Task Main(string[] args)
-    {
-        internal static void Main(string[] args)
     {
         // Create list of bots we want to process
         string[] botTypes = {
@@ -41,8 +35,7 @@ namespace Generator
         // Read raw bot dumps and turn into c# objects
         var workingPath = Directory.GetCurrentDirectory();
         var dumpPath = $"{workingPath}//dumps";
-            var botParser = new BotParser(dumpPath);
+        var parsedBots = await BotParser.ParseAsync(dumpPath);
-            var parsedBots = botParser.Parse();
 
         if (parsedBots.Count == 0)
         {
@@ -61,5 +54,4 @@ namespace Generator
         var jsonWriter = new JsonWriter(workingPath, "output");
         jsonWriter.WriteJson(bots.ToList());
     }
-    }
 }

UniqueTemplates/Program.cs

@@ -1,25 +1,23 @@
 using Common;
 using Common.Bots;
 using Common.Models.Input;
-using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using UniqueTemplates.Extensions;
+using System.Threading.Tasks;
 
-namespace UniqueTemplates
+namespace UniqueTemplates;
+
+public class Program
 {
-    public class Program
+    static async Task Main(string[] args)
-    {
-        static void Main(string[] args)
     {
         // Get the unique bot types that bsg generate from live dumps
 
         // Read raw bot dumps and turn into c# objects
         var workingPath = Directory.GetCurrentDirectory();
         var dumpPath = $"{workingPath}//dumps";
-            var botParser = new BotParser(dumpPath);
+        var parsedBots = await BotParser.ParseAsync(dumpPath);
-            var parsedBots = botParser.Parse();
 
         if (parsedBots.Count == 0)
         {
@@ -74,10 +72,4 @@ namespace UniqueTemplates
             jsonWriter.WriteJson(botList, group.Key);
         }
     }
-
-
-
-
-
-    }
 }