Removed 7z preprocessing functionality and simplified file processing

source/LootDumpProcessor/Config/config.json

@@ -9,13 +9,6 @@
     "fileDataStorageTempLocation": "M:\\Development\\spt\\dumps\\cache"
   },
   "readerConfig": {
-    "preProcessorConfig": {
-      "preProcessors": [
-        "SevenZip"
-      ],
-      "preProcessorTempFolder": "M:\\Development\\spt\\dumps\\temp",
-      "cleanupTempFolderAfterProcess": true
-    },
     "intakeReaderConfig": {
       "maxDumpsPerMap": 5000,
       "ignoredDumpLocations": [

source/LootDumpProcessor/LootDumpProcessor.csproj

@@ -11,10 +11,8 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="7z.Libs" Version="21.7.0"/>
         <PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0"/>
         <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0"/>
-        <PackageReference Include="SevenZipSharp.Interop" Version="19.1.0"/>
         <PackageReference Include="System.Configuration.ConfigurationManager" Version="7.0.0"/>
         <PackageReference Include="YamlDotNet" Version="13.0.0"/>
     </ItemGroup>

source/LootDumpProcessor/Model/Config/PreProcessorConfig.cs

@@ -1,12 +0,0 @@
-using System.Text.Json.Serialization;
-
-
-namespace LootDumpProcessor.Model.Config;
-
-public class PreProcessorConfig
-{
-    [JsonPropertyName("preProcessorTempFolder")] public string? PreProcessorTempFolder { get; set; }
-
-
-    [JsonPropertyName("cleanupTempFolderAfterProcess")] public bool CleanupTempFolderAfterProcess { get; set; } = true;
-}

source/LootDumpProcessor/Model/Config/ReaderConfig.cs

@@ -5,17 +5,8 @@ namespace LootDumpProcessor.Model.Config;
 
 public class ReaderConfig
 {
-    [JsonPropertyName("intakeReaderConfig")] public IntakeReaderConfig? IntakeReaderConfig { get; set; }
+    public IntakeReaderConfig? IntakeReaderConfig { get; set; }
-
+    public List<string>? DumpFilesLocation { get; set; }
-
+    public string? ThresholdDate { get; set; }
-    [JsonPropertyName("preProcessorConfig")] public PreProcessorConfig? PreProcessorConfig { get; set; }
+    public bool ProcessSubFolders { get; set; }
-
-
-    [JsonPropertyName("dumpFilesLocation")] public List<string>? DumpFilesLocation { get; set; }
-
-
-    [JsonPropertyName("thresholdDate")] public string? ThresholdDate { get; set; }
-
-
-    [JsonPropertyName("processSubFolders")] public bool ProcessSubFolders { get; set; }
 }

source/LootDumpProcessor/Process/QueuePipeline.cs

@@ -7,7 +7,6 @@ using LootDumpProcessor.Process.Processor.DumpProcessor;
 using LootDumpProcessor.Process.Processor.FileProcessor;
 using LootDumpProcessor.Process.Reader.Filters;
 using LootDumpProcessor.Process.Reader.Intake;
-using LootDumpProcessor.Process.Reader.PreProcess;
 using LootDumpProcessor.Process.Writer;
 using LootDumpProcessor.Serializers.Json;
 using LootDumpProcessor.Storage;
@@ -17,8 +16,8 @@ using Microsoft.Extensions.Logging;
 namespace LootDumpProcessor.Process;
 
 public class QueuePipeline(
-    IFileProcessor fileProcessor, IDumpProcessor dumpProcessor, ILogger<QueuePipeline> logger,
+    IFileProcessor fileProcessor, IDumpProcessor dumpProcessor, ILogger<QueuePipeline> logger, IFileFilter fileFilter,
-    IPreProcessReader preProcessReader, IFileFilter fileFilter, IIntakeReader intakeReader
+    IIntakeReader intakeReader
 )
     : IPipeline
 {
@@ -30,9 +29,6 @@ public class QueuePipeline(
 
     private readonly ILogger<QueuePipeline> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
 
-    private readonly IPreProcessReader _preProcessReader =
-        preProcessReader ?? throw new ArgumentNullException(nameof(preProcessReader));
-
     private readonly IFileFilter _fileFilter = fileFilter ?? throw new ArgumentNullException(nameof(fileFilter));
 
     private readonly IIntakeReader
@@ -60,7 +56,6 @@ public class QueuePipeline(
         }
         finally
         {
-            _preProcessReader.Dispose();
             stopwatch.Stop();
             _logger.LogInformation("Dumps processed in {@Time}", stopwatch.Elapsed);
         }
@@ -92,24 +87,9 @@ public class QueuePipeline(
             var extensionFull = Path.GetExtension(file);
             if (extensionFull.Length > 1)
             {
-                // if the preprocessor found something new to process or generated new files, add them to the queue
+                if (_fileFilter.Accept(file)) gatheredFiles.Add(file);
-                if (extensionFull == "7z" &&
-                    _preProcessReader.TryPreProcess(file, out var outputFiles, out var outputDirectories))
-                {
-                    // all new directories need to be processed as well
-                    GetFileQueue(outputDirectories).ToList().ForEach(queuedFilesToProcess.Enqueue);
-                    // all output files need to be queued as well
-                    outputFiles.ForEach(queuedFilesToProcess.Enqueue);
-                }
 
-                else
+                else gatheredFiles.Add(file);
-                {
-                    // if there is no preprocessor for the file, means its ready to filter or accept
-
-                    if (_fileFilter.Accept(file)) gatheredFiles.Add(file);
-
-                    else gatheredFiles.Add(file);
-                }
             }
             else
             {

source/LootDumpProcessor/Process/Reader/PreProcess/AbstractPreProcessReader.cs

@@ -1,40 +0,0 @@
-using Microsoft.Extensions.Logging;
-
-namespace LootDumpProcessor.Process.Reader.PreProcess;
-
-public abstract class AbstractPreProcessReader : IPreProcessReader
-{
-    protected readonly string _tempFolder;
-
-    public AbstractPreProcessReader(ILogger logger)
-    {
-        var tempFolder = LootDumpProcessorContext.GetConfig().ReaderConfig.PreProcessorConfig?.PreProcessorTempFolder;
-        if (string.IsNullOrEmpty(tempFolder))
-        {
-            tempFolder = GetBaseDirectory();
-            logger.LogWarning(
-                "No temp folder was assigned preProcessorTempFolder in PreProcessorConfig, defaulting to {tempFolder}",
-                tempFolder
-            );
-        }
-
-        // Cleanup the temp directory before starting the process
-        if (Directory.Exists(tempFolder)) Directory.Delete(tempFolder, true);
-
-        Directory.CreateDirectory(tempFolder);
-
-        _tempFolder = tempFolder;
-    }
-
-    public abstract string GetHandleExtension();
-    public abstract bool TryPreProcess(string file, out List<string> files, out List<string> directories);
-
-    protected string GetBaseDirectory() =>
-        $@"{Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)}\SPT\tmp\PreProcessor";
-
-    public void Dispose()
-    {
-        if (LootDumpProcessorContext.GetConfig().ReaderConfig.PreProcessorConfig?.CleanupTempFolderAfterProcess ?? true)
-            Directory.Delete(_tempFolder, true);
-    }
-}

source/LootDumpProcessor/Process/Reader/PreProcess/IPreProcessReader.cs

@@ -1,11 +0,0 @@
-namespace LootDumpProcessor.Process.Reader.PreProcess;
-
-public interface IPreProcessReader
-{
-    string GetHandleExtension();
-
-    bool TryPreProcess(string file, out List<string> files, out List<string> directories);
-
-    // Custom dispose, not IDisposable
-    void Dispose();
-}

source/LootDumpProcessor/Process/Reader/PreProcess/PreProcessReaderTypes.cs

@@ -1,6 +0,0 @@
-namespace LootDumpProcessor.Process.Reader.PreProcess;
-
-public enum PreProcessReaderTypes
-{
-    SevenZip
-}

source/LootDumpProcessor/Process/Reader/PreProcess/SevenZipPreProcessReader.cs

@@ -1,48 +0,0 @@
-using LootDumpProcessor.Process.Reader.PreProcess;
-using SevenZip;
-using Microsoft.Extensions.Logging;
-
-public class SevenZipPreProcessReader : AbstractPreProcessReader
-{
-    private readonly ILogger<SevenZipPreProcessReader> _logger;
-
-    public SevenZipPreProcessReader(ILogger<SevenZipPreProcessReader> logger) : base(logger)
-    {
-        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
-    }
-
-    public override string GetHandleExtension() => "7z";
-
-    static SevenZipPreProcessReader()
-    {
-        SevenZipBase.SetLibraryPath("./x64/7z.dll");
-    }
-
-    public override bool TryPreProcess(string file, out List<string> files, out List<string> directories)
-    {
-        files = new List<string>();
-        directories = new List<string>();
-
-        var fileRaw = Path.GetFileNameWithoutExtension(file);
-        // SevenZip library doesn't handle forward slashes properly
-        var outPath = $"{_tempFolder}\\{fileRaw}".Replace("/", "\\");
-
-        _logger.LogInformation("Unzipping {File} into temp path {OutPath}, this may take a while...", file, outPath);
-
-        var extractor = new SevenZipExtractor(file);
-
-        // Log progress in debug mode
-        extractor.Extracting += (_, args) =>
-        {
-            if (args.PercentDone % 10 == 0) _logger.LogDebug("Unzip progress: {PercentDone}%", args.PercentDone);
-        };
-
-        extractor.ExtractArchive(outPath);
-        _logger.LogInformation("Finished unzipping {File} into temp path {OutPath}", file, outPath);
-
-        files = Directory.GetFiles(outPath).ToList();
-        directories = Directory.GetDirectories(outPath).ToList();
-
-        return true;
-    }
-}

source/LootDumpProcessor/Program.cs

@@ -7,7 +7,6 @@ using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
 using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
 using LootDumpProcessor.Process.Reader.Filters;
 using LootDumpProcessor.Process.Reader.Intake;
-using LootDumpProcessor.Process.Reader.PreProcess;
 using LootDumpProcessor.Storage;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -46,7 +45,6 @@ public static class Program
 
         services.AddTransient<IIntakeReader, JsonFileIntakeReader>();
         services.AddTransient<IFileFilter, JsonDumpFileFilter>();
-        services.AddTransient<IPreProcessReader, SevenZipPreProcessReader>();
         services.AddTransient<IFileProcessor, FileProcessor>();
         services.AddTransient<IDumpProcessor, MultithreadSteppedDumpProcessor>();
         services.AddTransient<IPipeline, QueuePipeline>();