diff --git a/source/LootDumpProcessor/Config/config.json b/source/LootDumpProcessor/Config/config.json index 5b8a4fc..7f737e4 100644 --- a/source/LootDumpProcessor/Config/config.json +++ b/source/LootDumpProcessor/Config/config.json @@ -9,13 +9,6 @@ "fileDataStorageTempLocation": "M:\\Development\\spt\\dumps\\cache" }, "readerConfig": { - "preProcessorConfig": { - "preProcessors": [ - "SevenZip" - ], - "preProcessorTempFolder": "M:\\Development\\spt\\dumps\\temp", - "cleanupTempFolderAfterProcess": true - }, "intakeReaderConfig": { "maxDumpsPerMap": 5000, "ignoredDumpLocations": [ diff --git a/source/LootDumpProcessor/LootDumpProcessor.csproj b/source/LootDumpProcessor/LootDumpProcessor.csproj index 36fac6d..3679abe 100644 --- a/source/LootDumpProcessor/LootDumpProcessor.csproj +++ b/source/LootDumpProcessor/LootDumpProcessor.csproj @@ -11,10 +11,8 @@ - - diff --git a/source/LootDumpProcessor/Model/Config/PreProcessorConfig.cs b/source/LootDumpProcessor/Model/Config/PreProcessorConfig.cs deleted file mode 100644 index 23bdfac..0000000 --- a/source/LootDumpProcessor/Model/Config/PreProcessorConfig.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.Text.Json.Serialization; - - -namespace LootDumpProcessor.Model.Config; - -public class PreProcessorConfig -{ - [JsonPropertyName("preProcessorTempFolder")] public string? PreProcessorTempFolder { get; set; } - - - [JsonPropertyName("cleanupTempFolderAfterProcess")] public bool CleanupTempFolderAfterProcess { get; set; } = true; -} \ No newline at end of file diff --git a/source/LootDumpProcessor/Model/Config/ReaderConfig.cs b/source/LootDumpProcessor/Model/Config/ReaderConfig.cs index f51bf33..3690508 100644 --- a/source/LootDumpProcessor/Model/Config/ReaderConfig.cs +++ b/source/LootDumpProcessor/Model/Config/ReaderConfig.cs @@ -5,17 +5,8 @@ namespace LootDumpProcessor.Model.Config; public class ReaderConfig { - [JsonPropertyName("intakeReaderConfig")] public IntakeReaderConfig? IntakeReaderConfig { get; set; } - - - [JsonPropertyName("preProcessorConfig")] public PreProcessorConfig? PreProcessorConfig { get; set; } - - - [JsonPropertyName("dumpFilesLocation")] public List? DumpFilesLocation { get; set; } - - - [JsonPropertyName("thresholdDate")] public string? ThresholdDate { get; set; } - - - [JsonPropertyName("processSubFolders")] public bool ProcessSubFolders { get; set; } + public IntakeReaderConfig? IntakeReaderConfig { get; set; } + public List? DumpFilesLocation { get; set; } + public string? ThresholdDate { get; set; } + public bool ProcessSubFolders { get; set; } } \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/QueuePipeline.cs b/source/LootDumpProcessor/Process/QueuePipeline.cs index 2d13919..de911d4 100644 --- a/source/LootDumpProcessor/Process/QueuePipeline.cs +++ b/source/LootDumpProcessor/Process/QueuePipeline.cs @@ -7,7 +7,6 @@ using LootDumpProcessor.Process.Processor.DumpProcessor; using LootDumpProcessor.Process.Processor.FileProcessor; using LootDumpProcessor.Process.Reader.Filters; using LootDumpProcessor.Process.Reader.Intake; -using LootDumpProcessor.Process.Reader.PreProcess; using LootDumpProcessor.Process.Writer; using LootDumpProcessor.Serializers.Json; using LootDumpProcessor.Storage; @@ -17,8 +16,8 @@ using Microsoft.Extensions.Logging; namespace LootDumpProcessor.Process; public class QueuePipeline( - IFileProcessor fileProcessor, IDumpProcessor dumpProcessor, ILogger logger, - IPreProcessReader preProcessReader, IFileFilter fileFilter, IIntakeReader intakeReader + IFileProcessor fileProcessor, IDumpProcessor dumpProcessor, ILogger logger, IFileFilter fileFilter, + IIntakeReader intakeReader ) : IPipeline { @@ -30,9 +29,6 @@ public class QueuePipeline( private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - private readonly IPreProcessReader _preProcessReader = - preProcessReader ?? throw new ArgumentNullException(nameof(preProcessReader)); - private readonly IFileFilter _fileFilter = fileFilter ?? throw new ArgumentNullException(nameof(fileFilter)); private readonly IIntakeReader @@ -60,7 +56,6 @@ public class QueuePipeline( } finally { - _preProcessReader.Dispose(); stopwatch.Stop(); _logger.LogInformation("Dumps processed in {@Time}", stopwatch.Elapsed); } @@ -92,24 +87,9 @@ public class QueuePipeline( var extensionFull = Path.GetExtension(file); if (extensionFull.Length > 1) { - // if the preprocessor found something new to process or generated new files, add them to the queue - if (extensionFull == "7z" && - _preProcessReader.TryPreProcess(file, out var outputFiles, out var outputDirectories)) - { - // all new directories need to be processed as well - GetFileQueue(outputDirectories).ToList().ForEach(queuedFilesToProcess.Enqueue); - // all output files need to be queued as well - outputFiles.ForEach(queuedFilesToProcess.Enqueue); - } + if (_fileFilter.Accept(file)) gatheredFiles.Add(file); - else - { - // if there is no preprocessor for the file, means its ready to filter or accept - - if (_fileFilter.Accept(file)) gatheredFiles.Add(file); - - else gatheredFiles.Add(file); - } + else gatheredFiles.Add(file); } else { diff --git a/source/LootDumpProcessor/Process/Reader/PreProcess/AbstractPreProcessReader.cs b/source/LootDumpProcessor/Process/Reader/PreProcess/AbstractPreProcessReader.cs deleted file mode 100644 index 0af0ba4..0000000 --- a/source/LootDumpProcessor/Process/Reader/PreProcess/AbstractPreProcessReader.cs +++ /dev/null @@ -1,40 +0,0 @@ -using Microsoft.Extensions.Logging; - -namespace LootDumpProcessor.Process.Reader.PreProcess; - -public abstract class AbstractPreProcessReader : IPreProcessReader -{ - protected readonly string _tempFolder; - - public AbstractPreProcessReader(ILogger logger) - { - var tempFolder = LootDumpProcessorContext.GetConfig().ReaderConfig.PreProcessorConfig?.PreProcessorTempFolder; - if (string.IsNullOrEmpty(tempFolder)) - { - tempFolder = GetBaseDirectory(); - logger.LogWarning( - "No temp folder was assigned preProcessorTempFolder in PreProcessorConfig, defaulting to {tempFolder}", - tempFolder - ); - } - - // Cleanup the temp directory before starting the process - if (Directory.Exists(tempFolder)) Directory.Delete(tempFolder, true); - - Directory.CreateDirectory(tempFolder); - - _tempFolder = tempFolder; - } - - public abstract string GetHandleExtension(); - public abstract bool TryPreProcess(string file, out List files, out List directories); - - protected string GetBaseDirectory() => - $@"{Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)}\SPT\tmp\PreProcessor"; - - public void Dispose() - { - if (LootDumpProcessorContext.GetConfig().ReaderConfig.PreProcessorConfig?.CleanupTempFolderAfterProcess ?? true) - Directory.Delete(_tempFolder, true); - } -} \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/Reader/PreProcess/IPreProcessReader.cs b/source/LootDumpProcessor/Process/Reader/PreProcess/IPreProcessReader.cs deleted file mode 100644 index bee44c4..0000000 --- a/source/LootDumpProcessor/Process/Reader/PreProcess/IPreProcessReader.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace LootDumpProcessor.Process.Reader.PreProcess; - -public interface IPreProcessReader -{ - string GetHandleExtension(); - - bool TryPreProcess(string file, out List files, out List directories); - - // Custom dispose, not IDisposable - void Dispose(); -} \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/Reader/PreProcess/PreProcessReaderTypes.cs b/source/LootDumpProcessor/Process/Reader/PreProcess/PreProcessReaderTypes.cs deleted file mode 100644 index 6cca332..0000000 --- a/source/LootDumpProcessor/Process/Reader/PreProcess/PreProcessReaderTypes.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace LootDumpProcessor.Process.Reader.PreProcess; - -public enum PreProcessReaderTypes -{ - SevenZip -} \ No newline at end of file diff --git a/source/LootDumpProcessor/Process/Reader/PreProcess/SevenZipPreProcessReader.cs b/source/LootDumpProcessor/Process/Reader/PreProcess/SevenZipPreProcessReader.cs deleted file mode 100644 index e68812f..0000000 --- a/source/LootDumpProcessor/Process/Reader/PreProcess/SevenZipPreProcessReader.cs +++ /dev/null @@ -1,48 +0,0 @@ -using LootDumpProcessor.Process.Reader.PreProcess; -using SevenZip; -using Microsoft.Extensions.Logging; - -public class SevenZipPreProcessReader : AbstractPreProcessReader -{ - private readonly ILogger _logger; - - public SevenZipPreProcessReader(ILogger logger) : base(logger) - { - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public override string GetHandleExtension() => "7z"; - - static SevenZipPreProcessReader() - { - SevenZipBase.SetLibraryPath("./x64/7z.dll"); - } - - public override bool TryPreProcess(string file, out List files, out List directories) - { - files = new List(); - directories = new List(); - - var fileRaw = Path.GetFileNameWithoutExtension(file); - // SevenZip library doesn't handle forward slashes properly - var outPath = $"{_tempFolder}\\{fileRaw}".Replace("/", "\\"); - - _logger.LogInformation("Unzipping {File} into temp path {OutPath}, this may take a while...", file, outPath); - - var extractor = new SevenZipExtractor(file); - - // Log progress in debug mode - extractor.Extracting += (_, args) => - { - if (args.PercentDone % 10 == 0) _logger.LogDebug("Unzip progress: {PercentDone}%", args.PercentDone); - }; - - extractor.ExtractArchive(outPath); - _logger.LogInformation("Finished unzipping {File} into temp path {OutPath}", file, outPath); - - files = Directory.GetFiles(outPath).ToList(); - directories = Directory.GetDirectories(outPath).ToList(); - - return true; - } -} \ No newline at end of file diff --git a/source/LootDumpProcessor/Program.cs b/source/LootDumpProcessor/Program.cs index bd4dcaf..4c1e7ce 100644 --- a/source/LootDumpProcessor/Program.cs +++ b/source/LootDumpProcessor/Program.cs @@ -7,7 +7,6 @@ using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor; using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor; using LootDumpProcessor.Process.Reader.Filters; using LootDumpProcessor.Process.Reader.Intake; -using LootDumpProcessor.Process.Reader.PreProcess; using LootDumpProcessor.Storage; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -46,7 +45,6 @@ public static class Program services.AddTransient(); services.AddTransient(); - services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient();