mirror of
https://github.com/sp-tarkov/loot-dump-processor.git
synced 2025-02-13 02:30:45 -05:00
Removed 7z preprocessing functionality and simplified file processing
This commit is contained in:
parent
e52a8b2fad
commit
50e290d491
@ -9,13 +9,6 @@
|
||||
"fileDataStorageTempLocation": "M:\\Development\\spt\\dumps\\cache"
|
||||
},
|
||||
"readerConfig": {
|
||||
"preProcessorConfig": {
|
||||
"preProcessors": [
|
||||
"SevenZip"
|
||||
],
|
||||
"preProcessorTempFolder": "M:\\Development\\spt\\dumps\\temp",
|
||||
"cleanupTempFolderAfterProcess": true
|
||||
},
|
||||
"intakeReaderConfig": {
|
||||
"maxDumpsPerMap": 5000,
|
||||
"ignoredDumpLocations": [
|
||||
|
||||
@ -11,10 +11,8 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="7z.Libs" Version="21.7.0"/>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0"/>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0"/>
|
||||
<PackageReference Include="SevenZipSharp.Interop" Version="19.1.0"/>
|
||||
<PackageReference Include="System.Configuration.ConfigurationManager" Version="7.0.0"/>
|
||||
<PackageReference Include="YamlDotNet" Version="13.0.0"/>
|
||||
</ItemGroup>
|
||||
|
||||
@ -1,12 +0,0 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
|
||||
namespace LootDumpProcessor.Model.Config;
|
||||
|
||||
public class PreProcessorConfig
|
||||
{
|
||||
[JsonPropertyName("preProcessorTempFolder")] public string? PreProcessorTempFolder { get; set; }
|
||||
|
||||
|
||||
[JsonPropertyName("cleanupTempFolderAfterProcess")] public bool CleanupTempFolderAfterProcess { get; set; } = true;
|
||||
}
|
||||
@ -5,17 +5,8 @@ namespace LootDumpProcessor.Model.Config;
|
||||
|
||||
public class ReaderConfig
|
||||
{
|
||||
[JsonPropertyName("intakeReaderConfig")] public IntakeReaderConfig? IntakeReaderConfig { get; set; }
|
||||
|
||||
|
||||
[JsonPropertyName("preProcessorConfig")] public PreProcessorConfig? PreProcessorConfig { get; set; }
|
||||
|
||||
|
||||
[JsonPropertyName("dumpFilesLocation")] public List<string>? DumpFilesLocation { get; set; }
|
||||
|
||||
|
||||
[JsonPropertyName("thresholdDate")] public string? ThresholdDate { get; set; }
|
||||
|
||||
|
||||
[JsonPropertyName("processSubFolders")] public bool ProcessSubFolders { get; set; }
|
||||
public IntakeReaderConfig? IntakeReaderConfig { get; set; }
|
||||
public List<string>? DumpFilesLocation { get; set; }
|
||||
public string? ThresholdDate { get; set; }
|
||||
public bool ProcessSubFolders { get; set; }
|
||||
}
|
||||
@ -7,7 +7,6 @@ using LootDumpProcessor.Process.Processor.DumpProcessor;
|
||||
using LootDumpProcessor.Process.Processor.FileProcessor;
|
||||
using LootDumpProcessor.Process.Reader.Filters;
|
||||
using LootDumpProcessor.Process.Reader.Intake;
|
||||
using LootDumpProcessor.Process.Reader.PreProcess;
|
||||
using LootDumpProcessor.Process.Writer;
|
||||
using LootDumpProcessor.Serializers.Json;
|
||||
using LootDumpProcessor.Storage;
|
||||
@ -17,8 +16,8 @@ using Microsoft.Extensions.Logging;
|
||||
namespace LootDumpProcessor.Process;
|
||||
|
||||
public class QueuePipeline(
|
||||
IFileProcessor fileProcessor, IDumpProcessor dumpProcessor, ILogger<QueuePipeline> logger,
|
||||
IPreProcessReader preProcessReader, IFileFilter fileFilter, IIntakeReader intakeReader
|
||||
IFileProcessor fileProcessor, IDumpProcessor dumpProcessor, ILogger<QueuePipeline> logger, IFileFilter fileFilter,
|
||||
IIntakeReader intakeReader
|
||||
)
|
||||
: IPipeline
|
||||
{
|
||||
@ -30,9 +29,6 @@ public class QueuePipeline(
|
||||
|
||||
private readonly ILogger<QueuePipeline> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
private readonly IPreProcessReader _preProcessReader =
|
||||
preProcessReader ?? throw new ArgumentNullException(nameof(preProcessReader));
|
||||
|
||||
private readonly IFileFilter _fileFilter = fileFilter ?? throw new ArgumentNullException(nameof(fileFilter));
|
||||
|
||||
private readonly IIntakeReader
|
||||
@ -60,7 +56,6 @@ public class QueuePipeline(
|
||||
}
|
||||
finally
|
||||
{
|
||||
_preProcessReader.Dispose();
|
||||
stopwatch.Stop();
|
||||
_logger.LogInformation("Dumps processed in {@Time}", stopwatch.Elapsed);
|
||||
}
|
||||
@ -92,24 +87,9 @@ public class QueuePipeline(
|
||||
var extensionFull = Path.GetExtension(file);
|
||||
if (extensionFull.Length > 1)
|
||||
{
|
||||
// if the preprocessor found something new to process or generated new files, add them to the queue
|
||||
if (extensionFull == "7z" &&
|
||||
_preProcessReader.TryPreProcess(file, out var outputFiles, out var outputDirectories))
|
||||
{
|
||||
// all new directories need to be processed as well
|
||||
GetFileQueue(outputDirectories).ToList().ForEach(queuedFilesToProcess.Enqueue);
|
||||
// all output files need to be queued as well
|
||||
outputFiles.ForEach(queuedFilesToProcess.Enqueue);
|
||||
}
|
||||
if (_fileFilter.Accept(file)) gatheredFiles.Add(file);
|
||||
|
||||
else
|
||||
{
|
||||
// if there is no preprocessor for the file, means its ready to filter or accept
|
||||
|
||||
if (_fileFilter.Accept(file)) gatheredFiles.Add(file);
|
||||
|
||||
else gatheredFiles.Add(file);
|
||||
}
|
||||
else gatheredFiles.Add(file);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -1,40 +0,0 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace LootDumpProcessor.Process.Reader.PreProcess;
|
||||
|
||||
public abstract class AbstractPreProcessReader : IPreProcessReader
|
||||
{
|
||||
protected readonly string _tempFolder;
|
||||
|
||||
public AbstractPreProcessReader(ILogger logger)
|
||||
{
|
||||
var tempFolder = LootDumpProcessorContext.GetConfig().ReaderConfig.PreProcessorConfig?.PreProcessorTempFolder;
|
||||
if (string.IsNullOrEmpty(tempFolder))
|
||||
{
|
||||
tempFolder = GetBaseDirectory();
|
||||
logger.LogWarning(
|
||||
"No temp folder was assigned preProcessorTempFolder in PreProcessorConfig, defaulting to {tempFolder}",
|
||||
tempFolder
|
||||
);
|
||||
}
|
||||
|
||||
// Cleanup the temp directory before starting the process
|
||||
if (Directory.Exists(tempFolder)) Directory.Delete(tempFolder, true);
|
||||
|
||||
Directory.CreateDirectory(tempFolder);
|
||||
|
||||
_tempFolder = tempFolder;
|
||||
}
|
||||
|
||||
public abstract string GetHandleExtension();
|
||||
public abstract bool TryPreProcess(string file, out List<string> files, out List<string> directories);
|
||||
|
||||
protected string GetBaseDirectory() =>
|
||||
$@"{Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)}\SPT\tmp\PreProcessor";
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (LootDumpProcessorContext.GetConfig().ReaderConfig.PreProcessorConfig?.CleanupTempFolderAfterProcess ?? true)
|
||||
Directory.Delete(_tempFolder, true);
|
||||
}
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
namespace LootDumpProcessor.Process.Reader.PreProcess;
|
||||
|
||||
public interface IPreProcessReader
|
||||
{
|
||||
string GetHandleExtension();
|
||||
|
||||
bool TryPreProcess(string file, out List<string> files, out List<string> directories);
|
||||
|
||||
// Custom dispose, not IDisposable
|
||||
void Dispose();
|
||||
}
|
||||
@ -1,6 +0,0 @@
|
||||
namespace LootDumpProcessor.Process.Reader.PreProcess;
|
||||
|
||||
public enum PreProcessReaderTypes
|
||||
{
|
||||
SevenZip
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
using LootDumpProcessor.Process.Reader.PreProcess;
|
||||
using SevenZip;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
public class SevenZipPreProcessReader : AbstractPreProcessReader
|
||||
{
|
||||
private readonly ILogger<SevenZipPreProcessReader> _logger;
|
||||
|
||||
public SevenZipPreProcessReader(ILogger<SevenZipPreProcessReader> logger) : base(logger)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public override string GetHandleExtension() => "7z";
|
||||
|
||||
static SevenZipPreProcessReader()
|
||||
{
|
||||
SevenZipBase.SetLibraryPath("./x64/7z.dll");
|
||||
}
|
||||
|
||||
public override bool TryPreProcess(string file, out List<string> files, out List<string> directories)
|
||||
{
|
||||
files = new List<string>();
|
||||
directories = new List<string>();
|
||||
|
||||
var fileRaw = Path.GetFileNameWithoutExtension(file);
|
||||
// SevenZip library doesn't handle forward slashes properly
|
||||
var outPath = $"{_tempFolder}\\{fileRaw}".Replace("/", "\\");
|
||||
|
||||
_logger.LogInformation("Unzipping {File} into temp path {OutPath}, this may take a while...", file, outPath);
|
||||
|
||||
var extractor = new SevenZipExtractor(file);
|
||||
|
||||
// Log progress in debug mode
|
||||
extractor.Extracting += (_, args) =>
|
||||
{
|
||||
if (args.PercentDone % 10 == 0) _logger.LogDebug("Unzip progress: {PercentDone}%", args.PercentDone);
|
||||
};
|
||||
|
||||
extractor.ExtractArchive(outPath);
|
||||
_logger.LogInformation("Finished unzipping {File} into temp path {OutPath}", file, outPath);
|
||||
|
||||
files = Directory.GetFiles(outPath).ToList();
|
||||
directories = Directory.GetDirectories(outPath).ToList();
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -7,7 +7,6 @@ using LootDumpProcessor.Process.Processor.v2.StaticContainersProcessor;
|
||||
using LootDumpProcessor.Process.Processor.v2.StaticLootProcessor;
|
||||
using LootDumpProcessor.Process.Reader.Filters;
|
||||
using LootDumpProcessor.Process.Reader.Intake;
|
||||
using LootDumpProcessor.Process.Reader.PreProcess;
|
||||
using LootDumpProcessor.Storage;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@ -46,7 +45,6 @@ public static class Program
|
||||
|
||||
services.AddTransient<IIntakeReader, JsonFileIntakeReader>();
|
||||
services.AddTransient<IFileFilter, JsonDumpFileFilter>();
|
||||
services.AddTransient<IPreProcessReader, SevenZipPreProcessReader>();
|
||||
services.AddTransient<IFileProcessor, FileProcessor>();
|
||||
services.AddTransient<IDumpProcessor, MultithreadSteppedDumpProcessor>();
|
||||
services.AddTransient<IPipeline, QueuePipeline>();
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user