Compare commits

...

5 Commits

139 changed files with 597 additions and 495 deletions

View File

@ -1,4 +1,4 @@
{
{
"serverLocation": "E:\\spt\\Server",
"threads": 10,
"threadPoolingTimeoutMs": 1000,
@ -47,6 +47,11 @@
"spawnPointToleranceForForced": 99.5,
"looseLootCountTolerancePercentage": 75
},
"collectorConfig": {
"collectorType": "Dump",
"maxEntitiesBeforeDumping": 1000,
"dumpLocation": "C:\\Users\\USER\\SPT\\tmp"
},
"writerConfig": {
"outputLocation": "E:\\spt\\dumps\\output"
},

View File

@ -1,6 +1,6 @@
namespace LootDumpProcessor;
namespace LootDumpProcessor;
public class GCHandler
public static class GCHandler
{
public static void Collect()
{

View File

@ -1,8 +1,9 @@
namespace LootDumpProcessor.Logger;
namespace LootDumpProcessor.Logger;
public interface ILogger
{
void Setup();
void Log(string message, LogLevel level);
bool CanBeLogged(LogLevel level);
void Stop();
}

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Logger;
namespace LootDumpProcessor.Logger;
public enum LogLevel
{

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Logger;
namespace LootDumpProcessor.Logger;
public static class LoggerFactory
{

View File

@ -1,15 +1,15 @@
using System.Collections.Concurrent;
using System.Collections.Concurrent;
namespace LootDumpProcessor.Logger;
public class QueueLogger : ILogger
{
private BlockingCollection<LoggedMessage> queuedMessages = new BlockingCollection<LoggedMessage>();
private readonly BlockingCollection<LoggedMessage> queuedMessages = new();
private Task? loggerThread;
private bool isRunning;
private int logLevel;
private static readonly int _logTerminationTimeoutMs = 1000;
private static readonly int _logTerminationRetryCount = 3;
private const int LogTerminationTimeoutMs = 1000;
private const int LogTerminationRetryCount = 3;
public void Setup()
{
@ -56,19 +56,14 @@ public class QueueLogger : ILogger
private int GetLogLevel(LogLevel level)
{
switch (level)
return level switch
{
case LogLevel.Error:
return 1;
case LogLevel.Warning:
return 2;
case LogLevel.Info:
return 3;
case LogLevel.Debug:
return 4;
default:
throw new ArgumentOutOfRangeException();
}
LogLevel.Error => 1,
LogLevel.Warning => 2,
LogLevel.Info => 3,
LogLevel.Debug => 4,
_ => throw new ArgumentOutOfRangeException()
};
}
public void Log(string message, LogLevel level)
@ -77,6 +72,11 @@ public class QueueLogger : ILogger
queuedMessages.Add(new LoggedMessage { Message = message, LogLevel = level });
}
public bool CanBeLogged(LogLevel level)
{
return GetLogLevel(level) <= logLevel;
}
// Wait for graceful termination of the logging thread
public void Stop()
{
@ -84,24 +84,24 @@ public class QueueLogger : ILogger
if (loggerThread != null)
{
Console.ResetColor();
int retryCount = 0;
var retryCount = 0;
while (!loggerThread.IsCompleted)
{
if (retryCount == _logTerminationRetryCount)
if (retryCount == LogTerminationRetryCount)
{
Console.WriteLine(
$"Logger thread did not terminate by itself after {retryCount} retries. Some log messages may be lost.");
break;
}
Console.WriteLine($"Waiting {_logTerminationTimeoutMs}ms for logger termination");
Thread.Sleep(_logTerminationTimeoutMs);
Console.WriteLine($"Waiting {LogTerminationTimeoutMs}ms for logger termination");
Thread.Sleep(LogTerminationTimeoutMs);
retryCount++;
}
}
}
class LoggedMessage
private class LoggedMessage
{
public string Message { get; init; }
public LogLevel LogLevel { get; init; }

View File

@ -1,4 +1,4 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.4.33213.308

View File

@ -1,4 +1,4 @@
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Process;
using LootDumpProcessor.Serializers.Json;
@ -6,22 +6,22 @@ using LootDumpProcessor.Serializers.Yaml;
namespace LootDumpProcessor;
public class LootDumpProcessorContext
public static class LootDumpProcessorContext
{
private static Config? _config;
private static readonly object _configLock = new object();
private static readonly object _configLock = new();
private static ForcedStatic? _forcedStatic;
private static readonly object _forcedStaticLock = new object();
private static readonly object _forcedStaticLock = new();
private static Dictionary<string, MapDirectoryMapping>? _mapDirectoryMappings;
private static readonly object _mapDirectoryMappingsLock = new object();
private static readonly object _mapDirectoryMappingsLock = new();
private static HashSet<string>? _staticWeaponIds;
private static readonly object _staticWeaponIdsLock = new object();
private static readonly object _staticWeaponIdsLock = new();
private static Dictionary<string, List<StaticForced>>? _forcedItems;
private static readonly object _forcedItemsLock = new object();
private static readonly object _forcedItemsLock = new();
private static Dictionary<string, HashSet<string>>? _forcedLoose;
private static readonly object _forcedLooseLock = new object();
private static readonly object _forcedLooseLock = new();
private static TarkovItems? _tarkovItems;
private static readonly object _tarkovItemsLock = new object();
private static readonly object _tarkovItemsLock = new();
public static Config GetConfig()
{

View File

@ -1,5 +1,6 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Utils;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model;
@ -26,7 +27,7 @@ public class ComposedKey
.Cast<string>()
.Select(i => (double)i.GetHashCode())
.Sum()
.ToString() ?? Guid.NewGuid().ToString();
.ToString() ?? KeyGenerator.GetNextKey();
FirstItem = items?[0];
}
@ -34,7 +35,7 @@ public class ComposedKey
{
if (obj is not ComposedKey key)
return false;
return this.Key == key.Key;
return Key == key.Key;
}
public override int GetHashCode() => Key.GetHashCode();

View File

@ -0,0 +1,20 @@
using System.Text.Json.Serialization;
using LootDumpProcessor.Process.Collector;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Config;
public class CollectorConfig
{
[JsonProperty("collectorType")]
[JsonPropertyName("collectorType")]
public CollectorType CollectorType { get; set; }
[JsonProperty("maxEntitiesBeforeDumping")]
[JsonPropertyName("maxEntitiesBeforeDumping")]
public int MaxEntitiesBeforeDumping { get; set; }
[JsonProperty("dumpLocation")]
[JsonPropertyName("dumpLocation")]
public string DumpLocation { get; set; }
}

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using LootDumpProcessor.Serializers.Json;
using Newtonsoft.Json;
@ -50,6 +50,10 @@ public class Config
[JsonPropertyName("writerConfig")]
public WriterConfig WriterConfig { get; set; }
[JsonProperty("collectorConfig")]
[JsonPropertyName("collectorConfig")]
public CollectorConfig CollectorConfig { get; set; }
[JsonProperty("containerIgnoreList")]
[JsonPropertyName("containerIgnoreList")]
public Dictionary<string, string[]> ContainerIgnoreList { get; set; }

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using LootDumpProcessor.Storage;
using Newtonsoft.Json;

View File

@ -1,4 +1,4 @@
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Output.StaticContainer;
using YamlDotNet.Serialization;
namespace LootDumpProcessor.Model.Config;

View File

@ -1,5 +1,6 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using LootDumpProcessor.Process.Reader;
using LootDumpProcessor.Process.Reader.Intake;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Config;
@ -17,5 +18,5 @@ public class IntakeReaderConfig
[JsonProperty("ignoredDumpLocations")]
[JsonPropertyName("ignoredDumpLocations")]
public List<string> IgnoredDumpLocations { get; set; } = new List<string>();
public List<string> IgnoredDumpLocations { get; set; } = new();
}

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using LootDumpProcessor.Logger;
using Newtonsoft.Json;

View File

@ -1,4 +1,4 @@
using YamlDotNet.Serialization;
using YamlDotNet.Serialization;
namespace LootDumpProcessor.Model.Config;

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using LootDumpProcessor.Process.Reader.PreProcess;
using Newtonsoft.Json;

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Config;

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using LootDumpProcessor.Process.Reader.Filters;
using Newtonsoft.Json;
@ -24,7 +24,7 @@ public class ReaderConfig
[JsonProperty("acceptedFileExtensions")]
[JsonPropertyName("acceptedFileExtensions")]
public List<string> AcceptedFileExtensions { get; set; } = new List<string>();
public List<string> AcceptedFileExtensions { get; set; } = new();
[JsonProperty("processSubFolders")]
[JsonPropertyName("processSubFolders")]

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Config;

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model
@ -13,7 +13,7 @@ namespace LootDumpProcessor.Model
{
return new FireMode
{
FireModeType = this.FireModeType
FireModeType = FireModeType
};
}
}

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model
@ -13,7 +13,7 @@ namespace LootDumpProcessor.Model
{
return new Foldable
{
Folded = this.Folded
Folded = Folded
};
}
}

View File

@ -1,5 +1,5 @@
using System.Text.Json.Serialization;
using LootDumpProcessor.Process.Processor;
using System.Text.Json.Serialization;
using LootDumpProcessor.Utils;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model
@ -26,10 +26,10 @@ namespace LootDumpProcessor.Model
{
return new GroupPosition
{
Name = this.Name,
Weight = this.Weight,
Position = ProcessorUtil.Copy(this.Position),
Rotation = ProcessorUtil.Copy(this.Rotation)
Name = Name,
Weight = Weight,
Position = ProcessorUtil.Copy(Position),
Rotation = ProcessorUtil.Copy(Rotation)
};
}
}

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Input

View File

@ -1,5 +1,5 @@
using System.Text.Json.Serialization;
using LootDumpProcessor.Process.Processor;
using System.Text.Json.Serialization;
using LootDumpProcessor.Utils;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model
@ -34,24 +34,24 @@ namespace LootDumpProcessor.Model
{
if (obj is not Item parsed)
return false;
return parsed.Tpl == this.Tpl && parsed.ParentId == this.ParentId;
return parsed.Tpl == Tpl && parsed.ParentId == ParentId;
}
public override int GetHashCode()
{
return (this.Tpl?.GetHashCode() + this.ParentId?.GetHashCode()) ?? base.GetHashCode();
return (Tpl?.GetHashCode() + ParentId?.GetHashCode()) ?? base.GetHashCode();
}
public object Clone()
{
return new Item
{
Id = this.Id,
Tpl = this.Tpl,
ParentId = this.ParentId,
SlotId = this.SlotId,
Location = this.Location,
Upd = ProcessorUtil.Copy(this.Upd)
Id = Id,
Tpl = Tpl,
ParentId = ParentId,
SlotId = SlotId,
Location = Location,
Upd = ProcessorUtil.Copy(Upd)
};
}
}

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output;

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output;

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output.LooseLoot

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output.LooseLoot

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output.LooseLoot

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output.LooseLoot

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output.StaticContainer

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output.StaticContainer

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output.StaticContainer;

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
using YamlDotNet.Serialization;

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output;

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Output

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Model.Processing;
public static class BaseClasses
{

View File

@ -1,4 +1,4 @@
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Input;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Processing;

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Model.Processing;
public class CaliberTemplateCount
{

View File

@ -1,4 +1,4 @@
using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage;
namespace LootDumpProcessor.Model.Processing;

View File

@ -1,5 +1,6 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Utils;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Processing;
@ -8,7 +9,7 @@ public class LooseLootCounts : IKeyable
{
[JsonProperty("__id__")]
[JsonPropertyName("__id__")]
public string __ID { get; set; } = Guid.NewGuid().ToString();
public string __ID { get; set; } = KeyGenerator.GetNextKey();
public IKey Counts { get; set; }

View File

@ -1,11 +1,11 @@
using System.Text.RegularExpressions;
using System.Text.RegularExpressions;
using LootDumpProcessor.Storage;
namespace LootDumpProcessor.Model.Processing;
public class ParsedDump : IKeyable
{
private static readonly Regex _hashRegex = new Regex("([^a-zA-Z0-9])");
private static readonly Regex _hashRegex = new("([^a-zA-Z0-9])");
public BasicInfo BasicInfo { get; set; }
public PreProcessedLooseLoot LooseLoot { get; set; }
public List<PreProcessedStaticLoot> Containers { get; set; }
@ -13,13 +13,13 @@ public class ParsedDump : IKeyable
public override bool Equals(object? obj)
{
if (obj is ParsedDump dump)
return dump.BasicInfo.Equals(this.BasicInfo);
return dump.BasicInfo.Equals(BasicInfo);
return false;
}
public override int GetHashCode()
{
return this.BasicInfo.GetHashCode();
return BasicInfo.GetHashCode();
}
public IKey GetKey()

View File

@ -1,4 +1,4 @@
using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage;
namespace LootDumpProcessor.Model.Processing;
@ -10,12 +10,12 @@ public class PartialData
public override bool Equals(object? obj)
{
if (obj is ParsedDump dump)
return dump.BasicInfo.Equals(this.BasicInfo);
return dump.BasicInfo.Equals(BasicInfo);
return false;
}
public override int GetHashCode()
{
return this.BasicInfo.GetHashCode();
return BasicInfo.GetHashCode();
}
}

View File

@ -1,4 +1,4 @@
using LootDumpProcessor.Serializers.Json.Converters;
using LootDumpProcessor.Serializers.Json.Converters;
using LootDumpProcessor.Storage;
namespace LootDumpProcessor.Model.Processing;

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Model.Processing;
public class PreProcessedStaticLoot
{

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model
@ -17,8 +17,8 @@ namespace LootDumpProcessor.Model
{
return new Repairable
{
Durability = this.Durability,
MaxDurability = this.MaxDurability
Durability = Durability,
MaxDurability = MaxDurability
};
}
}

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Model.Tarkov;
namespace LootDumpProcessor.Model.Tarkov;
public class Category
{

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model.Tarkov;

View File

@ -1,6 +1,6 @@
using System.Text.Json.Serialization;
using LootDumpProcessor.Process.Processor;
using System.Text.Json.Serialization;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Utils;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model
@ -9,7 +9,7 @@ namespace LootDumpProcessor.Model
{
[Newtonsoft.Json.JsonIgnore]
[System.Text.Json.Serialization.JsonIgnore]
public string __ID { get; } = Guid.NewGuid().ToString();
public string __ID { get; } = KeyGenerator.GetNextKey();
[JsonProperty("Id", NullValueHandling = NullValueHandling.Ignore)]
[JsonPropertyName("Id")]
@ -82,17 +82,17 @@ namespace LootDumpProcessor.Model
{
return new Template
{
Id = this.Id,
IsContainer = this.IsContainer,
UseGravity = this.UseGravity,
RandomRotation = this.RandomRotation,
Position = ProcessorUtil.Copy(this.Position),
Rotation = ProcessorUtil.Copy(this.Rotation),
IsGroupPosition = this.IsGroupPosition,
GroupPositions = ProcessorUtil.Copy(this.GroupPositions),
IsAlwaysSpawn = this.IsAlwaysSpawn,
Root = this.Root,
Items = ProcessorUtil.Copy(this.Items)
Id = Id,
IsContainer = IsContainer,
UseGravity = UseGravity,
RandomRotation = RandomRotation,
Position = ProcessorUtil.Copy(Position),
Rotation = ProcessorUtil.Copy(Rotation),
IsGroupPosition = IsGroupPosition,
GroupPositions = ProcessorUtil.Copy(GroupPositions),
IsAlwaysSpawn = IsAlwaysSpawn,
Root = Root,
Items = ProcessorUtil.Copy(Items)
};
}
}

View File

@ -1,5 +1,6 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using LootDumpProcessor.Process.Processor;
using LootDumpProcessor.Utils;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model
@ -26,10 +27,10 @@ namespace LootDumpProcessor.Model
{
return new Upd
{
StackObjectsCount = this.StackObjectsCount,
FireMode = ProcessorUtil.Copy(this.FireMode),
Foldable = ProcessorUtil.Copy(this.Foldable),
Repairable = ProcessorUtil.Copy(this.Repairable)
StackObjectsCount = StackObjectsCount,
FireMode = ProcessorUtil.Copy(FireMode),
Foldable = ProcessorUtil.Copy(Foldable),
Repairable = ProcessorUtil.Copy(Repairable)
};
}
}

View File

@ -1,4 +1,4 @@
using System.Text.Json.Serialization;
using System.Text.Json.Serialization;
using Newtonsoft.Json;
namespace LootDumpProcessor.Model
@ -21,9 +21,9 @@ namespace LootDumpProcessor.Model
{
return new Vector3
{
X = this.X,
Y = this.Y,
Z = this.Z
X = X,
Y = Y,
Z = Z
};
}
}

View File

@ -1,10 +1,17 @@
namespace LootDumpProcessor.Process.Collector;
namespace LootDumpProcessor.Process.Collector;
public static class CollectorFactory
{
private static ICollector? _collector;
public static ICollector GetInstance()
{
// TODO: implement real factory
return new HashSetCollector();
if (_collector == null)
_collector = LootDumpProcessorContext.GetConfig().CollectorConfig.CollectorType switch
{
CollectorType.Memory => new HashSetCollector(),
CollectorType.Dump => new DumpCollector(),
_ => throw new ArgumentOutOfRangeException()
};
return _collector;
}
}

View File

@ -0,0 +1,7 @@
namespace LootDumpProcessor.Process.Collector;
public enum CollectorType
{
Memory,
Dump
}

View File

@ -0,0 +1,45 @@
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Serializers.Json;
namespace LootDumpProcessor.Process.Collector;
public class DumpCollector : ICollector
{
private static readonly string DumpLocation = $"{LootDumpProcessorContext.GetConfig().CollectorConfig.DumpLocation}/collector/";
private readonly List<PartialData> processedDumps = new(LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping + 50);
private readonly object lockObject = new();
public void Setup()
{
if (Directory.Exists(DumpLocation))
{
Directory.Delete(DumpLocation, true);
}
Directory.CreateDirectory(DumpLocation);
}
public void Hold(PartialData parsedDump)
{
lock (lockObject)
{
processedDumps.Add(parsedDump);
if (processedDumps.Count > LootDumpProcessorContext.GetConfig().CollectorConfig.MaxEntitiesBeforeDumping)
{
var fileName = $"collector-{DateTime.Now.ToString("yyyyMMddHHmmssfffff")}.json";
File.WriteAllText($"{DumpLocation}{fileName}", JsonSerializerFactory.GetInstance().Serialize(processedDumps));
processedDumps.Clear();
}
}
}
public List<PartialData> Retrieve()
{
foreach (var file in Directory.GetFiles(DumpLocation))
{
processedDumps.AddRange(JsonSerializerFactory.GetInstance().Deserialize<List<PartialData>>(File.ReadAllText(file)));
}
return processedDumps;
}
}

View File

@ -1,13 +1,11 @@
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Process.Collector;
public class HashSetCollector : ICollector
{
private HashSet<PartialData> processedDumps = new HashSet<PartialData>();
private object lockObject = new object();
private readonly HashSet<PartialData> processedDumps = new(100_000);
private readonly object lockObject = new();
public void Setup()
{

View File

@ -1,4 +1,3 @@
using LootDumpProcessor.Model.Config;
using LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Process.Collector;

View File

@ -1,6 +0,0 @@
namespace LootDumpProcessor.Process;
public class FilesGatherer
{
}

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process;
namespace LootDumpProcessor.Process;
public interface IPipeline
{

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process;
namespace LootDumpProcessor.Process;
public enum OutputFileType
{

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process;
namespace LootDumpProcessor.Process;
public static class PipelineFactory
{

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process.Processor.DumpProcessor;
namespace LootDumpProcessor.Process.Processor.DumpProcessor;
public static class DumpProcessorFactory
{

View File

@ -1,4 +1,4 @@
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Process.Processor.DumpProcessor;

View File

@ -1,4 +1,4 @@
using System.Collections.Concurrent;
using System.Collections.Concurrent;
using LootDumpProcessor.Logger;
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
@ -7,6 +7,7 @@ using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage.Collections;
using LootDumpProcessor.Utils;
namespace LootDumpProcessor.Process.Processor.DumpProcessor;
@ -25,11 +26,13 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
public Dictionary<OutputFileType, object> ProcessDumps(List<PartialData> dumps)
{
LoggerFactory.GetInstance().Log("Starting final dump processing", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Starting final dump processing", LogLevel.Info);
var output = new Dictionary<OutputFileType, object>();
var dumpProcessData = GetDumpProcessData(dumps);
LoggerFactory.GetInstance().Log("Heavy processing done!", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Heavy processing done!", LogLevel.Info);
var staticContainers = new Dictionary<string, MapStaticLoot>();
var staticContainersLock = new object();
@ -42,13 +45,15 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
Runners.Clear();
// BSG changed the map data so static containers are now dynamic, so we need to scan all dumps for the static containers.
LoggerFactory.GetInstance().Log("Queuing dumps for static data processing", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Queuing dumps for static data processing", LogLevel.Info);
foreach (var dumped in dumps)
{
Runners.Add(
Task.Factory.StartNew(() =>
{
LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", LogLevel.Debug);
var data = _jsonSerializer.Deserialize<RootData>(File.ReadAllText(dumped.BasicInfo.FileName));
// the if statement below takes care of processing "forced" or real static data for each map, we only need
// to do this once per map, so we dont care about doing it again
@ -56,7 +61,8 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
{
if (!staticContainers.ContainsKey(data.Data.Name))
{
LoggerFactory.GetInstance().Log($"Doing first time process for map {data.Data.Name} of real static data", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log($"Doing first time process for map {data.Data.Name} of real static data", LogLevel.Info);
var mapStaticLoot = StaticLootProcessor.CreateRealStaticContainers(data);
staticContainers[mapStaticLoot.Item1] = mapStaticLoot.Item2;
}
@ -113,7 +119,8 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
}
Task.WaitAll(Runners.ToArray());
LoggerFactory.GetInstance().Log("All static data processing threads finished", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("All static data processing threads finished", LogLevel.Info);
// Aggregate and calculate the probability of a static container
mapStaticContainersAggregated.ToDictionary(
kv => kv.Key,
@ -128,35 +135,39 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
// Static containers
output.Add(OutputFileType.StaticContainer, staticContainers);
LoggerFactory.GetInstance().Log("Processing ammo distribution", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Processing ammo distribution", LogLevel.Info);
// Ammo distribution
output.Add(
OutputFileType.StaticAmmo,
StaticLootProcessor.CreateAmmoDistribution(dumpProcessData.ContainerCounts)
);
LoggerFactory.GetInstance().Log("Processing static loot distribution", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Processing static loot distribution", LogLevel.Info);
// Static loot distribution
output.Add(
OutputFileType.StaticLoot,
StaticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts)
);
LoggerFactory.GetInstance().Log("Processing loose loot distribution", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Processing loose loot distribution", LogLevel.Info);
// Loose loot distribution
var looseLootDistribution = LooseLootProcessor.CreateLooseLootDistribution(
dumpProcessData.MapCounts,
dumpProcessData.LooseLootCounts
);
LoggerFactory.GetInstance().Log("Collecting loose loot distribution information", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Collecting loose loot distribution information", LogLevel.Info);
var loot = dumpProcessData.MapCounts
.Select(mapCount => mapCount.Key)
.ToDictionary(mi => mi, mi => looseLootDistribution[mi]);
output.Add(OutputFileType.LooseLoot, loot);
LoggerFactory.GetInstance().Log("Dump processing fully completed!", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Dump processing fully completed!", LogLevel.Info);
return output;
}
@ -175,10 +186,11 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
{
var mapi = tuple.Key;
var g = tuple.ToList();
LoggerFactory.GetInstance().Log(
$"Processing map {mapi}, total dump data to process: {g.Count}",
LogLevel.Info
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log(
$"Processing map {mapi}, total dump data to process: {g.Count}",
LogLevel.Info
);
dumpProcessData.MapCounts[mapi] = g.Count;
var lockObjectContainerCounts = new object();
@ -274,18 +286,16 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
lock (lockObjectCounts)
{
counts.MapSpawnpointCount.AddRange(new List<int>
{
dumpData.LooseLoot.MapSpawnpointCount
});
counts.MapSpawnpointCount.Add(dumpData.LooseLoot.MapSpawnpointCount);
}
}
catch (Exception e)
{
LoggerFactory.GetInstance().Log(
$"ERROR OCCURRED:{e.Message}\n{e.StackTrace}",
LogLevel.Error
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log(
$"ERROR OCCURRED:{e.Message}\n{e.StackTrace}",
LogLevel.Error
);
}
}
},
@ -296,10 +306,11 @@ public class MultithreadSteppedDumpProcessor : IDumpProcessor
// Wait until all runners are done processing
while (!Runners.All(r => r.IsCompleted))
{
LoggerFactory.GetInstance().Log(
$"One or more file processors are still processing files. Waiting {LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs}ms before checking again",
LogLevel.Info
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log(
$"One or more file processors are still processing files. Waiting {LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs}ms before checking again",
LogLevel.Info
);
Thread.Sleep(
TimeSpan.FromMilliseconds(LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs));
}

View File

@ -1,16 +1,16 @@
using LootDumpProcessor.Logger;
using LootDumpProcessor.Logger;
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Processor;
using LootDumpProcessor.Storage;
namespace LootDumpProcessor.Process.Impl;
namespace LootDumpProcessor.Process.Processor.FileProcessor;
public class FileProcessor : IFileProcessor
{
public PartialData Process(BasicInfo parsedData)
{
LoggerFactory.GetInstance().Log($"Processing file {parsedData.FileName}...", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"Processing file {parsedData.FileName}...", LogLevel.Debug);
List<Template> looseLoot = new List<Template>();
List<Template> staticLoot = new List<Template>();
@ -29,7 +29,7 @@ public class FileProcessor : IFileProcessor
BasicInfo = parsedData
};
PartialData data = new PartialData
var data = new PartialData
{
BasicInfo = parsedData,
ParsedDumpKey = (AbstractKey)dumpData.GetKey()
@ -37,16 +37,17 @@ public class FileProcessor : IFileProcessor
if (!DataStorageFactory.GetInstance().Exists(dumpData.GetKey()))
{
LoggerFactory.GetInstance().Log(
$"Cached not found for {string.Join("/", dumpData.GetKey().GetLookupIndex())} processing.",
LogLevel.Info
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log(
$"Cached not found for {string.Join("/", dumpData.GetKey().GetLookupIndex())} processing.",
LogLevel.Debug
);
dumpData.Containers = StaticLootProcessor.PreProcessStaticLoot(staticLoot);
dumpData.LooseLoot = LooseLootProcessor.PreProcessLooseLoot(looseLoot);
DataStorageFactory.GetInstance().Store(dumpData);
}
LoggerFactory.GetInstance().Log($"File {parsedData.FileName} finished processing!", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"File {parsedData.FileName} finished processing!", LogLevel.Debug);
return data;
}
}

View File

@ -1,12 +1,13 @@
using LootDumpProcessor.Process.Impl;
namespace LootDumpProcessor.Process.Processor;
namespace LootDumpProcessor.Process.Processor.FileProcessor;
public static class FileProcessorFactory
{
private static IFileProcessor? _fileProcessor;
public static IFileProcessor GetInstance()
{
// implement actual factory someday
return new FileProcessor();
// TODO: implement actual factory someday
if (_fileProcessor == null)
_fileProcessor = new FileProcessor();
return _fileProcessor;
}
}

View File

@ -1,6 +1,6 @@
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Process;
namespace LootDumpProcessor.Process.Processor.FileProcessor;
public interface IFileProcessor
{

View File

@ -1,15 +1,16 @@
using LootDumpProcessor.Logger;
using LootDumpProcessor.Logger;
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Storage;
using LootDumpProcessor.Storage.Collections;
using LootDumpProcessor.Utils;
using NumSharp;
namespace LootDumpProcessor.Process.Processor;
public class LooseLootProcessor
public static class LooseLootProcessor
{
public static PreProcessedLooseLoot PreProcessLooseLoot(List<Template> looseloot)
{
@ -163,10 +164,11 @@ public class LooseLootProcessor
Template = template
};
looseLootDistribution[mapName].SpawnPointsForced.Add(spawnPointToAdd);
LoggerFactory.GetInstance().Log(
$"Item: {template.Id} has > {LootDumpProcessorContext.GetConfig().ProcessorConfig.SpawnPointToleranceForForced}% spawn chance in spawn point: {spawnPointToAdd.LocationId} but isn't in forced loot, adding to forced",
LogLevel.Warning
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
LoggerFactory.GetInstance().Log(
$"Item: {template.Id} has > {LootDumpProcessorContext.GetConfig().ProcessorConfig.SpawnPointToleranceForForced}% spawn chance in spawn point: {spawnPointToAdd.LocationId} but isn't in forced loot, adding to forced",
LogLevel.Warning
);
}
else // Normal spawn point, add to non-forced spawnpoint array
{
@ -204,10 +206,11 @@ public class LooseLootProcessor
}
else
{
LoggerFactory.GetInstance().Log(
$"Item template {distribution.ComposedKey?.FirstItem?.Tpl} was on loose loot distribution for spawn point {template.Id} but the spawn points didnt contain a template matching it.",
LogLevel.Error
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log(
$"Item template {distribution.ComposedKey?.FirstItem?.Tpl} was on loose loot distribution for spawn point {template.Id} but the spawn points didnt contain a template matching it.",
LogLevel.Error
);
}
}
@ -248,10 +251,11 @@ public class LooseLootProcessor
{
if (!forcedTplsFound.Contains(itemTpl))
{
LoggerFactory.GetInstance().Log(
$"Expected item: {itemTpl} defined in forced_loose.yaml config not found in forced loot",
LogLevel.Error
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log(
$"Expected item: {itemTpl} defined in forced_loose.yaml config not found in forced loot",
LogLevel.Error
);
}
}
@ -260,10 +264,11 @@ public class LooseLootProcessor
{
if (!forcedTplsInConfig.Contains(itemTpl))
{
LoggerFactory.GetInstance().Log(
$"Map: {mapName} Item: {itemTpl} not defined in forced_loose.yaml config but was flagged as forced by code",
LogLevel.Warning
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
LoggerFactory.GetInstance().Log(
$"Map: {mapName} Item: {itemTpl} not defined in forced_loose.yaml config but was flagged as forced by code",
LogLevel.Warning
);
}
}
}

View File

@ -1,12 +1,13 @@
using LootDumpProcessor.Model;
using LootDumpProcessor.Model;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Utils;
namespace LootDumpProcessor.Process.Processor;
public class StaticLootProcessor
public static class StaticLootProcessor
{
public static List<PreProcessedStaticLoot> PreProcessStaticLoot(List<Template> staticloot)
{
@ -30,7 +31,6 @@ public class StaticLootProcessor
public static Tuple<string, MapStaticLoot> CreateRealStaticContainers(RootData rawMapDump)
{
List<StaticForced> forcedStaticItems;
var mapName = rawMapDump.Data.Name;
var staticLootPositions = (from li in rawMapDump.Data.Loot
where li.IsContainer ?? false
@ -46,7 +46,7 @@ public class StaticLootProcessor
}
}
forcedStaticItems = LootDumpProcessorContext.GetForcedItems().ContainsKey(mapName)
var forcedStaticItems = LootDumpProcessorContext.GetForcedItems().ContainsKey(mapName)
? LootDumpProcessorContext.GetForcedItems()[mapName]
: new List<StaticForced>();
@ -145,15 +145,12 @@ public class StaticLootProcessor
var itemsHitCounts = new Dictionary<string, int>();
foreach (var ci in container_counts_selected)
{
foreach (var cii in ci.Items)
foreach (var cii in ci.Items.Where(cii => cii.ParentId == ci.ContainerId))
{
if (cii.ParentId == ci.ContainerId)
{
if (itemsHitCounts.ContainsKey(cii.Tpl))
itemsHitCounts[cii.Tpl] += 1;
else
itemsHitCounts[cii.Tpl] = 1;
}
if (itemsHitCounts.ContainsKey(cii.Tpl))
itemsHitCounts[cii.Tpl] += 1;
else
itemsHitCounts[cii.Tpl] = 1;
}
}

View File

@ -1,12 +1,15 @@
using System.Collections.Concurrent;
using System.Collections.Concurrent;
using LootDumpProcessor.Logger;
using LootDumpProcessor.Process.Collector;
using LootDumpProcessor.Process.Processor;
using LootDumpProcessor.Process.Processor.DumpProcessor;
using LootDumpProcessor.Process.Processor.FileProcessor;
using LootDumpProcessor.Process.Reader;
using LootDumpProcessor.Process.Reader.Filters;
using LootDumpProcessor.Process.Reader.Intake;
using LootDumpProcessor.Process.Reader.PreProcess;
using LootDumpProcessor.Process.Writer;
using LootDumpProcessor.Utils;
namespace LootDumpProcessor.Process;
@ -38,7 +41,8 @@ public class QueuePipeline : IPipeline
// We add 2 more threads to the total count to account for subprocesses and others
int threads = LootDumpProcessorContext.GetConfig().Threads;
ThreadPool.SetMaxThreads(threads + 2, threads + 2);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Gathering files to begin processing", LogLevel.Info);
try
{
// Gather all files, sort them by date descending and then add them into the processing queue
@ -49,9 +53,14 @@ public class QueuePipeline : IPipeline
}
).ToList().ForEach(f => _filesToProcess.Add(f));
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Files sorted and ready to begin pre-processing", LogLevel.Info);
// We startup all the threads and collect them into a runners list
for (int i = 0; i < threads; i++)
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Creating pre-processing threads", LogLevel.Info);
Runners.Add(
Task.Factory.StartNew(
() =>
@ -67,9 +76,10 @@ public class QueuePipeline : IPipeline
}
catch (Exception e)
{
LoggerFactory.GetInstance().Log(
$"Error occurred while processing file {file}\n{e.Message}\n{e.StackTrace}",
LogLevel.Error);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log(
$"Error occurred while processing file {file}\n{e.Message}\n{e.StackTrace}",
LogLevel.Error);
}
}
},
@ -80,12 +90,14 @@ public class QueuePipeline : IPipeline
// Wait until all runners are done processing
while (!Runners.All(r => r.IsCompleted))
{
LoggerFactory.GetInstance().Log(
$"One or more file processors are still processing files. Waiting {LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs}ms before checking again",
LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log(
$"One or more file processors are still processing files. Waiting {LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs}ms before checking again",
LogLevel.Info);
Thread.Sleep(TimeSpan.FromMilliseconds(LootDumpProcessorContext.GetConfig().ThreadPoolingTimeoutMs));
}
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log("Pre-processing finished", LogLevel.Info);
// Single writer instance to collect results
var writer = WriterFactory.GetInstance();
// Single collector instance to collect results

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process.Reader.Filters;
namespace LootDumpProcessor.Process.Reader.Filters;
public static class FileFilterFactory
{

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process.Reader.Filters;
namespace LootDumpProcessor.Process.Reader.Filters;
public enum FileFilterTypes
{

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process.Reader.Filters;
namespace LootDumpProcessor.Process.Reader.Filters;
public interface IFileFilter
{

View File

@ -1,4 +1,4 @@
using System.Globalization;
using System.Globalization;
using System.Text.RegularExpressions;
using LootDumpProcessor.Logger;
@ -6,22 +6,23 @@ namespace LootDumpProcessor.Process.Reader.Filters;
public class JsonDumpFileFilter : IFileFilter
{
private static Regex FileNameDateRegex = new("([0-9]{4}(-[0-9]{2}){2}_((-){0,1}[0-9]{2}){3})");
private static DateTime parsedThresholdDate;
private static readonly Regex _fileNameDateRegex = new("([0-9]{4}(-[0-9]{2}){2}_((-){0,1}[0-9]{2}){3})");
private static readonly DateTime _parsedThresholdDate;
static JsonDumpFileFilter()
{
// Calculate parsed date from config threshold
if (string.IsNullOrEmpty(LootDumpProcessorContext.GetConfig().ReaderConfig.ThresholdDate))
{
LoggerFactory.GetInstance()
.Log($"ThresholdDate is null or empty in configs, defaulting to current day minus 30 days",
LogLevel.Warning);
parsedThresholdDate = (DateTime.Now - TimeSpan.FromDays(30));
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
LoggerFactory.GetInstance()
.Log($"ThresholdDate is null or empty in configs, defaulting to current day minus 30 days",
LogLevel.Warning);
_parsedThresholdDate = (DateTime.Now - TimeSpan.FromDays(30));
}
else
{
parsedThresholdDate = DateTime.ParseExact(
_parsedThresholdDate = DateTime.ParseExact(
LootDumpProcessorContext.GetConfig().ReaderConfig.ThresholdDate,
"yyyy-MM-dd",
CultureInfo.InvariantCulture
@ -33,8 +34,8 @@ public class JsonDumpFileFilter : IFileFilter
public bool Accept(string filename)
{
var unparsedDate = FileNameDateRegex.Match(filename).Groups[1].Value;
var unparsedDate = _fileNameDateRegex.Match(filename).Groups[1].Value;
var date = DateTime.ParseExact(unparsedDate, "yyyy-MM-dd_HH-mm-ss", CultureInfo.InvariantCulture);
return date > parsedThresholdDate;
return date > _parsedThresholdDate;
}
}

View File

@ -1,6 +1,6 @@
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Model.Processing;
namespace LootDumpProcessor.Process;
namespace LootDumpProcessor.Process.Reader.Intake;
public interface IIntakeReader
{

View File

@ -1,18 +1,28 @@
using LootDumpProcessor.Process.Impl;
namespace LootDumpProcessor.Process.Reader;
namespace LootDumpProcessor.Process.Reader.Intake;
public static class IntakeReaderFactory
{
private static readonly Dictionary<IntakeReaderTypes, IIntakeReader> Instances = new();
private static readonly object DictionaryLock = new();
public static IIntakeReader GetInstance()
{
return LootDumpProcessorContext.GetConfig().ReaderConfig.IntakeReaderConfig.IntakeReaderType switch
var type = LootDumpProcessorContext.GetConfig().ReaderConfig.IntakeReaderConfig?.IntakeReaderType ??
IntakeReaderTypes.Json;
lock (DictionaryLock)
{
IntakeReaderTypes.Json => new JsonFileIntakeReader(),
_ => throw new ArgumentOutOfRangeException(
"IntakeReaderType",
"Value was not defined on IntakeReaderConfig"
)
};
if (!Instances.TryGetValue(type, out var intakeReader))
{
intakeReader = type switch
{
IntakeReaderTypes.Json => new JsonFileIntakeReader(),
_ => throw new ArgumentOutOfRangeException(
"IntakeReaderType",
"Value was not defined on IntakeReaderConfig"
)
};
Instances.Add(type, intakeReader);
}
return intakeReader;
}
}
}

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process.Reader;
namespace LootDumpProcessor.Process.Reader.Intake;
public enum IntakeReaderTypes
{

View File

@ -1,11 +1,11 @@
using System.Collections.Concurrent;
using System.Collections.Concurrent;
using LootDumpProcessor.Logger;
using LootDumpProcessor.Model.Input;
using LootDumpProcessor.Model.Processing;
using LootDumpProcessor.Process.Processor;
using LootDumpProcessor.Serializers.Json;
using LootDumpProcessor.Utils;
namespace LootDumpProcessor.Process.Impl;
namespace LootDumpProcessor.Process.Reader.Intake;
public class JsonFileIntakeReader : IIntakeReader
{
@ -21,19 +21,21 @@ public class JsonFileIntakeReader : IIntakeReader
var fileData = File.ReadAllText(file);
// If the file format changes it may screw up this date parser
if (!FileDateParser.TryParseFileDate(file, out var date))
LoggerFactory.GetInstance().Log($"Couldnt parse date from file: {file}", LogLevel.Error);
{
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"Couldnt parse date from file: {file}", LogLevel.Error);
}
var fi = _jsonSerializer.Deserialize<RootData>(fileData);
if (fi.Data?.Name != null && (!_ignoredLocations?.Contains(fi.Data.Name) ?? true))
{
int counter;
if (!_totalMapDumpsCounter.TryGetValue(fi.Data.Name, out counter))
if (!_totalMapDumpsCounter.TryGetValue(fi.Data.Name, out var counter))
{
counter = 0;
_totalMapDumpsCounter[fi.Data.Name] = counter;
}
if (counter < LootDumpProcessorContext.GetConfig().ReaderConfig.IntakeReaderConfig.MaxDumpsPerMap)
if (counter < (LootDumpProcessorContext.GetConfig().ReaderConfig.IntakeReaderConfig?.MaxDumpsPerMap ?? 1500))
{
basicInfo = new BasicInfo
{
@ -44,16 +46,19 @@ public class JsonFileIntakeReader : IIntakeReader
FileName = file
};
_totalMapDumpsCounter[fi.Data.Name] += 1;
LoggerFactory.GetInstance().Log($"File {file} fully read, returning data", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"File {file} fully read, returning data", LogLevel.Debug);
return true;
}
LoggerFactory.GetInstance().Log($"Ignoring file {file} as the file cap for map {fi.Data.Name} has been reached", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
LoggerFactory.GetInstance().Log($"Ignoring file {file} as the file cap for map {fi.Data.Name} has been reached", LogLevel.Debug);
}
LoggerFactory.GetInstance().Log(
$"File {file} was not eligible for dump data, it did not contain a location name or it was on ignored locations config",
LogLevel.Info
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
LoggerFactory.GetInstance().Log(
$"File {file} was not eligible for dump data, it did not contain a location name or it was on ignored locations config",
LogLevel.Warning
);
basicInfo = null;
return false;
}

View File

@ -1,10 +1,10 @@
using LootDumpProcessor.Logger;
using LootDumpProcessor.Logger;
namespace LootDumpProcessor.Process.Reader.PreProcess;
public abstract class AbstractPreProcessReader : IPreProcessReader
{
protected string _tempFolder;
protected readonly string _tempFolder;
public AbstractPreProcessReader()
{
@ -12,11 +12,12 @@ public abstract class AbstractPreProcessReader : IPreProcessReader
if (string.IsNullOrEmpty(tempFolder))
{
tempFolder = GetBaseDirectory();
LoggerFactory.GetInstance()
.Log(
$"No temp folder was assigned preProcessorTempFolder in PreProcessorConfig, defaulting to {tempFolder}",
LogLevel.Warning
);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Warning))
LoggerFactory.GetInstance()
.Log(
$"No temp folder was assigned preProcessorTempFolder in PreProcessorConfig, defaulting to {tempFolder}",
LogLevel.Warning
);
}
// Cleanup the temp directory before starting the process
@ -35,7 +36,7 @@ public abstract class AbstractPreProcessReader : IPreProcessReader
protected string GetBaseDirectory()
{
return $"{Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)}\\SPT\\tmp\\PreProcessor";
return $@"{Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)}\SPT\tmp\PreProcessor";
}
public void Dispose()

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process.Reader.PreProcess;
namespace LootDumpProcessor.Process.Reader.PreProcess;
public interface IPreProcessReader
{

View File

@ -1,9 +1,8 @@
namespace LootDumpProcessor.Process.Reader.PreProcess;
namespace LootDumpProcessor.Process.Reader.PreProcess;
public static class PreProcessReaderFactory
{
private static readonly Dictionary<PreProcessReaderTypes, IPreProcessReader> _proProcessReaders = new();
private static object lockObject = new object();
public static IPreProcessReader GetInstance(PreProcessReaderTypes type)
{

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process.Reader.PreProcess;
namespace LootDumpProcessor.Process.Reader.PreProcess;
public enum PreProcessReaderTypes
{

View File

@ -1,4 +1,4 @@
using LootDumpProcessor.Logger;
using LootDumpProcessor.Logger;
using SevenZip;
using SevenZip.Sdk.Compression.Lzma;
@ -15,22 +15,26 @@ public class SevenZipPreProcessReader : AbstractPreProcessReader
public override bool TryPreProcess(string file, out List<string> files, out List<string> directories)
{
Decoder decoder = new Decoder();
var fileRaw = Path.GetFileNameWithoutExtension(file);
// SevenZip library doesnt like forward slashes for some reason
var outPath = $"{_tempFolder}\\{fileRaw}".Replace("/", "\\");
LoggerFactory.GetInstance().Log(
$"Unzipping {file} into temp path {outPath}, this may take a while...",
LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log(
$"Unzipping {file} into temp path {outPath}, this may take a while...",
LogLevel.Info);
var extractor = new SevenZipExtractor(file);
extractor.Extracting += (sender, args) =>
// Only log process on debug mode
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug))
{
if (args.PercentDone % 10 == 0)
LoggerFactory.GetInstance().Log($"Unzip progress: {args.PercentDone}%", LogLevel.Info);
};
extractor.Extracting += (_, args) =>
{
if (args.PercentDone % 10 == 0)
LoggerFactory.GetInstance().Log($"Unzip progress: {args.PercentDone}%", LogLevel.Debug);
};
}
extractor.ExtractArchive(outPath);
LoggerFactory.GetInstance().Log($"Finished unzipping {file} into temp path {outPath}", LogLevel.Info);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info))
LoggerFactory.GetInstance().Log($"Finished unzipping {file} into temp path {outPath}", LogLevel.Info);
files = Directory.GetFiles(outPath).ToList();
directories = Directory.GetDirectories(outPath).ToList();

View File

@ -1,25 +1,23 @@
using LootDumpProcessor.Logger;
using LootDumpProcessor.Logger;
using LootDumpProcessor.Model.Tarkov;
using LootDumpProcessor.Serializers.Json;
namespace LootDumpProcessor.Process;
public class TarkovItems
public class TarkovItems(string items)
{
private static readonly IJsonSerializer _jsonSerializer = JsonSerializerFactory.GetInstance();
private Dictionary<string, TemplateFileItem> _items;
public TarkovItems(string items)
{
_items = _jsonSerializer.Deserialize<Dictionary<string, TemplateFileItem>>(File.ReadAllText(items));
}
private readonly Dictionary<string, TemplateFileItem>? _items = _jsonSerializer.Deserialize<Dictionary<string, TemplateFileItem>>(File.ReadAllText(items));
public virtual bool IsBaseClass(string tpl, string baseclass_id)
{
if (_items == null)
throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
if (!_items.TryGetValue(tpl, out var item_template))
{
LoggerFactory.GetInstance().Log($"[IsBaseClass] Item template '{tpl}' with base class id '{baseclass_id}' was not found on the server items!", LogLevel.Error);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"[IsBaseClass] Item template '{tpl}' with base class id '{baseclass_id}' was not found on the server items!", LogLevel.Error);
return false;
}
@ -31,9 +29,12 @@ public class TarkovItems
public virtual bool IsQuestItem(string tpl)
{
if (_items == null)
throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
if (!_items.TryGetValue(tpl, out var item_template))
{
LoggerFactory.GetInstance().Log($"[IsQuestItem] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"[IsQuestItem] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
return false;
}
return item_template.Props.QuestItem;
@ -41,9 +42,12 @@ public class TarkovItems
public virtual string? MaxDurability(string tpl)
{
if (_items == null)
throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
if (!_items.TryGetValue(tpl, out var item_template))
{
LoggerFactory.GetInstance().Log($"[MaxDurability] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"[MaxDurability] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
return null;
}
return item_template.Props.MaxDurability?.ToString() ?? "";
@ -51,9 +55,12 @@ public class TarkovItems
public virtual string? AmmoCaliber(string tpl)
{
if (_items == null)
throw new Exception("The server items couldnt be found or loaded. Check server config is pointing to the correct place");
if (!_items.TryGetValue(tpl, out var item_template))
{
LoggerFactory.GetInstance().Log($"[AmmoCaliber] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Error))
LoggerFactory.GetInstance().Log($"[AmmoCaliber] Item template '{tpl}' was not found on the server items!", LogLevel.Error);
return null;
}
return item_template.Props.Caliber;

View File

@ -1,14 +1,14 @@
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output;
using LootDumpProcessor.Model.Output.LooseLoot;
using LootDumpProcessor.Model.Output.StaticContainer;
using LootDumpProcessor.Serializers.Json;
namespace LootDumpProcessor.Process.Impl;
namespace LootDumpProcessor.Process.Writer;
public class FileWriter : IWriter
{
private static readonly IJsonSerializer _jsonSerializer = JsonSerializerFactory.GetInstance();
private static string _outputPath;
private static readonly string _outputPath;
static FileWriter()
{
@ -47,9 +47,9 @@ public class FileWriter : IWriter
{
foreach (var s in LootDumpProcessorContext.GetDirectoryMappings()[key].Name)
{
if (!Directory.Exists($"{_outputPath}\\locations\\{s}"))
Directory.CreateDirectory($"{_outputPath}\\locations\\{s}");
File.WriteAllText($"{_outputPath}\\locations\\{s}\\looseLoot.json",
if (!Directory.Exists($@"{_outputPath}\locations\{s}"))
Directory.CreateDirectory($@"{_outputPath}\locations\{s}");
File.WriteAllText($@"{_outputPath}\locations\{s}\looseLoot.json",
_jsonSerializer.Serialize(value));
}
}
@ -57,17 +57,17 @@ public class FileWriter : IWriter
break;
case OutputFileType.StaticContainer:
var staticContainer = (Dictionary<string, MapStaticLoot>)data;
File.WriteAllText($"{_outputPath}\\loot\\staticContainers.json",
File.WriteAllText($@"{_outputPath}\loot\staticContainers.json",
_jsonSerializer.Serialize(staticContainer));
break;
case OutputFileType.StaticLoot:
var staticLoot = (Dictionary<string, StaticItemDistribution>)data;
File.WriteAllText($"{_outputPath}\\loot\\staticLoot.json",
File.WriteAllText($@"{_outputPath}\loot\staticLoot.json",
_jsonSerializer.Serialize(staticLoot));
break;
case OutputFileType.StaticAmmo:
var staticAmmo = (Dictionary<string, List<AmmoDistribution>>)data;
File.WriteAllText($"{_outputPath}\\loot\\staticAmmo.json",
File.WriteAllText($@"{_outputPath}\loot\staticAmmo.json",
_jsonSerializer.Serialize(staticAmmo));
break;
default:

View File

@ -1,4 +1,4 @@
namespace LootDumpProcessor.Process;
namespace LootDumpProcessor.Process.Writer;
public interface IWriter
{

View File

@ -1,5 +1,3 @@
using LootDumpProcessor.Process.Impl;
namespace LootDumpProcessor.Process.Writer;
public static class WriterFactory

Some files were not shown because too many files have changed in this diff Show More