waffle.lord/Patcher
0
0
Fork 0
Code Issues Pull Requests Projects Releases 22 Wiki Activity

impr/parallel-patching #17

Merged
waffle.lord merged 2 commits from impr/parallel-patching into main 2024-05-04 10:57:31 -04:00
Conversation 0 Commits 2 Files Changed 9 +264 -188
9 changed files with 264 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Patcher/PatchClient/PatchClient.csproj
View File

@ -4,8 +4,8 @@
<TargetFramework>net8.0</TargetFramework>
<IncludeNativeLibrariesForSelfExtract>true</IncludeNativeLibrariesForSelfExtract>
<Nullable>enable</Nullable>
<AssemblyVersion>2.12.0</AssemblyVersion>
<FileVersion>2.12.0</FileVersion>
<AssemblyVersion>2.15.0</AssemblyVersion>
<FileVersion>2.15.0</FileVersion>
</PropertyGroup>
<ItemGroup>
<AvaloniaResource Include="Assets\**" />

4
Patcher/PatchGenerator/PatchGenerator.csproj
View File

@ -4,8 +4,8 @@
<TargetFramework>net8.0</TargetFramework>
<IncludeNativeLibrariesForSelfExtract>true</IncludeNativeLibrariesForSelfExtract>
<Nullable>enable</Nullable>
<AssemblyVersion>2.12.0</AssemblyVersion>
<FileVersion>2.12.0</FileVersion>
<AssemblyVersion>2.15.0</AssemblyVersion>
<FileVersion>2.15.0</FileVersion>
</PropertyGroup>
<ItemGroup>

BIN
Patcher/PatchGenerator/Resources/PatchClient.exe
View File

Binary file not shown.

4
Patcher/PatchGenerator/ViewModels/MainWindowViewModel.cs
View File

@ -23,7 +23,6 @@ namespace PatchGenerator.ViewModels
{
this.WhenActivated((CompositeDisposable disposables) =>
{
if (genArgs != null && genArgs.ReadyToRun)
{
PatchGenInfo genInfo = new PatchGenInfo();
@ -31,7 +30,8 @@ namespace PatchGenerator.ViewModels
genInfo.TargetFolderPath = genArgs.TargetFolderPath;
genInfo.SourceFolderPath = genArgs.SourceFolderPath;
genInfo.PatchName = genArgs.OutputFolderName;
genInfo.AutoZip = genArgs.AutoZip;
// issues with auto zip, but it's not really used anymore so just disabling for now
genInfo.AutoZip = false;
genInfo.AutoClose = genArgs.AutoClose;
Router.Navigate.Execute(new PatchGenerationViewModel(this, genInfo));

61
Patcher/PatchGenerator/ViewModels/PatchGenerationViewModel.cs
View File

@ -14,9 +14,11 @@ using System.IO;
using System.Reactive.Disposables;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Timers;
using PatcherUtils.Model;
using Timer = System.Timers.Timer;
namespace PatchGenerator.ViewModels
{
@ -113,14 +115,16 @@ namespace PatchGenerator.ViewModels
var message = patcher.GeneratePatches();
if(message.ExitCode != PatcherExitCode.Success && generationInfo.AutoClose)
{
Environment.Exit((int)message.ExitCode);
}
patchGenStopwatch.Stop();
updateElapsedTimeTimer.Stop();
if(message.ExitCode != PatcherExitCode.Success && generationInfo.AutoClose)
{
PatchLogger.LogInfo("Exiting: Auto close on failure");
Environment.Exit((int)message.ExitCode);
}
PatchLogger.LogInfo("Printing summary info ...");
PrintSummary();
StringBuilder sb = new StringBuilder()
@ -133,29 +137,32 @@ namespace PatchGenerator.ViewModels
File.Copy(LazyOperations.PatcherClientPath, $"{generationInfo.PatchName.FromCwd()}\\patcher.exe", true);
if (generationInfo.AutoZip)
{
IndeterminateProgress = true;
PatchLogger.LogInfo("Copied patcher.exe to output folder");
PatchItemCollection.Add(new PatchItem("Allowing Time for files to unlock ..."));
System.Threading.Thread.Sleep(2000);
PatchItemCollection.Add(new PatchItem("Zipping patcher ..."));
ProgressMessage = "Zipping patcher";
IndeterminateProgress = false;
var progress = new Progress<int>(p =>
{
PatchPercent = p;
});
LazyOperations.CompressDirectory(generationInfo.PatchName.FromCwd(), $"{generationInfo.PatchName}.7z".FromCwd(), progress);
PatchItemCollection.Add(new PatchItem("Done"));
}
// if (generationInfo.AutoZip)
// {
// PatchLogger.LogInfo("AutoZipping");
// IndeterminateProgress = true;
//
// PatchItemCollection.Add(new PatchItem("Allowing Time for files to unlock ..."));
//
// Thread.Sleep(2000);
//
// PatchItemCollection.Add(new PatchItem("Zipping patcher ..."));
//
// ProgressMessage = "Zipping patcher";
//
// IndeterminateProgress = false;
//
// var progress = new Progress<int>(p =>
// {
// PatchPercent = p;
// });
//
// LazyOperations.CompressDirectory(generationInfo.PatchName.FromCwd(), $"{generationInfo.PatchName}.7z".FromCwd(), progress);
//
// PatchItemCollection.Add(new PatchItem("Done"));
// }
if (generationInfo.AutoClose)
{

5
Patcher/PatchGenerator/Views/OptionsView.axaml
View File

@ -16,8 +16,9 @@
Watermark="Output Folder Name"
/>
<CheckBox Content="Zip Generated Files" Grid.Row="4"
IsChecked="{Binding GenerationInfo.AutoZip}"/>
<!-- UNRESOLVED ISSUES: disabling for now -->
<!-- <CheckBox Content="Zip Generated Files" Grid.Row="4" -->
<!-- IsChecked="{Binding GenerationInfo.AutoZip}"/> -->
<Button Content="Generate Patches" Grid.ColumnSpan="3" Grid.Row="4"
HorizontalAlignment="Right"

30
Patcher/PatcherUtils/LazyOperations.cs
View File

@ -2,7 +2,6 @@
using PatcherUtils.Model;
using System.IO;
using System.Reflection;
using Aki.Common.Utils;
using SevenZip;
namespace PatcherUtils
@ -102,31 +101,44 @@ namespace PatcherUtils
public static void CompressDirectory(string SourceDirectoryPath, string DestinationFilePath, IProgress<int> progress)
{
try
{
PatchLogger.LogInfo($"Compressing: {SourceDirectoryPath}");
PatchLogger.LogInfo($"Output file: {DestinationFilePath}");
var outputFile = new FileInfo(DestinationFilePath);
SevenZipBase.SetLibraryPath(SevenZDllPath);
PatchLogger.LogInfo($"7z.dll set: {SevenZDllPath}");
var compressor = new SevenZipCompressor()
{
ArchiveFormat = OutArchiveFormat.SevenZip,
CompressionMethod = CompressionMethod.Lzma2,
CompressionLevel = CompressionLevel.Normal
CompressionLevel = CompressionLevel.Normal,
PreserveDirectoryRoot = true
};
compressor.Compressing += (_, args) =>
{
progress.Report(args.PercentDone);
};
compressor.Compressing += (_, args) => { progress.Report(args.PercentDone); };
using var outputStream = outputFile.OpenWrite();
PatchLogger.LogInfo("Starting compression");
compressor.CompressDirectory(SourceDirectoryPath, outputStream);
PatchLogger.LogInfo("Compression complete");
outputFile.Refresh();
// failed to compress data
if (!outputFile.Exists || outputFile.Length == 0)
{
Logger.LogError("Failed to compress patcher");
PatchLogger.LogError("Failed to compress patcher");
}
}
catch (Exception ex)
{
PatchLogger.LogException(ex);
}
}
@ -140,7 +152,7 @@ namespace PatcherUtils
if (dir.Exists)
{
dir.Delete(true);
PatchLogger.LogInfo("Temp directory delted");
PatchLogger.LogInfo("Temp directory deleted");
}
}
}

140
Patcher/PatcherUtils/PatchHelper.cs
View File

@ -1,11 +1,14 @@
using PatchClient.Models;
using PatcherUtils.Model;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using PatcherUtils.Helpers;
namespace PatcherUtils
@ -34,7 +37,8 @@ namespace PatcherUtils
/// <remarks>Includes an array of <see cref="LineItem"/> with details for each type of patch</remarks>
public event ProgressChangedHandler ProgressChanged;
protected virtual void RaiseProgressChanged(int progress, int total, string Message = "", params LineItem[] AdditionalLineItems)
protected virtual void RaiseProgressChanged(int progress, int total, string Message = "",
params LineItem[] AdditionalLineItems)
{
int percent = (int)Math.Floor((double)progress / total * 100);
@ -88,7 +92,8 @@ namespace PatcherUtils
bool matched = Enumerable.SequenceEqual(sourceHash, targetHash);
PatchLogger.LogInfo($"Hash Check: S({sourceInfo.Name}|{Convert.ToBase64String(sourceHash)}) - T({targetInfo.Name}|{Convert.ToBase64String(targetHash)}) - Match:{matched}");
PatchLogger.LogInfo(
$"Hash Check: S({sourceInfo.Name}|{Convert.ToBase64String(sourceHash)}) - T({targetInfo.Name}|{Convert.ToBase64String(targetHash)}) - Match:{matched}");
return matched;
}
@ -153,7 +158,7 @@ namespace PatcherUtils
{
Directory.CreateDirectory(deltaPath.Replace(sourceFileInfo.Name + ".delta", ""));
}
catch(Exception ex)
catch (Exception ex)
{
PatchLogger.LogException(ex);
}
@ -191,7 +196,7 @@ namespace PatcherUtils
{
Directory.CreateDirectory(deltaPath.Replace(sourceFileInfo.Name + ".del", ""));
}
catch(Exception ex)
catch (Exception ex)
{
PatchLogger.LogException(ex);
}
@ -201,7 +206,7 @@ namespace PatcherUtils
File.Create(deltaPath);
PatchLogger.LogInfo($"File Created [DEL]: {deltaPath}");
}
catch(Exception ex)
catch (Exception ex)
{
PatchLogger.LogException(ex);
}
@ -222,7 +227,7 @@ namespace PatcherUtils
{
Directory.CreateDirectory(deltaPath.Replace(targetSourceInfo.Name + ".new", ""));
}
catch(Exception ex)
catch (Exception ex)
{
PatchLogger.LogException(ex);
}
@ -232,7 +237,7 @@ namespace PatcherUtils
targetSourceInfo.CopyTo(deltaPath, true);
PatchLogger.LogInfo($"File Created [NEW]: {deltaPath}");
}
catch(Exception ex)
catch (Exception ex)
{
PatchLogger.LogException(ex);
}
@ -275,9 +280,11 @@ namespace PatcherUtils
LazyOperations.ExtractResourcesToTempDir();
List<FileInfo> SourceFiles = sourceDir.GetFiles("*", SearchOption.AllDirectories).ToList();
List<FileInfo> sourceFiles = sourceDir.GetFiles("*", SearchOption.AllDirectories).ToList();
var targetFiles = targetDir.GetFiles("*", SearchOption.AllDirectories).ToList();
ConcurrentQueue<FileInfo> foundFilesQueue = new ConcurrentQueue<FileInfo>();
fileCountTotal = SourceFiles.Count;
fileCountTotal = targetFiles.Count;
PatchLogger.LogInfo($"Total source files: {fileCountTotal}");
@ -291,10 +298,13 @@ namespace PatcherUtils
RaiseProgressChanged(0, fileCountTotal, "Generating deltas...");
foreach (FileInfo targetFile in targetDir.GetFiles("*", SearchOption.AllDirectories))
Parallel.ForEach(targetFiles,
new ParallelOptions() { MaxDegreeOfParallelism = 5 }, targetFile =>
{
//find a matching source file based on the relative path of the file
FileInfo sourceFile = SourceFiles.Find(f => f.FullName.Replace(sourceDir.FullName, "") == targetFile.FullName.Replace(targetDir.FullName, ""));
FileInfo sourceFile = sourceFiles.Find(f =>
f.FullName.Replace(sourceDir.FullName, "") ==
targetFile.FullName.Replace(targetDir.FullName, ""));
//if the target file doesn't exist in the source files, the target file needs to be added.
if (sourceFile == null)
@ -305,16 +315,19 @@ namespace PatcherUtils
newCount++;
filesProcessed++;
RaiseProgressChanged(filesProcessed, fileCountTotal, $"{targetFile.FullName.Replace(TargetFolder, "...")}.new", AdditionalInfo.ToArray());
RaiseProgressChanged(filesProcessed, fileCountTotal,
$"{targetFile.FullName.Replace(TargetFolder, "...")}.new", AdditionalInfo.ToArray());
continue;
return;
}
string extension = "";
//if a matching source file was found, check the file hashes and get the delta.
// if a matching source file was found, check the file hashes and get the delta.
// add it to the bag for removal later
if (!CompareFileHashes(sourceFile.FullName, targetFile.FullName))
{
foundFilesQueue.Enqueue(sourceFile);
PatchLogger.LogInfo("::: Creating .delta file :::");
CreateDelta(sourceFile.FullName, targetFile.FullName);
extension = ".delta";
@ -322,43 +335,55 @@ namespace PatcherUtils
}
else
{
foundFilesQueue.Enqueue(sourceFile);
PatchLogger.LogInfo("::: File Exists :::");
existCount++;
}
try
{
SourceFiles.Remove(sourceFile);
}
catch(Exception ex)
{
PatchLogger.LogException(ex);
}
filesProcessed++;
AdditionalInfo[0].ItemValue = deltaCount;
AdditionalInfo[1].ItemValue = newCount;
AdditionalInfo[3].ItemValue = existCount;
RaiseProgressChanged(filesProcessed, fileCountTotal, $"{targetFile.FullName.Replace(TargetFolder, "...")}{extension}", AdditionalInfo.ToArray());
RaiseProgressChanged(filesProcessed, fileCountTotal,
$"{targetFile.FullName.Replace(TargetFolder, "...")}{extension}", AdditionalInfo.ToArray());
});
// remove all queued files that were found in the source files list
PatchLogger.LogInfo(":: Updating Source List ::");
try
{
int processedQueueCount = 0;
int queueTotal = foundFilesQueue.Count;
foreach (var queuedFile in foundFilesQueue)
{
RaiseProgressChanged(processedQueueCount, queueTotal, $"Queued file removed: {queuedFile.Name}",
AdditionalInfo.ToArray());
sourceFiles.Remove(queuedFile);
}
}
catch (Exception ex)
{
PatchLogger.LogException(ex);
}
//Any remaining source files do not exist in the target folder and can be removed.
//reset progress info
if (SourceFiles.Count == 0)
if (sourceFiles.Count == 0)
{
PatchLogger.LogInfo("::: Patch Generation Complete :::");
return new PatchMessage("Generation Done", PatcherExitCode.Success);
}
RaiseProgressChanged(0, SourceFiles.Count, "Processing .del files...");
RaiseProgressChanged(0, sourceFiles.Count, "Processing .del files...");
filesProcessed = 0;
fileCountTotal = SourceFiles.Count;
fileCountTotal = sourceFiles.Count;
foreach (FileInfo delFile in SourceFiles)
foreach (FileInfo delFile in sourceFiles)
{
PatchLogger.LogInfo("::: Creating .del file :::");
CreateDelFile(delFile.FullName);
@ -368,7 +393,8 @@ namespace PatcherUtils
AdditionalInfo[2].ItemValue = delCount;
filesProcessed++;
RaiseProgressChanged(filesProcessed, fileCountTotal, $"{delFile.FullName.Replace(SourceFolder, "...")}.del", AdditionalInfo.ToArray());
RaiseProgressChanged(filesProcessed, fileCountTotal,
$"{delFile.FullName.Replace(SourceFolder, "...")}.del", AdditionalInfo.ToArray());
}
PatchLogger.LogInfo("::: Patch Generation Complete :::");
@ -398,7 +424,7 @@ namespace PatcherUtils
return new PatchMessage(message, PatcherExitCode.MissingDir);
}
if(!deltaDir.Exists)
if (!deltaDir.Exists)
{
string message = $"Could not find delta directory: {deltaDir.FullName}";
PatchLogger.LogError(message);
@ -424,30 +450,43 @@ namespace PatcherUtils
new LineItem("Files to Delete", delCount)
};
ConcurrentQueue<PatchMessage> errorsQueue = new ConcurrentQueue<PatchMessage>();
filesProcessed = 0;
fileCountTotal = deltaFiles.Count;
var patchingTokenSource = new CancellationTokenSource();
foreach (FileInfo deltaFile in deltaDir.GetFiles("*", SearchOption.AllDirectories))
// foreach (FileInfo deltaFile in deltaDir.GetFiles("*", SearchOption.AllDirectories))
Parallel.ForEach(deltaDir.GetFiles("*", SearchOption.AllDirectories).ToList(),
new ParallelOptions() { MaxDegreeOfParallelism = 5, CancellationToken = patchingTokenSource.Token},
deltaFile =>
{
switch (deltaFile.Extension)
{
case ".delta":
{
//apply delta
FileInfo sourceFile = SourceFiles.Find(f => f.FullName.Replace(sourceDir.FullName, "") == deltaFile.FullName.Replace(deltaDir.FullName, "").Replace(".delta", ""));
FileInfo sourceFile = SourceFiles.Find(f =>
f.FullName.Replace(sourceDir.FullName, "") == deltaFile.FullName
.Replace(deltaDir.FullName, "").Replace(".delta", ""));
if (sourceFile == null)
{
return new PatchMessage($"Failed to find matching source file for '{deltaFile.FullName}'", PatcherExitCode.MissingFile);
patchingTokenSource.Cancel();
errorsQueue.Enqueue(new PatchMessage(
$"Failed to find matching source file for '{deltaFile.FullName}'",
PatcherExitCode.MissingFile));
return;
}
PatchLogger.LogInfo("::: Applying Delta :::");
var result = ApplyDelta(sourceFile.FullName, deltaFile.FullName);
if(!result.Item1)
if (!result.Item1)
{
return new PatchMessage(result.Item2, PatcherExitCode.PatchFailed);
patchingTokenSource.Cancel();
errorsQueue.Enqueue(new PatchMessage(result.Item2, PatcherExitCode.PatchFailed));
return;
}
deltaCount--;
@ -457,7 +496,8 @@ namespace PatcherUtils
case ".new":
{
//copy new file
string destination = Path.Join(sourceDir.FullName, deltaFile.FullName.Replace(deltaDir.FullName, "").Replace(".new", ""));
string destination = Path.Join(sourceDir.FullName,
deltaFile.FullName.Replace(deltaDir.FullName, "").Replace(".new", ""));
PatchLogger.LogInfo("::: Adding New File :::");
@ -467,10 +507,12 @@ namespace PatcherUtils
File.Copy(deltaFile.FullName, destination, true);
PatchLogger.LogInfo($"File added: {destination}");
}
catch(Exception ex)
catch (Exception ex)
{
patchingTokenSource.Cancel();
PatchLogger.LogException(ex);
return new PatchMessage(ex.Message, PatcherExitCode.PatchFailed);
errorsQueue.Enqueue(new PatchMessage(ex.Message, PatcherExitCode.PatchFailed));
return;
}
newCount--;
@ -480,7 +522,8 @@ namespace PatcherUtils
case ".del":
{
//remove unneeded file
string delFilePath = Path.Join(sourceDir.FullName, deltaFile.FullName.Replace(deltaDir.FullName, "").Replace(".del", ""));
string delFilePath = Path.Join(sourceDir.FullName,
deltaFile.FullName.Replace(deltaDir.FullName, "").Replace(".del", ""));
PatchLogger.LogInfo("::: Removing Uneeded File :::");
@ -489,10 +532,12 @@ namespace PatcherUtils
File.Delete(delFilePath);
PatchLogger.LogInfo($"File removed: {delFilePath}");
}
catch(Exception ex)
catch (Exception ex)
{
patchingTokenSource.Cancel();
PatchLogger.LogException(ex);
return new PatchMessage(ex.Message, PatcherExitCode.PatchFailed);
errorsQueue.Enqueue(new PatchMessage(ex.Message, PatcherExitCode.PatchFailed));
return;
}
delCount--;
@ -507,10 +552,21 @@ namespace PatcherUtils
++filesProcessed;
RaiseProgressChanged(filesProcessed, fileCountTotal, deltaFile.Name, AdditionalInfo.ToArray());
});
if (errorsQueue.Count > 0)
{
if (!errorsQueue.TryDequeue(out PatchMessage error))
{
return new PatchMessage("Errors occurred during patching, but we couldn't dequeue them :(\n\nThere may be more information in the log", PatcherExitCode.PatchFailed);
}
return error;
}
PatchLogger.LogInfo("::: Patching Complete :::");
return new PatchMessage($"Patching Complete. You can delete the patcher.exe file.", PatcherExitCode.Success);
return new PatchMessage($"Patching Complete. You can delete the patcher.exe file.",
PatcherExitCode.Success);
}
}
}

2
Patcher/PatcherUtils/PatcherUtils.csproj
View File

@ -22,7 +22,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Squid-Box.SevenZipSharp" Version="1.6.1.23" />
<PackageReference Include="Squid-Box.SevenZipSharp" Version="1.6.2.24" />
</ItemGroup>
</Project>