From 64d0b4b35e4a2bdd75bdcd442f53f19350133d4a Mon Sep 17 00:00:00 2001 From: "waffle.lord" Date: Sat, 4 May 2024 16:18:49 -0400 Subject: [PATCH 1/3] use cache ttl for metadata files --- SPTInstaller/Helpers/DownloadCacheHelper.cs | 78 +++++++++++++++++-- SPTInstaller/Installer Tasks/DownloadTask.cs | 2 +- .../Installer Tasks/ReleaseCheckTask.cs | 9 ++- SPTInstaller/Models/InstallerUpdateInfo.cs | 4 +- SPTInstaller/ViewModels/PreChecksViewModel.cs | 6 +- 5 files changed, 84 insertions(+), 15 deletions(-) diff --git a/SPTInstaller/Helpers/DownloadCacheHelper.cs b/SPTInstaller/Helpers/DownloadCacheHelper.cs index 2866bb3..45ddb8b 100644 --- a/SPTInstaller/Helpers/DownloadCacheHelper.cs +++ b/SPTInstaller/Helpers/DownloadCacheHelper.cs @@ -8,7 +8,8 @@ namespace SPTInstaller.Helpers; public static class DownloadCacheHelper { private static HttpClient _httpClient = new() { Timeout = TimeSpan.FromMinutes(15) }; - + + public static TimeSpan SuggestedTtl = TimeSpan.FromHours(1); public static string CachePath = Path.Join(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "spt-installer/cache"); @@ -50,10 +51,10 @@ public static class DownloadCacheHelper /// The expected hash of the file in the cache /// The file found in the cache; null if no file is found /// True if the file is in the cache and its hash matches the expected hash, otherwise false - public static bool CheckCache(string fileName, string expectedHash, out FileInfo cachedFile) - => CheckCache(new FileInfo(Path.Join(CachePath, fileName)), expectedHash, out cachedFile); + public static bool CheckCacheHash(string fileName, string expectedHash, out FileInfo cachedFile) + => CheckCacheHash(new FileInfo(Path.Join(CachePath, fileName)), expectedHash, out cachedFile); - private static bool CheckCache(FileInfo cacheFile, string expectedHash, out FileInfo fileInCache) + private static bool CheckCacheHash(FileInfo cacheFile, string expectedHash, out FileInfo fileInCache) { fileInCache = cacheFile; @@ -85,6 +86,44 @@ public static class DownloadCacheHelper return false; } } + + /// + /// Gets a file in the cache based on a time-to-live from its last modified time + /// + /// The name of the file to look for in the cache + /// The time-to-live to check against + /// The file found in the cache if it exists + /// Returns true if the file was found in the cache, otherwise false + public static bool CheckCacheTTL(string fileName, TimeSpan ttl, out FileInfo cachedFile) => + CheckCacheTTL(new FileInfo(Path.Join(CachePath, fileName)), ttl, out cachedFile); + + private static bool CheckCacheTTL(FileInfo cacheFile, TimeSpan ttl, out FileInfo fileInCache) + { + fileInCache = cacheFile; + + try + { + cacheFile.Refresh(); + Directory.CreateDirectory(CachePath); + + if (!cacheFile.Exists) + { + Log.Information($"{cacheFile.Name} {(cacheFile.Exists ? "is in cache" : "NOT in cache")}"); + return false; + } + + var validTimeToLive = cacheFile.LastWriteTime.Add(ttl) > DateTime.Now; + + Log.Information($"{cacheFile.Name} TTL is {(validTimeToLive ? "OK" : "INVALID")}"); + + return validTimeToLive; + } + catch (Exception ex) + { + Log.Error(ex, "Something went wrong during hashing"); + return false; + } + } /// /// Download a file to the cache folder @@ -166,6 +205,33 @@ public static class DownloadCacheHelper return null; } } + + /// + /// Get or download a file using a time to live + /// + /// The file to get from cache + /// The link to use for the download + /// A progress object for reporting download progress + /// The time-to-live to check against in the cache + /// + public static async Task GetOrDownloadFileAsync(string fileName, string targetLink, + IProgress progress, TimeSpan timeToLive) + { + try + { + if (CheckCacheTTL(fileName, timeToLive, out FileInfo cachedFile)) + { + return cachedFile; + } + + return await DownloadFileAsync(fileName, targetLink, progress); + } + catch (Exception ex) + { + Log.Error(ex, $"Error while getting file: {fileName}"); + return null; + } + } /// /// Get the file from cache or download it @@ -181,7 +247,7 @@ public static class DownloadCacheHelper { try { - if (CheckCache(fileName, expectedHash, out var cacheFile)) + if (CheckCacheHash(fileName, expectedHash, out var cacheFile)) return cacheFile; return await DownloadFileAsync(fileName, targetLink, progress); @@ -206,7 +272,7 @@ public static class DownloadCacheHelper { try { - if (CheckCache(fileName, expectedHash, out var cacheFile)) + if (CheckCacheHash(fileName, expectedHash, out var cacheFile)) return cacheFile; return await DownloadFileAsync(fileName, fileDownloadStream); diff --git a/SPTInstaller/Installer Tasks/DownloadTask.cs b/SPTInstaller/Installer Tasks/DownloadTask.cs index af0785d..75875cc 100644 --- a/SPTInstaller/Installer Tasks/DownloadTask.cs +++ b/SPTInstaller/Installer Tasks/DownloadTask.cs @@ -45,7 +45,7 @@ public class DownloadTask : InstallerTaskBase { SetStatus("Downloading Patcher", "Verifying cached patcher ...", progressStyle: ProgressStyle.Indeterminate); - if (DownloadCacheHelper.CheckCache("patcher", _expectedPatcherHash, out var cacheFile)) + if (DownloadCacheHelper.CheckCacheHash("patcher", _expectedPatcherHash, out var cacheFile)) { _data.PatcherZipInfo = cacheFile; Log.Information("Using cached file {fileName} - Hash: {hash}", _data.PatcherZipInfo.Name, diff --git a/SPTInstaller/Installer Tasks/ReleaseCheckTask.cs b/SPTInstaller/Installer Tasks/ReleaseCheckTask.cs index 7add179..8857c55 100644 --- a/SPTInstaller/Installer Tasks/ReleaseCheckTask.cs +++ b/SPTInstaller/Installer Tasks/ReleaseCheckTask.cs @@ -25,8 +25,9 @@ public class ReleaseCheckTask : InstallerTaskBase var progress = new Progress((d) => { SetStatus(null, null, (int)Math.Floor(d)); }); var akiReleaseInfoFile = - await DownloadCacheHelper.DownloadFileAsync("release.json", DownloadCacheHelper.ReleaseMirrorUrl, - progress); + await DownloadCacheHelper.GetOrDownloadFileAsync("release.json", DownloadCacheHelper.ReleaseMirrorUrl, + progress, DownloadCacheHelper.SuggestedTtl); + if (akiReleaseInfoFile == null) { return Result.FromError("Failed to download release metadata"); @@ -38,8 +39,8 @@ public class ReleaseCheckTask : InstallerTaskBase SetStatus("Checking for Patches", "", null, ProgressStyle.Indeterminate); var akiPatchMirrorsFile = - await DownloadCacheHelper.DownloadFileAsync("mirrors.json", DownloadCacheHelper.PatchMirrorUrl, - progress); + await DownloadCacheHelper.GetOrDownloadFileAsync("mirrors.json", DownloadCacheHelper.PatchMirrorUrl, + progress, DownloadCacheHelper.SuggestedTtl); if (akiPatchMirrorsFile == null) { diff --git a/SPTInstaller/Models/InstallerUpdateInfo.cs b/SPTInstaller/Models/InstallerUpdateInfo.cs index e8575ce..cd58bc8 100644 --- a/SPTInstaller/Models/InstallerUpdateInfo.cs +++ b/SPTInstaller/Models/InstallerUpdateInfo.cs @@ -140,8 +140,8 @@ public class InstallerUpdateInfo : ReactiveObject try { var installerInfoFile = - await DownloadCacheHelper.DownloadFileAsync("installer.json", DownloadCacheHelper.InstallerInfoUrl, - null); + await DownloadCacheHelper.GetOrDownloadFileAsync("installer.json", DownloadCacheHelper.InstallerInfoUrl, null + , DownloadCacheHelper.SuggestedTtl); if (installerInfoFile == null) { diff --git a/SPTInstaller/ViewModels/PreChecksViewModel.cs b/SPTInstaller/ViewModels/PreChecksViewModel.cs index 182db76..7b111c6 100644 --- a/SPTInstaller/ViewModels/PreChecksViewModel.cs +++ b/SPTInstaller/ViewModels/PreChecksViewModel.cs @@ -286,9 +286,11 @@ public class PreChecksViewModel : ViewModelBase InstallButtonCheckState = StatusSpinner.SpinnerState.Running; var progress = new Progress((d) => { }); + var akiReleaseInfoFile = - await DownloadCacheHelper.DownloadFileAsync("release.json", DownloadCacheHelper.ReleaseMirrorUrl, - progress); + await DownloadCacheHelper.GetOrDownloadFileAsync("release.json", DownloadCacheHelper.ReleaseMirrorUrl, + progress, DownloadCacheHelper.SuggestedTtl); + if (akiReleaseInfoFile == null) { InstallButtonText = "Could not get SPT release metadata"; From 1bfb6e9946842fd87bc55c906faab803714708e7 Mon Sep 17 00:00:00 2001 From: "waffle.lord" Date: Sat, 4 May 2024 16:19:04 -0400 Subject: [PATCH 2/3] simplify file copies --- SPTInstaller/Helpers/FileHelper.cs | 146 ++++++++++------------------- SPTInstaller/Models/CopyInfo.cs | 24 +++++ 2 files changed, 73 insertions(+), 97 deletions(-) create mode 100644 SPTInstaller/Models/CopyInfo.cs diff --git a/SPTInstaller/Helpers/FileHelper.cs b/SPTInstaller/Helpers/FileHelper.cs index 18fa8b9..bcb962b 100644 --- a/SPTInstaller/Helpers/FileHelper.cs +++ b/SPTInstaller/Helpers/FileHelper.cs @@ -9,98 +9,6 @@ namespace SPTInstaller.Helpers; public static class FileHelper { - private static Result IterateDirectories(DirectoryInfo sourceDir, DirectoryInfo targetDir, string[] exclusions) - { - try - { - foreach (var dir in sourceDir.GetDirectories("*", SearchOption.AllDirectories)) - { - var exclude = false; - - foreach (var exclusion in exclusions) - { - var currentDirRelativePath = dir.FullName.Replace(sourceDir.FullName, ""); - - if (currentDirRelativePath.StartsWith(exclusion) || currentDirRelativePath == exclusion) - { - exclude = true; - Log.Debug( - $"EXCLUSION FOUND :: DIR\nExclusion: '{exclusion}'\nPath: '{currentDirRelativePath}'"); - break; - } - } - - if (exclude) - continue; - - Directory.CreateDirectory(dir.FullName.Replace(sourceDir.FullName, targetDir.FullName)); - } - - return Result.FromSuccess(); - } - catch (Exception ex) - { - Log.Error(ex, "Error while creating directories"); - return Result.FromError(ex.Message); - } - } - - private static Result IterateFiles(DirectoryInfo sourceDir, DirectoryInfo targetDir, string[] exclusions, - Action updateCallback = null) - { - try - { - int totalFiles = sourceDir.GetFiles("*.*", SearchOption.AllDirectories).Length; - int processedFiles = 0; - - foreach (var file in sourceDir.GetFiles("*.*", SearchOption.AllDirectories)) - { - var exclude = false; - - updateCallback?.Invoke(file.Name, (int)Math.Floor(((double)processedFiles / totalFiles) * 100)); - - foreach (var exclusion in exclusions) - { - var currentFileRelativePath = file.FullName.Replace(sourceDir.FullName, ""); - - if (currentFileRelativePath.StartsWith(exclusion) || currentFileRelativePath == exclusion) - { - exclude = true; - Log.Debug( - $"EXCLUSION FOUND :: FILE\nExclusion: '{exclusion}'\nPath: '{currentFileRelativePath}'"); - break; - } - - if (currentFileRelativePath.EndsWith(".bak")) - { - exclude = true; - Log.Debug($"EXCLUDING BAK FILE :: {currentFileRelativePath}"); - break; - } - } - - if (exclude) - continue; - - - var targetFile = file.FullName.Replace(sourceDir.FullName, targetDir.FullName); - - Log.Debug( - $"COPY\nSourceDir: '{sourceDir.FullName}'\nTargetDir: '{targetDir.FullName}'\nNewPath: '{targetFile}'"); - - File.Copy(file.FullName, targetFile, true); - processedFiles++; - } - - return Result.FromSuccess(); - } - catch (Exception ex) - { - Log.Error(ex, "Error while copying files"); - return Result.FromError(ex.Message); - } - } - public static string GetRedactedPath(string path) { var nameMatched = Regex.Match(path, @".:\\[uU]sers\\(?[^\\]+)"); @@ -123,13 +31,57 @@ public static class FileHelper { try { - var iterateDirectoriesResult = IterateDirectories(sourceDir, targetDir, exclusions ??= new string[0]); + var allFiles = sourceDir.GetFiles("*", SearchOption.AllDirectories); + var fileCopies = new List(); + int count = 0; - if (!iterateDirectoriesResult.Succeeded) return iterateDirectoriesResult; + // filter files before starting copy + foreach (var file in allFiles) + { + count++; + updateCallback?.Invoke("getting list of files to copy", (int)Math.Floor((double)count / allFiles.Length * 100)); + + var currentFileRelativePath = file.FullName.Replace(sourceDir.FullName, ""); + + if (exclusions != null) + { + // check exclusions + foreach (var exclusion in exclusions) + { + if (currentFileRelativePath.StartsWith(exclusion) || currentFileRelativePath == exclusion) + { + Log.Debug( + $"EXCLUSION FOUND :: FILE\nExclusion: '{exclusion}'\nPath: '{currentFileRelativePath}'"); + break; + } + } + } + + // don't copy .bak files + if (currentFileRelativePath.EndsWith(".bak")) + { + Log.Debug($"EXCLUDING BAK FILE :: {currentFileRelativePath}"); + break; + } + + fileCopies.Add(new CopyInfo(file.FullName, file.FullName.Replace(sourceDir.FullName, targetDir.FullName))); + } + + count = 0; - var iterateFilesResult = IterateFiles(sourceDir, targetDir, exclusions ??= new string[0], updateCallback); - - if (!iterateFilesResult.Succeeded) return iterateDirectoriesResult; + // process copy info for files that need to be copied + foreach (var copyInfo in fileCopies) + { + count++; + updateCallback?.Invoke(copyInfo.FileName, (int)Math.Floor((double)count / fileCopies.Count * 100)); + + var result = copyInfo.Copy(); + + if (!result.Succeeded) + { + return result; + } + } return Result.FromSuccess(); } diff --git a/SPTInstaller/Models/CopyInfo.cs b/SPTInstaller/Models/CopyInfo.cs new file mode 100644 index 0000000..0b1c2a5 --- /dev/null +++ b/SPTInstaller/Models/CopyInfo.cs @@ -0,0 +1,24 @@ +using Serilog; +using SPTInstaller.Helpers; + +namespace SPTInstaller.Models; + +class CopyInfo(string sourcePath, string targetPath) +{ + public string FileName => $"{Path.GetFileName(sourcePath)}"; + public Result Copy() + { + try + { + var directory = Path.GetDirectoryName(targetPath); + Directory.CreateDirectory(directory); + Log.Debug($"COPY\nSource: {FileHelper.GetRedactedPath(sourcePath)}\nTarget: {FileHelper.GetRedactedPath(targetPath)}"); + File.Copy(sourcePath, targetPath); + return Result.FromSuccess(); + } + catch (Exception ex) + { + return Result.FromError(ex.Message); + } + } +} \ No newline at end of file From b51c373e965aa0cbaaf4be3ab6589775e2e0182e Mon Sep 17 00:00:00 2001 From: "waffle.lord" Date: Sat, 4 May 2024 16:19:52 -0400 Subject: [PATCH 3/3] version bump --- SPTInstaller/SPTInstaller.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SPTInstaller/SPTInstaller.csproj b/SPTInstaller/SPTInstaller.csproj index 9b99f82..df64e08 100644 --- a/SPTInstaller/SPTInstaller.csproj +++ b/SPTInstaller/SPTInstaller.csproj @@ -9,8 +9,8 @@ icon.ico Assets\icon.ico Debug;Release;TEST - 2.62 - 2.62 + 2.63 + 2.63 SPT-AKI