diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ae84069..8b27182f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ### Changelog +#### Version - 2.5.3.19 - 6/4/2022 +* Fix a potential long standing problem with hash caching + #### Version - 2.5.3.18 - 6/1/2022 * Switch to a working version of Game Finder diff --git a/Wabbajack.Compiler.Test/SanityTests.cs b/Wabbajack.Compiler.Test/SanityTests.cs index 6c445160..5eda994b 100644 --- a/Wabbajack.Compiler.Test/SanityTests.cs +++ b/Wabbajack.Compiler.Test/SanityTests.cs @@ -117,9 +117,7 @@ public class CompilerSanityTests : IAsyncLifetime [Fact] public async Task CanRecreateBSAs() { - var bsa = _mod.FullPath.EnumerateFiles(Ext.Bsa) - .OrderBy(d => d.Size()) - .First(); + var bsa = _mod.FullPath.EnumerateFiles(Ext.Bsa).MinBy(d => d.Size()); await _fileExtractor.ExtractAll(bsa, _mod.FullPath, CancellationToken.None); var reader = await BSADispatch.Open(bsa); @@ -127,7 +125,7 @@ public class CompilerSanityTests : IAsyncLifetime var fileStates = reader.Files.Select(f => f.State).ToArray(); bsa.Delete(); - var creator = BSADispatch.CreateBuilder(bsaState, _manager); + await using var creator = BSADispatch.CreateBuilder(bsaState, _manager); await fileStates.Take(2).PDoAll(new Resource(), async f => await creator.AddFile(f, f.Path.RelativeTo(_mod.FullPath).Open(FileMode.Open), CancellationToken.None)); diff --git a/Wabbajack.Compiler/ACompiler.cs b/Wabbajack.Compiler/ACompiler.cs index d6079944..e1236d8a 100644 --- a/Wabbajack.Compiler/ACompiler.cs +++ b/Wabbajack.Compiler/ACompiler.cs @@ -454,11 +454,12 @@ public abstract class ACompiler foreach (var match in matches) { var destFile = FindDestFile(match.To); + _logger.LogInformation("Patching {from} {to}", destFile, match.To); // Build the patch await _vfs.Extract(new[] {destFile}.ToHashSet(), async (destvf, destsfn) => { - _logger.LogInformation("Patching {from} {to}", destFile, match.To); + await using var srcStream = await sf.GetStream(); await using var destStream = await destsfn.GetStream(); using var _ = await CompilerLimiter.Begin($"Patching {match.To}", 100, token); diff --git a/Wabbajack.Compiler/MO2Compiler.cs b/Wabbajack.Compiler/MO2Compiler.cs index 7225a731..742554a0 100644 --- a/Wabbajack.Compiler/MO2Compiler.cs +++ b/Wabbajack.Compiler/MO2Compiler.cs @@ -81,7 +81,7 @@ public class MO2Compiler : ACompiler NextStep("Initializing", "Add Roots"); await _vfs.AddRoots(roots, token); // Step 1 - await InferMetas(token); // Step 2 + //await InferMetas(token); // Step 2 NextStep("Initializing", "Add Download Roots"); await _vfs.AddRoot(Settings.Downloads, token); // Step 3 diff --git a/Wabbajack.Compression.BSA.Test/CompressionTests.cs b/Wabbajack.Compression.BSA.Test/CompressionTests.cs index 67bf3d3d..4dc09a99 100644 --- a/Wabbajack.Compression.BSA.Test/CompressionTests.cs +++ b/Wabbajack.Compression.BSA.Test/CompressionTests.cs @@ -68,7 +68,7 @@ public class CompressionTests var oldState = reader.State; - var build = BSADispatch.CreateBuilder(oldState, _tempManager); + await using var build = BSADispatch.CreateBuilder(oldState, _tempManager); await dataStates.PDoAll( async itm => { await build.AddFile(itm.State, itm.Stream, CancellationToken.None); }); diff --git a/Wabbajack.Compression.BSA/DiskSlabAllocator.cs b/Wabbajack.Compression.BSA/DiskSlabAllocator.cs index fe101883..5599ae17 100644 --- a/Wabbajack.Compression.BSA/DiskSlabAllocator.cs +++ b/Wabbajack.Compression.BSA/DiskSlabAllocator.cs @@ -26,7 +26,8 @@ public class DiskSlabAllocator foreach (var s in _streams) await s.DisposeAsync(); - foreach (var file in _files) file.Dispose(); + foreach (var file in _files) + await file.DisposeAsync(); } public Stream Allocate(long rLength) diff --git a/Wabbajack.Compression.BSA/Interfaces/IBuilder.cs b/Wabbajack.Compression.BSA/Interfaces/IBuilder.cs index 02b9ef6e..e0bb0df2 100644 --- a/Wabbajack.Compression.BSA/Interfaces/IBuilder.cs +++ b/Wabbajack.Compression.BSA/Interfaces/IBuilder.cs @@ -1,3 +1,4 @@ +using System; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -5,7 +6,7 @@ using Wabbajack.DTOs.BSA.FileStates; namespace Wabbajack.Compression.BSA.Interfaces; -public interface IBuilder +public interface IBuilder : IAsyncDisposable { ValueTask AddFile(AFile state, Stream src, CancellationToken token); ValueTask Build(Stream filename, CancellationToken token); diff --git a/Wabbajack.Compression.BSA/TES3Archive/Builder.cs b/Wabbajack.Compression.BSA/TES3Archive/Builder.cs index d4eaabcf..ec308932 100644 --- a/Wabbajack.Compression.BSA/TES3Archive/Builder.cs +++ b/Wabbajack.Compression.BSA/TES3Archive/Builder.cs @@ -69,4 +69,9 @@ public class Builder : IBuilder await data.DisposeAsync(); } } + + public async ValueTask DisposeAsync() + { + return; + } } \ No newline at end of file diff --git a/Wabbajack.Compression.Zip/ZipReader.cs b/Wabbajack.Compression.Zip/ZipReader.cs index 533f0d8d..b8b66d86 100644 --- a/Wabbajack.Compression.Zip/ZipReader.cs +++ b/Wabbajack.Compression.Zip/ZipReader.cs @@ -166,6 +166,10 @@ public class ZipReader : IAsyncDisposable } } + else + { + _rdr.Position += extraFieldLength; + } entries[i] = new ExtractedEntry { diff --git a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs index c111d712..e482867f 100644 --- a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs +++ b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs @@ -25,14 +25,16 @@ public class DownloadDispatcher private readonly IResource _limiter; private readonly ILogger _logger; private readonly Client _wjClient; + private readonly bool _useProxyCache; public DownloadDispatcher(ILogger logger, IEnumerable downloaders, - IResource limiter, Client wjClient) + IResource limiter, Client wjClient, bool useProxyCache = true) { _downloaders = downloaders.OrderBy(d => d.Priority).ToArray(); _logger = logger; _wjClient = wjClient; _limiter = limiter; + _useProxyCache = useProxyCache; } public async Task Download(Archive a, AbsolutePath dest, CancellationToken token) @@ -47,7 +49,26 @@ public class DownloadDispatcher if (!dest.Parent.DirectoryExists()) dest.Parent.CreateDirectory(); - var hash = await Downloader(a).Download(a, dest, job, token); + var downloader = Downloader(a); + if (_useProxyCache && downloader is IProxyable p) + { + var uri = p.UnParse(a.State); + var newUri = _wjClient.MakeProxyUrl(a, uri); + a = new Archive + { + Name = a.Name, + Size = a.Size, + Hash = a.Hash, + State = new DTOs.DownloadStates.Http() + { + Url = newUri + } + }; + downloader = Downloader(a); + _logger.LogInformation("Downloading Proxy ({Hash}) {Uri}", (await uri.ToString().Hash()).ToHex(), uri); + } + + var hash = await downloader.Download(a, dest, job, token); return hash; } diff --git a/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs b/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs index d3592f1f..90babc45 100644 --- a/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs +++ b/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs @@ -1,34 +1,62 @@ +using System.Collections.Generic; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Wabbajack.Downloaders.Bethesda; using Wabbajack.Downloaders.Http; +using Wabbajack.Downloaders.Interfaces; using Wabbajack.Downloaders.IPS4OAuth2Downloader; using Wabbajack.Downloaders.Manual; using Wabbajack.Downloaders.MediaFire; using Wabbajack.Downloaders.ModDB; using Wabbajack.DTOs.JsonConverters; using Wabbajack.Networking.WabbajackClientApi; +using Wabbajack.RateLimiter; namespace Wabbajack.Downloaders; public static class ServiceExtensions { - public static IServiceCollection AddDownloadDispatcher(this IServiceCollection services) + public static IServiceCollection AddDownloadDispatcher(this IServiceCollection services, bool useLoginDownloaders = true, bool useProxyCache = true) { - return services - .AddDTOConverters() - .AddDTOSerializer() - .AddGoogleDriveDownloader() - .AddHttpDownloader() - .AddMegaDownloader() - .AddMediaFireDownloader() - .AddModDBDownloader() - .AddNexusDownloader() - .AddIPS4OAuth2Downloaders() - .AddWabbajackCDNDownloader() - .AddGameFileDownloader() - .AddBethesdaDownloader() - .AddWabbajackClient() - .AddManualDownloader() - .AddSingleton(); + if (useLoginDownloaders) + { + services + .AddDTOConverters() + .AddDTOSerializer() + .AddGoogleDriveDownloader() + .AddHttpDownloader() + .AddMegaDownloader() + .AddMediaFireDownloader() + .AddModDBDownloader() + .AddNexusDownloader() + .AddIPS4OAuth2Downloaders() + .AddWabbajackCDNDownloader() + .AddGameFileDownloader() + .AddBethesdaDownloader() + .AddWabbajackClient() + .AddManualDownloader(); + } + else + { + services + .AddDTOConverters() + .AddDTOSerializer() + .AddGoogleDriveDownloader() + .AddHttpDownloader() + .AddMegaDownloader() + .AddMediaFireDownloader() + .AddModDBDownloader() + .AddWabbajackCDNDownloader() + .AddWabbajackClient(); + } + + services.AddSingleton(s => + new DownloadDispatcher(s.GetRequiredService>(), + s.GetRequiredService>(), + s.GetRequiredService>(), + s.GetRequiredService(), + useProxyCache)); + + return services; } } \ No newline at end of file diff --git a/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs b/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs index 58b981c0..508eb877 100644 --- a/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs +++ b/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs @@ -2,9 +2,12 @@ using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; +using System.Text.Encodings.Web; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using System.Web; +using HtmlAgilityPack; using Microsoft.Extensions.Logging; using Wabbajack.Downloaders.Interfaces; using Wabbajack.DTOs; @@ -18,7 +21,7 @@ using Wabbajack.RateLimiter; namespace Wabbajack.Downloaders.GoogleDrive; -public class GoogleDriveDownloader : ADownloader, IUrlDownloader +public class GoogleDriveDownloader : ADownloader, IUrlDownloader, IProxyable { private static readonly Regex GDriveRegex = new("((?<=id=)[a-zA-Z0-9_-]*)|(?<=\\/file\\/d\\/)[a-zA-Z0-9_-]*", RegexOptions.Compiled); @@ -98,8 +101,29 @@ public class GoogleDriveDownloader : ADownloader c.Key.StartsWith("download_warning_")); + + if (warning == default && response.Content.Headers.ContentType?.MediaType == "text/html") + { + var doc = new HtmlDocument(); + var txt = await response.Content.ReadAsStringAsync(token); + + doc.LoadHtml(txt); + + var action = doc.DocumentNode.DescendantsAndSelf() + .Where(d => d.Name == "form" && d.Id == "downloadForm" && + d.GetAttributeValue("method", "") == "post") + .Select(d => d.GetAttributeValue("action", "")) + .FirstOrDefault(); + + if (action != null) + warning = ("download_warning_", "t"); + + } response.Dispose(); - if (warning == default) return new HttpRequestMessage(HttpMethod.Get, initialUrl); + if (warning == default) + { + return new HttpRequestMessage(HttpMethod.Get, initialUrl); + } var url = $"https://drive.google.com/uc?export=download&confirm={warning.Value}&id={state.Id}"; var httpState = new HttpRequestMessage(HttpMethod.Get, url); diff --git a/Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj b/Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj index 2c70873e..824784e5 100644 --- a/Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj +++ b/Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj @@ -8,6 +8,7 @@ + diff --git a/Wabbajack.Downloaders.Interfaces/IProxyable.cs b/Wabbajack.Downloaders.Interfaces/IProxyable.cs new file mode 100644 index 00000000..261097e8 --- /dev/null +++ b/Wabbajack.Downloaders.Interfaces/IProxyable.cs @@ -0,0 +1,6 @@ +namespace Wabbajack.Downloaders.Interfaces; + +public interface IProxyable : IUrlDownloader +{ + +} \ No newline at end of file diff --git a/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs b/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs index fbd40827..e47e628d 100644 --- a/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs +++ b/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs @@ -17,7 +17,7 @@ using Wabbajack.RateLimiter; namespace Wabbajack.Downloaders.MediaFire; -public class MediaFireDownloader : ADownloader, IUrlDownloader +public class MediaFireDownloader : ADownloader, IUrlDownloader, IProxyable { private readonly IHttpDownloader _downloader; private readonly HttpClient _httpClient; diff --git a/Wabbajack.Downloaders.Mega/MegaDownloader.cs b/Wabbajack.Downloaders.Mega/MegaDownloader.cs index 568e59aa..572c88c9 100644 --- a/Wabbajack.Downloaders.Mega/MegaDownloader.cs +++ b/Wabbajack.Downloaders.Mega/MegaDownloader.cs @@ -17,7 +17,7 @@ using Wabbajack.RateLimiter; namespace Wabbajack.Downloaders.ModDB; -public class MegaDownloader : ADownloader, IUrlDownloader +public class MegaDownloader : ADownloader, IUrlDownloader, IProxyable { private const string MegaPrefix = "https://mega.nz/#!"; private const string MegaFilePrefix = "https://mega.nz/file/"; diff --git a/Wabbajack.FileExtractor/FileExtractor.cs b/Wabbajack.FileExtractor/FileExtractor.cs index 02c6a606..37720fd6 100644 --- a/Wabbajack.FileExtractor/FileExtractor.cs +++ b/Wabbajack.FileExtractor/FileExtractor.cs @@ -204,7 +204,7 @@ public class FileExtractor { var tmpFile = _manager.CreateFile(); await tmpFile.Path.WriteAllAsync(archive, CancellationToken.None); - var dest = _manager.CreateFolder(); + await using var dest = _manager.CreateFolder(); using var omod = new OMOD(tmpFile.Path.ToString()); @@ -364,7 +364,7 @@ public class FileExtractor job.Dispose(); var results = await dest.Path.EnumerateFiles() - .PMapAll(async f => + .SelectAsync(async f => { var path = f.RelativeTo(dest.Path); if (!shouldExtract(path)) return ((RelativePath, T)) default; @@ -375,6 +375,7 @@ public class FileExtractor }) .Where(d => d.Item1 != default) .ToDictionary(d => d.Item1, d => d.Item2); + return results; } diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index 66ddb4e9..70d82f2d 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -63,7 +63,7 @@ public abstract class AInstaller private readonly Stopwatch _updateStopWatch = new(); public Action? OnStatusUpdate; - private readonly IResource _limiter; + protected readonly IResource _limiter; private Func _statusFormatter = x => x.ToString(); @@ -98,6 +98,8 @@ public abstract class AInstaller public TemporaryPath ExtractedModlistFolder { get; set; } public ModList ModList => _configuration.ModList; + public Directive[] UnoptimizedDirectives { get; set; } + public Archive[] UnoptimizedArchives { get; set; } public void NextStep(string statusCategory, string statusText, long maxStepProgress, Func? formatter = null) { @@ -558,7 +560,11 @@ public abstract class AInstaller .Select(d => d.Key) .ToHashSet(); + UnoptimizedArchives = ModList.Archives; + UnoptimizedDirectives = ModList.Directives; ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToArray(); ModList.Directives = indexed.Values.ToArray(); } + + } \ No newline at end of file diff --git a/Wabbajack.Installer/StandardInstaller.cs b/Wabbajack.Installer/StandardInstaller.cs index 2bfd50e2..365fcb60 100644 --- a/Wabbajack.Installer/StandardInstaller.cs +++ b/Wabbajack.Installer/StandardInstaller.cs @@ -216,19 +216,38 @@ public class StandardInstaller : AInstaller private async Task InstallIncludedDownloadMetas(CancellationToken token) { - await ModList.Archives - .PDoAll(async archive => + _logger.LogInformation("Looking for downloads by size"); + var bySize = UnoptimizedArchives.ToLookup(x => x.Size); + + _logger.LogInformation("Writing Metas"); + await _configuration.Downloads.EnumerateFiles() + .PDoAll(async download => { - if (HashedArchives.TryGetValue(archive.Hash, out var paths)) + var found = bySize[download.Size()]; + var hash = await FileHashCache.FileHashCachedAsync(download, token); + var archive = found.FirstOrDefault(f => f.Hash == hash); + if (archive == default) return; + + var metaFile = download.WithExtension(Ext.Meta); + if (metaFile.FileExists()) { - var metaPath = paths.WithExtension(Ext.Meta); - if (archive.State is GameFileSource) return; - if (!metaPath.FileExists()) + try { - var meta = AddInstalled(_downloadDispatcher.MetaIni(archive)); - await metaPath.WriteAllLinesAsync(meta, token); + var parsed = metaFile.LoadIniFile(); + if (parsed["General"] != null && parsed["General"]["unknownArchive"] == null) + { + return; + } + } + catch (Exception) + { + // Ignore } } + + _logger.LogInformation("Writing {FileName}", metaFile.FileName); + var meta = AddInstalled(_downloadDispatcher.MetaIni(archive)); + await metaFile.WriteAllLinesAsync(meta, token); }); } @@ -253,11 +272,15 @@ public class StandardInstaller : AInstaller _logger.LogInformation("Building {bsaTo}", bsa.To.FileName); var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID); - var a = BSADispatch.CreateBuilder(bsa.State, _manager); + await using var a = BSADispatch.CreateBuilder(bsa.State, _manager); var streams = await bsa.FileStates.PMapAll(async state => { + using var job = await _limiter.Begin($"Adding {state.Path.FileName}", 0, token); var fs = sourceDir.Combine(state.Path).Open(FileMode.Open, FileAccess.Read, FileShare.Read); + var size = fs.Length; + job.Size = size; await a.AddFile(state, fs, token); + await job.Report((int)size, token); return fs; }).ToList(); diff --git a/Wabbajack.Launcher/ViewModels/MainWindowViewModel.cs b/Wabbajack.Launcher/ViewModels/MainWindowViewModel.cs index b36ed9e9..a7d58df9 100644 --- a/Wabbajack.Launcher/ViewModels/MainWindowViewModel.cs +++ b/Wabbajack.Launcher/ViewModels/MainWindowViewModel.cs @@ -85,6 +85,30 @@ public class MainWindowViewModel : ViewModelBase Status = $"Getting download Uri for {_version.Version}"; var uri = await _version.Uri(); + var archive = new Archive() + { + Name = $"{_version.Version}.zip", + Size = _version.Size, + State = new Http {Url = uri} + }; + + await using var stream = await _downloader.GetChunkedSeekableStream(archive, CancellationToken.None); + var rdr = new ZipReader(stream, true); + var entries = (await rdr.GetFiles()).OrderBy(d => d.FileOffset).ToArray(); + foreach (var file in entries) + { + if (file.FileName.EndsWith("/") || file.FileName.EndsWith("\\")) continue; + var relPath = file.FileName.ToRelativePath(); + Status = $"Extracting: {relPath.FileName}"; + var outPath = baseFolder.Combine(relPath); + if (!outPath.Parent.DirectoryExists()) + outPath.Parent.CreateDirectory(); + + await using var of = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); + await rdr.Extract(file, of, CancellationToken.None); + } + + var wc = new WebClient(); wc.DownloadProgressChanged += UpdateProgress; Status = $"Downloading {_version.Version} ..."; diff --git a/Wabbajack.Networking.WabbajackClientApi/Client.cs b/Wabbajack.Networking.WabbajackClientApi/Client.cs index c3db918d..d634d80b 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Client.cs +++ b/Wabbajack.Networking.WabbajackClientApi/Client.cs @@ -362,4 +362,9 @@ public class Client var url = $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}_steam_manifests.json"; return await _client.GetFromJsonAsync(url, _dtos.Options) ?? Array.Empty(); } + + public Uri MakeProxyUrl(Archive archive, Uri uri) + { + return new Uri($"{_configuration.BuildServerUrl}proxy?name={archive.Name}&hash={archive.Hash.ToHex()}&uri={uri}"); + } } \ No newline at end of file diff --git a/Wabbajack.Paths.IO/AbsolutePathExtensions.cs b/Wabbajack.Paths.IO/AbsolutePathExtensions.cs index b1d34ef4..c967602d 100644 --- a/Wabbajack.Paths.IO/AbsolutePathExtensions.cs +++ b/Wabbajack.Paths.IO/AbsolutePathExtensions.cs @@ -60,6 +60,11 @@ public static class AbsolutePathExtensions { return new FileInfo(file.ToNativePath()).LastWriteTime; } + + public static void Touch(this AbsolutePath file) + { + new FileInfo(file.ToNativePath()).LastWriteTime = DateTime.Now; + } public static byte[] ReadAllBytes(this AbsolutePath file) { @@ -220,7 +225,7 @@ public static class AbsolutePathExtensions var di = new DirectoryInfo(path.ToString()); if (di.Attributes.HasFlag(FileAttributes.ReadOnly)) di.Attributes &= ~FileAttributes.ReadOnly; - Directory.Delete(path.ToString(), true); + di.Delete(true); } catch (UnauthorizedAccessException) { diff --git a/Wabbajack.Paths.IO/TemporaryFileManager.cs b/Wabbajack.Paths.IO/TemporaryFileManager.cs index 79314fa6..5ca27c82 100644 --- a/Wabbajack.Paths.IO/TemporaryFileManager.cs +++ b/Wabbajack.Paths.IO/TemporaryFileManager.cs @@ -7,19 +7,22 @@ namespace Wabbajack.Paths.IO; public class TemporaryFileManager : IDisposable { private readonly AbsolutePath _basePath; + private readonly bool _deleteOnDispose; public TemporaryFileManager() : this(KnownFolders.EntryPoint.Combine("temp")) { } - public TemporaryFileManager(AbsolutePath basePath) + public TemporaryFileManager(AbsolutePath basePath, bool deleteOnDispose = true) { + _deleteOnDispose = deleteOnDispose; _basePath = basePath; _basePath.CreateDirectory(); } public void Dispose() { + if (!_deleteOnDispose) return; for (var retries = 0; retries < 10; retries++) try { @@ -34,7 +37,7 @@ public class TemporaryFileManager : IDisposable } } - public TemporaryPath CreateFile(Extension? ext = default) + public TemporaryPath CreateFile(Extension? ext = default, bool deleteOnDispose = true) { var path = _basePath.Combine(Guid.NewGuid().ToString()); if (path.Extension != default) diff --git a/Wabbajack.Server/AppSettings.cs b/Wabbajack.Server/AppSettings.cs index c16e8e2c..b4e3e4cb 100644 --- a/Wabbajack.Server/AppSettings.cs +++ b/Wabbajack.Server/AppSettings.cs @@ -16,6 +16,9 @@ public class AppSettings public string WabbajackBuildServerUri { get; set; } = "https://build.wabbajack.org/"; public string MetricsKeyHeader { get; set; } = "x-metrics-key"; public string TempFolder { get; set; } + + public string ProxyFolder { get; set; } + public AbsolutePath ProxyPath => (AbsolutePath) ProxyFolder; public AbsolutePath TempPath => (AbsolutePath) TempFolder; public string SpamWebHook { get; set; } = null; public string HamWebHook { get; set; } = null; diff --git a/Wabbajack.Server/Controllers/Proxy.cs b/Wabbajack.Server/Controllers/Proxy.cs new file mode 100644 index 00000000..edada767 --- /dev/null +++ b/Wabbajack.Server/Controllers/Proxy.cs @@ -0,0 +1,124 @@ +using System.Text; +using FluentFTP.Helpers; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.Downloaders; +using Wabbajack.Downloaders.Interfaces; +using Wabbajack.DTOs; +using Wabbajack.DTOs.DownloadStates; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Paths.IO; +using Wabbajack.VFS; + +namespace Wabbajack.Server.Controllers; + +[ApiController] +[Route("/proxy")] +public class Proxy : ControllerBase +{ + private readonly ILogger _logger; + private readonly DownloadDispatcher _dispatcher; + private readonly TemporaryFileManager _tempFileManager; + private readonly AppSettings _appSettings; + private readonly FileHashCache _hashCache; + + public Proxy(ILogger logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager, FileHashCache hashCache, AppSettings appSettings) + { + _logger = logger; + _dispatcher = dispatcher; + _tempFileManager = tempFileManager; + _appSettings = appSettings; + _hashCache = hashCache; + } + + [HttpGet] + public async Task ProxyGet(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name, [FromQuery] string? hash) + { + var shouldMatch = hash != null ? Hash.FromHex(hash) : default; + + _logger.LogInformation("Got proxy request for {Uri}", uri); + var state = _dispatcher.Parse(uri); + var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex(); + var cacheFile = _appSettings.ProxyPath.Combine(cacheName); + + if (state == null) + { + return BadRequest(new {Type = "Could not get state from Uri", Uri = uri.ToString()}); + } + + var archive = new Archive + { + Name = name ?? "", + State = state, + Hash = shouldMatch + + }; + + var downloader = _dispatcher.Downloader(archive); + if (downloader is not IProxyable) + { + return BadRequest(new {Type = "Downloader is not IProxyable", Downloader = downloader.GetType().FullName}); + } + + if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(4)) + { + try + { + var verify = await _dispatcher.Verify(archive, token); + if (verify) + cacheFile.Touch(); + } + catch (Exception ex) + { + _logger.LogInformation("When trying to verify cached file"); + } + } + + if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(24)) + { + try + { + cacheFile.Delete(); + } + catch (Exception ex) + { + _logger.LogError(ex, "When trying to delete expired file"); + } + } + + + if (cacheFile.FileExists()) + { + if (hash != default) + { + var hashResult = await _hashCache.FileHashCachedAsync(cacheFile, token); + if (hashResult != shouldMatch) + return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = hashResult.ToHex()}); + } + var ret = new PhysicalFileResult(cacheFile.ToString(), "application/octet-stream"); + if (name != null) + ret.FileDownloadName = name; + return ret; + } + + _logger.LogInformation("Downloading proxy request for {Uri}", uri); + + var tempFile = _tempFileManager.CreateFile(deleteOnDispose:false); + + var result = await _dispatcher.Download(archive, tempFile.Path, token); + if (hash != default && result != shouldMatch) + { + if (tempFile.Path.FileExists()) + tempFile.Path.Delete(); + + return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = result.ToHex()}); + } + + await tempFile.Path.MoveToAsync(cacheFile, true, token); + + _logger.LogInformation("Returning proxy request for {Uri} {Size}", uri, cacheFile.Size().FileSizeToString()); + return new PhysicalFileResult(cacheFile.ToString(), "application/binary"); + } +} \ No newline at end of file diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 92451fa2..98542330 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -17,10 +17,12 @@ using Nettle.Compiler; using Newtonsoft.Json; using Octokit; using Wabbajack.BuildServer; +using Wabbajack.Downloaders; using Wabbajack.DTOs; using Wabbajack.DTOs.JsonConverters; using Wabbajack.DTOs.Logins; using Wabbajack.Networking.GitHub; +using Wabbajack.Networking.Http; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Networking.NexusApi; using Wabbajack.Paths; @@ -30,6 +32,8 @@ using Wabbajack.Server.Extensions; using Wabbajack.Server.Services; using Wabbajack.Services.OSIntegrated.TokenProviders; using Wabbajack.Networking.WabbajackClientApi; +using Wabbajack.Paths.IO; +using Wabbajack.VFS; using Client = Wabbajack.Networking.GitHub.Client; namespace Wabbajack.Server; @@ -79,6 +83,18 @@ public class Startup services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddAllSingleton(); + services.AddDownloadDispatcher(useLoginDownloaders:false, useProxyCache:false); + var tempBase = KnownFolders.EntryPoint.Combine("temp"); + services.AddTransient(s => + new TemporaryFileManager(tempBase.Combine(Environment.ProcessId + "_" + Guid.NewGuid()))); + services.AddAllSingleton, WabbajackApiTokenProvider>(); + services.AddAllSingleton>(s => new Resource("Downloads", 12)); + services.AddAllSingleton>(s => new Resource("File Hashing", 12)); + services.AddSingleton(s => + new FileHashCache(KnownFolders.AppDataLocal.Combine("Wabbajack", "GlobalHashCache.sqlite"), + s.GetService>()!)); + services.AddAllSingleton, NexusApiTokenProvider>(); services.AddAllSingleton>(s => new Resource("Web Requests", 12)); // Application Info diff --git a/Wabbajack.Server/appsettings.json b/Wabbajack.Server/appsettings.json index de6eb138..1a115752 100644 --- a/Wabbajack.Server/appsettings.json +++ b/Wabbajack.Server/appsettings.json @@ -14,6 +14,7 @@ "PatchesFilesFolder": "c:\\tmp\\patches", "MirrorFilesFolder": "c:\\tmp\\mirrors", "NexusCacheFolder": "c:\\tmp\\nexus-cache", + "ProxyFolder": "c:\\tmp\\proxy", "GitHubKey": "" }, "AllowedHosts": "*" diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index e9dccba1..b50dbee2 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; using System.Net.Http; using System.Runtime.InteropServices; using System.Threading.Tasks; @@ -41,8 +43,11 @@ public static class ServiceExtensions var options = new OSIntegratedOptions(); cfn?.Invoke(options); + var tempBase = KnownFolders.EntryPoint.Combine("temp"); service.AddTransient(s => - new TemporaryFileManager(KnownFolders.EntryPoint.Combine("temp", Guid.NewGuid().ToString()))); + new TemporaryFileManager(tempBase.Combine(Environment.ProcessId + "_" + Guid.NewGuid()))); + + Task.Run(() => CleanAllTempData(tempBase)); service.AddSingleton(s => options.UseLocalCache ? new FileHashCache(s.GetService()!.CreateFile().Path, @@ -176,6 +181,31 @@ public static class ServiceExtensions return service; } + private static void CleanAllTempData(AbsolutePath path) + { + // Get directories first and cache them, this freezes the directories were looking at + // so any new ones don't show up in the middle of our deletes. + + var dirs = path.EnumerateDirectories().ToList(); + var processIds = Process.GetProcesses().Select(p => p.Id).ToHashSet(); + foreach (var dir in dirs) + { + var name = dir.FileName.ToString().Split("_"); + if (!int.TryParse(name[0], out var processId)) continue; + if (processIds.Contains(processId)) continue; + + try + { + dir.DeleteDirectory(); + } + catch (Exception) + { + // ignored + } + } + + } + public class OSIntegratedOptions { public bool UseLocalCache { get; set; } = false; diff --git a/Wabbajack.VFS/Context.cs b/Wabbajack.VFS/Context.cs index fa99d0b6..3d57bd31 100644 --- a/Wabbajack.VFS/Context.cs +++ b/Wabbajack.VFS/Context.cs @@ -58,11 +58,12 @@ public class Context var allFiles = await filesToIndex .PMapAll(async f => { + using var job = await Limiter.Begin($"Analyzing {f}", 0, token); if (byPath.TryGetValue(f, out var found)) if (found.LastModified == f.LastModifiedUtc().AsUnixTime() && found.Size == f.Size()) return found; - return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, token); + return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, token, job: job); }).ToList(); var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); @@ -143,7 +144,7 @@ public class Context token, fileNames.Keys.ToHashSet()); } - catch (Exception) + catch (Exception ex) { await using var stream = await sfn.GetStream(); var hash = await stream.HashingCopy(Stream.Null, token); diff --git a/Wabbajack.VFS/FileHashCache.cs b/Wabbajack.VFS/FileHashCache.cs index b9fdb680..d62038ee 100644 --- a/Wabbajack.VFS/FileHashCache.cs +++ b/Wabbajack.VFS/FileHashCache.cs @@ -90,7 +90,7 @@ public class FileHashCache if (!file.FileExists()) return false; var result = Get(file); - if (result == default) + if (result == default || result.Hash == default) return false; if (result.LastModified == file.LastModifiedUtc().ToFileTimeUtc()) diff --git a/Wabbajack.VFS/VFSCache.cs b/Wabbajack.VFS/VFSCache.cs index 45ec874d..59359f4a 100644 --- a/Wabbajack.VFS/VFSCache.cs +++ b/Wabbajack.VFS/VFSCache.cs @@ -41,6 +41,9 @@ public class VFSCache public bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, Hash hash, out VirtualFile found) { + if (hash == default) + throw new ArgumentException("Cannot cache default hashes"); + using var cmd = new SQLiteCommand(_conn); cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash"; cmd.Parameters.AddWithValue("@hash", (long) hash); diff --git a/Wabbajack.VFS/VirtualFile.cs b/Wabbajack.VFS/VirtualFile.cs index 79225512..5a8c217a 100644 --- a/Wabbajack.VFS/VirtualFile.cs +++ b/Wabbajack.VFS/VirtualFile.cs @@ -14,6 +14,7 @@ using Wabbajack.Hashing.PHash; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; using Wabbajack.Paths.IO; +using Wabbajack.RateLimiter; namespace Wabbajack.VFS; @@ -159,7 +160,7 @@ public class VirtualFile public static async Task Analyze(Context context, VirtualFile? parent, IStreamFactory extractedFile, - IPath relPath, CancellationToken token, int depth = 0) + IPath relPath, CancellationToken token, int depth = 0, IJob? job = null) { Hash hash; if (extractedFile is NativeFileStreamFactory) @@ -169,9 +170,9 @@ public class VirtualFile } else { - using var job = await context.HashLimiter.Begin("Hashing memory stream", 0, token); await using var hstream = await extractedFile.GetStream(); - job.Size = hstream.Length; + if (job != null) + job.Size += hstream.Length; hash = await hstream.HashingCopy(Stream.Null, token, job); } @@ -198,9 +199,13 @@ public class VirtualFile if (TextureExtensions.Contains(relPath.FileName.Extension) && await DDSSig.MatchesAsync(stream) != null) try { - using var job = await context.HashLimiter.Begin("Hashing image", 0, token); self.ImageState = await ImageLoader.Load(stream); - await job.Report((int) self.Size, token); + if (job != null) + { + job.Size += self.Size; + await job.Report((int) self.Size, token); + } + stream.Position = 0; } catch (Exception) @@ -222,7 +227,7 @@ public class VirtualFile { var list = await context.Extractor.GatheringExtract(extractedFile, _ => true, - async (path, sfactory) => await Analyze(context, self, sfactory, path, token, depth + 1), + async (path, sfactory) => await Analyze(context, self, sfactory, path, token, depth + 1, job), token); self.Children = list.Values.ToImmutableList();