Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement Proxy support for unstable downloaders

Browse Source
This commit is contained in:
Timothy Baldridge 2022-06-07 21:48:13 -06:00
parent 48483d1d74
commit b29bed24da
32 changed files with 400 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -1,5 +1,8 @@
### Changelog
#### Version - 2.5.3.19 - 6/4/2022
* Fix a potential long standing problem with hash caching
#### Version - 2.5.3.18 - 6/1/2022
* Switch to a working version of Game Finder

6
Wabbajack.Compiler.Test/SanityTests.cs
View File

@ -117,9 +117,7 @@ public class CompilerSanityTests : IAsyncLifetime
[Fact]
public async Task CanRecreateBSAs()
{
var bsa = _mod.FullPath.EnumerateFiles(Ext.Bsa)
.OrderBy(d => d.Size())
.First();
var bsa = _mod.FullPath.EnumerateFiles(Ext.Bsa).MinBy(d => d.Size());
await _fileExtractor.ExtractAll(bsa, _mod.FullPath, CancellationToken.None);
var reader = await BSADispatch.Open(bsa);
@ -127,7 +125,7 @@ public class CompilerSanityTests : IAsyncLifetime
var fileStates = reader.Files.Select(f => f.State).ToArray();
bsa.Delete();
var creator = BSADispatch.CreateBuilder(bsaState, _manager);
await using var creator = BSADispatch.CreateBuilder(bsaState, _manager);
await fileStates.Take(2).PDoAll(new Resource<CompilerSanityTests>(),
async f => await creator.AddFile(f, f.Path.RelativeTo(_mod.FullPath).Open(FileMode.Open),
CancellationToken.None));

3
Wabbajack.Compiler/ACompiler.cs
View File

@ -454,11 +454,12 @@ public abstract class ACompiler
foreach (var match in matches)
{
var destFile = FindDestFile(match.To);
_logger.LogInformation("Patching {from} {to}", destFile, match.To);
// Build the patch
await _vfs.Extract(new[] {destFile}.ToHashSet(),
async (destvf, destsfn) =>
{
_logger.LogInformation("Patching {from} {to}", destFile, match.To);
await using var srcStream = await sf.GetStream();
await using var destStream = await destsfn.GetStream();
using var _ = await CompilerLimiter.Begin($"Patching {match.To}", 100, token);

2
Wabbajack.Compiler/MO2Compiler.cs
View File

@ -81,7 +81,7 @@ public class MO2Compiler : ACompiler
NextStep("Initializing", "Add Roots");
await _vfs.AddRoots(roots, token); // Step 1
await InferMetas(token); // Step 2
//await InferMetas(token); // Step 2
NextStep("Initializing", "Add Download Roots");
await _vfs.AddRoot(Settings.Downloads, token); // Step 3

2
Wabbajack.Compression.BSA.Test/CompressionTests.cs
View File

@ -68,7 +68,7 @@ public class CompressionTests
var oldState = reader.State;
var build = BSADispatch.CreateBuilder(oldState, _tempManager);
await using var build = BSADispatch.CreateBuilder(oldState, _tempManager);
await dataStates.PDoAll(
async itm => { await build.AddFile(itm.State, itm.Stream, CancellationToken.None); });

3
Wabbajack.Compression.BSA/DiskSlabAllocator.cs
View File

@ -26,7 +26,8 @@ public class DiskSlabAllocator
foreach (var s in _streams)
await s.DisposeAsync();
foreach (var file in _files) file.Dispose();
foreach (var file in _files)
await file.DisposeAsync();
}
public Stream Allocate(long rLength)

3
Wabbajack.Compression.BSA/Interfaces/IBuilder.cs
View File

@ -1,3 +1,4 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
@ -5,7 +6,7 @@ using Wabbajack.DTOs.BSA.FileStates;
namespace Wabbajack.Compression.BSA.Interfaces;
public interface IBuilder
public interface IBuilder : IAsyncDisposable
{
ValueTask AddFile(AFile state, Stream src, CancellationToken token);
ValueTask Build(Stream filename, CancellationToken token);

5
Wabbajack.Compression.BSA/TES3Archive/Builder.cs
View File

@ -69,4 +69,9 @@ public class Builder : IBuilder
await data.DisposeAsync();
}
}
public async ValueTask DisposeAsync()
{
return;
}
}

4
Wabbajack.Compression.Zip/ZipReader.cs
View File

@ -166,6 +166,10 @@ public class ZipReader : IAsyncDisposable
}
}
else
{
_rdr.Position += extraFieldLength;
}
entries[i] = new ExtractedEntry
{

25
Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs
View File

@ -25,14 +25,16 @@ public class DownloadDispatcher
private readonly IResource<DownloadDispatcher> _limiter;
private readonly ILogger<DownloadDispatcher> _logger;
private readonly Client _wjClient;
private readonly bool _useProxyCache;
public DownloadDispatcher(ILogger<DownloadDispatcher> logger, IEnumerable<IDownloader> downloaders,
IResource<DownloadDispatcher> limiter, Client wjClient)
IResource<DownloadDispatcher> limiter, Client wjClient, bool useProxyCache = true)
{
_downloaders = downloaders.OrderBy(d => d.Priority).ToArray();
_logger = logger;
_wjClient = wjClient;
_limiter = limiter;
_useProxyCache = useProxyCache;
}
public async Task<Hash> Download(Archive a, AbsolutePath dest, CancellationToken token)
@ -47,7 +49,26 @@ public class DownloadDispatcher
if (!dest.Parent.DirectoryExists())
dest.Parent.CreateDirectory();
var hash = await Downloader(a).Download(a, dest, job, token);
var downloader = Downloader(a);
if (_useProxyCache && downloader is IProxyable p)
{
var uri = p.UnParse(a.State);
var newUri = _wjClient.MakeProxyUrl(a, uri);
a = new Archive
{
Name = a.Name,
Size = a.Size,
Hash = a.Hash,
State = new DTOs.DownloadStates.Http()
{
Url = newUri
}
};
downloader = Downloader(a);
_logger.LogInformation("Downloading Proxy ({Hash}) {Uri}", (await uri.ToString().Hash()).ToHex(), uri);
}
var hash = await downloader.Download(a, dest, job, token);
return hash;
}

62
Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs
View File

@ -1,34 +1,62 @@
using System.Collections.Generic;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Wabbajack.Downloaders.Bethesda;
using Wabbajack.Downloaders.Http;
using Wabbajack.Downloaders.Interfaces;
using Wabbajack.Downloaders.IPS4OAuth2Downloader;
using Wabbajack.Downloaders.Manual;
using Wabbajack.Downloaders.MediaFire;
using Wabbajack.Downloaders.ModDB;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.RateLimiter;
namespace Wabbajack.Downloaders;
public static class ServiceExtensions
{
public static IServiceCollection AddDownloadDispatcher(this IServiceCollection services)
public static IServiceCollection AddDownloadDispatcher(this IServiceCollection services, bool useLoginDownloaders = true, bool useProxyCache = true)
{
return services
.AddDTOConverters()
.AddDTOSerializer()
.AddGoogleDriveDownloader()
.AddHttpDownloader()
.AddMegaDownloader()
.AddMediaFireDownloader()
.AddModDBDownloader()
.AddNexusDownloader()
.AddIPS4OAuth2Downloaders()
.AddWabbajackCDNDownloader()
.AddGameFileDownloader()
.AddBethesdaDownloader()
.AddWabbajackClient()
.AddManualDownloader()
.AddSingleton<DownloadDispatcher>();
if (useLoginDownloaders)
{
services
.AddDTOConverters()
.AddDTOSerializer()
.AddGoogleDriveDownloader()
.AddHttpDownloader()
.AddMegaDownloader()
.AddMediaFireDownloader()
.AddModDBDownloader()
.AddNexusDownloader()
.AddIPS4OAuth2Downloaders()
.AddWabbajackCDNDownloader()
.AddGameFileDownloader()
.AddBethesdaDownloader()
.AddWabbajackClient()
.AddManualDownloader();
}
else
{
services
.AddDTOConverters()
.AddDTOSerializer()
.AddGoogleDriveDownloader()
.AddHttpDownloader()
.AddMegaDownloader()
.AddMediaFireDownloader()
.AddModDBDownloader()
.AddWabbajackCDNDownloader()
.AddWabbajackClient();
}
services.AddSingleton(s =>
new DownloadDispatcher(s.GetRequiredService<ILogger<DownloadDispatcher>>(),
s.GetRequiredService<IEnumerable<IDownloader>>(),
s.GetRequiredService<IResource<DownloadDispatcher>>(),
s.GetRequiredService<Client>(),
useProxyCache));
return services;
}
}

28
Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs
View File

@ -2,9 +2,12 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.Encodings.Web;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs;
@ -18,7 +21,7 @@ using Wabbajack.RateLimiter;
namespace Wabbajack.Downloaders.GoogleDrive;
public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive>, IUrlDownloader
public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive>, IUrlDownloader, IProxyable
{
private static readonly Regex GDriveRegex = new("((?<=id=)[a-zA-Z0-9_-]*)|(?<=\\/file\\/d\\/)[a-zA-Z0-9_-]*",
RegexOptions.Compiled);
@ -98,8 +101,29 @@ public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive
using var response = await _client.GetAsync(initialUrl, token);
var cookies = response.GetSetCookies();
var warning = cookies.FirstOrDefault(c => c.Key.StartsWith("download_warning_"));
if (warning == default && response.Content.Headers.ContentType?.MediaType == "text/html")
{
var doc = new HtmlDocument();
var txt = await response.Content.ReadAsStringAsync(token);
doc.LoadHtml(txt);
var action = doc.DocumentNode.DescendantsAndSelf()
.Where(d => d.Name == "form" && d.Id == "downloadForm" &&
d.GetAttributeValue("method", "") == "post")
.Select(d => d.GetAttributeValue("action", ""))
.FirstOrDefault();
if (action != null)
warning = ("download_warning_", "t");
}
response.Dispose();
if (warning == default) return new HttpRequestMessage(HttpMethod.Get, initialUrl);
if (warning == default)
{
return new HttpRequestMessage(HttpMethod.Get, initialUrl);
}
var url = $"https://drive.google.com/uc?export=download&confirm={warning.Value}&id={state.Id}";
var httpState = new HttpRequestMessage(HttpMethod.Get, url);

1
Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj
View File

@ -8,6 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.42" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.2-mauipre.1.22054.8" />
</ItemGroup>

6
Wabbajack.Downloaders.Interfaces/IProxyable.cs Normal file
View File

@ -0,0 +1,6 @@
namespace Wabbajack.Downloaders.Interfaces;
public interface IProxyable : IUrlDownloader
{
}

2
Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs
View File

@ -17,7 +17,7 @@ using Wabbajack.RateLimiter;
namespace Wabbajack.Downloaders.MediaFire;
public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, IUrlDownloader
public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, IUrlDownloader, IProxyable
{
private readonly IHttpDownloader _downloader;
private readonly HttpClient _httpClient;

2
Wabbajack.Downloaders.Mega/MegaDownloader.cs
View File

@ -17,7 +17,7 @@ using Wabbajack.RateLimiter;
namespace Wabbajack.Downloaders.ModDB;
public class MegaDownloader : ADownloader<Mega>, IUrlDownloader
public class MegaDownloader : ADownloader<Mega>, IUrlDownloader, IProxyable
{
private const string MegaPrefix = "https://mega.nz/#!";
private const string MegaFilePrefix = "https://mega.nz/file/";

5
Wabbajack.FileExtractor/FileExtractor.cs
View File

@ -204,7 +204,7 @@ public class FileExtractor
{
var tmpFile = _manager.CreateFile();
await tmpFile.Path.WriteAllAsync(archive, CancellationToken.None);
var dest = _manager.CreateFolder();
await using var dest = _manager.CreateFolder();
using var omod = new OMOD(tmpFile.Path.ToString());
@ -364,7 +364,7 @@ public class FileExtractor
job.Dispose();
var results = await dest.Path.EnumerateFiles()
.PMapAll(async f =>
.SelectAsync(async f =>
{
var path = f.RelativeTo(dest.Path);
if (!shouldExtract(path)) return ((RelativePath, T)) default;
@ -375,6 +375,7 @@ public class FileExtractor
})
.Where(d => d.Item1 != default)
.ToDictionary(d => d.Item1, d => d.Item2);
return results;
}

8
Wabbajack.Installer/AInstaller.cs
View File

@ -63,7 +63,7 @@ public abstract class AInstaller<T>
private readonly Stopwatch _updateStopWatch = new();
public Action<StatusUpdate>? OnStatusUpdate;
private readonly IResource<IInstaller> _limiter;
protected readonly IResource<IInstaller> _limiter;
private Func<long, string> _statusFormatter = x => x.ToString();
@ -98,6 +98,8 @@ public abstract class AInstaller<T>
public TemporaryPath ExtractedModlistFolder { get; set; }
public ModList ModList => _configuration.ModList;
public Directive[] UnoptimizedDirectives { get; set; }
public Archive[] UnoptimizedArchives { get; set; }
public void NextStep(string statusCategory, string statusText, long maxStepProgress, Func<long, string>? formatter = null)
{
@ -558,7 +560,11 @@ public abstract class AInstaller<T>
.Select(d => d.Key)
.ToHashSet();
UnoptimizedArchives = ModList.Archives;
UnoptimizedDirectives = ModList.Directives;
ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToArray();
ModList.Directives = indexed.Values.ToArray();
}
}

41
Wabbajack.Installer/StandardInstaller.cs
View File

@ -216,19 +216,38 @@ public class StandardInstaller : AInstaller<StandardInstaller>
private async Task InstallIncludedDownloadMetas(CancellationToken token)
{
await ModList.Archives
.PDoAll(async archive =>
_logger.LogInformation("Looking for downloads by size");
var bySize = UnoptimizedArchives.ToLookup(x => x.Size);
_logger.LogInformation("Writing Metas");
await _configuration.Downloads.EnumerateFiles()
.PDoAll(async download =>
{
if (HashedArchives.TryGetValue(archive.Hash, out var paths))
var found = bySize[download.Size()];
var hash = await FileHashCache.FileHashCachedAsync(download, token);
var archive = found.FirstOrDefault(f => f.Hash == hash);
if (archive == default) return;
var metaFile = download.WithExtension(Ext.Meta);
if (metaFile.FileExists())
{
var metaPath = paths.WithExtension(Ext.Meta);
if (archive.State is GameFileSource) return;
if (!metaPath.FileExists())
try
{
var meta = AddInstalled(_downloadDispatcher.MetaIni(archive));
await metaPath.WriteAllLinesAsync(meta, token);
var parsed = metaFile.LoadIniFile();
if (parsed["General"] != null && parsed["General"]["unknownArchive"] == null)
{
return;
}
}
catch (Exception)
{
// Ignore
}
}
_logger.LogInformation("Writing {FileName}", metaFile.FileName);
var meta = AddInstalled(_downloadDispatcher.MetaIni(archive));
await metaFile.WriteAllLinesAsync(meta, token);
});
}
@ -253,11 +272,15 @@ public class StandardInstaller : AInstaller<StandardInstaller>
_logger.LogInformation("Building {bsaTo}", bsa.To.FileName);
var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID);
var a = BSADispatch.CreateBuilder(bsa.State, _manager);
await using var a = BSADispatch.CreateBuilder(bsa.State, _manager);
var streams = await bsa.FileStates.PMapAll(async state =>
{
using var job = await _limiter.Begin($"Adding {state.Path.FileName}", 0, token);
var fs = sourceDir.Combine(state.Path).Open(FileMode.Open, FileAccess.Read, FileShare.Read);
var size = fs.Length;
job.Size = size;
await a.AddFile(state, fs, token);
await job.Report((int)size, token);
return fs;
}).ToList();

24
Wabbajack.Launcher/ViewModels/MainWindowViewModel.cs
View File

@ -85,6 +85,30 @@ public class MainWindowViewModel : ViewModelBase
Status = $"Getting download Uri for {_version.Version}";
var uri = await _version.Uri();
var archive = new Archive()
{
Name = $"{_version.Version}.zip",
Size = _version.Size,
State = new Http {Url = uri}
};
await using var stream = await _downloader.GetChunkedSeekableStream(archive, CancellationToken.None);
var rdr = new ZipReader(stream, true);
var entries = (await rdr.GetFiles()).OrderBy(d => d.FileOffset).ToArray();
foreach (var file in entries)
{
if (file.FileName.EndsWith("/") || file.FileName.EndsWith("\\")) continue;
var relPath = file.FileName.ToRelativePath();
Status = $"Extracting: {relPath.FileName}";
var outPath = baseFolder.Combine(relPath);
if (!outPath.Parent.DirectoryExists())
outPath.Parent.CreateDirectory();
await using var of = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None);
await rdr.Extract(file, of, CancellationToken.None);
}
var wc = new WebClient();
wc.DownloadProgressChanged += UpdateProgress;
Status = $"Downloading {_version.Version} ...";

5
Wabbajack.Networking.WabbajackClientApi/Client.cs
View File

@ -362,4 +362,9 @@ public class Client
var url = $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}_steam_manifests.json";
return await _client.GetFromJsonAsync<SteamManifest[]>(url, _dtos.Options) ?? Array.Empty<SteamManifest>();
}
public Uri MakeProxyUrl(Archive archive, Uri uri)
{
return new Uri($"{_configuration.BuildServerUrl}proxy?name={archive.Name}&hash={archive.Hash.ToHex()}&uri={uri}");
}
}

7
Wabbajack.Paths.IO/AbsolutePathExtensions.cs
View File

@ -60,6 +60,11 @@ public static class AbsolutePathExtensions
{
return new FileInfo(file.ToNativePath()).LastWriteTime;
}
public static void Touch(this AbsolutePath file)
{
new FileInfo(file.ToNativePath()).LastWriteTime = DateTime.Now;
}
public static byte[] ReadAllBytes(this AbsolutePath file)
{
@ -220,7 +225,7 @@ public static class AbsolutePathExtensions
var di = new DirectoryInfo(path.ToString());
if (di.Attributes.HasFlag(FileAttributes.ReadOnly))
di.Attributes &= ~FileAttributes.ReadOnly;
Directory.Delete(path.ToString(), true);
di.Delete(true);
}
catch (UnauthorizedAccessException)
{

7
Wabbajack.Paths.IO/TemporaryFileManager.cs
View File

@ -7,19 +7,22 @@ namespace Wabbajack.Paths.IO;
public class TemporaryFileManager : IDisposable
{
private readonly AbsolutePath _basePath;
private readonly bool _deleteOnDispose;
public TemporaryFileManager() : this(KnownFolders.EntryPoint.Combine("temp"))
{
}
public TemporaryFileManager(AbsolutePath basePath)
public TemporaryFileManager(AbsolutePath basePath, bool deleteOnDispose = true)
{
_deleteOnDispose = deleteOnDispose;
_basePath = basePath;
_basePath.CreateDirectory();
}
public void Dispose()
{
if (!_deleteOnDispose) return;
for (var retries = 0; retries < 10; retries++)
try
{
@ -34,7 +37,7 @@ public class TemporaryFileManager : IDisposable
}
}
public TemporaryPath CreateFile(Extension? ext = default)
public TemporaryPath CreateFile(Extension? ext = default, bool deleteOnDispose = true)
{
var path = _basePath.Combine(Guid.NewGuid().ToString());
if (path.Extension != default)

3
Wabbajack.Server/AppSettings.cs
View File

@ -16,6 +16,9 @@ public class AppSettings
public string WabbajackBuildServerUri { get; set; } = "https://build.wabbajack.org/";
public string MetricsKeyHeader { get; set; } = "x-metrics-key";
public string TempFolder { get; set; }
public string ProxyFolder { get; set; }
public AbsolutePath ProxyPath => (AbsolutePath) ProxyFolder;
public AbsolutePath TempPath => (AbsolutePath) TempFolder;
public string SpamWebHook { get; set; } = null;
public string HamWebHook { get; set; } = null;

124
Wabbajack.Server/Controllers/Proxy.cs Normal file
View File

@ -0,0 +1,124 @@
using System.Text;
using FluentFTP.Helpers;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths.IO;
using Wabbajack.VFS;
namespace Wabbajack.Server.Controllers;
[ApiController]
[Route("/proxy")]
public class Proxy : ControllerBase
{
private readonly ILogger<Proxy> _logger;
private readonly DownloadDispatcher _dispatcher;
private readonly TemporaryFileManager _tempFileManager;
private readonly AppSettings _appSettings;
private readonly FileHashCache _hashCache;
public Proxy(ILogger<Proxy> logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager, FileHashCache hashCache, AppSettings appSettings)
{
_logger = logger;
_dispatcher = dispatcher;
_tempFileManager = tempFileManager;
_appSettings = appSettings;
_hashCache = hashCache;
}
[HttpGet]
public async Task<IActionResult> ProxyGet(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name, [FromQuery] string? hash)
{
var shouldMatch = hash != null ? Hash.FromHex(hash) : default;
_logger.LogInformation("Got proxy request for {Uri}", uri);
var state = _dispatcher.Parse(uri);
var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex();
var cacheFile = _appSettings.ProxyPath.Combine(cacheName);
if (state == null)
{
return BadRequest(new {Type = "Could not get state from Uri", Uri = uri.ToString()});
}
var archive = new Archive
{
Name = name ?? "",
State = state,
Hash = shouldMatch
};
var downloader = _dispatcher.Downloader(archive);
if (downloader is not IProxyable)
{
return BadRequest(new {Type = "Downloader is not IProxyable", Downloader = downloader.GetType().FullName});
}
if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(4))
{
try
{
var verify = await _dispatcher.Verify(archive, token);
if (verify)
cacheFile.Touch();
}
catch (Exception ex)
{
_logger.LogInformation("When trying to verify cached file");
}
}
if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(24))
{
try
{
cacheFile.Delete();
}
catch (Exception ex)
{
_logger.LogError(ex, "When trying to delete expired file");
}
}
if (cacheFile.FileExists())
{
if (hash != default)
{
var hashResult = await _hashCache.FileHashCachedAsync(cacheFile, token);
if (hashResult != shouldMatch)
return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = hashResult.ToHex()});
}
var ret = new PhysicalFileResult(cacheFile.ToString(), "application/octet-stream");
if (name != null)
ret.FileDownloadName = name;
return ret;
}
_logger.LogInformation("Downloading proxy request for {Uri}", uri);
var tempFile = _tempFileManager.CreateFile(deleteOnDispose:false);
var result = await _dispatcher.Download(archive, tempFile.Path, token);
if (hash != default && result != shouldMatch)
{
if (tempFile.Path.FileExists())
tempFile.Path.Delete();
return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = result.ToHex()});
}
await tempFile.Path.MoveToAsync(cacheFile, true, token);
_logger.LogInformation("Returning proxy request for {Uri} {Size}", uri, cacheFile.Size().FileSizeToString());
return new PhysicalFileResult(cacheFile.ToString(), "application/binary");
}
}

16
Wabbajack.Server/Startup.cs
View File

@ -17,10 +17,12 @@ using Nettle.Compiler;
using Newtonsoft.Json;
using Octokit;
using Wabbajack.BuildServer;
using Wabbajack.Downloaders;
using Wabbajack.DTOs;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.DTOs.Logins;
using Wabbajack.Networking.GitHub;
using Wabbajack.Networking.Http;
using Wabbajack.Networking.Http.Interfaces;
using Wabbajack.Networking.NexusApi;
using Wabbajack.Paths;
@ -30,6 +32,8 @@ using Wabbajack.Server.Extensions;
using Wabbajack.Server.Services;
using Wabbajack.Services.OSIntegrated.TokenProviders;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Paths.IO;
using Wabbajack.VFS;
using Client = Wabbajack.Networking.GitHub.Client;
namespace Wabbajack.Server;
@ -79,6 +83,18 @@ public class Startup
services.AddSingleton<NexusApi>();
services.AddSingleton<DiscordBackend>();
services.AddSingleton<TarLog>();
services.AddAllSingleton<IHttpDownloader, SingleThreadedDownloader>();
services.AddDownloadDispatcher(useLoginDownloaders:false, useProxyCache:false);
var tempBase = KnownFolders.EntryPoint.Combine("temp");
services.AddTransient(s =>
new TemporaryFileManager(tempBase.Combine(Environment.ProcessId + "_" + Guid.NewGuid())));
services.AddAllSingleton<ITokenProvider<WabbajackApiState>, WabbajackApiTokenProvider>();
services.AddAllSingleton<IResource, IResource<DownloadDispatcher>>(s => new Resource<DownloadDispatcher>("Downloads", 12));
services.AddAllSingleton<IResource, IResource<FileHashCache>>(s => new Resource<FileHashCache>("File Hashing", 12));
services.AddSingleton(s =>
new FileHashCache(KnownFolders.AppDataLocal.Combine("Wabbajack", "GlobalHashCache.sqlite"),
s.GetService<IResource<FileHashCache>>()!));
services.AddAllSingleton<ITokenProvider<NexusApiState>, NexusApiTokenProvider>();
services.AddAllSingleton<IResource, IResource<HttpClient>>(s => new Resource<HttpClient>("Web Requests", 12));
// Application Info

1
Wabbajack.Server/appsettings.json
View File

@ -14,6 +14,7 @@
"PatchesFilesFolder": "c:\\tmp\\patches",
"MirrorFilesFolder": "c:\\tmp\\mirrors",
"NexusCacheFolder": "c:\\tmp\\nexus-cache",
"ProxyFolder": "c:\\tmp\\proxy",
"GitHubKey": ""
},
"AllowedHosts": "*"

32
Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
View File

@ -1,5 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net.Http;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
@ -41,8 +43,11 @@ public static class ServiceExtensions
var options = new OSIntegratedOptions();
cfn?.Invoke(options);
var tempBase = KnownFolders.EntryPoint.Combine("temp");
service.AddTransient(s =>
new TemporaryFileManager(KnownFolders.EntryPoint.Combine("temp", Guid.NewGuid().ToString())));
new TemporaryFileManager(tempBase.Combine(Environment.ProcessId + "_" + Guid.NewGuid())));
Task.Run(() => CleanAllTempData(tempBase));
service.AddSingleton(s => options.UseLocalCache
? new FileHashCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path,
@ -176,6 +181,31 @@ public static class ServiceExtensions
return service;
}
private static void CleanAllTempData(AbsolutePath path)
{
// Get directories first and cache them, this freezes the directories were looking at
// so any new ones don't show up in the middle of our deletes.
var dirs = path.EnumerateDirectories().ToList();
var processIds = Process.GetProcesses().Select(p => p.Id).ToHashSet();
foreach (var dir in dirs)
{
var name = dir.FileName.ToString().Split("_");
if (!int.TryParse(name[0], out var processId)) continue;
if (processIds.Contains(processId)) continue;
try
{
dir.DeleteDirectory();
}
catch (Exception)
{
// ignored
}
}
}
public class OSIntegratedOptions
{
public bool UseLocalCache { get; set; } = false;

5
Wabbajack.VFS/Context.cs
View File

@ -58,11 +58,12 @@ public class Context
var allFiles = await filesToIndex
.PMapAll(async f =>
{
using var job = await Limiter.Begin($"Analyzing {f}", 0, token);
if (byPath.TryGetValue(f, out var found))
if (found.LastModified == f.LastModifiedUtc().AsUnixTime() && found.Size == f.Size())
return found;
return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, token);
return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, token, job: job);
}).ToList();
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -143,7 +144,7 @@ public class Context
token,
fileNames.Keys.ToHashSet());
}
catch (Exception)
catch (Exception ex)
{
await using var stream = await sfn.GetStream();
var hash = await stream.HashingCopy(Stream.Null, token);

2
Wabbajack.VFS/FileHashCache.cs
View File

@ -90,7 +90,7 @@ public class FileHashCache
if (!file.FileExists()) return false;
var result = Get(file);
if (result == default)
if (result == default || result.Hash == default)
return false;
if (result.LastModified == file.LastModifiedUtc().ToFileTimeUtc())

3
Wabbajack.VFS/VFSCache.cs
View File

@ -41,6 +41,9 @@ public class VFSCache
public bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile,
Hash hash, out VirtualFile found)
{
if (hash == default)
throw new ArgumentException("Cannot cache default hashes");
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash";
cmd.Parameters.AddWithValue("@hash", (long) hash);

17
Wabbajack.VFS/VirtualFile.cs
View File

@ -14,6 +14,7 @@ using Wabbajack.Hashing.PHash;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
namespace Wabbajack.VFS;
@ -159,7 +160,7 @@ public class VirtualFile
public static async Task<VirtualFile> Analyze(Context context, VirtualFile? parent,
IStreamFactory extractedFile,
IPath relPath, CancellationToken token, int depth = 0)
IPath relPath, CancellationToken token, int depth = 0, IJob? job = null)
{
Hash hash;
if (extractedFile is NativeFileStreamFactory)
@ -169,9 +170,9 @@ public class VirtualFile
}
else
{
using var job = await context.HashLimiter.Begin("Hashing memory stream", 0, token);
await using var hstream = await extractedFile.GetStream();
job.Size = hstream.Length;
if (job != null)
job.Size += hstream.Length;
hash = await hstream.HashingCopy(Stream.Null, token, job);
}
@ -198,9 +199,13 @@ public class VirtualFile
if (TextureExtensions.Contains(relPath.FileName.Extension) && await DDSSig.MatchesAsync(stream) != null)
try
{
using var job = await context.HashLimiter.Begin("Hashing image", 0, token);
self.ImageState = await ImageLoader.Load(stream);
await job.Report((int) self.Size, token);
if (job != null)
{
job.Size += self.Size;
await job.Report((int) self.Size, token);
}
stream.Position = 0;
}
catch (Exception)
@ -222,7 +227,7 @@ public class VirtualFile
{
var list = await context.Extractor.GatheringExtract(extractedFile,
_ => true,
async (path, sfactory) => await Analyze(context, self, sfactory, path, token, depth + 1),
async (path, sfactory) => await Analyze(context, self, sfactory, path, token, depth + 1, job),
token);
self.Children = list.Values.ToImmutableList();