Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2140 from wabbajack-tools/verify-modlist-install

Browse Source 
Verify modlist install
This commit is contained in:
Timothy Baldridge 2022-10-24 21:29:55 -06:00 committed by GitHub
commit 3c5525f1b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 316 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.CLI/VerbRegistration.cs
View File

@ -51,6 +51,8 @@ CommandLineBuilder.RegisterCommand<UploadToNexus>(UploadToNexus.Definition, c =>
services.AddSingleton<UploadToNexus>();
CommandLineBuilder.RegisterCommand<ValidateLists>(ValidateLists.Definition, c => ((ValidateLists)c).Run);
services.AddSingleton<ValidateLists>();
CommandLineBuilder.RegisterCommand<VerifyModlistInstall>(VerifyModlistInstall.Definition, c => ((VerifyModlistInstall)c).Run);
services.AddSingleton<VerifyModlistInstall>();
CommandLineBuilder.RegisterCommand<VFSIndex>(VFSIndex.Definition, c => ((VFSIndex)c).Run);
services.AddSingleton<VFSIndex>();
}

2
Wabbajack.CLI/Verbs/ModlistReport.cs
View File

@ -70,7 +70,7 @@ public class ModlistReport
string FixupTo(RelativePath path)
{
if (path.GetPart(0) != StandardInstaller.BSACreationDir.ToString()) return path.ToString();
if (path.GetPart(0) != Consts.BSACreationDir.ToString()) return path.ToString();
var bsaId = path.GetPart(1);
if (!bsas.TryGetValue(bsaId, out var bsa))

173
Wabbajack.CLI/Verbs/VerifyModlistInstall.cs Normal file
View File

@ -0,0 +1,173 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.CLI.Builder;
using Wabbajack.Common;
using Wabbajack.Compression.BSA;
using Wabbajack.DTOs;
using Wabbajack.DTOs.BSA.FileStates;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Installer;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
using Wabbajack.VFS;
using AbsolutePathExtensions = Wabbajack.Common.AbsolutePathExtensions;
namespace Wabbajack.CLI.Verbs;
public class VerifyModlistInstall
{
private readonly DTOSerializer _dtos;
private readonly ILogger<VerifyModlistInstall> _logger;
public VerifyModlistInstall(ILogger<VerifyModlistInstall> logger, DTOSerializer dtos, IResource<FileHashCache> limiter)
{
_limiter = limiter;
_logger = logger;
_dtos = dtos;
}
public static VerbDefinition Definition = new("verify-modlist-install", "Verify a modlist installed correctly",
new[]
{
new OptionDefinition(typeof(AbsolutePath), "m", "modlistLocation",
"The .wabbajack file used to install the modlist"),
new OptionDefinition(typeof(AbsolutePath), "i", "installFolder", "The installation folder of the modlist")
});
private readonly IResource<FileHashCache> _limiter;
public async Task<int> Run(AbsolutePath modlistLocation, AbsolutePath installFolder, CancellationToken token)
{
_logger.LogInformation("Loading modlist {ModList}", modlistLocation);
var list = await StandardInstaller.LoadFromFile(_dtos, modlistLocation);
_logger.LogInformation("Indexing files");
var byTo = list.Directives.ToDictionary(d => d.To);
_logger.LogInformation("Scanning files");
var errors = await list.Directives.PMapAllBatchedAsync(_limiter, async directive =>
{
if (directive is ArchiveMeta)
return null;
if (directive is RemappedInlineFile)
return null;
if (directive.To.InFolder(Consts.BSACreationDir))
return null;
var dest = directive.To.RelativeTo(installFolder);
if (!dest.FileExists())
{
return new Result
{
Path = directive.To,
Message = $"File does not exist directive {directive.GetType()}"
};
}
if (Consts.KnownModifiedFiles.Contains(directive.To.FileName))
return null;
if (directive is CreateBSA bsa)
{
return await VerifyBSA(dest, bsa, byTo, token);
}
if (dest.Size() != directive.Size)
{
return new Result
{
Path = directive.To,
Message = $"Sizes do not match got {dest.Size()} expected {directive.Size}"
};
}
if (directive.Size > (1024 * 1024 * 128))
{
_logger.LogInformation("Hashing {Size} file at {Path}", directive.Size.ToFileSizeString(),
directive.To);
}
var hash = await AbsolutePathExtensions.Hash(dest, token);
if (hash != directive.Hash)
{
return new Result
{
Path = directive.To,
Message = $"Hashes do not match, got {hash} expected {directive.Hash}"
};
}
return null;
}).Where(r => r != null)
.ToList();
_logger.LogInformation("Found {Count} errors", errors.Count);
foreach (var error in errors)
{
_logger.LogError("{File} | {Message}", error.Path, error.Message);
}
return 0;
}
private async Task<Result?> VerifyBSA(AbsolutePath dest, CreateBSA bsa, Dictionary<RelativePath, Directive> byTo, CancellationToken token)
{
_logger.LogInformation("Verifying Created BSA {To}", bsa.To);
var archive = await BSADispatch.Open(dest);
var filesIndexed = archive.Files.ToDictionary(d => d.Path);
if (dest.Extension == Ext.Bsa && dest.Size() >= 1024L * 1024 * 1024 * 2)
{
return new Result()
{
Path = bsa.To,
Message = $"BSA is over 2GB in size, this will cause crashes : {bsa.To}"
};
}
foreach (var file in bsa.FileStates)
{
if (file is BA2DX10File) continue;
var state = filesIndexed[file.Path];
var sf = await state.GetStreamFactory(token);
await using var stream = await sf.GetStream();
var hash = await stream.Hash(token);
var astate = bsa.FileStates.First(f => f.Path == state.Path);
var srcDirective = byTo[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
if (srcDirective.Hash != hash)
{
return new Result
{
Path = bsa.To,
Message =
$"BSA {bsa.To} contents do not match at {file.Path} got {hash} expected {srcDirective.Hash}"
};
}
}
return null;
}
public class Result
{
public RelativePath Path { get; set; }
public string Message { get; set; }
}
}

21
Wabbajack.Common/AbsolutePathExtensions.cs
View File

@ -21,4 +21,25 @@ public static class AbsolutePathExtensions
await using var fs = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
return await fs.FromJson<T>(dtos);
}
public static async Task<Hash> WriteAllHashedAsync(this AbsolutePath file, Stream srcStream, CancellationToken token,
bool closeWhenDone = true)
{
try
{
await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None);
return await srcStream.HashingCopy(dest, token);
}
finally
{
if (closeWhenDone)
srcStream.Close();
}
}
public static async Task<Hash> WriteAllHashedAsync(this AbsolutePath file, byte[] data, CancellationToken token)
{
await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None);
return await new MemoryStream(data).HashingCopy(dest, token);
}
}

19
Wabbajack.Downloaders.Nexus/NexusDownloader.cs
View File

@ -129,8 +129,23 @@ public class NexusDownloader : ADownloader<Nexus>, IUrlDownloader
var urls = await _api.DownloadLink(state.Game.MetaData().NexusName!, state.ModID, state.FileID, token);
_logger.LogInformation("Downloading Nexus File: {game}|{modid}|{fileid}", state.Game, state.ModID,
state.FileID);
var message = new HttpRequestMessage(HttpMethod.Get, urls.info.First().URI);
return await _downloader.Download(message, destination, job, token);
foreach (var link in urls.info)
{
try
{
var message = new HttpRequestMessage(HttpMethod.Get, link.URI);
return await _downloader.Download(message, destination, job, token);
}
catch (Exception ex)
{
if (link.URI == urls.info.Last().URI)
throw;
_logger.LogInformation(ex, "While downloading {URI}, trying another link", link.URI);
}
}
// Should never be hit
throw new NotImplementedException();
}
catch (HttpRequestException ex)
{

21
Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs
View File

@ -1,6 +1,8 @@
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.DTOs.Streams;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
namespace Wabbajack.FileExtractor.ExtractedFiles;
@ -16,4 +18,23 @@ public interface IExtractedFile : IStreamFactory
/// <param name="newPath">destination to move the entry to</param>
/// <returns></returns>
public ValueTask Move(AbsolutePath newPath, CancellationToken token);
}
public static class IExtractedFileExtensions
{
public static async Task<Hash> MoveHashedAsync(this IExtractedFile file, AbsolutePath destPath, CancellationToken token)
{
if (file.CanMove)
{
await file.Move(destPath, token);
return await destPath.Hash(token);
}
else
{
await using var s = await file.GetStream();
return await destPath.WriteAllHashedAsync(s, token, false);
}
}
}

36
Wabbajack.Installer/AInstaller.cs
View File

@ -12,9 +12,11 @@ using Wabbajack.Common;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.BSA.FileStates;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.FileExtractor.ExtractedFiles;
using Wabbajack.Hashing.PHash;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Installer.Utilities;
@ -39,7 +41,7 @@ public abstract class AInstaller<T>
where T : AInstaller<T>
{
private const int _limitMS = 100;
public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath();
private static readonly Regex NoDeleteRegex = new(@"(?i)[\\\/]\[NoDelete\]", RegexOptions.Compiled);
protected readonly InstallerConfiguration _configuration;
@ -243,7 +245,8 @@ public abstract class AInstaller<T>
await using var patchDataStream = await InlinedFileStream(pfa.PatchID);
{
await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
await BinaryPatching.ApplyPatch(s, patchDataStream, os);
var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os);
ThrowOnNonMatchingHash(file, hash);
}
}
break;
@ -263,12 +266,14 @@ public abstract class AInstaller<T>
case FromArchive _:
if (grouped[vf].Count() == 1)
{
await sf.Move(destPath, token);
var hash = await sf.MoveHashedAsync(destPath, token);
ThrowOnNonMatchingHash(file, hash);
}
else
{
await using var s = await sf.GetStream();
await destPath.WriteAllAsync(s, token, false);
var hash = await destPath.WriteAllHashedAsync(s, token, false);
ThrowOnNonMatchingHash(file, hash);
}
break;
@ -282,6 +287,25 @@ public abstract class AInstaller<T>
}, token);
}
protected void ThrowOnNonMatchingHash(Directive file, Hash gotHash)
{
if (file.Hash != gotHash)
ThrowNonMatchingError(file, gotHash);
}
private void ThrowNonMatchingError(Directive file, Hash gotHash)
{
_logger.LogError("Hashes for {Path} did not match, expected {Expected} got {Got}", file.To, file.Hash, gotHash);
throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}");
}
protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash)
{
if (hash == directive.Hash) return;
_logger.LogError("Hashes for BSA don't match after extraction, {BSA}, {Directive}, {ExpectedHash}, {Hash}", bsa.To, directive.To, directive.Hash, hash);
throw new Exception($"Hashes for {bsa.To} file {directive.To} did not match, expected {directive.Hash} got {hash}");
}
public async Task DownloadArchives(CancellationToken token)
{
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
@ -474,8 +498,8 @@ public abstract class AInstaller<T>
return d switch
{
CreateBSA bsa => !bsasToNotBuild.Contains(bsa.TempID),
FromArchive a when a.To.StartsWith($"{BSACreationDir}") => !bsasToNotBuild.Any(b =>
a.To.RelativeTo(_configuration.Install).InFolder(_configuration.Install.Combine(BSACreationDir, b))),
FromArchive a when a.To.StartsWith($"{Consts.BSACreationDir}") => !bsasToNotBuild.Any(b =>
a.To.RelativeTo(_configuration.Install).InFolder(_configuration.Install.Combine(Consts.BSACreationDir, b))),
_ => true
};
}).ToDictionary(d => d.To);

9
Wabbajack.Installer/Consts.cs
View File

@ -1,3 +1,5 @@
using System.Collections.Generic;
using System.Linq;
using Wabbajack.Paths;
namespace Wabbajack.Installer;
@ -26,4 +28,11 @@ public static class Consts
public const string StepDownloading = "Downloading";
public const string StepHashing = "Hashing";
public const string StepFinished = "Finished";
public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath();
public static HashSet<RelativePath> KnownModifiedFiles = new[]
{
"modlist.txt",
"SkyrimPrefs.ini"
}.Select(r => r.ToRelativePath()).ToHashSet();
}

48
Wabbajack.Installer/StandardInstaller.cs
View File

@ -19,9 +19,11 @@ using Wabbajack.Compression.Zip;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.BSA.FileStates;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Installer.Utilities;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Paths;
@ -271,6 +273,8 @@ public class StandardInstaller : AInstaller<StandardInstaller>
private async Task BuildBSAs(CancellationToken token)
{
var bsas = ModList.Directives.OfType<CreateBSA>().ToList();
_logger.LogInformation("Generating debug caches");
var indexedByDestination = UnoptimizedDirectives.ToDictionary(d => d.To);
_logger.LogInformation("Building {bsasCount} bsa files", bsas.Count);
NextStep("Installing", "Building BSAs", bsas.Count);
@ -278,34 +282,50 @@ public class StandardInstaller : AInstaller<StandardInstaller>
{
UpdateProgress(1);
_logger.LogInformation("Building {bsaTo}", bsa.To.FileName);
var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID);
var sourceDir = _configuration.Install.Combine(Consts.BSACreationDir, bsa.TempID);
await using var a = BSADispatch.CreateBuilder(bsa.State, _manager);
var streams = await bsa.FileStates.PMapAll(async state =>
var streams = await bsa.FileStates.PMapAllBatchedAsync(_limiter, async state =>
{
using var job = await _limiter.Begin($"Adding {state.Path.FileName}", 0, token);
var fs = sourceDir.Combine(state.Path).Open(FileMode.Open, FileAccess.Read, FileShare.Read);
var size = fs.Length;
job.Size = size;
await a.AddFile(state, fs, token);
await job.Report((int)size, token);
return fs;
}).ToList();
_logger.LogInformation("Writing {bsaTo}", bsa.To);
var outPath = _configuration.Install.Combine(bsa.To);
await using var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None);
await a.Build(outStream, token);
await using (var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None))
{
await a.Build(outStream, token);
}
streams.Do(s => s.Dispose());
await FileHashCache.FileHashWriteCache(outPath, bsa.Hash);
sourceDir.DeleteDirectory();
_logger.LogInformation("Verifying {bsaTo}", bsa.To);
var reader = await BSADispatch.Open(outPath);
var results = await reader.Files.PMapAllBatchedAsync(_limiter, async state =>
{
var sf = await state.GetStreamFactory(token);
await using var stream = await sf.GetStream();
var hash = await stream.Hash(token);
var astate = bsa.FileStates.First(f => f.Path == state.Path);
var srcDirective = indexedByDestination[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
//DX10Files are lossy
if (astate is not BA2DX10File)
ThrowOnNonMatchingHash(bsa, srcDirective, astate, hash);
return (srcDirective, hash);
}).ToHashSet();
}
var bsaDir = _configuration.Install.Combine(BSACreationDir);
var bsaDir = _configuration.Install.Combine(Consts.BSACreationDir);
if (bsaDir.DirectoryExists())
{
_logger.LogInformation("Removing temp folder {bsaCreationDir}", BSACreationDir);
_logger.LogInformation("Removing temp folder {bsaCreationDir}", Consts.BSACreationDir);
bsaDir.DeleteDirectory();
}
}
@ -329,7 +349,10 @@ public class StandardInstaller : AInstaller<StandardInstaller>
await FileHashCache.FileHashCachedAsync(outPath, token);
break;
default:
await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID), token);
var hash = await outPath.WriteAllHashedAsync(await LoadBytesFromPath(directive.SourceDataID), token);
if (!Consts.KnownModifiedFiles.Contains(directive.To.FileName))
ThrowOnNonMatchingHash(directive, hash);
await FileHashCache.FileHashWriteCache(outPath, directive.Hash);
break;
}
@ -481,7 +504,8 @@ public class StandardInstaller : AInstaller<StandardInstaller>
.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
try
{
await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs);
var hash = await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs);
ThrowOnNonMatchingHash(m, hash);
}
catch (Exception ex)
{

7
Wabbajack.Installer/Utilities/BinaryPatching.cs
View File

@ -1,16 +1,19 @@
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Octodiff.Core;
using Octodiff.Diagnostics;
using Wabbajack.Hashing.xxHash64;
namespace Wabbajack.Installer.Utilities;
public class BinaryPatching
{
public static ValueTask ApplyPatch(Stream input, Stream deltaStream, Stream output)
public static async ValueTask<Hash> ApplyPatch(Stream input, Stream deltaStream, Stream output, CancellationToken? token = null)
{
var deltaApplier = new DeltaApplier();
deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, new NullProgressReporter()), output);
return ValueTask.CompletedTask;
output.Position = 0;
return await output.Hash(token ?? CancellationToken.None);
}
}

2
Wabbajack.Installer/Wabbajack.Installer.csproj
View File

@ -24,7 +24,7 @@
<ItemGroup>
<PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
<PackageReference Include="Octodiff" Version="1.3.23" />
<PackageReference Include="Octopus.Octodiff" Version="1.3.93" />
</ItemGroup>
</Project>