Finished verification code

This commit is contained in:
Halgari 2022-10-24 20:12:14 -06:00
parent 0bd79a40cd
commit 38c355adba
3 changed files with 107 additions and 22 deletions

View File

@ -1,13 +1,23 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.CLI.Builder;
using Wabbajack.Common;
using Wabbajack.Compression.BSA;
using Wabbajack.DTOs;
using Wabbajack.DTOs.BSA.FileStates;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Installer;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
using Wabbajack.VFS;
using AbsolutePathExtensions = Wabbajack.Common.AbsolutePathExtensions;
namespace Wabbajack.CLI.Verbs;
@ -16,8 +26,9 @@ public class VerifyModlistInstall
private readonly DTOSerializer _dtos;
private readonly ILogger<VerifyModlistInstall> _logger;
public VerifyModlistInstall(ILogger<VerifyModlistInstall> logger, DTOSerializer dtos)
public VerifyModlistInstall(ILogger<VerifyModlistInstall> logger, DTOSerializer dtos, IResource<FileHashCache> limiter)
{
_limiter = limiter;
_logger = logger;
_dtos = dtos;
}
@ -30,47 +41,76 @@ public class VerifyModlistInstall
new OptionDefinition(typeof(AbsolutePath), "i", "installFolder", "The installation folder of the modlist")
});
private readonly IResource<FileHashCache> _limiter;
public async Task<int> Run(AbsolutePath modlistLocation, AbsolutePath installFolder)
public async Task<int> Run(AbsolutePath modlistLocation, AbsolutePath installFolder, CancellationToken token)
{
_logger.LogInformation("Loading modlist {ModList}", modlistLocation);
var list = await StandardInstaller.LoadFromFile(_dtos, modlistLocation);
var errors = new List<Result>();
_logger.LogInformation("Indexing files");
var byTo = list.Directives.ToDictionary(d => d.To);
_logger.LogInformation("Scanning files");
foreach (var directive in list.Directives)
var errors = await list.Directives.PMapAllBatchedAsync(_limiter, async directive =>
{
if (directive is ArchiveMeta)
continue;
return null;
if (directive is RemappedInlineFile)
continue;
return null;
if (directive.To.InFolder(Consts.BSACreationDir))
continue;
return null;
var dest = directive.To.RelativeTo(installFolder);
if (!dest.FileExists())
{
errors.Add(new Result
return new Result
{
Path = directive.To,
Message = $"File does not exist directive {directive.GetType()}"
});
continue;
};
}
if (Consts.KnownModifiedFiles.Contains(directive.To.FileName))
return null;
if (directive is CreateBSA bsa)
{
return await VerifyBSA(dest, bsa, byTo, token);
}
if (dest.Size() != directive.Size)
{
errors.Add(new Result
return new Result
{
Path = directive.To,
Message = $"Sizes do not match got {dest.Size()} expected {directive.Size}"
});
};
}
}
if (directive.Size > (1024 * 1024 * 128))
{
_logger.LogInformation("Hashing {Size} file at {Path}", directive.Size.ToFileSizeString(),
directive.To);
}
var hash = await AbsolutePathExtensions.Hash(dest, token);
if (hash != directive.Hash)
{
return new Result
{
Path = directive.To,
Message = $"Hashes do not match, got {hash} expected {directive.Hash}"
};
}
return null;
}).Where(r => r != null)
.ToList();
_logger.LogInformation("Found {Count} errors", errors.Count);
@ -84,6 +124,47 @@ public class VerifyModlistInstall
return 0;
}
private async Task<Result?> VerifyBSA(AbsolutePath dest, CreateBSA bsa, Dictionary<RelativePath, Directive> byTo, CancellationToken token)
{
_logger.LogInformation("Verifying Created BSA {To}", bsa.To);
var archive = await BSADispatch.Open(dest);
var filesIndexed = archive.Files.ToDictionary(d => d.Path);
if (dest.Extension == Ext.Bsa && dest.Size() >= 1024L * 1024 * 1024 * 2)
{
return new Result()
{
Path = bsa.To,
Message = $"BSA is over 2GB in size, this will cause crashes : {bsa.To}"
};
}
foreach (var file in bsa.FileStates)
{
if (file is BA2DX10File) continue;
var state = filesIndexed[file.Path];
var sf = await state.GetStreamFactory(token);
await using var stream = await sf.GetStream();
var hash = await stream.Hash(token);
var astate = bsa.FileStates.First(f => f.Path == state.Path);
var srcDirective = byTo[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
if (srcDirective.Hash != hash)
{
return new Result
{
Path = bsa.To,
Message =
$"BSA {bsa.To} contents do not match at {file.Path} got {hash} expected {srcDirective.Hash}"
};
}
}
return null;
}
public class Result
{
public RelativePath Path { get; set; }

View File

@ -1,3 +1,5 @@
using System.Collections.Generic;
using System.Linq;
using Wabbajack.Paths;
namespace Wabbajack.Installer;
@ -27,4 +29,10 @@ public static class Consts
public const string StepHashing = "Hashing";
public const string StepFinished = "Finished";
public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath();
public static HashSet<RelativePath> KnownModifiedFiles = new[]
{
"modlist.txt",
"SkyrimPrefs.ini"
}.Select(r => r.ToRelativePath()).ToHashSet();
}

View File

@ -330,10 +330,6 @@ public class StandardInstaller : AInstaller<StandardInstaller>
}
}
private static HashSet<RelativePath> KnownModifiedFiles = new[]
{
"modlist.txt"
}.Select(r => r.ToRelativePath()).ToHashSet();
private async Task InstallIncludedFiles(CancellationToken token)
{
_logger.LogInformation("Writing inline files");
@ -354,7 +350,7 @@ public class StandardInstaller : AInstaller<StandardInstaller>
break;
default:
var hash = await outPath.WriteAllHashedAsync(await LoadBytesFromPath(directive.SourceDataID), token);
if (!KnownModifiedFiles.Contains(directive.To.FileName))
if (!Consts.KnownModifiedFiles.Contains(directive.To.FileName))
ThrowOnNonMatchingHash(directive, hash);
await FileHashCache.FileHashWriteCache(outPath, directive.Hash);