Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP on the compiler, can compile most of SMEFT

Browse Source
This commit is contained in:
Timothy Baldridge 2021-10-12 06:20:34 -06:00
parent 92240aab49
commit 3e3299c303
9 changed files with 40 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.Compiler/ACompiler.cs
View File

@ -547,7 +547,7 @@ namespace Wabbajack.Compiler
if (!await _dispatcher.Verify(result, token.Token))
{
_logger.LogWarning(
"Unable to resolve link for {archive}. If this is hosted on the Nexus the file may have been removed.", archive);
"Unable to resolve link for {Archive}. If this is hosted on the Nexus the file may have been removed.", archive.State!.PrimaryKeyString);
}
result.Meta = "[General]\n" + string.Join("\n", _dispatcher.MetaIni(result));

11
Wabbajack.Compiler/CompilationSteps/IgnoreDisabledMods.cs
View File

@ -16,16 +16,17 @@ namespace Wabbajack.Compiler.CompilationSteps
public IgnoreDisabledMods(ACompiler compiler) : base(compiler)
{
_mo2Compiler = (MO2Compiler) compiler;
var alwaysEnabled = _mo2Compiler.ModInis.Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_ENABLE)).Select(f => f.Key).Distinct();
var alwaysDisabled = _mo2Compiler.ModInis
.Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_DISABLE)).Select(f => f.Key).Distinct();
//var alwaysEnabled = _mo2Compiler.ModInis.Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_ENABLE)).Select(f => f.Key).Distinct();
// TODO: Re-enable this
//var alwaysDisabled = _mo2Compiler.ModInis
// .Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_DISABLE)).Select(f => f.Key).Distinct();
_allEnabledMods = _mo2Compiler._settings.SelectedProfiles
.SelectMany(p => _mo2Compiler._settings.Source.Combine("profiles", p, "modlist.txt").ReadAllLines())
.Where(line => line.StartsWith("+") || line.EndsWith("_separator"))
.Select(line => line[1..].ToRelativePath().RelativeTo(_mo2Compiler.MO2ModsFolder))
.Concat(alwaysEnabled)
.Except(alwaysDisabled)
//.Concat(alwaysEnabled)
//.Except(alwaysDisabled)
.ToList();
}

2
Wabbajack.Compiler/Consts.cs
View File

@ -35,7 +35,7 @@ namespace Wabbajack.Compiler
public static readonly HashSet<Extension> SupportedBSAs = new[] {".bsa", ".ba2"}
.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml"}.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml", ".yaml"}.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> ESPFileExtensions = new []{ ".esp", ".esm", ".esl"}.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> AssetFileExtensions = new[] {".dds", ".tga", ".nif", ".psc", ".pex"}.Select(s => new Extension(s)).ToHashSet();

4
Wabbajack.Compiler/MO2Compiler.cs
View File

@ -275,6 +275,10 @@ namespace Wabbajack.Compiler
new IncludeRegex(this, ".*\\.txt"),
new IgnorePathContains(this,@"\Edit Scripts\Export\"),
new IgnoreExtension(this, new Extension(".CACHE")),
// Misc
new IncludeRegex(this, "modlist-image\\.png"),
new DropAll(this)
};

5
Wabbajack.Installer/AInstaller.cs
View File

@ -256,15 +256,18 @@ namespace Wabbajack.Installer
await Task.WhenAll(dispatchers.Select(d => d.Prepare()));
_logger.LogInformation("Downloading validation data");
var validationData = await _wjClient.LoadDownloadAllowList();
_logger.LogInformation("Validating Archives");
foreach (var archive in missing.Where(archive =>
!_downloadDispatcher.Downloader(archive).IsAllowed(validationData, archive.State)))
{
_logger.LogCritical("File {primaryKeyString} failed validation", archive.State.PrimaryKeyString);
return;
}
_logger.LogInformation("Downloading missing archives");
await DownloadMissingArchives(missing, token);
}

1
Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
View File

@ -59,6 +59,7 @@ namespace Wabbajack.Services.OSIntegrated
service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});
service.AddAllSingleton<IResource, IResource<DownloadDispatcher>>(s => new Resource<DownloadDispatcher>(12));
service.AddAllSingleton<IResource, IResource<HttpClient>>(s => new Resource<HttpClient>(12));
service.AddAllSingleton<IResource, IResource<Context>>(s => new Resource<Context>(12));
service.AddAllSingleton<IResource, IResource<FileExtractor.FileExtractor>>(s =>
new Resource<FileExtractor.FileExtractor>(12));

7
Wabbajack.VFS/Context.cs
View File

@ -10,6 +10,7 @@ using Wabbajack.FileExtractor.ExtractedFiles;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
namespace Wabbajack.VFS
{
@ -25,10 +26,12 @@ namespace Wabbajack.VFS
public readonly FileHashCache HashCache;
public readonly ILogger<Context> Logger;
public readonly VFSCache VfsCache;
public readonly IResource<Context> Limiter;
public Context(ILogger<Context> logger, ParallelOptions parallelOptions, TemporaryFileManager manager, VFSCache vfsCache,
FileHashCache hashCache, FileExtractor.FileExtractor extractor)
FileHashCache hashCache, IResource<Context> limiter, FileExtractor.FileExtractor extractor)
{
Limiter = limiter;
Logger = logger;
_manager = manager;
Extractor = extractor;
@ -49,7 +52,7 @@ namespace Wabbajack.VFS
var filesToIndex = root.EnumerateFiles().Distinct().ToList();
var allFiles = await filesToIndex
.PMap(_parallelOptions, async f =>
.PMapAll(async f =>
{
if (byPath.TryGetValue(f, out var found))
if (found.LastModified == f.LastModifiedUtc().AsUnixTime() && found.Size == f.Size())

5
Wabbajack.VFS/FileHashCache.cs
View File

@ -5,6 +5,7 @@ using System.Threading.Tasks;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
namespace Wabbajack.VFS
{
@ -113,13 +114,13 @@ namespace Wabbajack.VFS
WriteHashCache(file, hash);
}
public async Task<Hash> FileHashCachedAsync(AbsolutePath file, CancellationToken token)
public async Task<Hash> FileHashCachedAsync(AbsolutePath file, CancellationToken token, IJob? job = null)
{
if (TryGetHashCache(file, out var foundHash)) return foundHash;
await using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
var hash = await fs.HashingCopy(Stream.Null, token);
var hash = await fs.HashingCopy(Stream.Null, token, job);
if (hash != default)
WriteHashCache(file, hash);
return hash;

22
Wabbajack.VFS/VirtualFile.cs
View File

@ -162,14 +162,20 @@ namespace Wabbajack.VFS
IPath relPath, CancellationToken token, int depth = 0)
{
Hash hash;
if (extractedFile is NativeFileStreamFactory)
using (var job = await context.Limiter.Begin("Hash file", 0, token))
{
hash = await context.HashCache.FileHashCachedAsync((AbsolutePath)extractedFile.Name, token);
}
else
{
await using var hstream = await extractedFile.GetStream();
hash = await hstream.HashingCopy(Stream.Null, token);
if (extractedFile is NativeFileStreamFactory)
{
var absPath = (AbsolutePath)extractedFile.Name;
job.Size = absPath.Size();
hash = await context.HashCache.FileHashCachedAsync(absPath, token, job);
}
else
{
await using var hstream = await extractedFile.GetStream();
job.Size = hstream.Length;
hash = await hstream.HashingCopy(Stream.Null, token, job);
}
}
if (context.VfsCache.TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
@ -195,7 +201,9 @@ namespace Wabbajack.VFS
if (TextureExtensions.Contains(relPath.FileName.Extension) && await DDSSig.MatchesAsync(stream) != null)
try
{
using var job = await context.Limiter.Begin("Perceptual hash", self.Size, token);
self.ImageState = await ImageLoader.Load(stream);
await job.Report((int)self.Size, token);
stream.Position = 0;
}
catch (Exception)