Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2051 from wabbajack-tools/3.0.0.5-fixes

Browse Source 
3.0.0.5 fixes
This commit is contained in:
Timothy Baldridge 2022-08-22 20:53:24 -06:00 committed by GitHub
commit 8af1ab65e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 114 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -1,5 +1,11 @@
### Changelog
#### Version - 3.0.0.5 - 8/22/2022
* No longer rehashes files on every compile (faster Add Roots step during compilation)
* Editing paths in the install/compile settings won't crash the app
* Fix for .refcache files not being ignored during compilation
* Greatly reduce the amount of UI hangups during compilation
#### Version - 3.0.0.4 - 8/20/2022
* Fix for: when some optional game files weren't present (like CreationKit.exe), the app would refuse to recognize any files from that game

2
Wabbajack.App.Wpf/Converters/AbsolutePathToStringConverter.cs
View File

@ -74,7 +74,7 @@ namespace Wabbajack
public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
{
return (AbsolutePath)(value as string);
return AbsolutePath.ConvertNoFailure((string) value);
}
}
}

4
Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs
View File

@ -278,7 +278,9 @@ namespace Wabbajack
try
{
await compiler.Begin(token);
var result = await compiler.Begin(token);
if (!result)
throw new Exception("Compilation Failed");
}
finally
{

5
Wabbajack.Common/AsyncParallelExtensions.cs
View File

@ -29,7 +29,10 @@ public static class AsyncParallelExtensions
public static async IAsyncEnumerable<TOut> PMapAll<TIn, TOut>(this IEnumerable<TIn> coll,
Func<TIn, Task<TOut>> mapFn)
{
var tasks = coll.Select(mapFn).ToList();
var tasks = coll.Select(async x =>
{
return await Task.Run(() => mapFn(x));
}).ToList();
foreach (var itm in tasks) yield return await itm;
}

5
Wabbajack.Common/StringExtensions.cs
View File

@ -24,4 +24,9 @@ public static class StringExtensions
{
return src.Contains(contains, StringComparison.InvariantCultureIgnoreCase);
}
public static string CleanIniString(this string src)
{
return src.TrimStart('\"').TrimEnd('\"');
}
}

26
Wabbajack.Compiler.Test/Startup.cs
View File

@ -1,5 +1,8 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Wabbajack.DTOs.Interventions;
using Wabbajack.Networking.Steam.UserInterventions;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Services.OSIntegrated;
using Xunit.DependencyInjection;
@ -19,10 +22,33 @@ public class Startup
});
service.AddScoped<ModListHarness>();
service.AddSingleton<IUserInterventionHandler, UserInterventionHandler>();
}
public void Configure(ILoggerFactory loggerFactory, ITestOutputHelperAccessor accessor)
{
loggerFactory.AddProvider(new XunitTestOutputLoggerProvider(accessor, delegate { return true; }));
}
public class UserInterventionHandler : IUserInterventionHandler
{
public void Raise(IUserIntervention intervention)
{
if (intervention is GetAuthCode gac)
{
switch (gac.Type)
{
case GetAuthCode.AuthType.EmailCode:
Console.WriteLine("Please enter the Steam code that was just emailed to you");
break;
case GetAuthCode.AuthType.TwoFactorAuth:
Console.WriteLine("Please enter your 2FA code for Steam");
break;
default:
throw new ArgumentOutOfRangeException();
}
gac.Finish(Console.ReadLine()!.Trim());
}
}
}
}

4
Wabbajack.Compiler/ACompiler.cs
View File

@ -447,6 +447,8 @@ public abstract class ACompiler
/// </summary>
protected async Task BuildPatches(CancellationToken token)
{
NextStep("Compiling","Looking for patches");
var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => _patchOptions.GetValueOrDefault(p, Array.Empty<VirtualFile>()).Length > 0)
.SelectMany(p => _patchOptions[p].Select(c => new PatchedFromArchive
@ -487,7 +489,7 @@ public abstract class ACompiler
_logger.LogInformation("Patch size {patchSize} for {to}", patchSize, match.To);
}, token);
}
}, token);
}, token, runInParallel: false);
// Load in the patches
await InstallDirectives.OfType<PatchedFromArchive>()

6
Wabbajack.Compiler/CompilationSteps/IgnoreEndsWith.cs
View File

@ -7,18 +7,18 @@ namespace Wabbajack.Compiler.CompilationSteps;
public class IgnoreFilename : ACompilationStep
{
private readonly RelativePath _postfix;
private readonly string _postfix;
private readonly string _reason;
public IgnoreFilename(ACompiler compiler, RelativePath postfix) : base(compiler)
{
_postfix = postfix;
_postfix = postfix.FileName.ToString();
_reason = $"Ignored because path ends with {postfix}";
}
public override async ValueTask<Directive?> Run(RawSourceFile source)
{
if (source.Path.FileName != _postfix.FileName) return null;
if (!source.Path.EndsWith(_postfix)) return null;
var result = source.EvolveTo<IgnoredDirectly>();
result.Reason = _reason;
return result;

12
Wabbajack.Compiler/MO2Compiler.cs
View File

@ -77,15 +77,11 @@ public class MO2Compiler : ACompiler
var roots = new List<AbsolutePath> {Settings.Source, Settings.Downloads};
roots.AddRange(Settings.OtherGames.Append(Settings.Game).Select(g => _locator.GameLocation(g)));
roots.Add(Settings.Downloads);
NextStep("Initializing", "Add Roots");
await _vfs.AddRoots(roots, token, async (cur, max) => UpdateProgressAbsolute(cur, max)); // Step 1
//await InferMetas(token); // Step 2
NextStep("Initializing", "Add Download Roots");
await _vfs.AddRoot(Settings.Downloads, token); // Step 3
// Find all Downloads
IndexedArchives = await Settings.Downloads.EnumerateFiles()
.Where(f => f.WithExtension(Ext.Meta).FileExists())
@ -103,7 +99,7 @@ public class MO2Compiler : ACompiler
await CleanInvalidArchivesAndFillState();
NextStep("Initializing", "Indexing Data");
var mo2Files = Settings.Source.EnumerateFiles()
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(_vfs.Index.ByRootPath[p], p.RelativeTo(Settings.Source)));
@ -164,6 +160,7 @@ public class MO2Compiler : ACompiler
InstallDirectives = results.Where(i => i is not IgnoredDirectly).ToList();
NextStep("Compiling", "Verifying zEdit Merges (if any)");
zEditIntegration.VerifyMerges(this);
await BuildPatches(token);
@ -242,6 +239,7 @@ public class MO2Compiler : ACompiler
/// <returns></returns>
public override IEnumerable<ICompilationStep> MakeStack()
{
NextStep("Initialization", "Generating Compilation Stack");
_logger.LogInformation("Generating compilation stack");
var steps = new List<ICompilationStep>
{

2
Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs
View File

@ -111,7 +111,7 @@ public class BethesdaDownloader : ADownloader<DTOs.DownloadStates.Bethesda>, IUr
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri))
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri))
{
return Parse(uri);
}

3
Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs
View File

@ -10,6 +10,7 @@ using System.Threading.Tasks;
using System.Web;
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
@ -67,7 +68,7 @@ public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri))
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri))
return Parse(uri);
return null;
}

1
Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj
View File

@ -13,6 +13,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
<ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" />

2
Wabbajack.Downloaders.Http/HttpDownloader.cs
View File

@ -53,7 +53,7 @@ public class HttpDownloader : ADownloader<DTOs.DownloadStates.Http>, IUrlDownloa
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri))
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri))
{
var state = new DTOs.DownloadStates.Http
{

2
Wabbajack.Downloaders.Manual/ManualDownloader.cs
View File

@ -71,7 +71,7 @@ public class ManualDownloader : ADownloader<DTOs.DownloadStates.Manual>, IProxya
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{
if (iniData.ContainsKey("manualURL") && Uri.TryCreate(iniData["manualURL"], UriKind.Absolute, out var uri))
if (iniData.ContainsKey("manualURL") && Uri.TryCreate(iniData["manualURL"].CleanIniString(), UriKind.Absolute, out var uri))
{
iniData.TryGetValue("prompt", out var prompt);

3
Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs
View File

@ -7,6 +7,7 @@ using System.Threading;
using System.Threading.Tasks;
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
@ -45,7 +46,7 @@ public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, I
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{
if (iniData.ContainsKey("directURL") &&
Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri)
Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri)
&& uri.Host == "www.mediafire.com")
{
var state = new DTOs.DownloadStates.MediaFire

1
Wabbajack.Downloaders.MediaFire/Wabbajack.Downloaders.MediaFire.csproj
View File

@ -12,6 +12,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" />
</ItemGroup>

3
Wabbajack.Downloaders.Mega/MegaDownloader.cs
View File

@ -6,6 +6,7 @@ using System.Threading;
using System.Threading.Tasks;
using CG.Web.MegaApiClient;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
@ -45,7 +46,7 @@ public class MegaDownloader : ADownloader<Mega>, IUrlDownloader, IProxyable
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{
return iniData.ContainsKey("directURL") ? GetDownloaderState(iniData["directURL"]) : null;
return iniData.ContainsKey("directURL") ? GetDownloaderState(iniData["directURL"].CleanIniString()) : null;
}
public override Priority Priority => Priority.Normal;

1
Wabbajack.Downloaders.Mega/Wabbajack.Downloaders.Mega.csproj
View File

@ -6,6 +6,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
</ItemGroup>

5
Wabbajack.Downloaders.ModDB/ModDBDownloader.cs
View File

@ -7,6 +7,7 @@ using System.Threading;
using System.Threading.Tasks;
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
@ -45,8 +46,8 @@ public class ModDBDownloader : ADownloader<DTOs.DownloadStates.ModDB>, IUrlDownl
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{
if (iniData.ContainsKey("directURL") &&
iniData["directURL"].StartsWith("https://www.moddb.com/downloads/start") &&
Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri))
iniData["directURL"].CleanIniString().StartsWith("https://www.moddb.com/downloads/start") &&
Uri.TryCreate(iniData["directURL"].CleanIniString().CleanIniString(), UriKind.Absolute, out var uri))
{
var state = new DTOs.DownloadStates.ModDB
{

1
Wabbajack.Downloaders.ModDB/Wabbajack.Downloaders.ModDB.csproj
View File

@ -6,6 +6,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.Networking.Http\Wabbajack.Networking.Http.csproj" />

2
Wabbajack.Downloaders.WabbajackCDN/WabbajackCDNDownloader.cs
View File

@ -56,7 +56,7 @@ public class WabbajackCDNDownloader : ADownloader<WabbajackCDN>, IUrlDownloader,
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri))
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri))
return Parse(uri);
return null;
}

20
Wabbajack.Paths/AbsolutePath.cs
View File

@ -47,7 +47,7 @@ public struct AbsolutePath : IPath, IComparable<AbsolutePath>, IEquatable<Absolu
if (path.StartsWith(@"\\"))
return PathFormat.Windows;
if (DriveLetters.Contains(path[0]) && path[1] == ':')
if (path.Length >= 2 && DriveLetters.Contains(path[0]) && path[1] == ':')
return PathFormat.Windows;
throw new PathException($"Invalid Path format: {path}");
@ -95,8 +95,10 @@ public struct AbsolutePath : IPath, IComparable<AbsolutePath>, IEquatable<Absolu
public override int GetHashCode()
{
if (_hashCode != 0) return _hashCode;
_hashCode = Parts.Aggregate(0,
(current, part) => current ^ part.GetHashCode(StringComparison.CurrentCultureIgnoreCase));
var result = 0;
foreach (var part in Parts)
result = result ^ part.GetHashCode(StringComparison.CurrentCultureIgnoreCase);
_hashCode = result;
return _hashCode;
}
@ -190,4 +192,16 @@ public struct AbsolutePath : IPath, IComparable<AbsolutePath>, IEquatable<Absolu
return Parent.Combine((FileName.WithoutExtension() + append).ToRelativePath()
.WithExtension(Extension));
}
public static AbsolutePath ConvertNoFailure(string value)
{
try
{
return (AbsolutePath) value;
}
catch (Exception ex)
{
return default;
}
}
}

2
Wabbajack.Paths/RelativePath.cs
View File

@ -175,7 +175,7 @@ public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<Relati
public RelativePath FileNameWithoutExtension => Parts[^1][..Extension.ToString().Length].ToRelativePath();
public int Level => Parts.Length;
public bool EndsWith(string postfix)
public readonly bool EndsWith(string postfix)
{
return Parts[^1].EndsWith(postfix);
}

2
Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
View File

@ -70,7 +70,7 @@ public static class ServiceExtensions
service.AddSingleton<IBinaryPatchCache>(s => options.UseLocalCache
? new BinaryPatchCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path)
: new BinaryPatchCache(KnownFolders.EntryPoint.Combine("patchCache.sqlite")));
: new BinaryPatchCache(KnownFolders.WabbajackAppLocal.Combine("patchCache.sqlite")));
service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});

57
Wabbajack.VFS/Context.cs
View File

@ -55,43 +55,10 @@ public class Context
public async Task<IndexRoot> AddRoot(AbsolutePath root, CancellationToken token)
{
var filtered = Index.AllFiles.Where(file => file.IsNative && ((AbsolutePath) file.Name).FileExists())
.ToList();
var byPath = filtered.ToDictionary(f => f.Name);
var filesToIndex = root.EnumerateFiles().Distinct().ToList();
var allFiles = await filesToIndex
.PMapAll(async f =>
{
using var job = await Limiter.Begin($"Analyzing {f}", 0, token);
if (byPath.TryGetValue(f, out var found))
if (found.LastModified == f.LastModifiedUtc().AsUnixTime() && found.Size == f.Size())
return found;
try
{
return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, token, job: job);
}
catch (Exception ex)
{
Logger.LogError(ex, "While analyzing {File}", f);
throw;
}
}).ToList();
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
lock (this)
{
Index = newIndex;
}
return newIndex;
return await AddRoots(new[] {root}, token);
}
public async Task<IndexRoot> AddRoots(List<AbsolutePath> roots, CancellationToken token, Func<long, long, Task>? updateFunction = null)
public async Task<IndexRoot> AddRoots(IEnumerable<AbsolutePath> roots, CancellationToken token, Func<long, long, Task>? updateFunction = null)
{
var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.FullPath.Base);
@ -120,7 +87,7 @@ public class Context
Index = newIndex;
}
VfsCache.Clean();
await VfsCache.Clean();
return newIndex;
}
@ -132,11 +99,11 @@ public class Context
/// <param name="files">Predefined list of files to extract, all others will be skipped</param>
/// <param name="callback">Func called for each file extracted</param>
/// <param name="tempFolder">Optional: folder to use for temporary storage</param>
/// <param name="updateTracker">Optional: Status update tracker</param>
/// <param name="runInParallel">Optional: run `callback`s in parallel</param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
public async Task Extract(HashSet<VirtualFile> files, Func<VirtualFile, IExtractedFile, ValueTask> callback,
CancellationToken token, AbsolutePath? tempFolder = null)
CancellationToken token, AbsolutePath? tempFolder = null, bool runInParallel = true)
{
var top = new VirtualFile();
var filesByParent = files.SelectMany(f => f.FilesInFullPath)
@ -177,8 +144,18 @@ public class Context
}
}
await filesByParent[top].PDoAll(
async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName) {CanMove = false}));
if (runInParallel)
{
await filesByParent[top].PDoAll(
async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName) {CanMove = false}));
}
else
{
foreach (var file in filesByParent[top])
{
await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName) {CanMove = false});
}
}
}
#region KnownFiles

6
Wabbajack.VFS/FileHashCache.cs
View File

@ -43,7 +43,7 @@ public class FileHashCache
{
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = "SELECT LastModified, Hash FROM HashCache WHERE Path = @path";
cmd.Parameters.AddWithValue("@path", path.ToString());
cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant());
cmd.PrepareAsync();
using var reader = cmd.ExecuteReader();
@ -56,7 +56,7 @@ public class FileHashCache
{
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = "DELETE FROM HashCache WHERE Path = @path";
cmd.Parameters.AddWithValue("@path", path.ToString());
cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant());
cmd.PrepareAsync();
cmd.ExecuteNonQuery();
@ -67,7 +67,7 @@ public class FileHashCache
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"INSERT INTO HashCache (Path, LastModified, Hash) VALUES (@path, @lastModified, @hash)
ON CONFLICT(Path) DO UPDATE SET LastModified = @lastModified, Hash = @hash";
cmd.Parameters.AddWithValue("@path", path.ToString());
cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant());
cmd.Parameters.AddWithValue("@lastModified", lastModified);
cmd.Parameters.AddWithValue("@hash", (long) hash);
cmd.PrepareAsync();

4
Wabbajack.VFS/VFSCache.cs
View File

@ -50,8 +50,8 @@ public class VFSDiskCache : IVfsCache
cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash";
cmd.Parameters.AddWithValue("@hash", (long) hash);
await using var rdr = cmd.ExecuteReader();
while (rdr.Read())
await using var rdr = await cmd.ExecuteReaderAsync(token);
while (await rdr.ReadAsync(token))
{
var data = IndexedVirtualFileExtensions.Read(rdr.GetStream(0));
return data;