Merge pull request #2051 from wabbajack-tools/3.0.0.5-fixes

3.0.0.5 fixes
This commit is contained in:
Timothy Baldridge 2022-08-22 20:53:24 -06:00 committed by GitHub
commit 8af1ab65e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 114 additions and 73 deletions

View File

@ -1,5 +1,11 @@
### Changelog ### Changelog
#### Version - 3.0.0.5 - 8/22/2022
* No longer rehashes files on every compile (faster Add Roots step during compilation)
* Editing paths in the install/compile settings won't crash the app
* Fix for .refcache files not being ignored during compilation
* Greatly reduce the amount of UI hangups during compilation
#### Version - 3.0.0.4 - 8/20/2022 #### Version - 3.0.0.4 - 8/20/2022
* Fix for: when some optional game files weren't present (like CreationKit.exe), the app would refuse to recognize any files from that game * Fix for: when some optional game files weren't present (like CreationKit.exe), the app would refuse to recognize any files from that game

View File

@ -74,7 +74,7 @@ namespace Wabbajack
public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture) public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
{ {
return (AbsolutePath)(value as string); return AbsolutePath.ConvertNoFailure((string) value);
} }
} }
} }

View File

@ -278,7 +278,9 @@ namespace Wabbajack
try try
{ {
await compiler.Begin(token); var result = await compiler.Begin(token);
if (!result)
throw new Exception("Compilation Failed");
} }
finally finally
{ {

View File

@ -29,7 +29,10 @@ public static class AsyncParallelExtensions
public static async IAsyncEnumerable<TOut> PMapAll<TIn, TOut>(this IEnumerable<TIn> coll, public static async IAsyncEnumerable<TOut> PMapAll<TIn, TOut>(this IEnumerable<TIn> coll,
Func<TIn, Task<TOut>> mapFn) Func<TIn, Task<TOut>> mapFn)
{ {
var tasks = coll.Select(mapFn).ToList(); var tasks = coll.Select(async x =>
{
return await Task.Run(() => mapFn(x));
}).ToList();
foreach (var itm in tasks) yield return await itm; foreach (var itm in tasks) yield return await itm;
} }

View File

@ -24,4 +24,9 @@ public static class StringExtensions
{ {
return src.Contains(contains, StringComparison.InvariantCultureIgnoreCase); return src.Contains(contains, StringComparison.InvariantCultureIgnoreCase);
} }
public static string CleanIniString(this string src)
{
return src.TrimStart('\"').TrimEnd('\"');
}
} }

View File

@ -1,5 +1,8 @@
using System;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Wabbajack.DTOs.Interventions;
using Wabbajack.Networking.Steam.UserInterventions;
using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Services.OSIntegrated; using Wabbajack.Services.OSIntegrated;
using Xunit.DependencyInjection; using Xunit.DependencyInjection;
@ -19,10 +22,33 @@ public class Startup
}); });
service.AddScoped<ModListHarness>(); service.AddScoped<ModListHarness>();
service.AddSingleton<IUserInterventionHandler, UserInterventionHandler>();
} }
public void Configure(ILoggerFactory loggerFactory, ITestOutputHelperAccessor accessor) public void Configure(ILoggerFactory loggerFactory, ITestOutputHelperAccessor accessor)
{ {
loggerFactory.AddProvider(new XunitTestOutputLoggerProvider(accessor, delegate { return true; })); loggerFactory.AddProvider(new XunitTestOutputLoggerProvider(accessor, delegate { return true; }));
} }
public class UserInterventionHandler : IUserInterventionHandler
{
public void Raise(IUserIntervention intervention)
{
if (intervention is GetAuthCode gac)
{
switch (gac.Type)
{
case GetAuthCode.AuthType.EmailCode:
Console.WriteLine("Please enter the Steam code that was just emailed to you");
break;
case GetAuthCode.AuthType.TwoFactorAuth:
Console.WriteLine("Please enter your 2FA code for Steam");
break;
default:
throw new ArgumentOutOfRangeException();
}
gac.Finish(Console.ReadLine()!.Trim());
}
}
}
} }

View File

@ -447,6 +447,8 @@ public abstract class ACompiler
/// </summary> /// </summary>
protected async Task BuildPatches(CancellationToken token) protected async Task BuildPatches(CancellationToken token)
{ {
NextStep("Compiling","Looking for patches");
var toBuild = InstallDirectives.OfType<PatchedFromArchive>() var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => _patchOptions.GetValueOrDefault(p, Array.Empty<VirtualFile>()).Length > 0) .Where(p => _patchOptions.GetValueOrDefault(p, Array.Empty<VirtualFile>()).Length > 0)
.SelectMany(p => _patchOptions[p].Select(c => new PatchedFromArchive .SelectMany(p => _patchOptions[p].Select(c => new PatchedFromArchive
@ -487,7 +489,7 @@ public abstract class ACompiler
_logger.LogInformation("Patch size {patchSize} for {to}", patchSize, match.To); _logger.LogInformation("Patch size {patchSize} for {to}", patchSize, match.To);
}, token); }, token);
} }
}, token); }, token, runInParallel: false);
// Load in the patches // Load in the patches
await InstallDirectives.OfType<PatchedFromArchive>() await InstallDirectives.OfType<PatchedFromArchive>()

View File

@ -7,18 +7,18 @@ namespace Wabbajack.Compiler.CompilationSteps;
public class IgnoreFilename : ACompilationStep public class IgnoreFilename : ACompilationStep
{ {
private readonly RelativePath _postfix; private readonly string _postfix;
private readonly string _reason; private readonly string _reason;
public IgnoreFilename(ACompiler compiler, RelativePath postfix) : base(compiler) public IgnoreFilename(ACompiler compiler, RelativePath postfix) : base(compiler)
{ {
_postfix = postfix; _postfix = postfix.FileName.ToString();
_reason = $"Ignored because path ends with {postfix}"; _reason = $"Ignored because path ends with {postfix}";
} }
public override async ValueTask<Directive?> Run(RawSourceFile source) public override async ValueTask<Directive?> Run(RawSourceFile source)
{ {
if (source.Path.FileName != _postfix.FileName) return null; if (!source.Path.EndsWith(_postfix)) return null;
var result = source.EvolveTo<IgnoredDirectly>(); var result = source.EvolveTo<IgnoredDirectly>();
result.Reason = _reason; result.Reason = _reason;
return result; return result;

View File

@ -77,15 +77,11 @@ public class MO2Compiler : ACompiler
var roots = new List<AbsolutePath> {Settings.Source, Settings.Downloads}; var roots = new List<AbsolutePath> {Settings.Source, Settings.Downloads};
roots.AddRange(Settings.OtherGames.Append(Settings.Game).Select(g => _locator.GameLocation(g))); roots.AddRange(Settings.OtherGames.Append(Settings.Game).Select(g => _locator.GameLocation(g)));
roots.Add(Settings.Downloads);
NextStep("Initializing", "Add Roots"); NextStep("Initializing", "Add Roots");
await _vfs.AddRoots(roots, token, async (cur, max) => UpdateProgressAbsolute(cur, max)); // Step 1 await _vfs.AddRoots(roots, token, async (cur, max) => UpdateProgressAbsolute(cur, max)); // Step 1
//await InferMetas(token); // Step 2
NextStep("Initializing", "Add Download Roots");
await _vfs.AddRoot(Settings.Downloads, token); // Step 3
// Find all Downloads // Find all Downloads
IndexedArchives = await Settings.Downloads.EnumerateFiles() IndexedArchives = await Settings.Downloads.EnumerateFiles()
.Where(f => f.WithExtension(Ext.Meta).FileExists()) .Where(f => f.WithExtension(Ext.Meta).FileExists())
@ -103,7 +99,7 @@ public class MO2Compiler : ACompiler
await CleanInvalidArchivesAndFillState(); await CleanInvalidArchivesAndFillState();
NextStep("Initializing", "Indexing Data");
var mo2Files = Settings.Source.EnumerateFiles() var mo2Files = Settings.Source.EnumerateFiles()
.Where(p => p.FileExists()) .Where(p => p.FileExists())
.Select(p => new RawSourceFile(_vfs.Index.ByRootPath[p], p.RelativeTo(Settings.Source))); .Select(p => new RawSourceFile(_vfs.Index.ByRootPath[p], p.RelativeTo(Settings.Source)));
@ -164,6 +160,7 @@ public class MO2Compiler : ACompiler
InstallDirectives = results.Where(i => i is not IgnoredDirectly).ToList(); InstallDirectives = results.Where(i => i is not IgnoredDirectly).ToList();
NextStep("Compiling", "Verifying zEdit Merges (if any)");
zEditIntegration.VerifyMerges(this); zEditIntegration.VerifyMerges(this);
await BuildPatches(token); await BuildPatches(token);
@ -242,6 +239,7 @@ public class MO2Compiler : ACompiler
/// <returns></returns> /// <returns></returns>
public override IEnumerable<ICompilationStep> MakeStack() public override IEnumerable<ICompilationStep> MakeStack()
{ {
NextStep("Initialization", "Generating Compilation Stack");
_logger.LogInformation("Generating compilation stack"); _logger.LogInformation("Generating compilation stack");
var steps = new List<ICompilationStep> var steps = new List<ICompilationStep>
{ {

View File

@ -111,7 +111,7 @@ public class BethesdaDownloader : ADownloader<DTOs.DownloadStates.Bethesda>, IUr
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData) public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{ {
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri)) if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri))
{ {
return Parse(uri); return Parse(uri);
} }

View File

@ -10,6 +10,7 @@ using System.Threading.Tasks;
using System.Web; using System.Web;
using HtmlAgilityPack; using HtmlAgilityPack;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Downloaders.Interfaces; using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs; using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.DownloadStates;
@ -67,7 +68,7 @@ public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData) public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{ {
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri)) if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri))
return Parse(uri); return Parse(uri);
return null; return null;
} }

View File

@ -13,6 +13,7 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" /> <ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" /> <ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
<ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" /> <ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" />

View File

@ -53,7 +53,7 @@ public class HttpDownloader : ADownloader<DTOs.DownloadStates.Http>, IUrlDownloa
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData) public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{ {
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri)) if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri))
{ {
var state = new DTOs.DownloadStates.Http var state = new DTOs.DownloadStates.Http
{ {

View File

@ -71,7 +71,7 @@ public class ManualDownloader : ADownloader<DTOs.DownloadStates.Manual>, IProxya
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData) public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{ {
if (iniData.ContainsKey("manualURL") && Uri.TryCreate(iniData["manualURL"], UriKind.Absolute, out var uri)) if (iniData.ContainsKey("manualURL") && Uri.TryCreate(iniData["manualURL"].CleanIniString(), UriKind.Absolute, out var uri))
{ {
iniData.TryGetValue("prompt", out var prompt); iniData.TryGetValue("prompt", out var prompt);

View File

@ -7,6 +7,7 @@ using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using HtmlAgilityPack; using HtmlAgilityPack;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Downloaders.Interfaces; using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs; using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.DownloadStates;
@ -45,7 +46,7 @@ public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, I
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData) public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{ {
if (iniData.ContainsKey("directURL") && if (iniData.ContainsKey("directURL") &&
Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri) Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri)
&& uri.Host == "www.mediafire.com") && uri.Host == "www.mediafire.com")
{ {
var state = new DTOs.DownloadStates.MediaFire var state = new DTOs.DownloadStates.MediaFire

View File

@ -12,6 +12,7 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" /> <ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" /> <ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" />
</ItemGroup> </ItemGroup>

View File

@ -6,6 +6,7 @@ using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using CG.Web.MegaApiClient; using CG.Web.MegaApiClient;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Downloaders.Interfaces; using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs; using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.DownloadStates;
@ -45,7 +46,7 @@ public class MegaDownloader : ADownloader<Mega>, IUrlDownloader, IProxyable
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData) public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{ {
return iniData.ContainsKey("directURL") ? GetDownloaderState(iniData["directURL"]) : null; return iniData.ContainsKey("directURL") ? GetDownloaderState(iniData["directURL"].CleanIniString()) : null;
} }
public override Priority Priority => Priority.Normal; public override Priority Priority => Priority.Normal;

View File

@ -6,6 +6,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" /> <ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" /> <ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
</ItemGroup> </ItemGroup>

View File

@ -7,6 +7,7 @@ using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using HtmlAgilityPack; using HtmlAgilityPack;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Downloaders.Interfaces; using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs; using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.DownloadStates;
@ -45,8 +46,8 @@ public class ModDBDownloader : ADownloader<DTOs.DownloadStates.ModDB>, IUrlDownl
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData) public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{ {
if (iniData.ContainsKey("directURL") && if (iniData.ContainsKey("directURL") &&
iniData["directURL"].StartsWith("https://www.moddb.com/downloads/start") && iniData["directURL"].CleanIniString().StartsWith("https://www.moddb.com/downloads/start") &&
Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri)) Uri.TryCreate(iniData["directURL"].CleanIniString().CleanIniString(), UriKind.Absolute, out var uri))
{ {
var state = new DTOs.DownloadStates.ModDB var state = new DTOs.DownloadStates.ModDB
{ {

View File

@ -6,6 +6,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" /> <ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" /> <ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.Networking.Http\Wabbajack.Networking.Http.csproj" /> <ProjectReference Include="..\Wabbajack.Networking.Http\Wabbajack.Networking.Http.csproj" />

View File

@ -56,7 +56,7 @@ public class WabbajackCDNDownloader : ADownloader<WabbajackCDN>, IUrlDownloader,
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData) public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{ {
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri)) if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri))
return Parse(uri); return Parse(uri);
return null; return null;
} }

View File

@ -47,7 +47,7 @@ public struct AbsolutePath : IPath, IComparable<AbsolutePath>, IEquatable<Absolu
if (path.StartsWith(@"\\")) if (path.StartsWith(@"\\"))
return PathFormat.Windows; return PathFormat.Windows;
if (DriveLetters.Contains(path[0]) && path[1] == ':') if (path.Length >= 2 && DriveLetters.Contains(path[0]) && path[1] == ':')
return PathFormat.Windows; return PathFormat.Windows;
throw new PathException($"Invalid Path format: {path}"); throw new PathException($"Invalid Path format: {path}");
@ -95,8 +95,10 @@ public struct AbsolutePath : IPath, IComparable<AbsolutePath>, IEquatable<Absolu
public override int GetHashCode() public override int GetHashCode()
{ {
if (_hashCode != 0) return _hashCode; if (_hashCode != 0) return _hashCode;
_hashCode = Parts.Aggregate(0, var result = 0;
(current, part) => current ^ part.GetHashCode(StringComparison.CurrentCultureIgnoreCase)); foreach (var part in Parts)
result = result ^ part.GetHashCode(StringComparison.CurrentCultureIgnoreCase);
_hashCode = result;
return _hashCode; return _hashCode;
} }
@ -190,4 +192,16 @@ public struct AbsolutePath : IPath, IComparable<AbsolutePath>, IEquatable<Absolu
return Parent.Combine((FileName.WithoutExtension() + append).ToRelativePath() return Parent.Combine((FileName.WithoutExtension() + append).ToRelativePath()
.WithExtension(Extension)); .WithExtension(Extension));
} }
public static AbsolutePath ConvertNoFailure(string value)
{
try
{
return (AbsolutePath) value;
}
catch (Exception ex)
{
return default;
}
}
} }

View File

@ -175,7 +175,7 @@ public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<Relati
public RelativePath FileNameWithoutExtension => Parts[^1][..Extension.ToString().Length].ToRelativePath(); public RelativePath FileNameWithoutExtension => Parts[^1][..Extension.ToString().Length].ToRelativePath();
public int Level => Parts.Length; public int Level => Parts.Length;
public bool EndsWith(string postfix) public readonly bool EndsWith(string postfix)
{ {
return Parts[^1].EndsWith(postfix); return Parts[^1].EndsWith(postfix);
} }

View File

@ -70,7 +70,7 @@ public static class ServiceExtensions
service.AddSingleton<IBinaryPatchCache>(s => options.UseLocalCache service.AddSingleton<IBinaryPatchCache>(s => options.UseLocalCache
? new BinaryPatchCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path) ? new BinaryPatchCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path)
: new BinaryPatchCache(KnownFolders.EntryPoint.Combine("patchCache.sqlite"))); : new BinaryPatchCache(KnownFolders.WabbajackAppLocal.Combine("patchCache.sqlite")));
service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount}); service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});

View File

@ -55,43 +55,10 @@ public class Context
public async Task<IndexRoot> AddRoot(AbsolutePath root, CancellationToken token) public async Task<IndexRoot> AddRoot(AbsolutePath root, CancellationToken token)
{ {
var filtered = Index.AllFiles.Where(file => file.IsNative && ((AbsolutePath) file.Name).FileExists()) return await AddRoots(new[] {root}, token);
.ToList();
var byPath = filtered.ToDictionary(f => f.Name);
var filesToIndex = root.EnumerateFiles().Distinct().ToList();
var allFiles = await filesToIndex
.PMapAll(async f =>
{
using var job = await Limiter.Begin($"Analyzing {f}", 0, token);
if (byPath.TryGetValue(f, out var found))
if (found.LastModified == f.LastModifiedUtc().AsUnixTime() && found.Size == f.Size())
return found;
try
{
return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, token, job: job);
}
catch (Exception ex)
{
Logger.LogError(ex, "While analyzing {File}", f);
throw;
}
}).ToList();
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
lock (this)
{
Index = newIndex;
}
return newIndex;
} }
public async Task<IndexRoot> AddRoots(List<AbsolutePath> roots, CancellationToken token, Func<long, long, Task>? updateFunction = null) public async Task<IndexRoot> AddRoots(IEnumerable<AbsolutePath> roots, CancellationToken token, Func<long, long, Task>? updateFunction = null)
{ {
var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.FullPath.Base); var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.FullPath.Base);
@ -120,7 +87,7 @@ public class Context
Index = newIndex; Index = newIndex;
} }
VfsCache.Clean(); await VfsCache.Clean();
return newIndex; return newIndex;
} }
@ -132,11 +99,11 @@ public class Context
/// <param name="files">Predefined list of files to extract, all others will be skipped</param> /// <param name="files">Predefined list of files to extract, all others will be skipped</param>
/// <param name="callback">Func called for each file extracted</param> /// <param name="callback">Func called for each file extracted</param>
/// <param name="tempFolder">Optional: folder to use for temporary storage</param> /// <param name="tempFolder">Optional: folder to use for temporary storage</param>
/// <param name="updateTracker">Optional: Status update tracker</param> /// <param name="runInParallel">Optional: run `callback`s in parallel</param>
/// <returns></returns> /// <returns></returns>
/// <exception cref="Exception"></exception> /// <exception cref="Exception"></exception>
public async Task Extract(HashSet<VirtualFile> files, Func<VirtualFile, IExtractedFile, ValueTask> callback, public async Task Extract(HashSet<VirtualFile> files, Func<VirtualFile, IExtractedFile, ValueTask> callback,
CancellationToken token, AbsolutePath? tempFolder = null) CancellationToken token, AbsolutePath? tempFolder = null, bool runInParallel = true)
{ {
var top = new VirtualFile(); var top = new VirtualFile();
var filesByParent = files.SelectMany(f => f.FilesInFullPath) var filesByParent = files.SelectMany(f => f.FilesInFullPath)
@ -177,8 +144,18 @@ public class Context
} }
} }
await filesByParent[top].PDoAll( if (runInParallel)
async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName) {CanMove = false})); {
await filesByParent[top].PDoAll(
async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName) {CanMove = false}));
}
else
{
foreach (var file in filesByParent[top])
{
await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName) {CanMove = false});
}
}
} }
#region KnownFiles #region KnownFiles

View File

@ -43,7 +43,7 @@ public class FileHashCache
{ {
using var cmd = new SQLiteCommand(_conn); using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = "SELECT LastModified, Hash FROM HashCache WHERE Path = @path"; cmd.CommandText = "SELECT LastModified, Hash FROM HashCache WHERE Path = @path";
cmd.Parameters.AddWithValue("@path", path.ToString()); cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant());
cmd.PrepareAsync(); cmd.PrepareAsync();
using var reader = cmd.ExecuteReader(); using var reader = cmd.ExecuteReader();
@ -56,7 +56,7 @@ public class FileHashCache
{ {
using var cmd = new SQLiteCommand(_conn); using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = "DELETE FROM HashCache WHERE Path = @path"; cmd.CommandText = "DELETE FROM HashCache WHERE Path = @path";
cmd.Parameters.AddWithValue("@path", path.ToString()); cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant());
cmd.PrepareAsync(); cmd.PrepareAsync();
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
@ -67,7 +67,7 @@ public class FileHashCache
using var cmd = new SQLiteCommand(_conn); using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"INSERT INTO HashCache (Path, LastModified, Hash) VALUES (@path, @lastModified, @hash) cmd.CommandText = @"INSERT INTO HashCache (Path, LastModified, Hash) VALUES (@path, @lastModified, @hash)
ON CONFLICT(Path) DO UPDATE SET LastModified = @lastModified, Hash = @hash"; ON CONFLICT(Path) DO UPDATE SET LastModified = @lastModified, Hash = @hash";
cmd.Parameters.AddWithValue("@path", path.ToString()); cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant());
cmd.Parameters.AddWithValue("@lastModified", lastModified); cmd.Parameters.AddWithValue("@lastModified", lastModified);
cmd.Parameters.AddWithValue("@hash", (long) hash); cmd.Parameters.AddWithValue("@hash", (long) hash);
cmd.PrepareAsync(); cmd.PrepareAsync();

View File

@ -50,8 +50,8 @@ public class VFSDiskCache : IVfsCache
cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash"; cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash";
cmd.Parameters.AddWithValue("@hash", (long) hash); cmd.Parameters.AddWithValue("@hash", (long) hash);
await using var rdr = cmd.ExecuteReader(); await using var rdr = await cmd.ExecuteReaderAsync(token);
while (rdr.Read()) while (await rdr.ReadAsync(token))
{ {
var data = IndexedVirtualFileExtensions.Read(rdr.GetStream(0)); var data = IndexedVirtualFileExtensions.Read(rdr.GetStream(0));
return data; return data;