mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
Implement BSA install optimization
This commit is contained in:
parent
83cf5a3436
commit
f93f52b1c9
@ -65,7 +65,6 @@ public class ResourceMonitor : IDisposable
|
||||
var used = new HashSet<ulong>();
|
||||
foreach (var resource in _resources)
|
||||
{
|
||||
_logger.LogInformation("Resource {Name}: {Jobs}", resource.Name, resource.Jobs.Count());
|
||||
foreach (var job in resource.Jobs)
|
||||
{
|
||||
used.Add(job.ID);
|
||||
|
@ -39,12 +39,13 @@ public abstract class AInstaller<T>
|
||||
where T : AInstaller<T>
|
||||
{
|
||||
private const int _limitMS = 100;
|
||||
public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath();
|
||||
private static readonly Regex NoDeleteRegex = new(@"(?i)[\\\/]\[NoDelete\]", RegexOptions.Compiled);
|
||||
|
||||
protected readonly InstallerConfiguration _configuration;
|
||||
protected readonly DownloadDispatcher _downloadDispatcher;
|
||||
private readonly FileExtractor.FileExtractor _extractor;
|
||||
private readonly FileHashCache _fileHashCache;
|
||||
protected readonly FileHashCache FileHashCache;
|
||||
protected readonly IGameLocator _gameLocator;
|
||||
private readonly DTOSerializer _jsonSerializer;
|
||||
protected readonly ILogger<T> _logger;
|
||||
@ -81,7 +82,7 @@ public abstract class AInstaller<T>
|
||||
_extractor = extractor;
|
||||
_jsonSerializer = jsonSerializer;
|
||||
_vfs = vfs;
|
||||
_fileHashCache = fileHashCache;
|
||||
FileHashCache = fileHashCache;
|
||||
_downloadDispatcher = downloadDispatcher;
|
||||
_parallelOptions = parallelOptions;
|
||||
_gameLocator = gameLocator;
|
||||
@ -114,7 +115,9 @@ public abstract class AInstaller<T>
|
||||
{
|
||||
Interlocked.Add(ref _currentStepProgress, stepProgress);
|
||||
|
||||
OnStatusUpdate?.Invoke(new StatusUpdate(_statusCategory, _statusText, Percent.FactoryPutInRange(_currentStep, MaxSteps), Percent.FactoryPutInRange(_currentStepProgress, MaxStepProgress)));
|
||||
OnStatusUpdate?.Invoke(new StatusUpdate(_statusCategory, _statusText,
|
||||
Percent.FactoryPutInRange(_currentStep, MaxSteps),
|
||||
Percent.FactoryPutInRange(_currentStepProgress, MaxStepProgress)));
|
||||
}
|
||||
|
||||
public abstract Task<bool> Begin(CancellationToken token);
|
||||
@ -218,7 +221,8 @@ public abstract class AInstaller<T>
|
||||
await _vfs.Extract(grouped.Keys.ToHashSet(), async (vf, sf) =>
|
||||
{
|
||||
var directives = grouped[vf];
|
||||
using var job = await _limiter.Begin($"Installing files from {vf.Name}", directives.Sum(f => f.VF.Size), token);
|
||||
using var job = await _limiter.Begin($"Installing files from {vf.Name}", directives.Sum(f => f.VF.Size),
|
||||
token);
|
||||
foreach (var directive in directives)
|
||||
{
|
||||
var file = directive.Directive;
|
||||
@ -357,7 +361,8 @@ public abstract class AInstaller<T>
|
||||
|
||||
if (hash != archive.Hash)
|
||||
{
|
||||
_logger.LogError("Downloaded hash {Downloaded} does not match expected hash: {Expected}", hash, archive.Hash);
|
||||
_logger.LogError("Downloaded hash {Downloaded} does not match expected hash: {Expected}", hash,
|
||||
archive.Hash);
|
||||
if (destination!.Value.FileExists())
|
||||
{
|
||||
destination!.Value.Delete();
|
||||
@ -367,7 +372,7 @@ public abstract class AInstaller<T>
|
||||
}
|
||||
|
||||
if (hash != default)
|
||||
_fileHashCache.FileHashWriteCache(destination.Value, hash);
|
||||
FileHashCache.FileHashWriteCache(destination.Value, hash);
|
||||
|
||||
if (result == DownloadResult.Update)
|
||||
await destination.Value.MoveToAsync(destination.Value.Parent.Combine(archive.Hash.ToHex()), true,
|
||||
@ -406,7 +411,7 @@ public abstract class AInstaller<T>
|
||||
.PMapAll(async e =>
|
||||
{
|
||||
UpdateProgress(1);
|
||||
return (await _fileHashCache.FileHashCachedAsync(e, token), e);
|
||||
return (await FileHashCache.FileHashCachedAsync(e, token), e);
|
||||
})
|
||||
.ToList();
|
||||
|
||||
@ -427,19 +432,46 @@ public abstract class AInstaller<T>
|
||||
{
|
||||
_logger.LogInformation("Optimizing ModList directives");
|
||||
|
||||
|
||||
var indexed = ModList.Directives.ToDictionary(d => d.To);
|
||||
|
||||
var bsasToBuild = await ModList.Directives
|
||||
.OfType<CreateBSA>()
|
||||
.PMapAll(async b =>
|
||||
{
|
||||
var file = _configuration.Install.Combine(b.To);
|
||||
if (!file.FileExists())
|
||||
return (true, b);
|
||||
return (b.Hash != await FileHashCache.FileHashCachedAsync(file, token), b);
|
||||
})
|
||||
.ToArray();
|
||||
|
||||
var bsasToNotBuild = bsasToBuild
|
||||
.Where(b => b.Item1 == false).Select(t => t.b.TempID).ToHashSet();
|
||||
|
||||
var bsaPathsToNotBuild = bsasToBuild
|
||||
.Where(b => b.Item1 == false).Select(t => t.b.To.RelativeTo(_configuration.Install))
|
||||
.ToHashSet();
|
||||
|
||||
indexed = indexed.Values
|
||||
.Where(d =>
|
||||
{
|
||||
return d switch
|
||||
{
|
||||
CreateBSA bsa => !bsasToNotBuild.Contains(bsa.TempID),
|
||||
FromArchive a when a.To.StartsWith($"{BSACreationDir}") => !bsasToNotBuild.Any(b =>
|
||||
a.To.RelativeTo(_configuration.Install).InFolder(_configuration.Install.Combine(BSACreationDir, b))),
|
||||
_ => true
|
||||
};
|
||||
}).ToDictionary(d => d.To);
|
||||
|
||||
|
||||
var profileFolder = _configuration.Install.Combine("profiles");
|
||||
var savePath = (RelativePath) "saves";
|
||||
|
||||
var existingFiles = _configuration.Install.EnumerateFiles().ToList();
|
||||
NextStep(Consts.StepPreparing, "Looking for files to delete", existingFiles.Count);
|
||||
await existingFiles
|
||||
NextStep(Consts.StepPreparing, "Looking for files to delete", 0);
|
||||
await _configuration.Install.EnumerateFiles()
|
||||
.PDoAll(async f =>
|
||||
{
|
||||
UpdateProgress(1);
|
||||
var relativeTo = f.RelativeTo(_configuration.Install);
|
||||
if (indexed.ContainsKey(relativeTo) || f.InFolder(_configuration.Downloads))
|
||||
return;
|
||||
@ -449,17 +481,18 @@ public abstract class AInstaller<T>
|
||||
if (NoDeleteRegex.IsMatch(f.ToString()))
|
||||
return;
|
||||
|
||||
_logger.LogTrace("Deleting {relativeTo} it's not part of this ModList", relativeTo);
|
||||
if (bsaPathsToNotBuild.Contains(f))
|
||||
return;
|
||||
|
||||
_logger.LogInformation("Deleting {RelativePath} it's not part of this ModList", relativeTo);
|
||||
f.Delete();
|
||||
});
|
||||
|
||||
_logger.LogInformation("Cleaning empty folders");
|
||||
NextStep(Consts.StepPreparing, "Cleaning empty folders", indexed.Keys.Count);
|
||||
var expectedFolders = (indexed.Keys
|
||||
var expectedFolders = indexed.Keys
|
||||
.Select(f => f.RelativeTo(_configuration.Install))
|
||||
// We ignore the last part of the path, so we need a dummy file name
|
||||
.Append(_configuration.Downloads.Combine("_"))
|
||||
.OnEach(_ => UpdateProgress(1))
|
||||
.Where(f => f.InFolder(_configuration.Install))
|
||||
.SelectMany(path =>
|
||||
{
|
||||
@ -468,28 +501,30 @@ public abstract class AInstaller<T>
|
||||
var split = ((string) path.RelativeTo(_configuration.Install)).Split('\\');
|
||||
return Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t)));
|
||||
})
|
||||
.ToList())
|
||||
.Distinct()
|
||||
.Select(p => _configuration.Install.Combine(p))
|
||||
.ToHashSet();
|
||||
|
||||
try
|
||||
{
|
||||
var toDelete = _configuration.Install.EnumerateDirectories()
|
||||
var toDelete = _configuration.Install.EnumerateDirectories(true)
|
||||
.Where(p => !expectedFolders.Contains(p))
|
||||
.OrderByDescending(p => p.ToString().Length)
|
||||
.ToList();
|
||||
foreach (var dir in toDelete) dir.DeleteDirectory(true);
|
||||
foreach (var dir in toDelete)
|
||||
{
|
||||
dir.DeleteDirectory(dontDeleteIfNotEmpty: true);
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// ignored because it's not worth throwing a fit over
|
||||
_logger.LogWarning("Error when trying to clean empty folders. This doesn't really matter.");
|
||||
_logger.LogInformation("Error when trying to clean empty folders. This doesn't really matter.");
|
||||
}
|
||||
|
||||
var existingfiles = _configuration.Install.EnumerateFiles().ToHashSet();
|
||||
|
||||
NextStep(Consts.StepPreparing, "Removing redundant directives", indexed.Count);
|
||||
NextStep(Consts.StepPreparing, "Looking for unmodified files", 0);
|
||||
await indexed.Values.PMapAll<Directive, Directive?>(async d =>
|
||||
{
|
||||
// Bit backwards, but we want to return null for
|
||||
@ -498,17 +533,18 @@ public abstract class AInstaller<T>
|
||||
var path = _configuration.Install.Combine(d.To);
|
||||
if (!existingfiles.Contains(path)) return null;
|
||||
|
||||
return await _fileHashCache.FileHashCachedAsync(path, token) == d.Hash ? d : null;
|
||||
return await FileHashCache.FileHashCachedAsync(path, token) == d.Hash ? d : null;
|
||||
})
|
||||
.Do(d =>
|
||||
{
|
||||
UpdateProgress(1);
|
||||
if (d != null) indexed.Remove(d.To);
|
||||
if (d != null)
|
||||
{
|
||||
indexed.Remove(d.To);
|
||||
}
|
||||
});
|
||||
|
||||
_logger.LogInformation("Optimized {optimized} directives to {indexed} required", ModList.Directives.Length,
|
||||
indexed.Count);
|
||||
NextStep(Consts.StepPreparing, "Finalizing modlist optimization", 0);
|
||||
NextStep(Consts.StepPreparing, "Updating ModList", 0);
|
||||
_logger.LogInformation("Optimized {From} directives to {To} required", ModList.Directives.Length, indexed.Count);
|
||||
var requiredArchives = indexed.Values.OfType<FromArchive>()
|
||||
.GroupBy(d => d.ArchiveHashPath.Hash)
|
||||
.Select(d => d.Key)
|
||||
|
@ -33,7 +33,6 @@ namespace Wabbajack.Installer;
|
||||
|
||||
public class StandardInstaller : AInstaller<StandardInstaller>
|
||||
{
|
||||
public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath();
|
||||
|
||||
public StandardInstaller(ILogger<StandardInstaller> logger,
|
||||
InstallerConfiguration config,
|
||||
@ -260,11 +259,14 @@ public class StandardInstaller : AInstaller<StandardInstaller>
|
||||
}).ToList();
|
||||
|
||||
_logger.LogInformation("Writing {bsaTo}", bsa.To);
|
||||
await using var outStream = _configuration.Install.Combine(bsa.To)
|
||||
.Open(FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
var outPath = _configuration.Install.Combine(bsa.To);
|
||||
await using var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
await a.Build(outStream, token);
|
||||
streams.Do(s => s.Dispose());
|
||||
|
||||
FileHashCache.FileHashWriteCache(outPath, bsa.Hash);
|
||||
|
||||
|
||||
sourceDir.DeleteDirectory();
|
||||
}
|
||||
|
||||
|
@ -178,4 +178,9 @@ public readonly struct RelativePath : IPath, IEquatable<RelativePath>, IComparab
|
||||
{
|
||||
return Parts[^1].StartsWith(mrkinn);
|
||||
}
|
||||
|
||||
public bool StartsWith(string s)
|
||||
{
|
||||
return ToString().StartsWith(s);
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user