Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Several fixes and performance improvements

Browse Source
This commit is contained in:
Timothy Baldridge 2019-11-15 16:13:27 -07:00
parent 815295b7d2
commit a8dd59227d
5 changed files with 130 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Compression.BSA/Utils.cs
View File

@ -144,7 +144,7 @@ namespace Compression.BSA
public static async Task CopyToLimitAsync(this Stream frm, Stream tw, int limit)
{
var buff = new byte[1024];
var buff = new byte[1024 * 16];
while (limit > 0)
{
var to_read = Math.Min(buff.Length, limit);
@ -153,7 +153,7 @@ namespace Compression.BSA
limit -= read;
}
tw.Flush();
await tw.FlushAsync();
}
public static void CopyToLimit(this Stream frm, Stream tw, int limit)

16
Wabbajack.Lib/ACompiler.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Subjects;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
@ -11,7 +12,16 @@ namespace Wabbajack.Lib
{
public abstract class ACompiler
{
protected static string _vfsCacheName = "vfs_compile_cache.bin";
/// <summary>
/// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task
/// and the current progress.
/// </summary>
public IObservable<(string, float)> ProgressUpdates => _progressUpdates;
protected readonly Subject<(string, float)> _progressUpdates = new Subject<(string, float)>();
public Context VFS { get; internal set; } = new Context();
public ModManager ModManager;
public string GamePath;
@ -39,5 +49,11 @@ namespace Wabbajack.Lib
public abstract Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source);
public abstract IEnumerable<ICompilationStep> GetStack();
public abstract IEnumerable<ICompilationStep> MakeStack();
protected ACompiler()
{
ProgressUpdates.Subscribe(itm => Utils.Log($"{itm.Item2} - {itm.Item1}"));
VFS.LogSpam.Subscribe(itm => Utils.Status(itm));
}
}
}

16
Wabbajack.Lib/Compiler.cs
View File

@ -28,6 +28,7 @@ namespace Wabbajack.Lib
{
public class Compiler : ACompiler
{
private string _mo2DownloadsFolder;
public Dictionary<string, IEnumerable<IndexedFileMatch>> DirectMatchIndex;
@ -49,6 +50,8 @@ namespace Wabbajack.Lib
ModListOutputFolder = "output_folder";
ModListOutputFile = MO2Profile + ExtensionManager.Extension;
VFS.ProgressUpdates.Debounce(new TimeSpan(0, 0, 0, 0, 100))
.Subscribe(itm => _progressUpdates.OnNext(itm));
}
public dynamic MO2Ini { get; }
@ -119,14 +122,25 @@ namespace Wabbajack.Lib
Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));
VFS.IntegrateFromFile(_vfsCacheName).Wait();
Info($"Indexing {MO2Folder}");
VFS.AddRoot(MO2Folder).Wait();
VFS.WriteToFile(_vfsCacheName).Wait();
Info($"Indexing {GamePath}");
VFS.AddRoot(GamePath).Wait();
VFS.WriteToFile(_vfsCacheName).Wait();
Info($"Indexing {MO2DownloadsFolder}");
VFS.AddRoot(MO2DownloadsFolder).Wait();
VFS.WriteToFile(_vfsCacheName).Wait();
Info("Cleaning output folder");
if (Directory.Exists(ModListOutputFolder))
Directory.Delete(ModListOutputFolder, true);
@ -151,6 +165,8 @@ namespace Wabbajack.Lib
{
Info($"Indexing {loot_path}");
VFS.AddRoot(loot_path).Wait();
VFS.WriteToFile(_vfsCacheName).Wait();
loot_files = Directory.EnumerateFiles(loot_path, "userlist.yaml", SearchOption.AllDirectories)
.Where(p => p.FileExists())

119
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -4,6 +4,7 @@ using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
@ -24,6 +25,22 @@ namespace Wabbajack.VirtualFileSystem
private readonly string _stagingFolder = "vfs_staging";
public IndexRoot Index { get; private set; } = IndexRoot.Empty;
/// <summary>
/// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task
/// and the current progress.
/// </summary>
public IObservable<(string, float)> ProgressUpdates => _progressUpdates;
private readonly Subject<(string, float)> _progressUpdates = new Subject<(string, float)>();
/// <summary>
/// A high throughput firehose of updates from the VFS. These go into more detail on the status
/// of what's happening in the context, but is probably too high bandwidth to tie driectly to the
/// UI.
/// </summary>
public IObservable<string> LogSpam => _logSpam;
internal readonly Subject<string> _logSpam = new Subject<string>();
public TemporaryDirectory GetTemporaryFolder()
{
return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString()));
@ -41,8 +58,11 @@ namespace Wabbajack.VirtualFileSystem
var byPath = filtered.ToImmutableDictionary(f => f.Name);
var results = Channel.Create<VirtualFile>(1024);
var pipeline = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive)
var filesToIndex = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive).ToList();
var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing {root}", filesToIndex.Count));
var pipeline = filesToIndex
.ToChannel()
.UnorderedPipeline(results, async f =>
{
@ -71,6 +91,25 @@ namespace Wabbajack.VirtualFileSystem
return newIndex;
}
class Box<T>
{
public T Value { get; set; }
}
private Func<IObservable<T>, IObservable<T>> ProgressUpdater<T>(string s, float totalCount)
{
if (totalCount == 0)
totalCount = 1;
var box = new Box<float>();
return sub => sub.Select(itm =>
{
box.Value += 1;
_progressUpdates.OnNext((s, box.Value / totalCount));
return itm;
});
}
public async Task WriteToFile(string filename)
{
using (var fs = File.OpenWrite(filename))
@ -105,40 +144,46 @@ namespace Wabbajack.VirtualFileSystem
public async Task IntegrateFromFile(string filename)
{
using (var fs = File.OpenRead(filename))
using (var br = new BinaryReader(fs, Encoding.UTF8, true))
try
{
var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length));
var fileVersion = br.ReadUInt64();
if (fileVersion != FileVersion || magic != magic)
throw new InvalidDataException("Bad Data Format");
var numFiles = br.ReadUInt64();
var input = Channel.Create<byte[]>(1024);
var pipeline = input.UnorderedPipelineSync(
data => VirtualFile.Read(this, data))
.TakeAll();
Utils.Log($"Loading {numFiles} files from {filename}");
for (ulong idx = 0; idx < numFiles; idx++)
using (var fs = File.OpenRead(filename))
using (var br = new BinaryReader(fs, Encoding.UTF8, true))
{
var size = br.ReadUInt64();
var bytes = new byte[size];
await br.BaseStream.ReadAsync(bytes, 0, (int) size);
await input.Put(bytes);
}
var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length));
var fileVersion = br.ReadUInt64();
if (fileVersion != FileVersion || magic != magic)
throw new InvalidDataException("Bad Data Format");
input.Close();
var numFiles = br.ReadUInt64();
var files = await pipeline;
var newIndex = await Index.Integrate(files);
lock (this)
{
Index = newIndex;
var input = Channel.Create(1024, ProgressUpdater<byte[]>("Loading VFS", numFiles));
var pipeline = input.UnorderedPipelineSync(
data => VirtualFile.Read(this, data))
.TakeAll();
for (ulong idx = 0; idx < numFiles; idx++)
{
var size = br.ReadUInt64();
var bytes = new byte[size];
await br.BaseStream.ReadAsync(bytes, 0, (int) size);
await input.Put(bytes);
}
input.Close();
var files = await pipeline;
var newIndex = await Index.Integrate(files);
lock (this)
{
Index = newIndex;
}
}
}
catch (IOException)
{
if (File.Exists(filename))
File.Delete(filename);
}
}
public async Task<Action> Stage(IEnumerable<VirtualFile> files)
@ -258,7 +303,6 @@ namespace Wabbajack.VirtualFileSystem
}
#endregion
}
public class KnownFile
@ -317,6 +361,7 @@ namespace Wabbajack.VirtualFileSystem
public async Task<IndexRoot> Integrate(List<VirtualFile> files)
{
Utils.Log($"Integrating");
var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList();
var byFullPath = Task.Run(() =>
@ -334,11 +379,13 @@ namespace Wabbajack.VirtualFileSystem
var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name));
return new IndexRoot(allFiles,
await byFullPath,
await byHash,
await byRootPath,
await byName);
var result = new IndexRoot(allFiles,
await byFullPath.ConfigureAwait(false),
await byHash.ConfigureAwait(false),
await byRootPath.ConfigureAwait(false),
await byName.ConfigureAwait(false));
Utils.Log($"Done integrating");
return result;
}
public VirtualFile FileForArchiveHashPath(string[] argArchiveHashPath)

17
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -120,8 +120,9 @@ namespace Wabbajack.VirtualFileSystem
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path,
string rel_path)
{
var hasher = abs_path.FileHashAsync();
var hasher = abs_path.FileHashAsync().ConfigureAwait(false);
var fi = new FileInfo(abs_path);
context._logSpam.OnNext($"Analyzing {rel_path}");
var self = new VirtualFile
{
Context = context,
@ -131,20 +132,28 @@ namespace Wabbajack.VirtualFileSystem
LastModified = fi.LastWriteTimeUtc.Ticks,
LastAnalyzed = DateTime.Now.Ticks
};
if (FileExtractor.CanExtract(Path.GetExtension(abs_path)))
{
context._logSpam.OnNext($"Extracting {rel_path}");
using (var tempFolder = context.GetTemporaryFolder())
{
await FileExtractor.ExtractAll(abs_path, tempFolder.FullName);
await FileExtractor.ExtractAll(abs_path, tempFolder.FullName).ConfigureAwait(false);
context._logSpam.OnNext($"Analyzing Contents {rel_path}");
var results = Channel.Create<VirtualFile>(1024);
var files = Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
.ToChannel()
.UnorderedPipeline(results,
async abs_src => await Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName)));
async abs_src =>
await Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName))
.ConfigureAwait(false));
self.Children = (await results.TakeAll()).ToImmutableList();
}
}
self.Hash = await hasher;
return self;
}