Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #719 from wabbajack-tools/extraction-improvements

Browse Source 
Extraction improvements
This commit is contained in:
Timothy Baldridge 2020-04-17 08:25:17 -06:00 committed by GitHub
commit a82b2d5094
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 380 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Compression.BSA.Test/BSATests.cs
View File

@ -81,7 +81,8 @@ namespace Compression.BSA.Test
var folder = _bsaFolder.Combine(game.ToString(), modid.ToString());
await folder.DeleteDirectory();
folder.CreateDirectory();
await FileExtractor.ExtractAll(Queue, filename, folder);
await using var files = await FileExtractor.ExtractAll(Queue, filename);
await files.MoveAllTo(folder);
foreach (var bsa in folder.EnumerateFiles().Where(f => Consts.SupportedBSAs.Contains(f.Extension)))
{
@ -94,7 +95,7 @@ namespace Compression.BSA.Test
var tempFile = ((RelativePath)"tmp.bsa").RelativeToEntryPoint();
var size = bsa.Size;
using var a = BSADispatch.OpenRead(bsa);
await using var a = BSADispatch.OpenRead(bsa);
await a.Files.PMap(Queue, file =>
{
var absName = _tempDir.Combine(file.Path);
@ -111,7 +112,7 @@ namespace Compression.BSA.Test
TestContext.WriteLine($"Building {bsa}");
using (var w = ViaJson(a.State).MakeBuilder(size))
await using (var w = ViaJson(a.State).MakeBuilder(size))
{
var streams = await a.Files.PMap(Queue, file =>
{
@ -125,7 +126,7 @@ namespace Compression.BSA.Test
}
TestContext.WriteLine($"Verifying {bsa}");
using var b = BSADispatch.OpenRead(tempFile);
await using var b = BSADispatch.OpenRead(tempFile);
TestContext.WriteLine($"Performing A/B tests on {bsa}");
Assert.Equal(a.State.ToJson(), b.State.ToJson());

3
Compression.BSA/BA2Builder.cs
View File

@ -4,6 +4,7 @@ using System.IO;
using System.IO.MemoryMappedFiles;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using Wabbajack.Common;
@ -34,7 +35,7 @@ namespace Compression.BSA
_slab = new DiskSlabAllocator(size);
}
public void Dispose()
public async ValueTask DisposeAsync()
{
_slab.Dispose();
}

3
Compression.BSA/BA2Reader.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
@ -99,7 +100,7 @@ namespace Compression.BSA
}
public void Dispose()
public async ValueTask DisposeAsync()
{
_stream?.Dispose();
_rdr?.Dispose();

3
Compression.BSA/BSABuilder.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Compression.LZ4;
using K4os.Compression.LZ4.Streams;
@ -75,7 +76,7 @@ namespace Compression.BSA
public bool HasNameBlobs => (_archiveFlags & 0x100) > 0;
public void Dispose()
public async ValueTask DisposeAsync()
{
_slab.Dispose();
}

5
Compression.BSA/BSAReader.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Compression.LZ4.Streams;
using Wabbajack.Common;
@ -49,7 +50,7 @@ namespace Compression.BSA
Miscellaneous = 0x100
}
public class BSAReader : IDisposable, IBSAReader
public class BSAReader : IAsyncDisposable, IBSAReader
{
internal uint _archiveFlags;
internal uint _fileCount;
@ -113,7 +114,7 @@ namespace Compression.BSA
}
}
public void Dispose()
public async ValueTask DisposeAsync()
{
_stream.Close();
}

4
Compression.BSA/IBSAReader.cs
View File

@ -5,7 +5,7 @@ using Wabbajack.Common;
namespace Compression.BSA
{
public interface IBSAReader : IDisposable
public interface IBSAReader : IAsyncDisposable
{
/// <summary>
/// The files defined by the archive
@ -15,7 +15,7 @@ namespace Compression.BSA
ArchiveStateObject State { get; }
}
public interface IBSABuilder : IDisposable
public interface IBSABuilder : IAsyncDisposable
{
void AddFile(FileStateObject state, Stream src);
void Build(AbsolutePath filename);

3
Compression.BSA/TES3Builder.cs
View File

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File;
@ -71,7 +72,7 @@ namespace Compression.BSA
}
}
public void Dispose()
public async ValueTask DisposeAsync()
{
}
}

3
Compression.BSA/TES3Reader.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
@ -61,7 +62,7 @@ namespace Compression.BSA
_dataOffset = br.BaseStream.Position;
}
public void Dispose()
public async ValueTask DisposeAsync()
{
}

2
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -117,7 +117,7 @@ namespace Wabbajack.BuildServer.Controllers
private static AsyncLock _findPatchLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive)
{
using var _ = await _findPatchLock.Wait();
using var _ = await _findPatchLock.WaitAsync();
var result = await _updater.GetAlternative(archive.Hash.ToHex());
return result switch

2
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -63,7 +63,7 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log($"Writing {ms.Length} at position {Offset} in ingest file {Key}");
long position;
using (var _ = await _writeLocks[Key].Wait())
using (var _ = await _writeLocks[Key].WaitAsync())
{
await using var file = _settings.TempPath.Combine(Key).WriteShared();
file.Position = Offset;

4
Wabbajack.CLI/Verbs/DownloadUrl.cs
View File

@ -32,11 +32,11 @@ namespace Wabbajack.CLI.Verbs
.Debounce(TimeSpan.FromSeconds(1))
.Subscribe(s => Console.WriteLine($"Downloading {s.ProgressPercent}"));
new[] {state}
await new[] {state}
.PMap(queue, async s =>
{
await s.Download(new Archive(state: null!) {Name = Path.GetFileName(Output)}, (AbsolutePath)Output);
}).Wait();
});
File.WriteAllLines(Output + ".meta", state.GetMetaIni());
return 0;

8
Wabbajack.Common.Test/AsyncLockTests.cs
View File

@ -13,7 +13,7 @@ namespace Wabbajack.Common.Test
bool firstRun = false;
var first = Task.Run(async () =>
{
using (await asyncLock.Wait())
using (await asyncLock.WaitAsync())
{
await Task.Delay(500);
firstRun = true;
@ -22,7 +22,7 @@ namespace Wabbajack.Common.Test
var second = Task.Run(async () =>
{
await Task.Delay(200);
using (await asyncLock.Wait())
using (await asyncLock.WaitAsync())
{
Assert.True(firstRun);
}
@ -41,7 +41,7 @@ namespace Wabbajack.Common.Test
{
return Task.Run(async () =>
{
using (await asyncLock.Wait())
using (await asyncLock.WaitAsync())
{
await Task.Delay(500);
firstRun = true;
@ -55,7 +55,7 @@ namespace Wabbajack.Common.Test
Task.Run(async () =>
{
await Task.Delay(200);
using (await asyncLock.Wait())
using (await asyncLock.WaitAsync())
{
Assert.True(firstRun);
secondRun = true;

11
Wabbajack.Common/Hash.cs
View File

@ -145,6 +145,17 @@ namespace Wabbajack.Common
var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
}
public static Hash xxHash(this Stream stream)
{
var hash = new xxHashConfig();
hash.HashSizeInBits = 64;
hash.Seed = 0x42;
var config = new xxHashConfig {HashSizeInBits = 64};
using var f = new StatusFileStream(stream, $"Hashing memory stream");
var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
}
public static Hash FileHashCached(this AbsolutePath file, bool nullOnIoError = false)
{
if (TryGetHashCache(file, out var foundHash)) return foundHash;

4
Wabbajack.Common/OctoDiff.cs
View File

@ -28,7 +28,7 @@ namespace Wabbajack.Common
return sigStream;
}
private static void CreateSignature(FileStream oldData, FileStream sigStream)
private static void CreateSignature(Stream oldData, FileStream sigStream)
{
Utils.Status("Creating Patch Signature");
var signatureBuilder = new SignatureBuilder();
@ -36,7 +36,7 @@ namespace Wabbajack.Common
sigStream.Position = 0;
}
public static void Create(FileStream oldData, FileStream newData, FileStream signature, FileStream output)
public static void Create(Stream oldData, FileStream newData, FileStream signature, FileStream output)
{
CreateSignature(oldData, signature);
var db = new DeltaBuilder {ProgressReporter = reporter};

2
Wabbajack.Common/Paths.cs
View File

@ -9,7 +9,7 @@ using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Directory = System.IO.Directory;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;

2
Wabbajack.Common/Util/AsyncLock.cs
View File

@ -12,7 +12,7 @@ namespace Wabbajack.Common
{
private readonly SemaphoreSlim _lock = new SemaphoreSlim(1, 1);
public async Task<IDisposable> Wait()
public async Task<IDisposable> WaitAsync()
{
await _lock.WaitAsync();
return Disposable.Create(() => _lock.Release());

2
Wabbajack.Common/Util/TempFolder.cs
View File

@ -14,7 +14,7 @@ namespace Wabbajack.Common
public TempFolder(bool deleteAfter = true)
{
Dir = new AbsolutePath(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()));
Dir = Path.Combine("tmp_files", Guid.NewGuid().ToString()).RelativeTo(AbsolutePath.EntryPoint);
if (!Dir.Exists)
Dir.CreateDirectory();
DeleteAfter = deleteAfter;

11
Wabbajack.Common/Utils.cs
View File

@ -427,6 +427,13 @@ namespace Wabbajack.Common
await ins.CopyToAsync(ms);
return ms.ToArray();
}
public static async Task<string> ReadAllTextAsync(this Stream ins)
{
await using var ms = new MemoryStream();
await ins.CopyToAsync(ms);
return Encoding.UTF8.GetString(ms.ToArray());
}
public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
Func<TI, TR> f)
@ -727,7 +734,7 @@ namespace Wabbajack.Common
}
}
public static async Task CreatePatch(FileStream srcStream, Hash srcHash, FileStream destStream, Hash destHash,
public static async Task CreatePatch(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash,
FileStream patchStream)
{
await using var sigFile = new TempStream();
@ -967,7 +974,7 @@ namespace Wabbajack.Common
Status($"Deleting: {p.Line}");
});
process.Start().Wait();
await process.Start();
await result;
}

4
Wabbajack.Common/WorkQueue.cs
View File

@ -93,7 +93,7 @@ namespace Wabbajack.Common
private async Task AddNewThreadsIfNeeded(int desired)
{
using (await _lock.Wait())
using (await _lock.WaitAsync())
{
DesiredNumWorkers = desired;
while (DesiredNumWorkers > _tasks.Count)
@ -141,7 +141,7 @@ namespace Wabbajack.Common
if (DesiredNumWorkers >= _tasks.Count) continue;
// Noticed that we may need to shut down, lock and check again
using (await _lock.Wait())
using (await _lock.WaitAsync())
{
// Check if another thread shut down before this one and got us back to the desired amount already
if (DesiredNumWorkers >= _tasks.Count) continue;

3
Wabbajack.Lib/AInstaller.cs
View File

@ -172,7 +172,8 @@ namespace Wabbajack.Lib
throw new ArgumentNullException("FromFile was null");
}
var firstDest = OutputFolder.Combine(group.First().To);
await CopyFile(group.Key.StagedPath, firstDest, true);
await group.Key.StagedFile.MoveTo(firstDest);
foreach (var copy in group.Skip(1))
{

2
Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs
View File

@ -77,7 +77,7 @@ namespace Wabbajack.Lib.CompilationSteps
}
CreateBSA directive;
using (var bsa = BSADispatch.OpenRead(source.AbsolutePath))
await using (var bsa = BSADispatch.OpenRead(source.AbsolutePath))
{
directive = new CreateBSA(
state: bsa.State,

10
Wabbajack.Lib/Data.cs
View File

@ -21,7 +21,15 @@ namespace Wabbajack.Lib
Path = path;
}
public AbsolutePath AbsolutePath => File.StagedPath;
public AbsolutePath AbsolutePath
{
get
{
if (!File.IsNative)
throw new InvalidDataException("Can't get the absolute path of a non-native file");
return File.FullPath.Base;
}
}
public VirtualFile File { get; }

18
Wabbajack.Lib/Downloaders/AFKModsDownloader.cs
View File

@ -1,18 +0,0 @@
using System;
namespace Wabbajack.Lib.Downloaders
{
public class AFKModsDownloader : AbstractIPS4Downloader<AFKModsDownloader, AFKModsDownloader.State>
{
#region INeedsDownload
public override string SiteName => "AFK Mods";
public override Uri SiteURL => new Uri("https://www.afkmods.com/index.php?");
public override Uri IconUri => new Uri("https://www.afkmods.com/favicon.ico");
#endregion
public AFKModsDownloader() : base(new Uri("https://www.afkmods.com/index.php?/login/"),
"afkmods", "www.afkmods.com"){}
public class State : State<AFKModsDownloader>{}
}
}

1
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -37,7 +37,6 @@ namespace Wabbajack.Lib.Downloaders
typeof(SteamWorkshopDownloader.State),
typeof(VectorPlexusDownloader.State),
typeof(DeadlyStreamDownloader.State),
typeof(AFKModsDownloader.State),
typeof(TESAllianceDownloader.State),
typeof(BethesdaNetDownloader.State),
typeof(YouTubeDownloader.State)

5
Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs
View File

@ -21,6 +21,9 @@ namespace Wabbajack.Lib.Downloaders
private readonly string _encryptedKeyName;
private readonly string _cookieDomain;
private readonly string _cookieName;
private bool _isPrepared;
// ToDo
// Remove null assignment. Either add nullability to type, or figure way to prepare it safely
public Common.Http.Client AuthedClient { get; private set; } = null!;
@ -100,7 +103,9 @@ namespace Wabbajack.Lib.Downloaders
public async Task Prepare()
{
if (_isPrepared) return;
AuthedClient = (await GetAuthedClient()) ?? throw new NotLoggedInError(this);
_isPrepared = true;
}
public class NotLoggedInError : Exception

9
Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs
View File

@ -29,6 +29,7 @@ namespace Wabbajack.Lib.Downloaders
{
public class BethesdaNetDownloader : IUrlDownloader, INeedsLogin
{
private bool _isPrepared;
public const string DataName = "bethesda-net-data";
public ReactiveCommand<Unit, Unit> TriggerLogin { get; }
@ -70,8 +71,14 @@ namespace Wabbajack.Lib.Downloaders
public async Task Prepare()
{
if (Utils.HaveEncryptedJson(DataName)) return;
if (_isPrepared) return;
if (Utils.HaveEncryptedJson(DataName))
{
_isPrepared = true;
return;
}
await Utils.Log(new RequestBethesdaNetLogin()).Task;
_isPrepared = true;
}
public static async Task<BethesdaNetData?> Login(Game game)

1
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -23,7 +23,6 @@ namespace Wabbajack.Lib.Downloaders
new VectorPlexusDownloader(),
new DeadlyStreamDownloader(),
new BethesdaNetDownloader(),
new AFKModsDownloader(),
new TESAllianceDownloader(),
new YouTubeDownloader(),
new HTTPDownloader(),

2
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -100,7 +100,7 @@ namespace Wabbajack.Lib.Downloaders
{
if (!_prepared)
{
using var _ = await _lock.Wait();
using var _ = await _lock.WaitAsync();
// Could have become prepared while we waited for the lock
if (!_prepared)
{

6
Wabbajack.Lib/MO2Compiler.cs
View File

@ -459,13 +459,13 @@ namespace Wabbajack.Lib
await using var srcStream = srcFile.OpenRead();
await using var outputStream = IncludeFile(out var id);
entry.PatchID = id;
await using var destStream = LoadDataForTo(entry.To, absolutePaths);
await using var destStream = await LoadDataForTo(entry.To, absolutePaths);
await Utils.CreatePatch(srcStream, srcFile.Hash, destStream, entry.Hash, outputStream);
Info($"Patch size {outputStream.Length} for {entry.To}");
});
}
private FileStream LoadDataForTo(RelativePath to, Dictionary<RelativePath, AbsolutePath> absolutePaths)
private async Task<FileStream> LoadDataForTo(RelativePath to, Dictionary<RelativePath, AbsolutePath> absolutePaths)
{
if (absolutePaths.TryGetValue(to, out var absolute))
return absolute.OpenRead();
@ -475,7 +475,7 @@ namespace Wabbajack.Lib
var bsaId = (RelativePath)((string)to).Split('\\')[1];
var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId);
using var a = BSADispatch.OpenRead(MO2Folder.Combine(bsa.To));
await using var a = BSADispatch.OpenRead(MO2Folder.Combine(bsa.To));
var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray());
var file = a.Files.First(e => e.Path == find);
var returnStream = new TempStream();

2
Wabbajack.Lib/MO2Installer.cs
View File

@ -259,7 +259,7 @@ namespace Wabbajack.Lib
var bsaSize = bsa.FileStates.Select(state => sourceDir.Combine(state.Path).Size).Sum();
using (var a = bsa.State.MakeBuilder(bsaSize))
await using (var a = bsa.State.MakeBuilder(bsaSize))
{
var streams = await bsa.FileStates.PMap(Queue, state =>
{

4
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -52,7 +52,7 @@ namespace Wabbajack.Lib.NexusApi
private static AsyncLock _getAPIKeyLock = new AsyncLock();
private static async Task<string> GetApiKey()
{
using (await _getAPIKeyLock.Wait())
using (await _getAPIKeyLock.WaitAsync())
{
// Clean up old location
if (File.Exists(API_KEY_CACHE_FILE))
@ -114,7 +114,7 @@ namespace Wabbajack.Lib.NexusApi
key = await browser.EvaluateJavaScript(
"document.querySelector(\"input[value=wabbajack]\").parentElement.parentElement.querySelector(\"textarea.application-key\").innerHTML");
}
catch (Exception ex)
catch (Exception)
{
// ignored
}

3
Wabbajack.Test/DownloaderTests.cs
View File

@ -46,7 +46,8 @@ namespace Wabbajack.Test
[Fact]
public async Task TestAllPrepares()
{
await Task.WhenAll(DownloadDispatcher.Downloaders.Select(d => d.Prepare()));
foreach (var downloader in DownloadDispatcher.Downloaders)
await downloader.Prepare();
}
[Fact]

7
Wabbajack.Test/EndToEndTests.cs
View File

@ -105,8 +105,8 @@ namespace Wabbajack.Test
await src.CopyToAsync(utils.DownloadsFolder.Combine(filename));
await FileExtractor.ExtractAll(Queue, src,
modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName));
await using var dest = await FileExtractor.ExtractAll(Queue, src);
await dest.MoveAllTo(modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName));
}
private async Task<(AbsolutePath Download, AbsolutePath ModFolder)> DownloadAndInstall(Game game, int modId, string modName)
@ -140,7 +140,8 @@ namespace Wabbajack.Test
await src.CopyToAsync(dest);
var modFolder = utils.ModsFolder.Combine(modName);
await FileExtractor.ExtractAll(Queue, src, modFolder);
await using var files = await FileExtractor.ExtractAll(Queue, src);
await files.MoveAllTo(modFolder);
await dest.WithExtension(Consts.MetaFileExtension).WriteAllTextAsync(ini);
return (dest, modFolder);

12
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using Xunit;
@ -141,7 +142,10 @@ namespace Wabbajack.VirtualFileSystem.Test
var file = context.Index.ByFullPath[res];
var cleanup = await context.Stage(new List<VirtualFile> {file});
Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync());
await using var stream = file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync());
await cleanup();
}
@ -164,7 +168,11 @@ namespace Wabbajack.VirtualFileSystem.Test
var cleanup = await context.Stage(files);
foreach (var file in files)
Assert.Equal("This is a test", await file.StagedPath.ReadAllTextAsync());
{
await using var stream = file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync());
}
await cleanup();
}

18
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -74,7 +74,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, f, f, 0);
return await VirtualFile.Analyze(this, null, new ExtractedDiskFile(f), f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -90,7 +90,7 @@ namespace Wabbajack.VirtualFileSystem
public async Task<IndexRoot> AddRoots(List<AbsolutePath> roots)
{
await _cleanupTask;
var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.StagedPath);
var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.FullPath.Base);
var filtered = Index.AllFiles.Where(file => ((AbsolutePath)file.Name).Exists).ToList();
@ -106,7 +106,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, f, f, 0);
return await VirtualFile.Analyze(this, null, new ExtractedDiskFile(f), f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -210,22 +210,22 @@ namespace Wabbajack.VirtualFileSystem
.OrderBy(f => f.Key?.NestingFactor ?? 0)
.ToList();
var paths = new List<AbsolutePath>();
var paths = new List<IAsyncDisposable>();
foreach (var group in grouped)
{
var tmpPath = ((RelativePath)Guid.NewGuid().ToString()).RelativeTo(StagingFolder);
await FileExtractor.ExtractAll(Queue, group.Key.StagedPath, tmpPath);
paths.Add(tmpPath);
var only = group.Select(f => f.RelativeName);
var extracted = await group.Key.StagedFile.ExtractAll(Queue, only);
paths.Add(extracted);
foreach (var file in group)
file.StagedPath = file.RelativeName.RelativeTo(tmpPath);
file.StagedFile = extracted[file.RelativeName];
}
return async () =>
{
foreach (var p in paths)
{
await p.DeleteDirectory();
await p.DisposeAsync();
}
};
}

51
Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs Normal file
View File

@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Compression.BSA;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public class ExtractedBSAFile : IExtractedFile
{
private readonly IFile _file;
public ExtractedBSAFile(IFile file)
{
_file = file;
}
public RelativePath Path => _file.Path;
public async Task<Hash> HashAsync()
{
await using var stream = OpenRead();
return stream.xxHash();
}
public DateTime LastModifiedUtc => DateTime.UtcNow;
public long Size => _file.Size;
public Stream OpenRead()
{
var ms = new MemoryStream();
_file.CopyDataTo(ms);
ms.Position = 0;
return ms;
}
public async Task<bool> CanExtract()
{
return false;
}
public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> OnlyFiles)
{
throw new Exception("BSAs can't contain archives");
}
public async Task MoveTo(AbsolutePath path)
{
await using var fs = path.Create();
_file.CopyDataTo(fs);
}
}
}

47
Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs Normal file
View File

@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public class ExtractedDiskFile : IExtractedFile
{
private AbsolutePath _path;
public ExtractedDiskFile(AbsolutePath path)
{
if (path == default)
throw new InvalidDataException("Path cannot be empty");
_path = path;
}
public async Task<Hash> HashAsync()
{
return await _path.FileHashAsync();
}
public DateTime LastModifiedUtc => _path.LastModifiedUtc;
public long Size => _path.Size;
public Stream OpenRead()
{
return _path.OpenRead();
}
public async Task<bool> CanExtract()
{
return await FileExtractor.CanExtract(_path);
}
public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> onlyFiles)
{
return FileExtractor.ExtractAll(queue, _path, onlyFiles);
}
public async Task MoveTo(AbsolutePath path)
{
await _path.MoveToAsync(path, true);
_path = path;
}
}
}

64
Wabbajack.VirtualFileSystem/ExtractedFiles.cs Normal file
View File

@ -0,0 +1,64 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public class ExtractedFiles : IAsyncDisposable, IEnumerable<KeyValuePair<RelativePath, IExtractedFile>>
{
private Dictionary<RelativePath, IExtractedFile> _files;
private IAsyncDisposable _disposable;
private AbsolutePath _tempFolder;
public ExtractedFiles(Dictionary<RelativePath, IExtractedFile> files, IAsyncDisposable disposeOther)
{
_files = files;
_disposable = disposeOther;
}
public ExtractedFiles(TempFolder tempPath)
{
_files = tempPath.Dir.EnumerateFiles().ToDictionary(f => f.RelativeTo(tempPath.Dir),
f => (IExtractedFile)new ExtractedDiskFile(f));
_disposable = tempPath;
}
public async ValueTask DisposeAsync()
{
if (_disposable != null)
{
await _disposable.DisposeAsync();
_disposable = null;
}
}
public bool ContainsKey(RelativePath key)
{
return _files.ContainsKey(key);
}
public int Count => _files.Count;
public IExtractedFile this[RelativePath key] => _files[key];
public IEnumerator<KeyValuePair<RelativePath, IExtractedFile>> GetEnumerator()
{
return _files.GetEnumerator();
}
public async Task MoveAllTo(AbsolutePath folder)
{
foreach (var (key, value) in this)
{
await value.MoveTo(key.RelativeTo(folder));
}
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}

88
Wabbajack.VirtualFileSystem/FileExtractor.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Reactive.Linq;
@ -16,42 +17,43 @@ namespace Wabbajack.VirtualFileSystem
{
public class FileExtractor
{
public static async Task ExtractAll(WorkQueue queue, AbsolutePath source, AbsolutePath dest)
public static async Task<ExtractedFiles> ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable<RelativePath> OnlyFiles = null)
{
try
{
if (Consts.SupportedBSAs.Contains(source.Extension))
await ExtractAllWithBSA(queue, source, dest);
return await ExtractAllWithBSA(queue, source);
else if (source.Extension == Consts.OMOD)
ExtractAllWithOMOD(source, dest);
return ExtractAllWithOMOD(source);
else if (source.Extension == Consts.EXE)
await ExtractAllExe(source, dest);
return await ExtractAllExe(source);
else
await ExtractAllWith7Zip(source, dest);
return await ExtractAllWith7Zip(source, OnlyFiles);
}
catch (Exception ex)
{
Utils.ErrorThrow(ex, $"Error while extracting {source}");
throw new Exception();
}
}
private static async Task ExtractAllExe(AbsolutePath source, AbsolutePath dest)
private static async Task<ExtractedFiles> ExtractAllExe(AbsolutePath source)
{
var isArchive = await TestWith7z(source);
if (isArchive)
{
await ExtractAllWith7Zip(source, dest);
return;
return await ExtractAllWith7Zip(source, null);
}
var dest = new TempFolder();
Utils.Log($"Extracting {(string)source.FileName}");
var process = new ProcessHelper
{
Path = @"Extractors\innounp.exe".RelativeTo(AbsolutePath.EntryPoint),
Arguments = new object[] {"-x", "-y", "-b", $"-d\"{dest}\"", source}
Arguments = new object[] {"-x", "-y", "-b", $"-d\"{dest.Dir}\"", source}
};
@ -69,7 +71,8 @@ namespace Wabbajack.VirtualFileSystem
Utils.Status($"Extracting {source.FileName} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d));
});
await process.Start();
}
return new ExtractedFiles(dest);
}
private class OMODProgress : ICodeProgress
{
@ -91,55 +94,70 @@ namespace Wabbajack.VirtualFileSystem
}
}
private static void ExtractAllWithOMOD(AbsolutePath source, AbsolutePath dest)
private static ExtractedFiles ExtractAllWithOMOD(AbsolutePath source)
{
var dest = new TempFolder();
Utils.Log($"Extracting {(string)source.FileName}");
Framework.Settings.TempPath = (string)dest;
Framework.Settings.TempPath = (string)dest.Dir;
Framework.Settings.CodeProgress = new OMODProgress();
var omod = new OMOD((string)source);
omod.GetDataFiles();
omod.GetPlugins();
return new ExtractedFiles(dest);
}
private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source, AbsolutePath dest)
private static async Task<ExtractedFiles> ExtractAllWithBSA(WorkQueue queue, AbsolutePath source)
{
try
{
using var arch = BSADispatch.OpenRead(source);
await arch.Files
.PMap(queue, f =>
{
Utils.Status($"Extracting {(string)f.Path}");
var outPath = f.Path.RelativeTo(dest);
var parent = outPath.Parent;
if (!parent.IsDirectory)
parent.CreateDirectory();
using var fs = outPath.Create();
f.CopyDataTo(fs);
});
await using var arch = BSADispatch.OpenRead(source);
var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile)new ExtractedBSAFile(f));
return new ExtractedFiles(files, arch);
}
catch (Exception ex)
{
Utils.ErrorThrow(ex, $"While Extracting {source}");
throw new Exception();
}
}
private static async Task ExtractAllWith7Zip(AbsolutePath source, AbsolutePath dest)
private static async Task<ExtractedFiles> ExtractAllWith7Zip(AbsolutePath source, IEnumerable<RelativePath> onlyFiles)
{
TempFile tmpFile = null;
var dest = new TempFolder();
Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}", $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe"));
var process = new ProcessHelper
{
Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),
Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest}\"", source, "-mmt=off"}
};
if (onlyFiles != null)
{
//It's stupid that we have to do this, but 7zip's file pattern matching isn't very fuzzy
IEnumerable<string> AllVariants(string input)
{
yield return input;
yield return "\\" + input;
}
tmpFile = new TempFile();
await tmpFile.Path.WriteAllLinesAsync(onlyFiles.SelectMany(f => AllVariants((string)f)).ToArray());
process.Arguments = new object[]
{
"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, $"@\"{tmpFile.Path}\"", "-mmt=off"
};
}
else
{
process.Arguments = new object[] {"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, "-mmt=off"};
}
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
@ -159,12 +177,15 @@ namespace Wabbajack.VirtualFileSystem
if (exitCode != 0)
{
Utils.Error(new _7zipReturnError(exitCode, source, dest, ""));
Utils.Error(new _7zipReturnError(exitCode, source, dest.Dir, ""));
}
else
{
Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
}
tmpFile?.Dispose();
return new ExtractedFiles(dest);
}
/// <summary>
@ -205,9 +226,8 @@ namespace Wabbajack.VirtualFileSystem
private static Extension _exeExtension = new Extension(".exe");
public static bool MightBeArchive(AbsolutePath path)
public static bool MightBeArchive(Extension ext)
{
var ext = path.Extension;
return ext == _exeExtension || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext);
}
}

24
Wabbajack.VirtualFileSystem/IExtractedFile.cs Normal file
View File

@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public interface IExtractedFile
{
public Task<Hash> HashAsync();
public DateTime LastModifiedUtc { get; }
public long Size { get; }
public Stream OpenRead();
public Task<bool> CanExtract();
public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> Only = null);
public Task MoveTo(AbsolutePath path);
}
}

57
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -38,22 +38,21 @@ namespace Wabbajack.VirtualFileSystem
public Context Context { get; set; }
public AbsolutePath StagedPath
private IExtractedFile _stagedFile = null;
public IExtractedFile StagedFile
{
get
{
if (IsNative)
return (AbsolutePath)Name;
if (_stagedPath == null)
throw new UnstagedFileException(FullPath);
return _stagedPath;
if (IsNative) return new ExtractedDiskFile(AbsoluteName);
if (_stagedFile == null)
throw new InvalidDataException("File is unstaged");
return _stagedFile;
}
internal set
set
{
if (IsNative)
throw new CannotStageNativeFile("Cannot stage a native file");
_stagedPath = value;
_stagedFile = value;
}
}
/// <summary>
@ -129,18 +128,18 @@ namespace Wabbajack.VirtualFileSystem
itm.ThisAndAllChildrenReduced(fn);
}
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, AbsolutePath absPath,
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile,
IPath relPath, int depth = 0)
{
var hash = absPath.FileHash();
var hash = await extractedFile.HashAsync();
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(absPath))
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
{
var result = await TryGetContentsFromServer(hash);
if (result != null)
{
Utils.Log($"Downloaded VFS data for {(string)absPath}");
Utils.Log($"Downloaded VFS data for {relPath.FileName}");
VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent)
{
@ -150,7 +149,7 @@ namespace Wabbajack.VirtualFileSystem
Name = path,
Parent = vparent,
Size = file.Size,
LastModified = absPath.LastModifiedUtc.AsUnixTime(),
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = file.Hash
};
@ -169,8 +168,8 @@ namespace Wabbajack.VirtualFileSystem
Context = context,
Name = relPath,
Parent = parent,
Size = absPath.Size,
LastModified = absPath.LastModifiedUtc.AsUnixTime(),
Size = extractedFile.Size,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash
};
@ -178,19 +177,17 @@ namespace Wabbajack.VirtualFileSystem
self.FillFullPath(depth);
if (context.UseExtendedHashes)
self.ExtendedHashes = ExtendedHashes.FromFile(absPath);
self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile);
if (await FileExtractor.CanExtract(absPath))
{
await using var tempFolder = Context.GetTemporaryFolder();
await FileExtractor.ExtractAll(context.Queue, absPath, tempFolder.FullName);
if (!await extractedFile.CanExtract()) return self;
var list = await tempFolder.FullName.EnumerateFiles()
.PMap(context.Queue,
absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), depth + 1));
await using var extracted = await extractedFile.ExtractAll(context.Queue);
self.Children = list.ToImmutableList();
}
var list = await extracted
.PMap(context.Queue,
file => Analyze(context, self, file.Value, file.Key, depth + 1));
self.Children = list.ToImmutableList();
return self;
}
@ -320,9 +317,9 @@ namespace Wabbajack.VirtualFileSystem
return path;
}
public FileStream OpenRead()
public Stream OpenRead()
{
return StagedPath.OpenRead();
return StagedFile.OpenRead();
}
}
@ -333,7 +330,7 @@ namespace Wabbajack.VirtualFileSystem
public string MD5 { get; set; }
public string CRC { get; set; }
public static ExtendedHashes FromFile(AbsolutePath file)
public static ExtendedHashes FromFile(IExtractedFile file)
{
var hashes = new ExtendedHashes();
using (var stream = file.OpenRead())

2
Wabbajack/View Models/MainWindowVM.cs
View File

@ -84,7 +84,7 @@ namespace Wabbajack
.ObserveOnGuiThread()
.SelectTask(async msg =>
{
using var _ = await singleton_lock.Wait();
using var _ = await singleton_lock.WaitAsync();
try
{
await UserInterventionHandlers.Handle(msg);