Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Re-add OMOD support

Browse Source
This commit is contained in:
Timothy Baldridge 2020-09-08 16:15:33 -06:00
parent 5d7bceb6dc
commit 548a0553d2
12 changed files with 161 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Compression.BSA/BA2Reader.cs
View File

@ -350,7 +350,7 @@ namespace Compression.BSA
var ms = new MemoryStream();
await CopyDataTo(ms);
ms.Position = 0;
return new MemoryStreamFactory(ms);
return new MemoryStreamFactory(ms, Path);
}
}
@ -517,7 +517,7 @@ namespace Compression.BSA
var ms = new MemoryStream();
await CopyDataTo(ms);
ms.Position = 0;
return new MemoryStreamFactory(ms);
return new MemoryStreamFactory(ms, Path);
}
}

2
Compression.BSA/BSA/Reader/FileRecord.cs
View File

@ -171,7 +171,7 @@ namespace Compression.BSA
var ms = new MemoryStream();
await CopyDataTo(ms);
ms.Position = 0;
return new MemoryStreamFactory(ms);
return new MemoryStreamFactory(ms, Path);
}
}
}

10
Compression.BSA/MemoryStreamFactory.cs
View File

@ -9,9 +9,10 @@ namespace Compression.BSA
{
private readonly MemoryStream _data;
public MemoryStreamFactory(MemoryStream data)
public MemoryStreamFactory(MemoryStream data, IPath path)
{
_data = data;
Name = path;
}
public async ValueTask<Stream> GetStream()
{
@ -19,7 +20,7 @@ namespace Compression.BSA
}
public DateTime LastModifiedUtc => DateTime.UtcNow;
public IPath Name => (RelativePath)"BSA Memory Stream";
public IPath Name { get; }
}
public class MemoryBufferFactory : IStreamFactory
@ -27,10 +28,11 @@ namespace Compression.BSA
private readonly byte[] _data;
private int _size;
public MemoryBufferFactory(byte[] data, int size)
public MemoryBufferFactory(byte[] data, int size, IPath path)
{
_data = data;
_size = size;
Name = path;
}
public async ValueTask<Stream> GetStream()
{
@ -38,6 +40,6 @@ namespace Compression.BSA
}
public DateTime LastModifiedUtc => DateTime.UtcNow;
public IPath Name => (RelativePath)"BSA Memory Stream";
public IPath Name { get; }
}
}

2
Compression.BSA/TES3Reader.cs
View File

@ -135,7 +135,7 @@ namespace Compression.BSA
var ms = new MemoryStream();
await CopyDataTo(ms);
ms.Position = 0;
return new MemoryStreamFactory(ms);
return new MemoryStreamFactory(ms, Path);
}

9
Wabbajack.Common/IStreamFactory.cs
View File

@ -18,9 +18,16 @@ namespace Wabbajack.Common
{
private AbsolutePath _file;
public NativeFileStreamFactory(AbsolutePath file, IPath path)
{
_file = file;
Name = path;
}
public NativeFileStreamFactory(AbsolutePath file)
{
_file = file;
Name = file;
}
public async ValueTask<Stream> GetStream()
{
@ -28,7 +35,7 @@ namespace Wabbajack.Common
}
public DateTime LastModifiedUtc => _file.LastModifiedUtc;
public IPath Name => _file;
public IPath Name { get; }
}
}

10
Wabbajack.Common/Paths/FileCompaction.cs
View File

@ -57,5 +57,15 @@ namespace Wabbajack.Common
return await proc.Start() == 0;
}
}
public static async Task CompactFolder(this AbsolutePath folder, WorkQueue queue, Algorithm algorithm)
{
await folder.EnumerateFiles(true)
.PMap(queue, async path =>
{
Utils.Status($"Compacting {path.FileName}");
await path.Compact(algorithm);
});
}
}
}

13
Wabbajack.Lib/MO2Installer.cs
View File

@ -40,7 +40,7 @@ namespace Wabbajack.Lib
outputFolder: outputFolder,
downloadFolder: downloadFolder,
parameters: parameters,
steps: 21,
steps: 22,
game: modList.GameType)
{
var gameExe = Consts.GameFolderFilesDir.Combine(modList.GameType.MetaData().MainExecutable!);
@ -178,6 +178,9 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep("Updating System-specific ini settings");
SetScreenSizeInPrefs();
UpdateTracker.NextStep("Compacting files");
await CompactFiles();
UpdateTracker.NextStep("Installation complete! You may exit the program.");
await ExtractedModlistFolder!.DisposeAsync();
@ -186,6 +189,14 @@ namespace Wabbajack.Lib
return true;
}
private async Task CompactFiles()
{
if (this.UseCompression)
{
await OutputFolder.CompactFolder(Queue, FileCompaction.Algorithm.XPRESS16K);
}
}
private void CreateOutputMods()
{
OutputFolder.Combine("profiles")

43
Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs
View File

@ -1,7 +1,10 @@
using System;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Xunit;
namespace Wabbajack.VirtualFileSystem.Test
@ -33,8 +36,26 @@ namespace Wabbajack.VirtualFileSystem.Test
{
Assert.Equal(await temp.Dir.Combine(path).FileHashAsync(), hash);
}
}
private static Extension OMODExtension = new Extension(".omod");
private static Extension CRCExtension = new Extension(".crc");
[Fact]
public async Task CanGatherDataFromOMODFiles()
{
var src = await DownloadMod(Game.Oblivion, 18498);
await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(src),
p => p.Extension == OMODExtension, async (path, sfn) =>
{
await FileExtractor2.GatheringExtract(sfn, _ => true, async (ipath, isfn) => {
// We shouldn't have any .crc files because this file should be recognized as a OMOD and extracted correctly
Assert.NotEqual(CRCExtension, ipath.Extension);
return 0;
});
return 0;
});
}
@ -59,5 +80,27 @@ namespace Wabbajack.VirtualFileSystem.Test
await folder.DeleteDirectory();
}
private static AbsolutePath _stagingFolder = ((RelativePath)"NexusDownloads").RelativeToEntryPoint();
private static async Task<AbsolutePath> DownloadMod(Game game, int mod)
{
using var client = await NexusApiClient.Get();
var results = await client.GetModFiles(game, mod);
var file = results.files.FirstOrDefault(f => f.is_primary) ??
results.files.OrderByDescending(f => f.uploaded_timestamp).First();
var src = _stagingFolder.Combine(file.file_name);
if (src.Exists) return src;
var state = new NexusDownloader.State
{
ModID = mod,
Game = game,
FileID = file.file_id
};
await state.Download(src);
return src;
}
}
}

1
Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj
View File

@ -15,6 +15,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Lib\Wabbajack.Lib.csproj" />
<ProjectReference Include="..\Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj" />
</ItemGroup>

94
Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs
View File

@ -4,11 +4,13 @@ using System.IO;
using System.Threading.Tasks;
using Compression.BSA;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using OMODFramework;
using SharpCompress.Archives.SevenZip;
using SharpCompress.Readers;
using Wabbajack.Common;
using Wabbajack.Common.FileSignatures;
using Wabbajack.VirtualFileSystem.SevenZipExtractor;
using Utils = Wabbajack.Common.Utils;
namespace Wabbajack.VirtualFileSystem
{
@ -22,6 +24,8 @@ namespace Wabbajack.VirtualFileSystem
Definitions.FileType.RAR_OLD,
Definitions.FileType.RAR_NEW,
Definitions.FileType._7Z);
private static Extension OMODExtension = new Extension(".omod");
public static async Task<Dictionary<RelativePath, T>> GatheringExtract<T>(IStreamFactory sFn,
@ -39,10 +43,21 @@ namespace Wabbajack.VirtualFileSystem
{
case Definitions.FileType.RAR_OLD:
case Definitions.FileType.RAR_NEW:
case Definitions.FileType._7Z:
case Definitions.FileType._7Z:
case Definitions.FileType.ZIP:
return await GatheringExtractWith7Zip<T>(archive, (Definitions.FileType)sig, shouldExtract, mapfn);
{
if (sFn.Name.FileName.Extension == OMODExtension)
{
return await GatheringExtractWithOMOD(archive, shouldExtract, mapfn);
}
else
{
return await GatheringExtractWith7Zip<T>(archive, (Definitions.FileType)sig, shouldExtract,
mapfn);
}
}
case Definitions.FileType.TES3:
case Definitions.FileType.BSA:
case Definitions.FileType.BA2:
@ -54,6 +69,54 @@ namespace Wabbajack.VirtualFileSystem
}
}
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithOMOD<T>(Stream archive, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
{
var tmpFile = new TempFile();
await tmpFile.Path.WriteAllAsync(archive);
var dest = await TempFolder.Create();
Utils.Log($"Extracting {(string)tmpFile.Path}");
Framework.Settings.TempPath = (string)dest.Dir;
Framework.Settings.CodeProgress = new OMODProgress();
var omod = new OMOD((string)tmpFile.Path);
omod.GetDataFiles();
omod.GetPlugins();
var results = new Dictionary<RelativePath, T>();
foreach (var file in dest.Dir.EnumerateFiles())
{
var path = file.RelativeTo(dest.Dir);
if (!shouldExtract(path)) continue;
var result = await mapfn(path, new NativeFileStreamFactory(file, path));
results.Add(path, result);
}
return results;
}
private class OMODProgress : ICodeProgress
{
private long _total;
public void SetProgress(long inSize, long outSize)
{
Utils.Status("Extracting OMOD", Percent.FactoryPutInRange(inSize, _total));
}
public void Init(long totalSize, bool compressing)
{
_total = totalSize;
}
public void Dispose()
{
//
}
}
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithBSA<T>(IStreamFactory sFn, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
{
var archive = await BSADispatch.OpenRead(sFn, sig);
@ -73,31 +136,6 @@ namespace Wabbajack.VirtualFileSystem
private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
{
return await new GatheringExtractor<T>(stream, sig, shouldExtract, mapfn).Extract();
/*
IReader reader;
if (sig == Definitions.FileType._7Z)
reader = SevenZipArchive.Open(stream).ExtractAllEntries();
else
{
reader = ReaderFactory.Open(stream);
}
var results = new Dictionary<RelativePath, T>();
while (reader.MoveToNextEntry())
{
var path = (RelativePath)reader.Entry.Key;
if (!reader.Entry.IsDirectory && shouldExtract(path))
{
var ms = new MemoryStream();
reader.WriteEntryTo(ms);
ms.Position = 0;
var result = await mapfn(path, new MemoryStreamFactory(ms));
results.Add(path, result);
}
}
return results;
*/
}
public static async Task ExtractAll(AbsolutePath src, AbsolutePath dest)

11
Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs
View File

@ -126,6 +126,11 @@ namespace Wabbajack.VirtualFileSystem
_diskCached = _totalSize >= 500_000_000;
}
private IPath GetPath()
{
return _extractor._indexes[_index].Item1;
}
public int Write(byte[] data, uint size, IntPtr processedSize)
{
try
@ -155,7 +160,7 @@ namespace Wabbajack.VirtualFileSystem
private void WriteSingleCall(byte[] data, in uint size)
{
var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryBufferFactory(data, (int)size)).Result;
var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryBufferFactory(data, (int)size, GetPath())).Result;
AddResult(result);
Cleanup();
}
@ -181,7 +186,7 @@ namespace Wabbajack.VirtualFileSystem
_tmpStream.Flush();
_tmpStream.Position = 0;
var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory((MemoryStream)_tmpStream)).Result;
var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory((MemoryStream)_tmpStream, GetPath())).Result;
AddResult(result);
Cleanup();
}
@ -201,7 +206,7 @@ namespace Wabbajack.VirtualFileSystem
_tmpStream.Flush();
_tmpStream.Close();
var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new NativeFileStreamFactory(_tmpFile.Path)).Result;
var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new NativeFileStreamFactory(_tmpFile.Path, GetPath())).Result;
AddResult(result);
Cleanup();
}

3
Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj
View File

@ -31,4 +31,7 @@
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<Folder Include="Extractors\OMOD" />
</ItemGroup>
</Project>