using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; using Wabbajack.VirtualFileSystem; using Xunit; using Xunit.Abstractions; namespace Compression.BSA.Test { public class BSATests : IAsyncLifetime { private static AbsolutePath _stagingFolder = ((RelativePath)"NexusDownloads").RelativeToEntryPoint(); private static AbsolutePath _bsaFolder = ((RelativePath)"BSAs").RelativeToEntryPoint(); private static AbsolutePath _testDir = ((RelativePath)"BSA Test Dir").RelativeToEntryPoint(); private static AbsolutePath _tempDir = ((RelativePath)"BSA Temp Dir").RelativeToEntryPoint(); private IDisposable _unsub; public ITestOutputHelper TestContext { get; } private static WorkQueue Queue { get; set; } public BSATests(ITestOutputHelper helper) { TestContext = helper; } public async Task InitializeAsync() { Queue = new WorkQueue(); _unsub = Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f.ShortDescription)); _stagingFolder.CreateDirectory(); await _bsaFolder.DeleteDirectory(); _bsaFolder.CreateDirectory(); } public async Task DisposeAsync() { await _bsaFolder.DeleteDirectory(); Queue.Dispose(); _unsub.Dispose(); } private static async Task DownloadMod(Game game, int mod) { using var client = await NexusApiClient.Get(); var results = await client.GetModFiles(game, mod); var file = results.files.FirstOrDefault(f => f.is_primary) ?? results.files.OrderByDescending(f => f.uploaded_timestamp).First(); var src = _stagingFolder.Combine(file.file_name); if (src.Exists) return src; var state = new NexusDownloader.State { ModID = mod, Game = game, FileID = file.file_id }; await state.Download(src); return src; } [Theory] [InlineData(Game.SkyrimSpecialEdition, 12604)] // SkyUI [InlineData(Game.Skyrim, 3863)] // SkyUI [InlineData(Game.Skyrim, 51473)] // INeed [InlineData(Game.Fallout4, 22223)] // 10mm SMG [InlineData(Game.Fallout4, 4472)] // True Storms [InlineData(Game.Morrowind, 44537)] [InlineData(Game.Fallout4, 43474)] // EM 2 Rifle public async Task BSACompressionRecompression(Game game, int modid) { var filename = await DownloadMod(game, modid); var folder = _bsaFolder.Combine(game.ToString(), modid.ToString()); await folder.DeleteDirectory(); folder.CreateDirectory(); await using var files = await FileExtractor.ExtractAll(Queue, filename); await files.MoveAllTo(folder); foreach (var bsa in folder.EnumerateFiles().Where(f => Consts.SupportedBSAs.Contains(f.Extension))) { TestContext.WriteLine($"From {bsa}"); TestContext.WriteLine("Cleaning Output Dir"); await _tempDir.DeleteDirectory(); _tempDir.CreateDirectory(); TestContext.WriteLine($"Reading {bsa}"); var tempFile = ((RelativePath)"tmp.bsa").RelativeToEntryPoint(); var size = bsa.Size; await using var a = await BSADispatch.OpenRead(bsa); await a.Files.PMap(Queue, async file => { var absName = _tempDir.Combine(file.Path); ViaJson(file.State); absName.Parent.CreateDirectory(); await using (var fs = await absName.Create()) { await file.CopyDataTo(fs); } Assert.Equal(file.Size, absName.Size); }); // Check Files should be case insensitive Assert.Equal(a.Files.Count(), a.Files.Select(f => f.Path).ToHashSet().Count); Assert.Equal(a.Files.Count(), a.Files.Select(f => f.Path.ToString().ToLowerInvariant()).ToHashSet().Count); TestContext.WriteLine($"Building {bsa}"); await using (var w = ViaJson(a.State).MakeBuilder(size)) { var streams = await a.Files.PMap(Queue, async file => { var absPath = _tempDir.Combine(file.Path); var str = await absPath.OpenRead(); await w.AddFile(ViaJson(file.State), str); return str; }); await w.Build(tempFile); streams.Do(s => s.Dispose()); } TestContext.WriteLine($"Verifying {bsa}"); await using var b = await BSADispatch.OpenRead(tempFile); TestContext.WriteLine($"Performing A/B tests on {bsa}"); Assert.Equal(a.State.ToJson(), b.State.ToJson()); // Check same number of files Assert.Equal(a.Files.Count(), b.Files.Count()); await a.Files.Zip(b.Files, (ai, bi) => (ai, bi)) .PMap(Queue, async pair => { Assert.Equal(pair.ai.State.ToJson(), pair.bi.State.ToJson()); //Console.WriteLine($" - {pair.ai.Path}"); Assert.Equal(pair.ai.Path, pair.bi.Path); //Equal(pair.ai.Compressed, pair.bi.Compressed); Assert.Equal(pair.ai.Size, pair.bi.Size); Assert.Equal(await GetData(pair.ai), await GetData(pair.bi)); }); } } private static async ValueTask GetData(IFile pairAi) { await using var ms = new MemoryStream(); await pairAi.CopyDataTo(ms); return ms.ToArray(); } private static T ViaJson(T i) { return i.ToJson().FromJsonString(); } } }