From ed41308f12f87aa68caee54751031a5f77f10dd1 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 9 Dec 2019 16:52:17 -0700 Subject: [PATCH 1/2] Can now "download" archives from the game folder. This is useful if installers need to repack game files, they can source the files from the main game archive and then build a new BSA modifying/adding files as needed --- .../Downloaders/DownloadDispatcher.cs | 1 + .../Downloaders/GameFileSourceDownloader.cs | 83 +++++++++++++++++++ Wabbajack.Lib/Wabbajack.Lib.csproj | 1 + Wabbajack.Test/DownloaderTests.cs | 28 ++++++- 4 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index c1e7c229..e7a016ea 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -9,6 +9,7 @@ namespace Wabbajack.Lib.Downloaders { public static readonly List Downloaders = new List() { + new GameFileSourceDownloader(), new MegaDownloader(), new DropboxDownloader(), new GoogleDriveDownloader(), diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs new file mode 100644 index 00000000..dea6961d --- /dev/null +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using Wabbajack.Common; +using Wabbajack.Lib.Validation; +using File = Alphaleonis.Win32.Filesystem.File; +using Game = Wabbajack.Common.Game; + +namespace Wabbajack.Lib.Downloaders +{ + public class GameFileSourceDownloader : IDownloader + { + public AbstractDownloadState GetDownloaderState(dynamic archiveINI) + { + var gameName = (string)archiveINI?.General?.gameName; + var gameFile = (string)archiveINI?.General?.gameFile; + var game = GameRegistry.GetByMO2ArchiveName(gameName); + if (gameFile == null || gameFile == null || game == null) + return null; + + var path = game.GameLocation(); + var filePath = Path.Combine(path, gameFile); + + if (!File.Exists(filePath)) + return null; + + var hash = filePath.FileHashCached(); + + return new State + { + Game = GameRegistry.GetByMO2ArchiveName(gameName).Game, + GameFile = gameFile, + Hash = hash, + }; + } + + public void Prepare() + { + } + + public class State : AbstractDownloadState + { + public Game Game { get; set; } + public string GameFile { get; set; } + public string Hash { get; set; } + + internal string SourcePath => Path.Combine(Game.MetaData().GameLocation(), GameFile); + + public override bool IsWhitelisted(ServerWhitelist whitelist) + { + return true; + } + + public override void Download(Archive a, string destination) + { + using(var src = File.OpenRead(SourcePath)) + using (var dest = File.OpenWrite(destination)) + { + var size = new FileInfo(SourcePath).Length; + src.CopyToWithStatus(size, dest, "Copying from Game folder"); + } + } + + public override bool Verify() + { + return File.Exists(SourcePath) && SourcePath.FileHash() == Hash; + } + + public override IDownloader GetDownloader() + { + return DownloadDispatcher.GetInstance(); + } + + public override string GetReportEntry(Archive a) + { + return $"* Game File {Game} - {GameFile}"; + } + } + } +} diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index ce951b6c..de25b312 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -116,6 +116,7 @@ + diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 0480f5a6..0a52f563 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Reactive.Disposables; using System.Reactive.Linq; +using Alphaleonis.Win32.Filesystem; using Microsoft.VisualStudio.TestTools.UnitTesting; using Wabbajack.Common; using Wabbajack.Common.StatusFeed; @@ -11,6 +12,7 @@ using Wabbajack.Lib.LibCefHelpers; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; using File = Alphaleonis.Win32.Filesystem.File; +using Game = Wabbajack.Common.Game; namespace Wabbajack.Test { @@ -298,8 +300,32 @@ namespace Wabbajack.Test Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename)); Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!"); - } + + [TestMethod] + public void GameFileSourceDownload() + { + DownloadDispatcher.GetInstance().Prepare(); + var ini = $@"[General] + gameName={Game.SkyrimSpecialEdition.MetaData().MO2ArchiveName} + gameFile=Data/Update.esm"; + + var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString()); + + Assert.IsNotNull(state); + + var converted = state.ViaJSON(); + Assert.IsTrue(converted.Verify()); + var filename = Guid.NewGuid().ToString(); + + Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); + + converted.Download(new Archive { Name = "Update.esm" }, filename); + + Assert.AreEqual("/DLG/LjdGXI=", Utils.FileHash(filename)); + CollectionAssert.AreEqual(File.ReadAllBytes(Path.Combine(Game.SkyrimSpecialEdition.MetaData().GameLocation(), "Data/Update.esm")), File.ReadAllBytes(filename)); + } + } From f6e9d672d2184b4fbdc5032d2b6f4c0cc404fb65 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 10 Dec 2019 05:26:49 -0700 Subject: [PATCH 2/2] Create .metas automatically for source-from-game files. --- Wabbajack.Common/BSDiff.cs | 2 +- Wabbajack.Common/Utils.cs | 1 + Wabbajack.Lib/CerasConfig.cs | 2 +- .../Downloaders/GameFileSourceDownloader.cs | 9 ++-- Wabbajack.Lib/MO2Compiler.cs | 53 ++++++++++++++++++- 5 files changed, 60 insertions(+), 7 deletions(-) diff --git a/Wabbajack.Common/BSDiff.cs b/Wabbajack.Common/BSDiff.cs index d0352127..e17f9184 100644 --- a/Wabbajack.Common/BSDiff.cs +++ b/Wabbajack.Common/BSDiff.cs @@ -677,4 +677,4 @@ namespace Wabbajack.Common return a < b ? b < c ? b : a < c ? c : a : b > c ? b : a > c ? c : a; } } -} \ No newline at end of file +} diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index b9a5a2d1..4a227f8c 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -665,6 +665,7 @@ namespace Wabbajack.Common using (var f = File.OpenWrite(tmpName)) { + Status("Creating Patch"); BSDiff.Create(a, b, f); } diff --git a/Wabbajack.Lib/CerasConfig.cs b/Wabbajack.Lib/CerasConfig.cs index 1b08961f..290bc843 100644 --- a/Wabbajack.Lib/CerasConfig.cs +++ b/Wabbajack.Lib/CerasConfig.cs @@ -24,7 +24,7 @@ namespace Wabbajack.Lib typeof(BSAStateObject), typeof(BSAFileStateObject), typeof(BA2StateObject), typeof(BA2DX10EntryState), typeof(BA2FileEntryState), typeof(MediaFireDownloader.State), typeof(ArchiveMeta), typeof(PropertyFile), typeof(SteamMeta), typeof(SteamWorkshopDownloader), typeof(SteamWorkshopDownloader.State), - typeof(LoversLabDownloader.State) + typeof(LoversLabDownloader.State), typeof(GameFileSourceDownloader.State) } }; diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs index dea6961d..430371dc 100644 --- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -17,10 +17,13 @@ namespace Wabbajack.Lib.Downloaders { var gameName = (string)archiveINI?.General?.gameName; var gameFile = (string)archiveINI?.General?.gameFile; - var game = GameRegistry.GetByMO2ArchiveName(gameName); - if (gameFile == null || gameFile == null || game == null) + + if (gameFile == null || gameFile == null) return null; + var game = GameRegistry.GetByMO2ArchiveName(gameName); + if (game == null) return null; + var path = game.GameLocation(); var filePath = Path.Combine(path, gameFile); @@ -66,7 +69,7 @@ namespace Wabbajack.Lib.Downloaders public override bool Verify() { - return File.Exists(SourcePath) && SourcePath.FileHash() == Hash; + return File.Exists(SourcePath) && SourcePath.FileHashCached() == Hash; } public override IDownloader GetDownloader() diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 952c2e25..a200c24d 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -5,12 +5,15 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; +using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; +using Game = Wabbajack.Common.Game; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib @@ -29,6 +32,8 @@ namespace Wabbajack.Lib public override string GamePath { get; } + public GameMetaData CompilingGame { get; set; } + public override string ModListOutputFolder => "output_folder"; public override string ModListOutputFile { get; } @@ -38,6 +43,8 @@ namespace Wabbajack.Lib MO2Folder = mo2Folder; MO2Profile = mo2Profile; MO2Ini = Path.Combine(MO2Folder, "ModOrganizer.ini").LoadIniFile(); + var mo2game = (string)MO2Ini.General.gameName; + CompilingGame = GameRegistry.Games.First(g => g.Value.MO2Name == mo2game).Value; GamePath = ((string)MO2Ini.General.gamePath).Replace("\\\\", "\\"); ModListOutputFile = outputFile; } @@ -72,7 +79,7 @@ namespace Wabbajack.Lib protected override bool _Begin() { - ConfigureProcessor(16); + ConfigureProcessor(18); UpdateTracker.Reset(); UpdateTracker.NextStep("Gathering information"); Info("Looking for other profiles"); @@ -116,6 +123,13 @@ namespace Wabbajack.Lib if (Directory.Exists(ModListOutputFolder)) Utils.DeleteDirectory(ModListOutputFolder); + UpdateTracker.NextStep("Inferring metas for game file downloads"); + InferMetas(); + + UpdateTracker.NextStep("Reindexing downloads after meta inferring"); + VFS.AddRoot(MO2DownloadsFolder); + VFS.WriteToFile(_vfsCacheName); + UpdateTracker.NextStep("Finding Install Files"); Directory.CreateDirectory(ModListOutputFolder); @@ -244,7 +258,7 @@ namespace Wabbajack.Lib ModList = new ModList { - GameType = GameRegistry.Games.Values.First(f => f.MO2Name == MO2Ini.General.gameName).Game, + GameType = CompilingGame.Game, WabbajackVersion = WabbajackVersion, Archives = SelectedArchives.ToList(), ModManager = ModManager.MO2, @@ -275,6 +289,41 @@ namespace Wabbajack.Lib return true; } + private void InferMetas() + { + var to_find = Directory.EnumerateFiles(MO2DownloadsFolder) + .Where(f => !f.EndsWith(".meta") && !f.EndsWith(Consts.HashFileExtension)) + .Where(f => !File.Exists(f + ".meta")) + .ToList(); + + if (to_find.Count == 0) return; + + var games = new[]{CompilingGame}.Concat(GameRegistry.Games.Values.Where(g => g != CompilingGame)); + var game_files = games + .Where(g => g.GameLocation() != null) + .SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", DirectoryEnumerationOptions.Recursive).Select(name => (game, name))) + .GroupBy(f => (Path.GetFileName(f.name), new FileInfo(f.name).Length)) + .ToDictionary(f => f.Key); + + to_find.PMap(Queue, f => + { + var vf = VFS.Index.ByFullPath[f]; + if (!game_files.TryGetValue((Path.GetFileName(f), vf.Size), out var found)) + return; + + var (game, name) = found.FirstOrDefault(ff => ff.name.FileHash() == vf.Hash); + if (name == null) + return; + + File.WriteAllLines(f+".meta", new[] + { + "[General]", + $"gameName={game.MO2ArchiveName}", + $"gameFile={name.RelativeTo(game.GameLocation()).Replace("\\", "/")}" + }); + }); + + } private void IncludeArchiveMetadata()