diff --git a/Wabbajack.Common/BSDiff.cs b/Wabbajack.Common/BSDiff.cs index d0352127..e17f9184 100644 --- a/Wabbajack.Common/BSDiff.cs +++ b/Wabbajack.Common/BSDiff.cs @@ -677,4 +677,4 @@ namespace Wabbajack.Common return a < b ? b < c ? b : a < c ? c : a : b > c ? b : a > c ? c : a; } } -} \ No newline at end of file +} diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index b9a5a2d1..4a227f8c 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -665,6 +665,7 @@ namespace Wabbajack.Common using (var f = File.OpenWrite(tmpName)) { + Status("Creating Patch"); BSDiff.Create(a, b, f); } diff --git a/Wabbajack.Lib/CerasConfig.cs b/Wabbajack.Lib/CerasConfig.cs index 1b08961f..290bc843 100644 --- a/Wabbajack.Lib/CerasConfig.cs +++ b/Wabbajack.Lib/CerasConfig.cs @@ -24,7 +24,7 @@ namespace Wabbajack.Lib typeof(BSAStateObject), typeof(BSAFileStateObject), typeof(BA2StateObject), typeof(BA2DX10EntryState), typeof(BA2FileEntryState), typeof(MediaFireDownloader.State), typeof(ArchiveMeta), typeof(PropertyFile), typeof(SteamMeta), typeof(SteamWorkshopDownloader), typeof(SteamWorkshopDownloader.State), - typeof(LoversLabDownloader.State) + typeof(LoversLabDownloader.State), typeof(GameFileSourceDownloader.State) } }; diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index c1e7c229..e7a016ea 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -9,6 +9,7 @@ namespace Wabbajack.Lib.Downloaders { public static readonly List Downloaders = new List() { + new GameFileSourceDownloader(), new MegaDownloader(), new DropboxDownloader(), new GoogleDriveDownloader(), diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs new file mode 100644 index 00000000..430371dc --- /dev/null +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using Wabbajack.Common; +using Wabbajack.Lib.Validation; +using File = Alphaleonis.Win32.Filesystem.File; +using Game = Wabbajack.Common.Game; + +namespace Wabbajack.Lib.Downloaders +{ + public class GameFileSourceDownloader : IDownloader + { + public AbstractDownloadState GetDownloaderState(dynamic archiveINI) + { + var gameName = (string)archiveINI?.General?.gameName; + var gameFile = (string)archiveINI?.General?.gameFile; + + if (gameFile == null || gameFile == null) + return null; + + var game = GameRegistry.GetByMO2ArchiveName(gameName); + if (game == null) return null; + + var path = game.GameLocation(); + var filePath = Path.Combine(path, gameFile); + + if (!File.Exists(filePath)) + return null; + + var hash = filePath.FileHashCached(); + + return new State + { + Game = GameRegistry.GetByMO2ArchiveName(gameName).Game, + GameFile = gameFile, + Hash = hash, + }; + } + + public void Prepare() + { + } + + public class State : AbstractDownloadState + { + public Game Game { get; set; } + public string GameFile { get; set; } + public string Hash { get; set; } + + internal string SourcePath => Path.Combine(Game.MetaData().GameLocation(), GameFile); + + public override bool IsWhitelisted(ServerWhitelist whitelist) + { + return true; + } + + public override void Download(Archive a, string destination) + { + using(var src = File.OpenRead(SourcePath)) + using (var dest = File.OpenWrite(destination)) + { + var size = new FileInfo(SourcePath).Length; + src.CopyToWithStatus(size, dest, "Copying from Game folder"); + } + } + + public override bool Verify() + { + return File.Exists(SourcePath) && SourcePath.FileHashCached() == Hash; + } + + public override IDownloader GetDownloader() + { + return DownloadDispatcher.GetInstance(); + } + + public override string GetReportEntry(Archive a) + { + return $"* Game File {Game} - {GameFile}"; + } + } + } +} diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 952c2e25..a200c24d 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -5,12 +5,15 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; +using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; +using Game = Wabbajack.Common.Game; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib @@ -29,6 +32,8 @@ namespace Wabbajack.Lib public override string GamePath { get; } + public GameMetaData CompilingGame { get; set; } + public override string ModListOutputFolder => "output_folder"; public override string ModListOutputFile { get; } @@ -38,6 +43,8 @@ namespace Wabbajack.Lib MO2Folder = mo2Folder; MO2Profile = mo2Profile; MO2Ini = Path.Combine(MO2Folder, "ModOrganizer.ini").LoadIniFile(); + var mo2game = (string)MO2Ini.General.gameName; + CompilingGame = GameRegistry.Games.First(g => g.Value.MO2Name == mo2game).Value; GamePath = ((string)MO2Ini.General.gamePath).Replace("\\\\", "\\"); ModListOutputFile = outputFile; } @@ -72,7 +79,7 @@ namespace Wabbajack.Lib protected override bool _Begin() { - ConfigureProcessor(16); + ConfigureProcessor(18); UpdateTracker.Reset(); UpdateTracker.NextStep("Gathering information"); Info("Looking for other profiles"); @@ -116,6 +123,13 @@ namespace Wabbajack.Lib if (Directory.Exists(ModListOutputFolder)) Utils.DeleteDirectory(ModListOutputFolder); + UpdateTracker.NextStep("Inferring metas for game file downloads"); + InferMetas(); + + UpdateTracker.NextStep("Reindexing downloads after meta inferring"); + VFS.AddRoot(MO2DownloadsFolder); + VFS.WriteToFile(_vfsCacheName); + UpdateTracker.NextStep("Finding Install Files"); Directory.CreateDirectory(ModListOutputFolder); @@ -244,7 +258,7 @@ namespace Wabbajack.Lib ModList = new ModList { - GameType = GameRegistry.Games.Values.First(f => f.MO2Name == MO2Ini.General.gameName).Game, + GameType = CompilingGame.Game, WabbajackVersion = WabbajackVersion, Archives = SelectedArchives.ToList(), ModManager = ModManager.MO2, @@ -275,6 +289,41 @@ namespace Wabbajack.Lib return true; } + private void InferMetas() + { + var to_find = Directory.EnumerateFiles(MO2DownloadsFolder) + .Where(f => !f.EndsWith(".meta") && !f.EndsWith(Consts.HashFileExtension)) + .Where(f => !File.Exists(f + ".meta")) + .ToList(); + + if (to_find.Count == 0) return; + + var games = new[]{CompilingGame}.Concat(GameRegistry.Games.Values.Where(g => g != CompilingGame)); + var game_files = games + .Where(g => g.GameLocation() != null) + .SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", DirectoryEnumerationOptions.Recursive).Select(name => (game, name))) + .GroupBy(f => (Path.GetFileName(f.name), new FileInfo(f.name).Length)) + .ToDictionary(f => f.Key); + + to_find.PMap(Queue, f => + { + var vf = VFS.Index.ByFullPath[f]; + if (!game_files.TryGetValue((Path.GetFileName(f), vf.Size), out var found)) + return; + + var (game, name) = found.FirstOrDefault(ff => ff.name.FileHash() == vf.Hash); + if (name == null) + return; + + File.WriteAllLines(f+".meta", new[] + { + "[General]", + $"gameName={game.MO2ArchiveName}", + $"gameFile={name.RelativeTo(game.GameLocation()).Replace("\\", "/")}" + }); + }); + + } private void IncludeArchiveMetadata() diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index ce951b6c..de25b312 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -116,6 +116,7 @@ + diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 0480f5a6..0a52f563 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Reactive.Disposables; using System.Reactive.Linq; +using Alphaleonis.Win32.Filesystem; using Microsoft.VisualStudio.TestTools.UnitTesting; using Wabbajack.Common; using Wabbajack.Common.StatusFeed; @@ -11,6 +12,7 @@ using Wabbajack.Lib.LibCefHelpers; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; using File = Alphaleonis.Win32.Filesystem.File; +using Game = Wabbajack.Common.Game; namespace Wabbajack.Test { @@ -298,8 +300,32 @@ namespace Wabbajack.Test Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename)); Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!"); - } + + [TestMethod] + public void GameFileSourceDownload() + { + DownloadDispatcher.GetInstance().Prepare(); + var ini = $@"[General] + gameName={Game.SkyrimSpecialEdition.MetaData().MO2ArchiveName} + gameFile=Data/Update.esm"; + + var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString()); + + Assert.IsNotNull(state); + + var converted = state.ViaJSON(); + Assert.IsTrue(converted.Verify()); + var filename = Guid.NewGuid().ToString(); + + Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); + + converted.Download(new Archive { Name = "Update.esm" }, filename); + + Assert.AreEqual("/DLG/LjdGXI=", Utils.FileHash(filename)); + CollectionAssert.AreEqual(File.ReadAllBytes(Path.Combine(Game.SkyrimSpecialEdition.MetaData().GameLocation(), "Data/Update.esm")), File.ReadAllBytes(filename)); + } + }