Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #472 from wabbajack-tools/patch-all

Browse Source 
Patch all support
This commit is contained in:
Timothy Baldridge 2020-02-05 06:01:15 -07:00 committed by GitHub
commit 3ef14c7c07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 149 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -3,6 +3,8 @@
* Auto update functionality added client-side.
* Slideshow now moves to next slide when users clicks, even if paused
* Installer now prints to log what modlist it is installing
* Adding `matchAll=<archive-name>` to a *mods's* `meta.ini` file will result in unconditional patching for all unmatching files or BSAs in
that mod (issue #465)
=======

1
Wabbajack.Common/Consts.cs
View File

@ -99,5 +99,6 @@ namespace Wabbajack.Common
public static string WabbajackCacheHostname = "build.wabbajack.org";
public static int WabbajackCachePort = 80;
public static int MaxHTTPRetries = 4;
public const string MO2ModFolderName = "mods";
}
}

31
Wabbajack.Common/Utils.cs
View File

@ -820,7 +820,7 @@ namespace Wabbajack.Common
return ToFileSizeString((long)byteCount);
}
public static void CreatePatch(byte[] a, byte[] b, Stream output)
public static async Task CreatePatch(byte[] a, byte[] b, Stream output)
{
var dataA = a.xxHash().FromBase64().ToHex();
var dataB = b.xxHash().FromBase64().ToHex();
@ -832,22 +832,32 @@ namespace Wabbajack.Common
{
if (File.Exists(cacheFile))
{
using (var f = File.OpenRead(cacheFile))
{
f.CopyTo(output);
}
await using var f = File.OpenRead(cacheFile);
await f.CopyToAsync(output);
}
else
{
var tmpName = Path.Combine("patch_cache", Guid.NewGuid() + ".tmp");
using (var f = File.Open(tmpName, System.IO.FileMode.Create))
await using (var f = File.Open(tmpName, System.IO.FileMode.Create))
{
Status("Creating Patch");
BSDiff.Create(a, b, f);
}
File.Move(tmpName, cacheFile, MoveOptions.ReplaceExisting);
RETRY:
try
{
File.Move(tmpName, cacheFile, MoveOptions.ReplaceExisting);
}
catch (UnauthorizedAccessException)
{
if (File.Exists(cacheFile))
continue;
await Task.Delay(1000);
goto RETRY;
}
continue;
}
@ -1224,5 +1234,12 @@ namespace Wabbajack.Common
random.NextBytes(bytes);
return bytes.ToHex();
}
public static async Task CopyFileAsync(string src, string dest)
{
await using var s = File.OpenRead(src);
await using var d = File.Create(dest);
await s.CopyToAsync(d);
}
}
}

17
Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs
View File

@ -7,14 +7,15 @@ using Compression.BSA;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed.Errors;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.Lib.CompilationSteps
{
public class DeconstructBSAs : ACompilationStep
{
private readonly IEnumerable<string> _includeDirectly;
private readonly List<ICompilationStep> _microstack;
private readonly List<ICompilationStep> _microstackWithInclude;
private readonly Func<VirtualFile, List<ICompilationStep>> _microstack;
private readonly Func<VirtualFile, List<ICompilationStep>> _microstackWithInclude;
private readonly MO2Compiler _mo2Compiler;
public DeconstructBSAs(ACompiler compiler) : base(compiler)
@ -30,17 +31,17 @@ namespace Wabbajack.Lib.CompilationSteps
.Select(kv => $"mods\\{kv.Key}\\")
.ToList();
_microstack = new List<ICompilationStep>
_microstack = bsa => new List<ICompilationStep>
{
new DirectMatch(_mo2Compiler),
new IncludePatches(_mo2Compiler),
new IncludePatches(_mo2Compiler, bsa),
new DropAll(_mo2Compiler)
};
_microstackWithInclude = new List<ICompilationStep>
_microstackWithInclude = bsa => new List<ICompilationStep>
{
new DirectMatch(_mo2Compiler),
new IncludePatches(_mo2Compiler),
new IncludePatches(_mo2Compiler, bsa),
new IncludeAll(_mo2Compiler)
};
}
@ -55,13 +56,13 @@ namespace Wabbajack.Lib.CompilationSteps
if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path).ToLower())) return null;
var defaultInclude = false;
if (source.Path.StartsWith("mods"))
if (source.Path.StartsWith(Consts.MO2ModFolderName))
if (_includeDirectly.Any(path => source.Path.StartsWith(path)))
defaultInclude = true;
var sourceFiles = source.File.Children;
var stack = defaultInclude ? _microstackWithInclude : _microstack;
var stack = defaultInclude ? _microstackWithInclude(source.File) : _microstack(source.File);
var id = Guid.NewGuid().ToString();

6
Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs
View File

@ -22,13 +22,13 @@ namespace Wabbajack.Lib.CompilationSteps
.Where(line => line.StartsWith("+") || line.EndsWith("_separator"))
.Select(line => line.Substring(1))
.Concat(alwaysEnabled)
.Select(line => Path.Combine("mods", line) + "\\")
.Select(line => Path.Combine(Consts.MO2ModFolderName, line) + "\\")
.ToList();
}
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!source.Path.StartsWith("mods") || _allEnabledMods.Any(mod => source.Path.StartsWith(mod)))
if (!source.Path.StartsWith(Consts.MO2ModFolderName) || _allEnabledMods.Any(mod => source.Path.StartsWith(mod)))
return null;
var r = source.EvolveTo<IgnoredDirectly>();
r.Reason = "Disabled Mod";
@ -50,7 +50,7 @@ namespace Wabbajack.Lib.CompilationSteps
Consts.WABBAJACK_ALWAYS_ENABLE))
return true;
if (data.General != null && data.General.comments != null &&
data.General.notes.Contains(Consts.WABBAJACK_ALWAYS_ENABLE))
data.General.comments.Contains(Consts.WABBAJACK_ALWAYS_ENABLE))
return true;
return false;
}

57
Wabbajack.Lib/CompilationSteps/IncludePatches.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
@ -11,13 +12,20 @@ namespace Wabbajack.Lib.CompilationSteps
public class IncludePatches : ACompilationStep
{
private readonly Dictionary<string, IGrouping<string, VirtualFile>> _indexed;
private VirtualFile _bsa;
private Dictionary<string, VirtualFile> _indexedByName;
public IncludePatches(ACompiler compiler) : base(compiler)
public IncludePatches(ACompiler compiler, VirtualFile constructingFromBSA = null) : base(compiler)
{
_bsa = constructingFromBSA;
_indexed = _compiler.IndexedFiles.Values
.SelectMany(f => f)
.GroupBy(f => Path.GetFileName(f.Name).ToLower())
.ToDictionary(f => f.Key);
_indexedByName = _indexed.Values
.SelectMany(s => s)
.Where(f => f.IsNative)
.ToDictionary(f => Path.GetFileName(f.FullPath));
}
public override async ValueTask<Directive> Run(RawSourceFile source)
@ -28,22 +36,53 @@ namespace Wabbajack.Lib.CompilationSteps
nameWithoutExt = Path.GetFileNameWithoutExtension(name);
if (!_indexed.TryGetValue(Path.GetFileName(name), out var choices))
if (!_indexed.TryGetValue(Path.GetFileName(nameWithoutExt), out choices))
return null;
_indexed.TryGetValue(Path.GetFileName(nameWithoutExt), out choices);
var mod_ini = ((MO2Compiler)_compiler).ModMetas.FirstOrDefault(f => source.Path.StartsWith(f.Key));
var installationFile = mod_ini.Value?.General?.installationFile;
dynamic mod_ini;
if (_bsa == null)
mod_ini = ((MO2Compiler)_compiler).ModMetas.FirstOrDefault(f => source.Path.StartsWith(f.Key)).Value;
else
{
var bsa_path = _bsa.FullPath.RelativeTo(((MO2Compiler)_compiler).MO2Folder);
mod_ini = ((MO2Compiler)_compiler).ModMetas.FirstOrDefault(f => bsa_path.StartsWith(f.Key)).Value;
}
var found = choices.FirstOrDefault(
f => Path.GetFileName(f.FilesInFullPath.First().Name) == installationFile);
var installationFile = mod_ini?.General?.installationFile;
if (found == null)
VirtualFile found = null;
// Find based on exact file name + ext
if (choices != null)
{
found = choices.FirstOrDefault(
f => Path.GetFileName(f.FilesInFullPath.First().Name) == installationFile);
}
// Find based on file name only (not ext)
if (found == null && choices != null)
{
found = choices.OrderBy(f => f.NestingFactor)
.ThenByDescending(f => (f.FilesInFullPath.First() ?? f).LastModified)
.First();
}
// Find based on matchAll=<archivename> in [General] in meta.ini
var matchAllName = (string)mod_ini?.General?.matchAll;
if (matchAllName != null)
{
matchAllName = matchAllName.Trim();
if (_indexedByName.TryGetValue(matchAllName, out var arch))
{
// Just match some file in the archive based on the smallest delta difference
found = arch.ThisAndAllChildren
.OrderBy(o => Math.Abs(o.Size - source.File.Size))
.First();
}
}
if (found == null)
return null;
var e = source.EvolveTo<PatchedFromArchive>();
e.FromHash = found.Hash;
e.ArchiveHashPath = found.MakeRelativePaths();

3
Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs
View File

@ -3,6 +3,7 @@ using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
using Wabbajack.Common;
namespace Wabbajack.Lib.CompilationSteps
{
@ -27,7 +28,7 @@ namespace Wabbajack.Lib.CompilationSteps
public override async ValueTask<Directive> Run(RawSourceFile source)
{
if (!source.Path.StartsWith("mods")) return null;
if (!source.Path.StartsWith(Consts.MO2ModFolderName)) return null;
foreach (var modpath in _includeDirectly)
{
if (!source.Path.StartsWith(modpath)) continue;

4
Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs
View File

@ -29,9 +29,9 @@ namespace Wabbajack.Lib.CompilationSteps
result.SourceESMHash = _compiler.VFS.Index.ByRootPath[gameFile].Hash;
Utils.Status($"Generating patch of {filename}");
using (var ms = new MemoryStream())
await using (var ms = new MemoryStream())
{
Utils.CreatePatch(File.ReadAllBytes(gameFile), File.ReadAllBytes(source.AbsolutePath), ms);
await Utils.CreatePatch(File.ReadAllBytes(gameFile), File.ReadAllBytes(source.AbsolutePath), ms);
var data = ms.ToArray();
result.SourceDataID = _compiler.IncludeFile(data);
Utils.Log($"Generated a {data.Length} byte patch for {filename}");

24
Wabbajack.Lib/MO2Compiler.cs
View File

@ -181,7 +181,7 @@ namespace Wabbajack.Lib
}
ModMetas = Directory.EnumerateDirectories(Path.Combine(MO2Folder, "mods"))
ModMetas = Directory.EnumerateDirectories(Path.Combine(MO2Folder, Consts.MO2ModFolderName))
.Keep(f =>
{
var path = Path.Combine(f, "meta.ini");
@ -222,7 +222,7 @@ namespace Wabbajack.Lib
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Loading INIs");
ModInis = Directory.EnumerateDirectories(Path.Combine(MO2Folder, "mods"))
ModInis = Directory.EnumerateDirectories(Path.Combine(MO2Folder, Consts.MO2ModFolderName))
.Select(f =>
{
var modName = Path.GetFileName(f);
@ -427,20 +427,18 @@ namespace Wabbajack.Lib
var byPath = files.GroupBy(f => string.Join("|", f.FilesInFullPath.Skip(1).Select(i => i.Name)))
.ToDictionary(f => f.Key, f => f.First());
// Now Create the patches
await group.PMap(Queue, entry =>
await group.PMap(Queue, async entry =>
{
Info($"Patching {entry.To}");
Status($"Patching {entry.To}");
using (var origin = byPath[string.Join("|", entry.ArchiveHashPath.Skip(1))].OpenRead())
using (var output = new MemoryStream())
{
var a = origin.ReadAll();
var b = LoadDataForTo(entry.To, absolutePaths);
Utils.CreatePatch(a, b, output);
entry.PatchID = IncludeFile(output.ToArray());
var fileSize = File.GetSize(Path.Combine(ModListOutputFolder, entry.PatchID));
Info($"Patch size {fileSize} for {entry.To}");
}
await using var origin = byPath[string.Join("|", entry.ArchiveHashPath.Skip(1))].OpenRead();
await using var output = new MemoryStream();
var a = origin.ReadAll();
var b = LoadDataForTo(entry.To, absolutePaths);
await Utils.CreatePatch(a, b, output);
entry.PatchID = IncludeFile(output.ToArray());
var fileSize = File.GetSize(Path.Combine(ModListOutputFolder, entry.PatchID));
Info($"Patch size {fileSize} for {entry.To}");
});
}
}

4
Wabbajack.Lib/MO2Installer.cs
View File

@ -73,7 +73,7 @@ namespace Wabbajack.Lib
Directory.CreateDirectory(OutputFolder);
Directory.CreateDirectory(DownloadFolder);
if (Directory.Exists(Path.Combine(OutputFolder, "mods")) && WarnOnOverwrite)
if (Directory.Exists(Path.Combine(OutputFolder, Consts.MO2ModFolderName)) && WarnOnOverwrite)
{
if ((await Utils.Log(new ConfirmUpdateOfExistingInstall { ModListName = ModList.Name, OutputFolder = OutputFolder }).Task) == ConfirmUpdateOfExistingInstall.Choice.Abort)
{
@ -175,7 +175,7 @@ namespace Wabbajack.Lib
data.Coll.Do(keyData =>
{
var v = keyData.Value;
var mod = Path.Combine(OutputFolder, "mods", v);
var mod = Path.Combine(OutputFolder, Consts.MO2ModFolderName, v);
if (!Directory.Exists(mod))
Directory.CreateDirectory(mod);

2
Wabbajack.Lib/VortexCompiler.cs
View File

@ -492,7 +492,7 @@ namespace Wabbajack.Lib
{
vortexFolderPath = vortexFolderPath ?? TypicalVortexFolder();
var gameName = game.MetaData().NexusName;
return Path.Combine(vortexFolderPath, gameName, "mods");
return Path.Combine(vortexFolderPath, gameName, Consts.MO2ModFolderName);
}
public static IErrorResponse IsValidBaseDownloadsFolder(string path)

6
Wabbajack.Lib/zEditIntegration.cs
View File

@ -67,7 +67,7 @@ namespace Wabbajack.Lib
_mergesIndexed =
merges.ToDictionary(
m => Path.Combine(_mo2Compiler.MO2Folder, "mods", m.Key.name, m.Key.filename),
m => Path.Combine(_mo2Compiler.MO2Folder, Consts.MO2ModFolderName, m.Key.name, m.Key.filename),
m => m.First());
}
@ -103,9 +103,9 @@ namespace Wabbajack.Lib
var dst_data = File.ReadAllBytes(source.AbsolutePath);
using (var ms = new MemoryStream())
await using (var ms = new MemoryStream())
{
Utils.CreatePatch(src_data, dst_data, ms);
await Utils.CreatePatch(src_data, dst_data, ms);
result.PatchID = _compiler.IncludeFile(ms.ToArray());
}

30
Wabbajack.Test/EndToEndTests.cs
View File

@ -61,11 +61,24 @@ namespace Wabbajack.Test
"directURL=https://github.com/ModOrganizer2/modorganizer/releases/download/v2.2.1/Mod.Organizer.2.2.1.7z"
});
await DownloadAndInstall(Game.SkyrimSpecialEdition, 12604, "SkyUI");
await DownloadAndInstall(Game.Fallout4, 11925, "Anti-Tank Rifle");
var modfiles = await Task.WhenAll(
DownloadAndInstall(Game.SkyrimSpecialEdition, 12604, "SkyUI"),
DownloadAndInstall(Game.Fallout4, 11925, "Anti-Tank Rifle"),
DownloadAndInstall(Game.SkyrimSpecialEdition, 4783, "Frost Armor UNP"),
DownloadAndInstall(Game.SkyrimSpecialEdition, 32359, "Frost Armor HDT"));
// We're going to fully patch this mod from another source.
File.Delete(modfiles[3].Download);
utils.Configure();
File.WriteAllLines(Path.Combine(modfiles[3].ModFolder, "meta.ini"), new []
{
"[General]",
$"matchAll= {Path.GetFileName(modfiles[2].Download)}"
});
var modlist = await CompileAndInstall(profile);
utils.VerifyAllFiles();
@ -97,18 +110,19 @@ namespace Wabbajack.Test
Directory.CreateDirectory(utils.DownloadsFolder);
}
File.Copy(src, Path.Combine(utils.DownloadsFolder, filename));
await Utils.CopyFileAsync(src, Path.Combine(utils.DownloadsFolder, filename));
await FileExtractor.ExtractAll(Queue, src,
mod_name == null ? utils.MO2Folder : Path.Combine(utils.ModsFolder, mod_name));
}
private async Task DownloadAndInstall(Game game, int modid, string mod_name)
private async Task<(string Download, string ModFolder)> DownloadAndInstall(Game game, int modid, string mod_name)
{
utils.AddMod(mod_name);
var client = await NexusApiClient.Get();
var resp = await client.GetModFiles(game, modid);
var file = resp.files.First(f => f.is_primary);
var file = resp.files.FirstOrDefault(f => f.is_primary) ?? resp.files.FirstOrDefault(f => !string.IsNullOrEmpty(f.category_name));
var src = Path.Combine(DOWNLOAD_FOLDER, file.file_name);
var ini = string.Join("\n",
@ -133,11 +147,13 @@ namespace Wabbajack.Test
}
var dest = Path.Combine(utils.DownloadsFolder, file.file_name);
File.Copy(src, dest);
await Utils.CopyFileAsync(src, dest);
await FileExtractor.ExtractAll(Queue, src, Path.Combine(utils.ModsFolder, mod_name));
var modFolder = Path.Combine(utils.ModsFolder, mod_name);
await FileExtractor.ExtractAll(Queue, src, modFolder);
File.WriteAllText(dest + Consts.MetaFileExtension, ini);
return (dest, modFolder);
}
private async Task<ModList> CompileAndInstall(string profile)

2
Wabbajack.Test/SanityTests.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Compression.BSA;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
using Wabbajack.Lib;
@ -254,6 +255,5 @@ namespace Wabbajack.Test
Assert.IsNotNull(directive);
Assert.IsInstanceOfType(directive, typeof(PatchedFromArchive));
}
}
}

23
Wabbajack.Test/TestUtils.cs
View File

@ -32,7 +32,7 @@ namespace Wabbajack.Test
public string GameFolder => Path.Combine(WorkingDirectory, ID, "game_folder");
public string MO2Folder => Path.Combine(WorkingDirectory, ID, "mo2_folder");
public string ModsFolder => Path.Combine(MO2Folder, "mods");
public string ModsFolder => Path.Combine(MO2Folder, Consts.MO2ModFolderName);
public string DownloadsFolder => Path.Combine(MO2Folder, "downloads");
public string InstallFolder => Path.Combine(TestFolder, "installed");
@ -71,12 +71,15 @@ namespace Wabbajack.Test
public string AddMod(string name = null)
{
string mod_name = name ?? RandomName();
var mod_folder = Path.Combine(MO2Folder, "mods", mod_name);
Directory.CreateDirectory(mod_folder);
File.WriteAllText(Path.Combine(mod_folder, "meta.ini"), "[General]");
Mods.Add(mod_name);
return mod_name;
lock (this)
{
string mod_name = name ?? RandomName();
var mod_folder = Path.Combine(MO2Folder, Consts.MO2ModFolderName, mod_name);
Directory.CreateDirectory(mod_folder);
File.WriteAllText(Path.Combine(mod_folder, "meta.ini"), "[General]");
Mods.Add(mod_name);
return mod_name;
}
}
/// <summary>
@ -160,10 +163,10 @@ namespace Wabbajack.Test
public void VerifyInstalledFile(string mod, string file)
{
var src = Path.Combine(MO2Folder, "mods", mod, file);
var src = Path.Combine(MO2Folder, Consts.MO2ModFolderName, mod, file);
Assert.IsTrue(File.Exists(src), src);
var dest = Path.Combine(InstallFolder, "mods", mod, file);
var dest = Path.Combine(InstallFolder, Consts.MO2ModFolderName, mod, file);
Assert.IsTrue(File.Exists(dest), dest);
var src_data = File.ReadAllBytes(src);
@ -199,7 +202,7 @@ namespace Wabbajack.Test
}
public string PathOfInstalledFile(string mod, string file)
{
return Path.Combine(InstallFolder, "mods", mod, file);
return Path.Combine(InstallFolder, Consts.MO2ModFolderName, mod, file);
}
public void VerifyAllFiles()

6
Wabbajack.Test/ZEditIntegrationTests.cs
View File

@ -38,17 +38,17 @@ namespace Wabbajack.Test
new zEditIntegration.zEditMergePlugin()
{
filename = "srca.esp",
dataFolder = Path.Combine(utils.MO2Folder, "mods", moda)
dataFolder = Path.Combine(utils.MO2Folder, Consts.MO2ModFolderName, moda)
},
new zEditIntegration.zEditMergePlugin()
{
filename = "srcb.esp",
dataFolder = Path.Combine(utils.MO2Folder, "mods", moda),
dataFolder = Path.Combine(utils.MO2Folder, Consts.MO2ModFolderName, moda),
},
new zEditIntegration.zEditMergePlugin()
{
filename = "srcc.esp",
dataFolder = Path.Combine(utils.MO2Folder, "mods", modb),
dataFolder = Path.Combine(utils.MO2Folder, Consts.MO2ModFolderName, modb),
}
}
}