Merge Master

This commit is contained in:
Timothy Baldridge
2020-04-04 16:06:14 -06:00
12 changed files with 210 additions and 88 deletions

View File

@ -126,6 +126,10 @@ namespace Wabbajack.Common
public static AbsolutePath SettingsFile => LocalAppDataPath.Combine("settings.json"); public static AbsolutePath SettingsFile => LocalAppDataPath.Combine("settings.json");
public static RelativePath SettingsIni = (RelativePath)"settings.ini"; public static RelativePath SettingsIni = (RelativePath)"settings.ini";
public static byte SettingsVersion => 1; public static byte SettingsVersion => 1;
public static Extension SeqExtension = new Extension(".seq");
public static RelativePath SettingsJson = (RelativePath)"settings.json";
public static Extension TempExtension = new Extension(".temp"); public static Extension TempExtension = new Extension(".temp");
public static Extension OctoSig = new Extension(".octo_sig"); public static Extension OctoSig = new Extension(".octo_sig");

View File

@ -105,6 +105,9 @@ namespace Wabbajack.Lib
{ {
if (a.State is IMetaState metaState) if (a.State is IMetaState metaState)
{ {
if (metaState.URL == null)
return;
var b = await metaState.LoadMetaData(); var b = await metaState.LoadMetaData();
Utils.Log(b Utils.Log(b
? $"Getting meta data for {a.Name} was successful!" ? $"Getting meta data for {a.Name} was successful!"

View File

@ -40,8 +40,7 @@ namespace Wabbajack.Lib.CompilationSteps
return new State(); return new State();
} }
public static bool IsAlwaysEnabled(dynamic data)
private static bool IsAlwaysEnabled(dynamic data)
{ {
if (data == null) if (data == null)
return false; return false;

View File

@ -11,7 +11,7 @@ namespace Wabbajack.Lib.CompilationSteps
public class IncludeThisProfile : ACompilationStep public class IncludeThisProfile : ACompilationStep
{ {
private readonly IEnumerable<AbsolutePath> _correctProfiles; private readonly IEnumerable<AbsolutePath> _correctProfiles;
private readonly MO2Compiler _mo2Compiler; private MO2Compiler _mo2Compiler;
public IncludeThisProfile(ACompiler compiler) : base(compiler) public IncludeThisProfile(ACompiler compiler) : base(compiler)
{ {
@ -21,8 +21,9 @@ namespace Wabbajack.Lib.CompilationSteps
public override async ValueTask<Directive> Run(RawSourceFile source) public override async ValueTask<Directive> Run(RawSourceFile source)
{ {
if (_correctProfiles.Any(p => source.AbsolutePath.InFolder(p))) if (!_correctProfiles.Any(p => source.AbsolutePath.InFolder(p)))
{ return null;
var data = source.Path.FileName == Consts.ModListTxt var data = source.Path.FileName == Consts.ModListTxt
? await ReadAndCleanModlist(source.AbsolutePath) ? await ReadAndCleanModlist(source.AbsolutePath)
: await source.AbsolutePath.ReadAllBytesAsync(); : await source.AbsolutePath.ReadAllBytesAsync();
@ -30,9 +31,7 @@ namespace Wabbajack.Lib.CompilationSteps
var e = source.EvolveTo<InlineFile>(); var e = source.EvolveTo<InlineFile>();
e.SourceDataID = await _compiler.IncludeFile(data); e.SourceDataID = await _compiler.IncludeFile(data);
return e; return e;
}
return null;
} }
public override IState GetState() public override IState GetState()
@ -43,8 +42,7 @@ namespace Wabbajack.Lib.CompilationSteps
private static async Task<byte[]> ReadAndCleanModlist(AbsolutePath absolutePath) private static async Task<byte[]> ReadAndCleanModlist(AbsolutePath absolutePath)
{ {
var lines = await absolutePath.ReadAllLinesAsync(); var lines = await absolutePath.ReadAllLinesAsync();
lines = lines.Where(line => !(line.StartsWith("-") && !line.EndsWith("_separator"))) lines = lines.Where(line => !(line.StartsWith("-") && !line.EndsWith("_separator"))).ToArray();
.ToArray();
return Encoding.UTF8.GetBytes(string.Join("\r\n", lines)); return Encoding.UTF8.GetBytes(string.Join("\r\n", lines));
} }

View File

@ -31,6 +31,15 @@ namespace Wabbajack.Lib.Downloaders
var absolute = true; var absolute = true;
if (url == null || url.Host != SiteURL.Host) return null; if (url == null || url.Host != SiteURL.Host) return null;
if (url.PathAndQuery.StartsWith("/applications/core/interface/file/attachment"))
{
return new TState
{
IsAttachment = true,
FullURL = url.ToString()
};
}
if (url.PathAndQuery.StartsWith("/index.php?")) if (url.PathAndQuery.StartsWith("/index.php?"))
{ {
var id2 = HttpUtility.ParseQueryString(url.Query)["r"]; var id2 = HttpUtility.ParseQueryString(url.Query)["r"];
@ -38,6 +47,7 @@ namespace Wabbajack.Lib.Downloaders
var name = parsed[null].Split("/", StringSplitOptions.RemoveEmptyEntries).Last(); var name = parsed[null].Split("/", StringSplitOptions.RemoveEmptyEntries).Last();
return new TState return new TState
{ {
FullURL = url.AbsolutePath,
FileID = id2, FileID = id2,
FileName = name FileName = name
}; };
@ -57,6 +67,7 @@ namespace Wabbajack.Lib.Downloaders
return new TState return new TState
{ {
FullURL = url.AbsolutePath,
FileID = id, FileID = id,
FileName = file FileName = file
}; };
@ -67,26 +78,30 @@ namespace Wabbajack.Lib.Downloaders
public class State<TStateDownloader> : AbstractDownloadState, IMetaState where TStateDownloader : IDownloader public class State<TStateDownloader> : AbstractDownloadState, IMetaState where TStateDownloader : IDownloader
{ {
[Key(0)] [Key(0)]
public string FullURL { get; set; }
[Key(1)]
public bool IsAttachment { get; set; }
[Key(2)]
public string FileID { get; set; } public string FileID { get; set; }
[Key(1)] [Key(3)]
public string FileName { get; set; } public string FileName { get; set; }
// from IMetaState // from IMetaState
[Key(2)] [Key(4)]
public Uri URL => new Uri($"{Site}/files/file/{FileName}"); public Uri URL => new Uri($"{Site}/files/file/{FileName}");
[Key(3)]
public string Name { get; set; }
[Key(4)]
public string Author { get; set; }
[Key(5)] [Key(5)]
public string Version { get; set; } public string Name { get; set; }
[Key(6)] [Key(6)]
public string ImageURL { get; set; } public string Author { get; set; }
[Key(7)] [Key(7)]
public virtual bool IsNSFW { get; set; } public string Version { get; set; }
[Key(8)] [Key(8)]
public string ImageURL { get; set; }
[Key(9)]
public virtual bool IsNSFW { get; set; }
[Key(10)]
public string Description { get; set; } public string Description { get; set; }
private static bool IsHTTPS => Downloader.SiteURL.AbsolutePath.StartsWith("https://"); private static bool IsHTTPS => Downloader.SiteURL.AbsolutePath.StartsWith("https://");
@ -105,7 +120,9 @@ namespace Wabbajack.Lib.Downloaders
get get
{ {
return FileID == null return FileID == null
? new object[] {Downloader.SiteURL, FileName} ? IsAttachment
? new object[] {Downloader.SiteURL, IsAttachment, FullURL}
: new object[] {Downloader.SiteURL, FileName}
: new object[] {Downloader.SiteURL, FileName, FileID}; : new object[] {Downloader.SiteURL, FileName, FileID};
} }
} }
@ -127,13 +144,18 @@ namespace Wabbajack.Lib.Downloaders
private async Task<Stream> ResolveDownloadStream() private async Task<Stream> ResolveDownloadStream()
{ {
//var downloader = (AbstractNeedsLoginDownloader)(object)DownloadDispatcher.GetInstance<TDownloader>();
TOP: TOP:
var csrfurl = FileID == null string url;
if (IsAttachment)
{
url = FullURL;
}
else
{
var csrfURL = FileID == null
? $"{Site}/files/file/{FileName}/?do=download" ? $"{Site}/files/file/{FileName}/?do=download"
: $"{Site}/files/file/{FileName}/?do=download&r={FileID}"; : $"{Site}/files/file/{FileName}/?do=download&r={FileID}";
var html = await Downloader.AuthedClient.GetStringAsync(csrfurl); var html = await Downloader.AuthedClient.GetStringAsync(csrfURL);
var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])"); var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])");
var matches = pattern.Matches(html).Cast<Match>(); var matches = pattern.Matches(html).Cast<Match>();
@ -144,10 +166,10 @@ namespace Wabbajack.Lib.Downloaders
return null; return null;
var sep = Site.EndsWith("?") ? "&" : "?"; var sep = Site.EndsWith("?") ? "&" : "?";
var url = FileID == null url = FileID == null
? $"{Site}/files/file/{FileName}/{sep}do=download&confirm=1&t=1&csrfKey={csrfKey}" ? $"{Site}/files/file/{FileName}/{sep}do=download&confirm=1&t=1&csrfKey={csrfKey}"
: $"{Site}/files/file/{FileName}/{sep}do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}"; : $"{Site}/files/file/{FileName}/{sep}do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
}
var streamResult = await Downloader.AuthedClient.GetAsync(url); var streamResult = await Downloader.AuthedClient.GetAsync(url);
if (streamResult.StatusCode != HttpStatusCode.OK) if (streamResult.StatusCode != HttpStatusCode.OK)
@ -198,13 +220,17 @@ namespace Wabbajack.Lib.Downloaders
public override string GetManifestURL(Archive a) public override string GetManifestURL(Archive a)
{ {
return $"{Site}/files/file/{FileName}/?do=download&r={FileID}"; return IsAttachment ? FullURL : $"{Site}/files/file/{FileName}/?do=download&r={FileID}";
} }
public override string[] GetMetaIni() public override string[] GetMetaIni()
{ {
if (FileID != null) if (IsAttachment)
{ return new[] {"[General]", $"directURL={FullURL}"};
if (FileID == null)
return new[] {"[General]", $"directURL={Site}/files/file/{FileName}"};
if (Site.EndsWith("?")) if (Site.EndsWith("?"))
{ {
return new[] return new[]
@ -218,14 +244,9 @@ namespace Wabbajack.Lib.Downloaders
{ {
"[General]", $"directURL={Site}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1" "[General]", $"directURL={Site}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1"
}; };
} }
return new[]
{
"[General]",
$"directURL={Site}/files/file/{FileName}"
};
}
public virtual async Task<bool> LoadMetaData() public virtual async Task<bool> LoadMetaData()
{ {
return false; return false;

View File

@ -1,5 +1,4 @@
using Alphaleonis.Win32.Filesystem; using Alphaleonis.Win32.Filesystem;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
@ -37,25 +36,80 @@ namespace Wabbajack.Lib
public class IncludeZEditPatches : ACompilationStep public class IncludeZEditPatches : ACompilationStep
{ {
private Dictionary<AbsolutePath, zEditMerge> _mergesIndexed; private readonly Dictionary<AbsolutePath, zEditMerge> _mergesIndexed;
private bool _disabled = true;
public IncludeZEditPatches(ACompiler compiler) : base(compiler) public IncludeZEditPatches(ACompiler compiler) : base(compiler)
{ {
var zEditPath = FindzEditPath(compiler); var zEditPath = FindzEditPath(compiler);
var havezEdit = zEditPath != default; var havezEdit = zEditPath != default;
Utils.Log(havezEdit ? $"Found zEdit at {zEditPath}" : $"zEdit not detected, disabling zEdit routines"); Utils.Log(havezEdit ? $"Found zEdit at {zEditPath}" : "zEdit not detected, disabling zEdit routines");
if (!havezEdit) if (!havezEdit)
{ {
_mergesIndexed = new Dictionary<AbsolutePath, zEditMerge>(); _mergesIndexed = new Dictionary<AbsolutePath, zEditMerge>();
return; return;
} }
_mo2Compiler = (MO2Compiler) compiler;
var merges = zEditPath.Combine("profiles").EnumerateFiles() var settingsFiles = zEditPath.Combine("profiles").EnumerateFiles(false)
.Where(f => f.FileName == (RelativePath)"merges.json") .Where(f => f.IsFile)
.SelectMany(f => f.FromJSON<List<zEditMerge>>()) .Where(f => f.FileName == Consts.SettingsJson)
.GroupBy(f => (f.name, f.filename)); .Where(f =>
{
var settings = f.FromJSON<zEditSettings>();
if (settings.modManager != "Mod Organizer 2")
{
Utils.Log($"zEdit settings file {f}: modManager is not Mod Organizer 2 but {settings.modManager}!");
return false;
}
if (settings.managerPath != _mo2Compiler.MO2Folder)
{
Utils.Log($"zEdit settings file {f}: managerPath is not {_mo2Compiler.MO2Folder} but {settings.managerPath}!");
return false;
}
if (settings.modsPath != _mo2Compiler.MO2Folder.Combine(Consts.MO2ModFolderName))
{
Utils.Log($"zEdit settings file {f}: modsPath is not {_mo2Compiler.MO2Folder}\\{Consts.MO2ModFolderName} but {settings.modsPath}!");
return false;
}
if (settings.mergePath != _mo2Compiler.MO2Folder.Combine(Consts.MO2ModFolderName))
{
Utils.Log($"zEdit settings file {f}: modsPath is not {_mo2Compiler.MO2Folder}\\{Consts.MO2ModFolderName} but {settings.modsPath}!");
return false;
}
return true;
}).ToList();
if (!settingsFiles.Any())
{
Utils.Log($"Found not acceptable settings.json file for zEdit!");
return;
}
var profileFolder =
settingsFiles.Where(x => x.Parent.Combine("merges.json").IsFile)
.Select(x => x == default ? x : x.Parent)
.FirstOrDefault();
if (profileFolder == default)
{
Utils.Log("Found no acceptable profiles folder for zEdit!");
return;
}
var mergeFile = profileFolder.Combine("merges.json");
Utils.Log($"Using merge file {mergeFile}");
var merges = mergeFile.FromJSON<List<zEditMerge>>().GroupBy(f => (f.name, f.filename)).ToArray();
merges.Where(m => m.Count() > 1) merges.Where(m => m.Count() > 1)
.Do(m => .Do(m =>
@ -68,11 +122,33 @@ namespace Wabbajack.Lib
merges.ToDictionary( merges.ToDictionary(
m => _mo2Compiler.MO2Folder.Combine((string)Consts.MO2ModFolderName, m.Key.name, m.Key.filename), m => _mo2Compiler.MO2Folder.Combine((string)Consts.MO2ModFolderName, m.Key.name, m.Key.filename),
m => m.First()); m => m.First());
_disabled = false;
} }
public override async ValueTask<Directive> Run(RawSourceFile source) public override async ValueTask<Directive> Run(RawSourceFile source)
{ {
if (!_mergesIndexed.TryGetValue(source.AbsolutePath, out var merge)) return null; if (_disabled) return null;
if (!_mergesIndexed.TryGetValue(source.AbsolutePath, out var merge))
{
if(source.AbsolutePath.Extension != Consts.SeqExtension)
return null;
var seqFolder = source.AbsolutePath.Parent;
if (seqFolder.FileName != (RelativePath)"seq")
return null;
var mergeFolder = seqFolder.Parent;
var mergeName = mergeFolder.FileName;
if (!mergeFolder.Combine(mergeName + ".esp").Exists)
return null;
var inline = source.EvolveTo<InlineFile>();
inline.SourceDataID = await _compiler.IncludeFile(await source.AbsolutePath.ReadAllBytesAsync());
return inline;
}
var result = source.EvolveTo<MergedPatch>(); var result = source.EvolveTo<MergedPatch>();
result.Sources = merge.plugins.Select(f => result.Sources = merge.plugins.Select(f =>
{ {
@ -138,6 +214,14 @@ namespace Wabbajack.Lib
} }
} }
public class zEditSettings
{
public string modManager;
public AbsolutePath managerPath;
public AbsolutePath modsPath;
public AbsolutePath mergePath;
}
public class zEditMerge public class zEditMerge
{ {
public string name; public string name;
@ -154,19 +238,18 @@ namespace Wabbajack.Lib
public static void VerifyMerges(MO2Compiler compiler) public static void VerifyMerges(MO2Compiler compiler)
{ {
var by_name = compiler.InstallDirectives.ToDictionary(f => f.To); var byName = compiler.InstallDirectives.ToDictionary(f => f.To);
foreach (var directive in compiler.InstallDirectives.OfType<MergedPatch>()) foreach (var directive in compiler.InstallDirectives.OfType<MergedPatch>())
{ {
foreach (var source in directive.Sources) foreach (var source in directive.Sources)
{ {
if (by_name.TryGetValue(source.RelativePath, out var result)) if (!byName.TryGetValue(source.RelativePath, out var result))
{ throw new InvalidDataException(
$"{source.RelativePath} is needed for merged patch {directive.To} but is not included in the install.");
if (result.Hash != source.Hash) if (result.Hash != source.Hash)
throw new InvalidDataException($"Hashes for {result.To} needed for zEdit merge sources don't match, this shouldn't happen"); throw new InvalidDataException($"Hashes for {result.To} needed for zEdit merge sources don't match, this shouldn't happen");
continue;
}
throw new InvalidDataException($"{source.RelativePath} is needed for merged patch {directive.To} but is not included in the install.");
} }
} }
} }

View File

@ -27,6 +27,18 @@ namespace Wabbajack.Test
Directory.CreateDirectory(Path.Combine(utils.MO2Folder, "tools", "mator", "bleh", "profiles", "myprofile")); Directory.CreateDirectory(Path.Combine(utils.MO2Folder, "tools", "mator", "bleh", "profiles", "myprofile"));
var settings = new zEditIntegration.zEditSettings()
{
modManager = "Mod Organizer 2",
managerPath = utils.MO2Folder,
modsPath = Path.Combine(utils.MO2Folder, Consts.MO2ModFolderName),
mergePath = Path.Combine(utils.MO2Folder, Consts.MO2ModFolderName)
};
settings.ToJSON(Path.Combine(utils.MO2Folder, "tools", "mator", "bleh", "profiles", "myprofile",
"settings.json"));
new List<zEditIntegration.zEditMerge>() new List<zEditIntegration.zEditMerge>()
{ {
new zEditIntegration.zEditMerge() new zEditIntegration.zEditMerge()

View File

@ -231,14 +231,14 @@ namespace Wabbajack
{ {
if (Completed?.Failed ?? false) if (Completed?.Failed ?? false)
{ {
Process.Start("explorer.exe", (string)Utils.LogFolder); Process.Start("explorer.exe", $"/select,\"{Utils.LogFolder}\"");
} }
else else
{ {
Process.Start("explorer.exe", Process.Start("explorer.exe",
(string)(OutputLocation.TargetPath == default OutputLocation.TargetPath == default
? AbsolutePath.EntryPoint ? $"/select,\"{Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location)}\""
: OutputLocation.TargetPath)); : $"/select,\"{OutputLocation.TargetPath}\"");
} }
}); });

View File

@ -101,24 +101,26 @@ namespace Wabbajack
using var fs = ModListPath.OpenShared(); using var fs = ModListPath.OpenShared();
using var ar = new ZipArchive(fs, ZipArchiveMode.Read); using var ar = new ZipArchive(fs, ZipArchiveMode.Read);
using var ms = new MemoryStream(); using var ms = new MemoryStream();
var entry = ar.GetEntry(Readme); var entry = ar.GetEntry(Readme);
if (entry == null) if (entry == null)
{ {
Utils.Log($"Tried to open a non-existent readme: {Readme}"); Utils.Log($"Tried to open a non-existent readme: {Readme}");
return; return;
} }
using (var e = entry.Open()) using (var e = entry.Open())
{ {
e.CopyTo(ms); e.CopyTo(ms);
} }
ms.Seek(0, SeekOrigin.Begin); ms.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(ms))
{ using var reader = new StreamReader(ms);
var viewer = new TextViewer(reader.ReadToEnd(), Name); var viewer = new TextViewer(reader.ReadToEnd(), Name);
viewer.Show(); viewer.Show();
} }
} }
}
public override void Dispose() public override void Dispose()
{ {

View File

@ -89,6 +89,7 @@ namespace Wabbajack
return modList.SourceModList.Archives return modList.SourceModList.Archives
.Select(m => m.State) .Select(m => m.State)
.OfType<IMetaState>() .OfType<IMetaState>()
.Where(x => x.URL != default && x.ImageURL != default)
.DistinctBy(x => x.URL) .DistinctBy(x => x.URL)
// Shuffle it // Shuffle it
.Shuffle(_random) .Shuffle(_random)

View File

@ -1,14 +1,15 @@
<Window <mah:MetroWindow
x:Class="Wabbajack.TextViewer" x:Class="Wabbajack.TextViewer"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008" xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:local="clr-namespace:Wabbajack"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:local="clr-namespace:Wabbajack"
xmlns:mah="http://metro.mahapps.com/winfx/xaml/controls"
Title="TextViewer" Title="TextViewer"
Width="800" Width="800"
Height="450" Height="450"
Icon="../Resources/Icons/wabbajack.ico" WindowTitleBrush="{StaticResource MahApps.Brushes.Accent}"
Style="{StaticResource {x:Type Window}}" Style="{StaticResource {x:Type Window}}"
mc:Ignorable="d"> mc:Ignorable="d">
<Grid> <Grid>
@ -17,4 +18,4 @@
FontSize="20" FontSize="20"
TextWrapping="Wrap" /> TextWrapping="Wrap" />
</Grid> </Grid>
</Window> </mah:MetroWindow>

View File

@ -1,8 +1,6 @@
using System.Windows; namespace Wabbajack
namespace Wabbajack
{ {
public partial class TextViewer : Window public partial class TextViewer
{ {
public TextViewer(string text, string title) public TextViewer(string text, string title)
{ {