Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

many improvements to accelerate compilation times

Browse Source
This commit is contained in:
Timothy Baldridge 2019-09-12 17:44:35 -06:00
parent 9347b99363
commit 3e6dd46803
15 changed files with 400 additions and 287 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -7,6 +7,10 @@
* Fixed a integer overflow resulting in a crash in very large BSA reading
* Fix a bug in BSA string encoding
* Add human friendly filesizes to the download header and file info sections in the Install Report
* Improve compilation times by caching BSDiff patches
* Detect when VFS root folders don't exist
* Only reauth against the Nexus every 3 days (instead of 12 hours)
* Optimize executable patching by switching to .NET serialization and LZ4 compression
#### Version 0.9.1 - 9/5/2019
* Fixed a bug where having only one profile selected would result in no profiles being selected

1
Compression.BSA/BSAReader.cs
View File

@ -146,6 +146,7 @@ namespace Compression.BSA
{
_fileName = filename;
}
public BSAReader(Stream stream)

3
VirtualFileSystem/VirtualFileSystem.cs
View File

@ -272,6 +272,7 @@ namespace VFS
/// <param name="path"></param>
public void AddRoot(string path)
{
if (!Directory.Exists(path)) return;
IndexPath(path);
RefreshIndexes();
}
@ -615,7 +616,7 @@ namespace VFS
}
/// <summary>
/// Calulate the file's SHA, size and last modified
/// Calculate the file's SHA, size and last modified
/// </summary>
internal void Analyze()
{

40
Wabbajack.Common/FileExtractor.cs
View File

@ -63,25 +63,33 @@ namespace Wabbajack.Common
private static void ExtractAllWithBSA(string source, string dest)
{
using (var arch = new BSAReader(source))
try
{
arch.Files.PMap(f =>
using (var arch = new BSAReader(source))
{
var path = f.Path;
if (f.Path.StartsWith("\\"))
path = f.Path.Substring(1);
Utils.Status($"Extracting {path}");
var out_path = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(out_path);
if (!Directory.Exists(parent))
Directory.CreateDirectory(parent);
using (var fs = File.OpenWrite(out_path))
arch.Files.PMap(f =>
{
f.CopyDataTo(fs);
}
});
var path = f.Path;
if (f.Path.StartsWith("\\"))
path = f.Path.Substring(1);
Utils.Status($"Extracting {path}");
var out_path = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(out_path);
if (!Directory.Exists(parent))
Directory.CreateDirectory(parent);
using (var fs = File.OpenWrite(out_path))
{
f.CopyDataTo(fs);
}
});
}
}
catch (Exception ex)
{
Utils.Log($"While Extracting {source}");
throw ex;
}
}

40
Wabbajack.Common/Utils.cs
View File

@ -312,7 +312,6 @@ namespace Wabbajack.Common
f(i);
return false;
});
return;
}
public static void DoProgress<T>(this IEnumerable<T> coll, String msg, Action<T> f)
@ -417,5 +416,44 @@ namespace Wabbajack.Common
return (Math.Sign(byteCount) * num).ToString() + Suffix[place];
}
public static void CreatePatch(byte[] a, byte[] b, Stream output)
{
var data_a = a.SHA256().FromBase64().ToHEX();
var data_b = b.SHA256().FromBase64().ToHEX();
var cache_file = Path.Combine("patch_cache", $"{data_a}_{data_b}.patch");
if (!Directory.Exists("patch_cache"))
Directory.CreateDirectory("patch_cache");
while (true)
{
if (File.Exists(cache_file))
{
using (var f = File.OpenRead(cache_file))
{
f.CopyTo(output);
}
}
else
{
var tmp_name = Path.Combine("patch_cache", Guid.NewGuid() + ".tmp");
using (var f = File.OpenWrite(tmp_name))
{
BSDiff.Create(a, b, f);
}
File.Move(tmp_name, cache_file);
continue;
}
break;
}
}
public static void TryGetPatch(string foundHash, string fileHash, out string ePatch)
{
var patch_name = Path.Combine("patch_cache", $"{foundHash.FromBase64().ToHEX()}_{fileHash.FromBase64().ToHEX()}.patch");
ePatch = File.Exists(patch_name) ? File.ReadAllBytes(patch_name).ToBase64() : null;
}
}
}

6
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -68,10 +68,16 @@
<Reference Include="Newtonsoft.Json.Bson, Version=1.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.Bson.1.0.2\lib\net45\Newtonsoft.Json.Bson.dll</HintPath>
</Reference>
<Reference Include="protobuf-net, Version=2.4.0.0, Culture=neutral, PublicKeyToken=257b51d87d2e4d67, processorArchitecture=MSIL">
<HintPath>..\packages\protobuf-net.2.4.0\lib\net40\protobuf-net.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Configuration" />
<Reference Include="System.Core" />
<Reference Include="System.IO.Compression" />
<Reference Include="System.Numerics" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.ServiceModel" />
<Reference Include="System.Transactions" />
<Reference Include="System.Xml.Linq" />
<Reference Include="Microsoft.CSharp" />

1
Wabbajack.Common/packages.config
View File

@ -5,6 +5,7 @@
<package id="murmurhash" version="1.0.3" targetFramework="net472" />
<package id="Newtonsoft.Json" version="12.0.2" targetFramework="net472" />
<package id="Newtonsoft.Json.Bson" version="1.0.2" targetFramework="net472" />
<package id="protobuf-net" version="2.4.0" targetFramework="net472" />
<package id="SharpZipLib" version="1.1.0" targetFramework="net472" />
<package id="System.Runtime.Numerics" version="4.3.0" targetFramework="net472" />
</packages>

4
Wabbajack/AppState.cs
View File

@ -215,9 +215,9 @@ namespace Wabbajack
}
}
internal void ConfigureForInstall(string modlist)
internal void ConfigureForInstall(ModList modlist)
{
_modList = modlist.FromJSONString<ModList>();
_modList = modlist;
Mode = "Installing";
ModListName = _modList.Name;
HTMLReport = _modList.ReportHTML;

488
Wabbajack/Compiler.cs
View File

File diff suppressed because it is too large Load Diff

36
Wabbajack/Data.cs
View File

@ -1,9 +1,6 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using VFS;
namespace Wabbajack
@ -44,20 +41,13 @@ namespace Wabbajack
}
}
[Serializable]
public class ModList
{
/// <summary>
/// Name of the ModList
/// </summary>
public string Name;
/// <summary>
/// Author of the Mod List
/// </summary>
public string Author;
/// <summary>
/// Version of this Mod List
/// </summary>
public string Version;
/// <summary>
/// Install directives
@ -75,6 +65,7 @@ namespace Wabbajack
public string ReportHTML;
}
[Serializable]
public class Directive
{
/// <summary>
@ -83,16 +74,19 @@ namespace Wabbajack
public string To;
}
[Serializable]
public class IgnoredDirectly : Directive
{
public string Reason;
}
[Serializable]
public class NoMatch : IgnoredDirectly
{
}
[Serializable]
public class InlineFile : Directive
{
/// <summary>
@ -101,6 +95,7 @@ namespace Wabbajack
public string SourceData;
}
[Serializable]
public class CleanedESM : InlineFile
{
public string SourceESMHash;
@ -109,10 +104,12 @@ namespace Wabbajack
/// <summary>
/// A file that has the game and MO2 folders remapped on installation
/// </summary>
[Serializable]
public class RemappedInlineFile : InlineFile
{
}
[Serializable]
public class FromArchive : Directive
{
/// <summary>
@ -121,9 +118,11 @@ namespace Wabbajack
public string[] ArchiveHashPath;
[JsonIgnore]
[NonSerialized]
public VirtualFile FromFile;
private string _fullPath = null;
[JsonIgnore]
public string FullPath
{
@ -137,6 +136,7 @@ namespace Wabbajack
}
}
[Serializable]
public class CreateBSA : Directive
{
public string TempID;
@ -150,6 +150,7 @@ namespace Wabbajack
public uint ArchiveFlags { get; set; }
}
[Serializable]
public class PatchedFromArchive : FromArchive
{
/// <summary>
@ -158,6 +159,7 @@ namespace Wabbajack
public string Patch;
}
[Serializable]
public class Archive
{
/// <summary>
@ -176,6 +178,7 @@ namespace Wabbajack
public long Size;
}
[Serializable]
public class NexusMod : Archive
{
public string GameName;
@ -187,6 +190,7 @@ namespace Wabbajack
public string Author;
}
[Serializable]
public class GoogleDriveMod : Archive
{
public string Id;
@ -195,6 +199,7 @@ namespace Wabbajack
/// <summary>
/// URL that can be downloaded directly without any additional options
/// </summary>
[Serializable]
public class DirectURLArchive : Archive
{
public string URL;
@ -206,6 +211,7 @@ namespace Wabbajack
/// <summary>
/// A URL that cannot be downloaded automatically and has to be downloaded by hand
/// </summary>
[Serializable]
public class ManualURLArchive : Archive
{
public string URL;
@ -214,6 +220,7 @@ namespace Wabbajack
/// <summary>
/// An archive that requires additional HTTP headers.
/// </summary>
[Serializable]
public class DirectURLArchiveEx : DirectURLArchive
{
public Dictionary<string, string> Headers;
@ -222,6 +229,7 @@ namespace Wabbajack
/// <summary>
/// Archive that comes from MEGA
/// </summary>
[Serializable]
public class MEGAArchive : DirectURLArchive
{
}
@ -229,6 +237,7 @@ namespace Wabbajack
/// <summary>
/// Archive that comes from MODDB
/// </summary>
[Serializable]
public class MODDBArchive : DirectURLArchive
{
}
@ -236,10 +245,12 @@ namespace Wabbajack
/// <summary>
/// Archive that comes from MediaFire
/// </summary>
[Serializable]
public class MediaFireArchive : DirectURLArchive
{
}
[Serializable]
public class IndexedArchive
{
public dynamic IniData;
@ -251,6 +262,7 @@ namespace Wabbajack
/// <summary>
/// A archive entry
/// </summary>
[Serializable]
public class IndexedEntry
{
/// <summary>
@ -267,6 +279,7 @@ namespace Wabbajack
public long Size;
}
[Serializable]
public class IndexedArchiveEntry : IndexedEntry
{
public string[] HashPath;
@ -275,6 +288,7 @@ namespace Wabbajack
/// <summary>
/// Data found inside a BSA file in an archive
/// </summary>
[Serializable]
public class BSAIndexedEntry : IndexedEntry
{
/// <summary>

19
Wabbajack/Installer.cs
View File

@ -7,10 +7,14 @@ using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Runtime.Serialization;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Windows;
using K4os.Compression.LZ4.Encoders;
using K4os.Compression.LZ4.Streams;
using VFS;
using Wabbajack.Common;
@ -630,7 +634,7 @@ namespace Wabbajack
return HashArchive(e);
}
public static string CheckForModPack()
public static ModList CheckForModPack()
{
Utils.Log("Looking for attached modlist");
using (var s = File.OpenRead(Assembly.GetExecutingAssembly().Location))
@ -650,13 +654,14 @@ namespace Wabbajack
s.Position = s.Length - magic_bytes.Length - 8;
var start_pos = br.ReadInt64();
s.Position = start_pos;
long length = br.ReadInt64();
Utils.Log("Modlist found, loading...");
var list = br.ReadBytes((int)length).BZip2String();
Utils.Log("Modlist loaded.");
return list;
using (var dc = LZ4Stream.Decode(br.BaseStream, leaveOpen: true))
{
IFormatter formatter = new BinaryFormatter();
var list = formatter.Deserialize(dc);
Utils.Log("Modlist loaded.");
return (ModList)list;
}
}
}
}

2
Wabbajack/MainWindow.xaml.cs
View File

@ -68,7 +68,7 @@ namespace Wabbajack
var modlist = compiler.ModList.ToJSON();
compiler = null;
context.ConfigureForInstall(modlist);
//context.ConfigureForInstall(modlist);
}).Start();
}

2
Wabbajack/NexusAPI.cs
View File

@ -19,7 +19,7 @@ namespace Wabbajack
public static string GetNexusAPIKey()
{
FileInfo fi = new FileInfo("nexus.key_cache");
if (fi.Exists && fi.LastWriteTime > DateTime.Now.AddHours(-12))
if (fi.Exists && fi.LastWriteTime > DateTime.Now.AddHours(-72))
{
return File.ReadAllText("nexus.key_cache");
}

31
Wabbajack/Wabbajack.csproj
View File

@ -102,12 +102,27 @@
<Reference Include="Costura, Version=4.0.0.0, Culture=neutral, PublicKeyToken=9919ef960d84173d, processorArchitecture=MSIL">
<HintPath>..\packages\Costura.Fody.4.0.0\lib\net40\Costura.dll</HintPath>
</Reference>
<Reference Include="ICSharpCode.SharpZipLib, Version=1.2.0.246, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
<HintPath>..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll</HintPath>
</Reference>
<Reference Include="K4os.Compression.LZ4, Version=1.1.11.0, Culture=neutral, PublicKeyToken=2186fa9121ef231d, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Compression.LZ4.1.1.11\lib\net46\K4os.Compression.LZ4.dll</HintPath>
</Reference>
<Reference Include="K4os.Compression.LZ4.Streams, Version=1.1.11.0, Culture=neutral, PublicKeyToken=2186fa9121ef231d, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Compression.LZ4.Streams.1.1.11\lib\net46\K4os.Compression.LZ4.Streams.dll</HintPath>
</Reference>
<Reference Include="K4os.Hash.xxHash, Version=1.0.6.0, Culture=neutral, PublicKeyToken=32cd54395057cec3, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Hash.xxHash.1.0.6\lib\net46\K4os.Hash.xxHash.dll</HintPath>
</Reference>
<Reference Include="MegaApiClient, Version=1.7.1.495, Culture=neutral, PublicKeyToken=0480d311efbeb4e2, processorArchitecture=MSIL">
<HintPath>..\packages\MegaApiClient.1.7.1\lib\net46\MegaApiClient.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json.Bson, Version=1.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.Bson.1.0.2\lib\net45\Newtonsoft.Json.Bson.dll</HintPath>
</Reference>
<Reference Include="Ookii.Dialogs.Wpf, Version=1.0.0.0, Culture=neutral, PublicKeyToken=66aa232afad40158, processorArchitecture=MSIL">
<HintPath>..\packages\Ookii.Dialogs.Wpf.1.1.0\lib\net45\Ookii.Dialogs.Wpf.dll</HintPath>
</Reference>
@ -115,10 +130,26 @@
<HintPath>..\packages\SharpCompress.0.23.0\lib\net45\SharpCompress.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Buffers, Version=4.0.2.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Buffers.4.4.0\lib\netstandard2.0\System.Buffers.dll</HintPath>
</Reference>
<Reference Include="System.Configuration" />
<Reference Include="System.Data" />
<Reference Include="System.Design" />
<Reference Include="System.Drawing" />
<Reference Include="System.Memory, Version=4.0.1.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Memory.4.5.3\lib\netstandard2.0\System.Memory.dll</HintPath>
</Reference>
<Reference Include="System.Numerics" />
<Reference Include="System.Numerics.Vectors, Version=4.1.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll</HintPath>
</Reference>
<Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
</Reference>
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.Security" />
<Reference Include="System.ServiceModel" />
<Reference Include="System.Web" />
<Reference Include="System.Windows.Forms" />
<Reference Include="System.Xml" />

10
Wabbajack/packages.config
View File

@ -4,10 +4,20 @@
<package id="Costura.Fody" version="4.0.0" targetFramework="net472" />
<package id="Fody" version="5.1.1" targetFramework="net472" developmentDependency="true" />
<package id="GitInfo" version="2.0.20" targetFramework="net472" developmentDependency="true" />
<package id="K4os.Compression.LZ4" version="1.1.11" targetFramework="net472" />
<package id="K4os.Compression.LZ4.Streams" version="1.1.11" targetFramework="net472" />
<package id="K4os.Hash.xxHash" version="1.0.6" targetFramework="net472" />
<package id="MegaApiClient" version="1.7.1" targetFramework="net472" />
<package id="Newtonsoft.Json" version="12.0.2" targetFramework="net472" />
<package id="Newtonsoft.Json.Bson" version="1.0.2" targetFramework="net472" />
<package id="Ookii.Dialogs.Wpf" version="1.1.0" targetFramework="net472" />
<package id="protobuf-net" version="2.4.0" targetFramework="net472" />
<package id="SharpCompress" version="0.23.0" targetFramework="net472" />
<package id="SharpZipLib" version="1.2.0" targetFramework="net472" />
<package id="System.Buffers" version="4.4.0" targetFramework="net472" />
<package id="System.Memory" version="4.5.3" targetFramework="net472" />
<package id="System.Numerics.Vectors" version="4.4.0" targetFramework="net472" />
<package id="System.Runtime.CompilerServices.Unsafe" version="4.5.2" targetFramework="net472" />
<package id="WebSocketSharpFork" version="1.0.4.0" targetFramework="net472" />
<package id="WPFThemes.DarkBlend" version="1.0.8" targetFramework="net472" />
</packages>