diff --git a/.gitignore b/.gitignore index 0ed7a9ec..02139e36 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,356 @@ /Wabbajack/obj/Debug /Wabbajack.Common/bin/Debug /Wabbajack.Common/obj/Debug + +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ diff --git a/Compression.BSA.Test/Program.cs b/Compression.BSA.Test/Program.cs index ecf26484..ad638ff4 100644 --- a/Compression.BSA.Test/Program.cs +++ b/Compression.BSA.Test/Program.cs @@ -9,11 +9,11 @@ namespace Compression.BSA.Test { class Program { - const string TestDir = "c:\\Mod Organizer 2\\mods"; + const string TestDir = "d:\\Personal YASHEd\\mods"; const string TempDir = "c:\\tmp\\out"; static void Main(string[] args) { - foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories)) + foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories).Skip(3)) { Console.WriteLine($"From {bsa}"); Console.WriteLine("Cleaning Output Dir"); @@ -84,7 +84,7 @@ namespace Compression.BSA.Test } } - //break; + break; } } } diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSABuilder.cs index 10210061..09f3f3e4 100644 --- a/Compression.BSA/BSABuilder.cs +++ b/Compression.BSA/BSABuilder.cs @@ -1,5 +1,4 @@ -using K4os.Compression.LZ4; -using K4os.Compression.LZ4.Streams; +using lz4; using System; using System.Collections.Generic; using System.IO; @@ -11,7 +10,6 @@ namespace Compression.BSA { public class BSABuilder : IDisposable { - internal LZ4Level _compressionLevel; internal byte[] _fileId; internal uint _version; internal uint _offset; @@ -27,7 +25,6 @@ namespace Compression.BSA public BSABuilder() { - _compressionLevel = LZ4Level.L10_OPT; _fileId = Encoding.ASCII.GetBytes("BSA\0"); _offset = 0x24; } @@ -299,6 +296,7 @@ namespace Compression.BSA internal ulong _hash; internal byte[] _nameBytes; internal byte[] _pathBytes; + private byte[] _pathBSBytes; internal byte[] _rawData; internal int _originalSize; private long _offsetOffset; @@ -311,6 +309,7 @@ namespace Compression.BSA _hash = _name.GetBSAHash(); _nameBytes = _name.ToTermString(); _pathBytes = _path.ToTermString(); + _pathBSBytes = _path.ToBSString(); _flipCompression = flipCompression; var ms = new MemoryStream(); @@ -328,8 +327,13 @@ namespace Compression.BSA if (_bsa.HeaderType == VersionType.SSE) { var r = new MemoryStream(); - using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings() { CompressionLevel = _bsa._compressionLevel })) - (new MemoryStream(_rawData)).CopyTo(w); + + using (var f = LZ4Stream.CreateCompressor(r, LZ4StreamMode.Write, LZ4FrameBlockMode.Independent, LZ4FrameBlockSize.Max4MB, LZ4FrameChecksumMode.Content, + highCompression: true, leaveInnerStreamOpen: true)) + { + + new MemoryStream(_rawData).CopyTo(f); + } _rawData = r.ToArray(); @@ -385,10 +389,19 @@ namespace Compression.BSA { wtr.Write(_hash); + var size = _rawData.Length; + if (_bsa.HasNameBlobs) + { + size += _pathBSBytes.Length; + } + if (Compressed) + { + size += 4; + } if (_flipCompression) - wtr.Write((uint)_rawData.Length | (0x1 << 30)); + wtr.Write((uint)size | (0x1 << 30)); else - wtr.Write((uint)_rawData.Length); + wtr.Write((uint)size); _offsetOffset = wtr.BaseStream.Position; wtr.Write((uint)0xDEADBEEF); @@ -405,7 +418,7 @@ namespace Compression.BSA { if (_bsa.HasNameBlobs) { - wtr.Write(_path.ToBSString()); + wtr.Write(_pathBSBytes); } wtr.Write((uint)_originalSize); wtr.Write(_rawData); @@ -414,7 +427,7 @@ namespace Compression.BSA { if (_bsa.HasNameBlobs) { - wtr.Write(_path.ToBSString()); + wtr.Write(_pathBSBytes); } wtr.Write(_rawData); } diff --git a/Compression.BSA/BSAReader.cs b/Compression.BSA/BSAReader.cs index dfc28496..cf7b3097 100644 --- a/Compression.BSA/BSAReader.cs +++ b/Compression.BSA/BSAReader.cs @@ -3,7 +3,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Text; -using K4os.Compression.LZ4.Streams; +using lz4; namespace Compression.BSA { @@ -359,10 +359,13 @@ namespace Compression.BSA } var original_size = rdr.ReadUInt32(); + file_size -= 4; if (_bsa.HeaderType == VersionType.SSE) { - var r = LZ4Stream.Decode(rdr.BaseStream); - r.CopyTo(output); + using (var dc = LZ4Stream.CreateDecompressor(output, LZ4StreamMode.Write, true)) + { + rdr.BaseStream.CopyToLimit(dc, file_size); + } } else { diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj index 69c3e49d..7a7ecddb 100644 --- a/Compression.BSA/Compression.BSA.csproj +++ b/Compression.BSA/Compression.BSA.csproj @@ -50,30 +50,12 @@ MinimumRecommendedRules.ruleset - - ..\packages\K4os.Compression.LZ4.1.1.11\lib\net46\K4os.Compression.LZ4.dll - - - ..\packages\K4os.Compression.LZ4.Streams.1.1.11\lib\net46\K4os.Compression.LZ4.Streams.dll - - - ..\packages\K4os.Hash.xxHash.1.0.6\lib\net46\K4os.Hash.xxHash.dll + + ..\packages\IonKiwi.lz4.net.1.0.12\lib\net472\lz4.AnyCPU.loader.dll - - ..\packages\System.Buffers.4.4.0\lib\netstandard2.0\System.Buffers.dll - - - ..\packages\System.Memory.4.5.3\lib\netstandard2.0\System.Memory.dll - - - ..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll - - - ..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll - diff --git a/Compression.BSA/Utils.cs b/Compression.BSA/Utils.cs index dd8cf209..6272c174 100644 --- a/Compression.BSA/Utils.cs +++ b/Compression.BSA/Utils.cs @@ -1,4 +1,5 @@ -using System; +using lz4.AnyCPU.loader; +using System; using System.Collections.Generic; using System.IO; using System.Text; @@ -8,6 +9,12 @@ namespace Compression.BSA internal static class Utils { private static Encoding Windows1251 = Encoding.GetEncoding(1251); + + static Utils () + { + LZ4Loader.DisableVCRuntimeDetection = true; + } + public static string ReadStringLen(this BinaryReader rdr) { var len = rdr.ReadByte(); diff --git a/Compression.BSA/packages.config b/Compression.BSA/packages.config index 7ad1ad47..2b4a3d68 100644 --- a/Compression.BSA/packages.config +++ b/Compression.BSA/packages.config @@ -1,11 +1,4 @@  - - - - - - - \ No newline at end of file diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 9854c621..47fcaf62 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -27,6 +27,8 @@ namespace Wabbajack.Common } } + public static string WABBAJACK_INCLUDE = "WABBAJACK_INCLUDE"; + public static String AppName = "Wabbajack"; } } diff --git a/Wabbajack.Common/DynamicIniData.cs b/Wabbajack.Common/DynamicIniData.cs index 13c63bf6..094630e9 100644 --- a/Wabbajack.Common/DynamicIniData.cs +++ b/Wabbajack.Common/DynamicIniData.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Dynamic; using System.Text; +using System.Text.RegularExpressions; namespace Wabbajack.Common { @@ -46,6 +47,10 @@ namespace Wabbajack.Common public override bool TryGetMember(GetMemberBinder binder, out object result) { result = _coll[binder.Name]; + if (result is string) + { + result = Regex.Unescape(((string)result).Trim('"')); + } return true; } } diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index fd4d9fe6..2e9658a4 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -166,8 +166,8 @@ namespace Wabbajack.Common public static List PMap(this IEnumerable coll, Func f) { var colllst = coll.ToList(); - WorkQueue.MaxQueueSize = colllst.Count; - WorkQueue.CurrentQueueSize = 0; + Interlocked.Add(ref WorkQueue.MaxQueueSize, colllst.Count); + //WorkQueue.CurrentQueueSize = 0; int remaining_tasks = colllst.Count; @@ -201,6 +201,12 @@ namespace Wabbajack.Common } } + if (WorkQueue.CurrentQueueSize == WorkQueue.MaxQueueSize) + { + WorkQueue.MaxQueueSize = 0; + WorkQueue.MaxQueueSize = 0; + } + return tasks.Select(t => { t.Wait(); diff --git a/Wabbajack/Compiler.cs b/Wabbajack/Compiler.cs index 2f077513..8fe770ab 100644 --- a/Wabbajack/Compiler.cs +++ b/Wabbajack/Compiler.cs @@ -57,6 +57,7 @@ namespace Wabbajack public List AllFiles { get; private set; } public ModList ModList { get; private set; } public ConcurrentBag ExtraFiles { get; private set; } + public Dictionary ModInis { get; private set; } public List IndexedArchives; @@ -252,6 +253,18 @@ namespace Wabbajack ExtraFiles = new ConcurrentBag(); + ModInis = Directory.EnumerateDirectories(Path.Combine(MO2Folder, "mods")) + .Select(f => + { + var mod_name = Path.GetFileName(f); + var meta_path = Path.Combine(f, "meta.ini"); + if (File.Exists(meta_path)) + return (mod_name, meta_path.LoadIniFile()); + return (null, null); + }) + .Where(f => f.Item2 != null) + .ToDictionary(f => f.Item1, f => f.Item2); + var stack = MakeStack(); Info("Running Compilation Stack"); @@ -373,6 +386,7 @@ namespace Wabbajack var b = LoadDataForTo(entry.To, absolute_paths); BSDiff.Create(a, b, output); entry.Patch = output.ToArray().ToBase64(); + Info($"Patch size {entry.Patch.Length} for {entry.To}"); } }); @@ -435,7 +449,7 @@ namespace Wabbajack Status($"Getting Nexus info for {found.Name}"); try { - var link = NexusAPI.GetNexusDownloadLink((NexusMod)result, NexusKey); + // var link = NexusAPI.GetNexusDownloadLink((NexusMod)result, NexusKey); } catch (Exception ex) { @@ -560,6 +574,7 @@ namespace Wabbajack IgnoreRegex(Consts.GameFolderFilesDir + "\\\\.*\\.bsa"), IncludeModIniData(), DirectMatch(), + IncludeTaggedFiles(), IncludePatches(), IncludeDummyESPs(), @@ -582,6 +597,43 @@ namespace Wabbajack } + /// + /// If a user includes WABBAJACK_INCLUDE directly in the notes or comments of a mod, the contents of that + /// mod will be inlined into the installer. USE WISELY. + /// + /// + private Func IncludeTaggedFiles() + { + var include_directly = ModInis.Where(kv => { + var general = kv.Value.General; + if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE)) + return true; + if (general.comments != null && general.comments.Contains(Consts.WABBAJACK_INCLUDE)) + return true; + return false; + }).Select(kv => $"mods\\{kv.Key}\\"); + + + return source => + { + + if (source.Path.StartsWith("mods")) + { + foreach (var modpath in include_directly) + { + if (source.Path.StartsWith(modpath)) + { + var result = source.EvolveTo(); + result.SourceData = File.ReadAllBytes(source.AbsolutePath).ToBase64(); + return result; + } + } + } + return null; + }; + } + + /// /// Some tools like the Cathedral Asset Optimizer will create dummy ESPs whos only existance is to make /// sure a BSA with the same name is loaded. We don't have a good way to detect these, but if an ESP is @@ -682,13 +734,13 @@ namespace Wabbajack var results = new List<(string, string)>(); using (var a = new BSAReader(absolutePath)) { - foreach (var entry in a.Files) + a.Files.PMap(entry => { Status($"Hashing BSA: {absolutePath} - {entry.Path}"); var data = entry.GetData(); results.Add((entry.Path, data.SHA256())); - } + }); } return results; } @@ -863,17 +915,7 @@ namespace Wabbajack .GroupBy(e => e.entry.Hash) .ToDictionary(e => e.Key); - var mod_inis = Directory.EnumerateDirectories(Path.Combine(MO2Folder, "mods")) - .Select(f => - { - var mod_name = Path.GetFileName(f); - var meta_path = Path.Combine(f, "meta.ini"); - if (File.Exists(meta_path)) - return (mod_name, meta_path.LoadIniFile()); - return (null, null); - }) - .Where(f => f.Item2 != null) - .ToDictionary(f => f.Item1, f => f.Item2); + return source => { diff --git a/Wabbajack/Installer.cs b/Wabbajack/Installer.cs index c627388b..5378f02a 100644 --- a/Wabbajack/Installer.cs +++ b/Wabbajack/Installer.cs @@ -131,6 +131,9 @@ namespace Wabbajack } }); + Info($"Removing temp folder {Consts.BSACreationDir}"); + Directory.Delete(Path.Combine(Outputfolder, Consts.BSACreationDir), true); + } private void InstallIncludedFiles() @@ -266,6 +269,7 @@ namespace Wabbajack if (output_path.FileExists()) File.Delete(output_path); + switch (archive) { case NexusMod a: Info($"Downloading Nexus Archive - {archive.Name} - {a.GameName} - {a.ModID} - {a.FileID}");