From 5562e15d8e5906712376164de50e255861204cdc Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 11 Sep 2020 20:49:53 -0600 Subject: [PATCH 1/4] Don't reupload mirrored files --- Wabbajack.Server/Services/MirrorUploader.cs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Wabbajack.Server/Services/MirrorUploader.cs b/Wabbajack.Server/Services/MirrorUploader.cs index 12e94aec..d0c5f9b4 100644 --- a/Wabbajack.Server/Services/MirrorUploader.cs +++ b/Wabbajack.Server/Services/MirrorUploader.cs @@ -1,10 +1,13 @@ using System; +using System.Collections.Generic; using System.IO; using System.IO.Compression; +using System.Linq; using System.Net; using System.Threading.Tasks; using FluentFTP; using Microsoft.Extensions.Logging; +using Org.BouncyCastle.Utilities.Collections; using Wabbajack.BuildServer; using Wabbajack.BuildServer.Controllers; using Wabbajack.Common; @@ -38,14 +41,28 @@ namespace Wabbajack.Server.Services try { + var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors); + using var queue = new WorkQueue(); if (_archives.TryGetPath(toUpload.Hash, out var path)) { _logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size.FileSizeToString()}"); + bool exists = false; + using (var client = await GetClient(creds)) + { + exists = await client.FileExistsAsync($"{toUpload.Hash.ToHex()}/definition.json.gz"); + } + + if (exists) + { + _logger.LogInformation($"Skipping {toUpload.Hash} it's already on the server"); + await toUpload.Finish(_sql); + goto TOP; + } + var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { }); - var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors); using (var client = await GetClient(creds)) { await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}"); From b6dbcc23683ec4a46545dd8f05f5e8de1710c65a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 12 Sep 2020 14:23:03 -0600 Subject: [PATCH 2/4] Rework performance profiles of WJ --- CHANGELOG.md | 5 + Wabbajack.Lib/ABatchProcessor.cs | 8 + Wabbajack.Lib/AInstaller.cs | 4 + Wabbajack.Lib/MO2Compiler.cs | 5 +- Wabbajack.Lib/MO2Installer.cs | 4 +- Wabbajack.Test/DownloaderTests.cs | 2 +- Wabbajack.VirtualFileSystem/Context.cs | 3 + .../FileExtractor2/FileExtractor.cs | 6 + .../FileExtractor2/GatheringExtractor.cs | 58 +++--- .../SevenZipExtractor/ArchiveFile.cs | 184 ++++++++++++++---- .../SevenZipExtractor/SevenZipInterface.cs | 27 ++- Wabbajack/Settings.cs | 40 ++-- .../View Models/Compilers/MO2CompilerVM.cs | 2 +- .../View Models/Installers/MO2InstallerVM.cs | 2 +- Wabbajack/Views/Common/CpuView.xaml.cs | 23 +-- .../Settings/PerformanceSettingsView.xaml | 96 ++++----- .../Settings/PerformanceSettingsView.xaml.cs | 39 +--- 17 files changed, 299 insertions(+), 209 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef95a148..077c91e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ * Reworked IPS4 integration to reduce download failures * Profiles can now contain an (optional) file `compiler_settings.json` that includes options for other games to be used during install. This is now the only way to include extra games in the install process, implicit inclusion has been removed. +* Number of download/install threads is now manually set (defaults to CPU cores) and is independently configurable +* Includes a "Favor performance over RAM" optional mode (defaults to off) that will use excessive amounts of RAM in exchange +for almost 1GB/sec install speed on the correct hardware. Don't enable this unless you have a fast SSD and at least 2.5GB of RAM for every +install thread. + #### Version - 2.2.2.0 - 8/31/2020 * Route CDN requests through a reverse proxy to improve reliability diff --git a/Wabbajack.Lib/ABatchProcessor.cs b/Wabbajack.Lib/ABatchProcessor.cs index b42d20d3..ba525e63 100644 --- a/Wabbajack.Lib/ABatchProcessor.cs +++ b/Wabbajack.Lib/ABatchProcessor.cs @@ -15,6 +15,10 @@ namespace Wabbajack.Lib { public WorkQueue Queue { get; } = new WorkQueue(); + public int DownloadThreads { get; set; } + public int DiskThreads { get; set; } + public bool FavorPerfOverRam { get; set; } + public Context VFS { get; } protected StatusUpdateTracker UpdateTracker { get; } @@ -51,6 +55,7 @@ namespace Wabbajack.Lib public BehaviorSubject ManualCoreLimit = new BehaviorSubject(true); public BehaviorSubject MaxCores = new BehaviorSubject(byte.MaxValue); public BehaviorSubject TargetUsagePercent = new BehaviorSubject(Percent.One); + public Subject DesiredThreads { get; set; } public ABatchProcessor(int steps) { @@ -62,8 +67,11 @@ namespace Wabbajack.Lib .DisposeWith(_subs); UpdateTracker.Progress.Subscribe(_percentCompleted); UpdateTracker.StepName.Subscribe(_textStatus); + DesiredThreads = new Subject(); + Queue.SetActiveThreadsObservable(DesiredThreads); } + /// /// Gets the recommended maximum number of threads that should be used for the current machine. /// This will either run a heavy processing job to do the measurement in the current folder, or refer to caches. diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 6924cefd..3654784e 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -216,6 +216,7 @@ namespace Wabbajack.Lib } } + DesiredThreads.OnNext(DownloadThreads); await missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State)) .PMap(Queue, async archive => { @@ -236,6 +237,9 @@ namespace Wabbajack.Lib return await DownloadArchive(archive, download, outputPath); }); + + DesiredThreads.OnNext(DiskThreads); + } public async Task DownloadArchive(Archive archive, bool download, AbsolutePath? destination = null) diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index c4fa92ca..1ad2d855 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -84,7 +84,10 @@ namespace Wabbajack.Lib { await Metrics.Send("begin_compiling", MO2Profile ?? "unknown"); if (cancel.IsCancellationRequested) return false; - Queue.SetActiveThreadsObservable(ConstructDynamicNumThreads(await RecommendQueueSize())); + + DesiredThreads.OnNext(DiskThreads); + FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam; + UpdateTracker.Reset(); UpdateTracker.NextStep("Gathering information"); diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index ad9ce598..cebcf504 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -22,6 +22,7 @@ using File = Alphaleonis.Win32.Filesystem.File; using Path = Alphaleonis.Win32.Filesystem.Path; using SectionData = Wabbajack.Common.SectionData; using System.Collections.Generic; +using Wabbajack.VirtualFileSystem; namespace Wabbajack.Lib { @@ -55,7 +56,8 @@ namespace Wabbajack.Lib await Metrics.Send(Metrics.BeginInstall, ModList.Name); Utils.Log("Configuring Processor"); - Queue.SetActiveThreadsObservable(ConstructDynamicNumThreads(await RecommendQueueSize())); + DesiredThreads.OnNext(DiskThreads); + FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam; if (GameFolder == null) GameFolder = Game.TryGetGameLocation(); diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 4c394c6f..0bd46741 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -569,7 +569,7 @@ namespace Wabbajack.Test public TestInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters parameters) : base(archive, modList, outputFolder, downloadFolder, parameters, steps: 1, modList.GameType) { - Queue.SetActiveThreadsObservable(Observable.Return(1)); + DesiredThreads.OnNext(1); } protected override Task _Begin(CancellationToken cancel) diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index e1609de5..3e93232b 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -42,6 +42,8 @@ namespace Wabbajack.VirtualFileSystem public WorkQueue Queue { get; } public bool UseExtendedHashes { get; set; } + + public bool FavorPerfOverRAM { get; set; } public Context(WorkQueue queue, bool extendedHashes = false) { @@ -234,6 +236,7 @@ namespace Wabbajack.VirtualFileSystem private List _knownFiles = new List(); private Dictionary _knownArchives = new Dictionary(); + public void AddKnown(IEnumerable known, Dictionary archives) { _knownFiles.AddRange(known); diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs index a7c1a7c5..0bf2da8d 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -26,6 +26,12 @@ namespace Wabbajack.VirtualFileSystem Definitions.FileType._7Z); private static Extension OMODExtension = new Extension(".omod"); + + /// + /// When true, will allow 7z to use multiple threads and cache more data in memory, potentially + /// using many GB of RAM during extraction but vastly reducing extraction times in the process. + /// + public static bool FavorPerfOverRAM { get; set; } public static async Task> GatheringExtract(IStreamFactory sFn, diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs index 1fd98d6d..7a568e9b 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs @@ -18,10 +18,10 @@ namespace Wabbajack.VirtualFileSystem private Predicate _shouldExtract; private Func> _mapFn; private Dictionary _results; - private Dictionary _indexes; private Stream _stream; private Definitions.FileType _sig; private Exception _killException; + private uint _itemsCount; public GatheringExtractor(Stream stream, Definitions.FileType sig, Predicate shouldExtract, Func> mapfn) { @@ -41,14 +41,9 @@ namespace Wabbajack.VirtualFileSystem try { _archive = ArchiveFile.Open(_stream, _sig).Result; - _indexes = _archive.Entries - .Select((entry, idx) => (entry, (uint)idx)) - .Where(f => !f.entry.IsFolder) - .Select(t => ((RelativePath)t.entry.FileName, t.Item2, t.entry.Size)) - .Where(t => _shouldExtract(t.Item1)) - .ToDictionary(t => t.Item2, t => (t.Item1, t.Size)); - - + ulong checkPos = 1024 * 32; + _archive._archive.Open(_archive._archiveStream, ref checkPos, null); + _itemsCount = _archive._archive.GetNumberOfItems(); _archive._archive.Extract(null, 0xFFFFFFFF, 0, this); _archive.Dispose(); if (_killException != null) @@ -86,21 +81,22 @@ namespace Wabbajack.VirtualFileSystem public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode) { - if (_indexes.ContainsKey(index)) + var entry = _archive.GetEntry(index); + var path = (RelativePath)entry.FileName; + if (!_shouldExtract(path)) { - var path = _indexes[index].Item1; - Utils.Status($"Extracting {path}", Percent.FactoryPutInRange(_results.Count, _indexes.Count)); - // Empty files are never extracted via a write call, so we have to fake that now - if (_indexes[index].Item2 == 0) - { - var result = _mapFn(path, new MemoryStreamFactory(new MemoryStream(), path)).Result; - _results.Add(path, result); - } - outStream = new GatheringExtractorStream(this, index); + outStream = null; return 0; } - outStream = null; + Utils.Status($"Extracting {path}", Percent.FactoryPutInRange(_results.Count, _itemsCount)); + // Empty files are never extracted via a write call, so we have to fake that now + if (entry.Size == 0) + { + var result = _mapFn(path, new MemoryStreamFactory(new MemoryStream(), path)).Result; + _results.Add(path, result); + } + outStream = new GatheringExtractorStream(this, entry, path); return 0; } @@ -117,25 +113,23 @@ namespace Wabbajack.VirtualFileSystem private class GatheringExtractorStream : ISequentialOutStream, IOutStream { private GatheringExtractor _extractor; - private uint _index; - private bool _written; private ulong _totalSize; private Stream _tmpStream; private TempFile _tmpFile; - private IStreamFactory _factory; private bool _diskCached; + private RelativePath _path; - public GatheringExtractorStream(GatheringExtractor extractor, uint index) + public GatheringExtractorStream(GatheringExtractor extractor, Entry entry, RelativePath path) { + _path = path; _extractor = extractor; - _index = index; - _totalSize = extractor._indexes[index].Item2; - _diskCached = _totalSize >= 500_000_000; + _totalSize = entry.Size; + _diskCached = _totalSize >= int.MaxValue - 1024; } private IPath GetPath() { - return _extractor._indexes[_index].Item1; + return _path; } public int Write(byte[] data, uint size, IntPtr processedSize) @@ -167,7 +161,7 @@ namespace Wabbajack.VirtualFileSystem private void WriteSingleCall(byte[] data, in uint size) { - var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryBufferFactory(data, (int)size, GetPath())).Result; + var result = _extractor._mapFn(_path, new MemoryBufferFactory(data, (int)size, GetPath())).Result; AddResult(result); Cleanup(); } @@ -180,7 +174,7 @@ namespace Wabbajack.VirtualFileSystem private void AddResult(T result) { - _extractor._results.Add(_extractor._indexes[_index].Item1, result); + _extractor._results.Add(_path, result); } private void WriteMemoryCached(byte[] data, in uint size) @@ -193,7 +187,7 @@ namespace Wabbajack.VirtualFileSystem _tmpStream.Flush(); _tmpStream.Position = 0; - var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory((MemoryStream)_tmpStream, GetPath())).Result; + var result = _extractor._mapFn(_path, new MemoryStreamFactory((MemoryStream)_tmpStream, GetPath())).Result; AddResult(result); Cleanup(); } @@ -213,7 +207,7 @@ namespace Wabbajack.VirtualFileSystem _tmpStream.Flush(); _tmpStream.Close(); - var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new NativeFileStreamFactory(_tmpFile.Path, GetPath())).Result; + var result = _extractor._mapFn(_path, new NativeFileStreamFactory(_tmpFile.Path, GetPath())).Result; AddResult(result); Cleanup(); } diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs index f2710986..438f163a 100644 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Runtime.InteropServices; @@ -13,21 +14,123 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor { private SevenZipHandle _sevenZipHandle; internal IInArchive _archive; - private InStreamWrapper _archiveStream; + public InStreamWrapper _archiveStream; private IList _entries; private static readonly AbsolutePath LibraryFilePath = @"Extractors\7z.dll".RelativeTo(AbsolutePath.EntryPoint); private static SignatureChecker _checker = new SignatureChecker(Formats.FileTypeGuidMapping.Keys.ToArray()); - + public static async Task Open(Stream archiveStream, Definitions.FileType format) { var self = new ArchiveFile(); self.InitializeAndValidateLibrary(); self._archive = self._sevenZipHandle.CreateInArchive(Formats.FileTypeGuidMapping[format]); + + if (!FileExtractor2.FavorPerfOverRAM) + { + self.SetCompressionProperties(new Dictionary() {{"mt", "off"}}); + } + + self._archiveStream = new InStreamWrapper(archiveStream); return self; } + /// + /// Sets the compression properties + /// + private void SetCompressionProperties(Dictionary CustomParameters) + { + + + { + ISetProperties setter; + try + { + setter = (ISetProperties)_archive; + } + catch (InvalidCastException _) + { + return; + } + + var names = new List(1 + CustomParameters.Count); + var values = new List(1 + CustomParameters.Count); + //var sp = new SecurityPermission(SecurityPermissionFlag.UnmanagedCode); + //sp.Demand(); + + #region Initialize compression properties + + names.Add(Marshal.StringToBSTR("x")); + values.Add(new PropVariant()); + + foreach (var pair in CustomParameters) + { + + names.Add(Marshal.StringToBSTR(pair.Key)); + var pv = new PropVariant(); + + #region List of parameters to cast as integers + + var integerParameters = new HashSet + { + "fb", + "pass", + "o", + "yx", + "a", + "mc", + "lc", + "lp", + "pb", + "cp" + }; + + #endregion + + if (integerParameters.Contains(pair.Key)) + { + pv.VarType = VarEnum.VT_UI4; + pv.UInt32Value = Convert.ToUInt32(pair.Value, CultureInfo.InvariantCulture); + } + else + { + pv.VarType = VarEnum.VT_BSTR; + pv.pointerValue = Marshal.StringToBSTR(pair.Value); + } + + values.Add(pv); + } + + #endregion + + #region Set compression level + + var clpv = values[0]; + clpv.VarType = VarEnum.VT_UI4; + clpv.UInt32Value = 0; + + values[0] = clpv; + + #endregion + + var namesHandle = GCHandle.Alloc(names.ToArray(), GCHandleType.Pinned); + var valuesHandle = GCHandle.Alloc(values.ToArray(), GCHandleType.Pinned); + + try + { + setter?.SetProperties(namesHandle.AddrOfPinnedObject(), valuesHandle.AddrOfPinnedObject(), + names.Count); + } + finally + { + namesHandle.Free(); + valuesHandle.Free(); + } + + } + } + public IList Entries { get @@ -51,47 +154,54 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor for (uint fileIndex = 0; fileIndex < itemsCount; fileIndex++) { - string fileName = this.GetProperty(fileIndex, ItemPropId.kpidPath); - bool isFolder = this.GetProperty(fileIndex, ItemPropId.kpidIsFolder); - bool isEncrypted = this.GetProperty(fileIndex, ItemPropId.kpidEncrypted); - ulong size = this.GetProperty(fileIndex, ItemPropId.kpidSize); - ulong packedSize = this.GetProperty(fileIndex, ItemPropId.kpidPackedSize); - DateTime creationTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidCreationTime); - DateTime lastWriteTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidLastWriteTime); - DateTime lastAccessTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidLastAccessTime); - uint crc = this.GetPropertySafe(fileIndex, ItemPropId.kpidCRC); - uint attributes = this.GetPropertySafe(fileIndex, ItemPropId.kpidAttributes); - string comment = this.GetPropertySafe(fileIndex, ItemPropId.kpidComment); - string hostOS = this.GetPropertySafe(fileIndex, ItemPropId.kpidHostOS); - string method = this.GetPropertySafe(fileIndex, ItemPropId.kpidMethod); - - bool isSplitBefore = this.GetPropertySafe(fileIndex, ItemPropId.kpidSplitBefore); - bool isSplitAfter = this.GetPropertySafe(fileIndex, ItemPropId.kpidSplitAfter); - - this._entries.Add(new Entry(this._archive, fileIndex) - { - FileName = fileName, - IsFolder = isFolder, - IsEncrypted = isEncrypted, - Size = size, - PackedSize = packedSize, - CreationTime = creationTime, - LastWriteTime = lastWriteTime, - LastAccessTime = lastAccessTime, - CRC = crc, - Attributes = attributes, - Comment = comment, - HostOS = hostOS, - Method = method, - IsSplitBefore = isSplitBefore, - IsSplitAfter = isSplitAfter - }); + var entry = GetEntry(fileIndex); + this._entries.Add(entry); } return this._entries; } } + internal Entry GetEntry(uint fileIndex) + { + string fileName = this.GetProperty(fileIndex, ItemPropId.kpidPath); + bool isFolder = this.GetProperty(fileIndex, ItemPropId.kpidIsFolder); + bool isEncrypted = this.GetProperty(fileIndex, ItemPropId.kpidEncrypted); + ulong size = this.GetProperty(fileIndex, ItemPropId.kpidSize); + ulong packedSize = this.GetProperty(fileIndex, ItemPropId.kpidPackedSize); + DateTime creationTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidCreationTime); + DateTime lastWriteTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidLastWriteTime); + DateTime lastAccessTime = this.GetPropertySafe(fileIndex, ItemPropId.kpidLastAccessTime); + uint crc = this.GetPropertySafe(fileIndex, ItemPropId.kpidCRC); + uint attributes = this.GetPropertySafe(fileIndex, ItemPropId.kpidAttributes); + string comment = this.GetPropertySafe(fileIndex, ItemPropId.kpidComment); + string hostOS = this.GetPropertySafe(fileIndex, ItemPropId.kpidHostOS); + string method = this.GetPropertySafe(fileIndex, ItemPropId.kpidMethod); + + bool isSplitBefore = this.GetPropertySafe(fileIndex, ItemPropId.kpidSplitBefore); + bool isSplitAfter = this.GetPropertySafe(fileIndex, ItemPropId.kpidSplitAfter); + + var entry = new Entry(this._archive, fileIndex) + { + FileName = fileName, + IsFolder = isFolder, + IsEncrypted = isEncrypted, + Size = size, + PackedSize = packedSize, + CreationTime = creationTime, + LastWriteTime = lastWriteTime, + LastAccessTime = lastAccessTime, + CRC = crc, + Attributes = attributes, + Comment = comment, + HostOS = hostOS, + Method = method, + IsSplitBefore = isSplitBefore, + IsSplitAfter = isSplitAfter + }; + return entry; + } + private T GetPropertySafe(uint fileIndex, ItemPropId name) { try diff --git a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs index f9b6811c..9f458415 100644 --- a/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs +++ b/Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs @@ -26,6 +26,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor [FieldOffset(8)] public IntPtr pointerValue; [FieldOffset(8)] public byte byteValue; [FieldOffset(8)] public long longValue; + [FieldOffset(8)] public UInt32 UInt32Value; [FieldOffset(8)] public System.Runtime.InteropServices.ComTypes.FILETIME filetime; [FieldOffset(8)] public PropArray propArray; @@ -35,6 +36,10 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor { return (VarEnum) this.vt; } + set + { + vt = (ushort)value; + } } public void Clear() @@ -304,6 +309,24 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor kpidUserDefined = 0x10000 } + + /// + /// 7-zip ISetProperties interface for setting various archive properties + /// + [ComImport] + [Guid("23170F69-40C1-278A-0000-000600030000")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + internal interface ISetProperties + { + /// + /// Sets the archive properties + /// + /// The names of the properties + /// The values of the properties + /// The properties count + /// + int SetProperties(IntPtr names, IntPtr values, int numProperties); + } [ComImport] @@ -395,7 +418,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor ArchivePropId propID, ref PropVariant value); // PROPVARIANT - internal class StreamWrapper : IDisposable + public class StreamWrapper : IDisposable { protected Stream BaseStream; @@ -420,7 +443,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor } } - internal class InStreamWrapper : StreamWrapper, ISequentialInStream, IInStream + public class InStreamWrapper : StreamWrapper, ISequentialInStream, IInStream { public InStreamWrapper(Stream baseStream) : base(baseStream) { diff --git a/Wabbajack/Settings.cs b/Wabbajack/Settings.cs index 50aac477..84736319 100644 --- a/Wabbajack/Settings.cs +++ b/Wabbajack/Settings.cs @@ -113,26 +113,28 @@ namespace Wabbajack [JsonObject(MemberSerialization.OptOut)] public class PerformanceSettings : ViewModel { - private bool _manual; - public bool Manual { get => _manual; set => RaiseAndSetIfChanged(ref _manual, value); } - - private byte _maxCores = byte.MaxValue; - public byte MaxCores { get => _maxCores; set => RaiseAndSetIfChanged(ref _maxCores, value); } - - private Percent _targetUsage = Percent.One; - public Percent TargetUsage { get => _targetUsage; set => RaiseAndSetIfChanged(ref _targetUsage, value); } - - public void AttachToBatchProcessor(ABatchProcessor processor) + public PerformanceSettings() { - processor.Add( - this.WhenAny(x => x.Manual) - .Subscribe(processor.ManualCoreLimit)); - processor.Add( - this.WhenAny(x => x.MaxCores) - .Subscribe(processor.MaxCores)); - processor.Add( - this.WhenAny(x => x.TargetUsage) - .Subscribe(processor.TargetUsagePercent)); + _favorPerfOverRam = false; + _diskThreads = Environment.ProcessorCount; + _downloadThreads = Environment.ProcessorCount; + } + + private int _downloadThreads; + public int DownloadThreads { get => _downloadThreads; set => RaiseAndSetIfChanged(ref _downloadThreads, value); } + + private int _diskThreads; + public int DiskThreads { get => _diskThreads; set => RaiseAndSetIfChanged(ref _diskThreads, value); } + + private bool _favorPerfOverRam; + public bool FavorPerfOverRam { get => _favorPerfOverRam; set => RaiseAndSetIfChanged(ref _favorPerfOverRam, value); } + + + public void SetProcessorSettings(ABatchProcessor processor) + { + processor.DownloadThreads = DownloadThreads; + processor.DiskThreads = DiskThreads; + processor.FavorPerfOverRam = FavorPerfOverRam; } } diff --git a/Wabbajack/View Models/Compilers/MO2CompilerVM.cs b/Wabbajack/View Models/Compilers/MO2CompilerVM.cs index 7d0c252d..c6d4c296 100644 --- a/Wabbajack/View Models/Compilers/MO2CompilerVM.cs +++ b/Wabbajack/View Models/Compilers/MO2CompilerVM.cs @@ -195,7 +195,7 @@ namespace Wabbajack ModlistIsNSFW = ModlistSettings.IsNSFW }) { - Parent.MWVM.Settings.Performance.AttachToBatchProcessor(ActiveCompilation); + Parent.MWVM.Settings.Performance.SetProcessorSettings(ActiveCompilation); var success = await ActiveCompilation.Begin(); return GetResponse.Create(success, ActiveCompilation.ModList); diff --git a/Wabbajack/View Models/Installers/MO2InstallerVM.cs b/Wabbajack/View Models/Installers/MO2InstallerVM.cs index 7f0118d0..ff463bc7 100644 --- a/Wabbajack/View Models/Installers/MO2InstallerVM.cs +++ b/Wabbajack/View Models/Installers/MO2InstallerVM.cs @@ -157,7 +157,7 @@ namespace Wabbajack parameters: SystemParametersConstructor.Create())) { installer.UseCompression = Parent.MWVM.Settings.Filters.UseCompression; - Parent.MWVM.Settings.Performance.AttachToBatchProcessor(installer); + Parent.MWVM.Settings.Performance.SetProcessorSettings(installer); return await Task.Run(async () => { diff --git a/Wabbajack/Views/Common/CpuView.xaml.cs b/Wabbajack/Views/Common/CpuView.xaml.cs index d631f954..fe5e639d 100644 --- a/Wabbajack/Views/Common/CpuView.xaml.cs +++ b/Wabbajack/Views/Common/CpuView.xaml.cs @@ -51,32 +51,11 @@ namespace Wabbajack InitializeComponent(); this.WhenActivated(disposable => { - Observable.CombineLatest( - this.WhenAny(x => x.ControlGrid.IsMouseOver), - this.WhenAny(x => x.SettingsHook.Performance.Manual) - .StartWith(true), - resultSelector: (over, manual) => over && !manual) - .Select(showing => showing ? Visibility.Visible : Visibility.Collapsed) - .BindToStrict(this, x => x.SettingsBar.Visibility) - .DisposeWith(disposable); - + this.WhenAny(x => x.ViewModel.StatusList) .BindToStrict(this, x => x.CpuListControl.ItemsSource) .DisposeWith(disposable); - this.BindStrict( - this.ViewModel, - x => x.MWVM.Settings.Performance.TargetUsage, - x => x.TargetPercentageSlider.Value, - vmToViewConverter: p => p.Value, - viewToVmConverter: d => new Percent(d)) - .DisposeWith(disposable); - - this.WhenAny(x => x.ViewModel.MWVM.Settings.Performance.TargetUsage) - .Select(p => p.ToString(0)) - .BindToStrict(this, x => x.PercentageText.Text) - .DisposeWith(disposable); - this.WhenAny(x => x.ViewModel.CurrentCpuCount) .DistinctUntilChanged() .Select(x => $"{x.CurrentCPUs} / {x.DesiredCPUs}") diff --git a/Wabbajack/Views/Settings/PerformanceSettingsView.xaml b/Wabbajack/Views/Settings/PerformanceSettingsView.xaml index e1c25625..45c2f730 100644 --- a/Wabbajack/Views/Settings/PerformanceSettingsView.xaml +++ b/Wabbajack/Views/Settings/PerformanceSettingsView.xaml @@ -25,6 +25,7 @@ + @@ -37,67 +38,44 @@ FontSize="20" FontWeight="Bold" Text="Performance" /> - - - - - - - - - -