Merge pull request #981 from wabbajack-tools/disk-size-fixes

Fix #980, 2.1.3.4, and deps bumps

CHANGELOG.md

@@ -1,5 +1,13 @@
 ### Changelog
 
+#### Version - 2.1.3.4 - 7/27/2020
+* Fixes for Tar Files (for realz this time)
+* Watch disk usage, throw an error if disk usage gets too high
+* Added error icon triangle under install play button if there are blocking problems.
+* Added tooltip styling to limit width to 500.
+* Adjusted error text for MO2Installer unexpected files.
+* Added filepicker error glow
+
 #### Version - 2.1.3.3 - 7/22/2020
 * Relax the RAR signature so it works with RAR 5 and RAR 4 formats
 

Wabbajack.CLI/Verbs/HashFile.cs

@@ -15,7 +15,8 @@ namespace Wabbajack.CLI.Verbs
         protected override async Task<ExitCode> Run()
         {
             var abs = (AbsolutePath)Input;
-            Console.WriteLine($"{abs} hash: {await abs.FileHashAsync()}");
+            var hash = await abs.FileHashAsync();
+            Console.WriteLine($"{abs} hash: {hash} {hash.ToHex()} {(long)hash}");
             return ExitCode.Ok;
         }
     }

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -6,8 +6,8 @@
         <AssemblyName>wabbajack-cli</AssemblyName>
         <Company>Wabbajack</Company>
         <Platforms>x64</Platforms>
-        <AssemblyVersion>2.1.3.3</AssemblyVersion>
+        <AssemblyVersion>2.1.3.4</AssemblyVersion>
-        <FileVersion>2.1.3.3</FileVersion>
+        <FileVersion>2.1.3.4</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>An automated ModList installer</Description>
         <PublishReadyToRun>true</PublishReadyToRun>        

Wabbajack.Common/Paths.cs

@@ -12,6 +12,7 @@ using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
+using DriveInfo = Alphaleonis.Win32.Filesystem.DriveInfo;
 using File = Alphaleonis.Win32.Filesystem.File;
 using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
 using Path = Alphaleonis.Win32.Filesystem.Path;
@@ -85,6 +86,11 @@ namespace Wabbajack.Common
             return _path.Replace("/", "\\").TrimEnd('\\');
         }
 
+        public DriveInfo DriveInfo()
+        {
+            return new DriveInfo(Path.GetPathRoot(_path));
+        }
+
         public Extension Extension => Extension.FromPath(_path);
 
         public ValueTask<FileStream> OpenRead()

Wabbajack.Launcher/Wabbajack.Launcher.csproj

@@ -4,8 +4,8 @@
         <OutputType>WinExe</OutputType>
         <TargetFramework>netcoreapp3.1</TargetFramework>
         <UseWPF>true</UseWPF>
-        <AssemblyVersion>2.1.3.3</AssemblyVersion>
+        <AssemblyVersion>2.1.3.4</AssemblyVersion>
-        <FileVersion>2.1.3.3</FileVersion>
+        <FileVersion>2.1.3.4</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Application Launcher</Description>
         <PublishReadyToRun>true</PublishReadyToRun>

Wabbajack.Lib/ABatchProcessor.cs

@@ -181,6 +181,11 @@ namespace Wabbajack.Lib
             _isRunning.OnNext(false);
         }
 
+        public void Abort()
+        {
+            _cancel.Cancel();
+        }
+
         public void Add(IDisposable disposable) => _subs.Add(disposable);
     }
 }

Wabbajack.Lib/DiskSpaceWatcher.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib
+{
+    public class DiskSpaceWatcher
+    {
+        private CancellationToken _token;
+        private long _minSpace;
+        private Action<DriveInfo> _onFailure;
+        private DriveInfo[] _drives;
+
+        public DiskSpaceWatcher(CancellationToken token, IEnumerable<AbsolutePath> paths, long minSpace, Action<DriveInfo> onFailure)
+        {
+            _token = token;
+            _minSpace = minSpace;
+            _onFailure = onFailure;
+            _drives = paths.Select(p => p.DriveInfo()).DistinctBy(d => d.Name).ToArray();
+        }
+
+        public async Task Start()
+        {
+            Utils.Log(
+                $"Drive watcher is starting. Will warn at {(_minSpace * 2).ToFileSizeString()} and error at {_minSpace.ToFileSizeString()}");
+            foreach (var drive in _drives)
+            {
+                Utils.Log(
+                    $"Starting Drive watcher on {drive.Name} currently {drive.AvailableFreeSpace.ToFileSizeString()} free out of {drive.TotalSize.ToFileSizeString()}");
+            }
+
+            
+            while (true)
+            {
+                foreach (var drive in _drives)
+                {
+                    var used = drive.AvailableFreeSpace;
+                    if (used < _minSpace)
+                    {
+                        _onFailure(drive);
+                        Utils.ErrorThrow(new Exception($"Out of space on drive {drive.Name}"));
+                    }
+
+                    if (used < _minSpace * 2)
+                    {
+                        Utils.Log(
+                            $"Warning! Drive {drive.Name} only has {used.ToFileSizeString()} of free space left, processing will stop when you only have {used.ToFileSizeString()}");
+                    }
+                }
+
+                if (_token.IsCancellationRequested)
+                {
+                    break;
+                }
+
+                await Task.Delay(1000, _token);
+            }
+        }
+
+    }
+}

Wabbajack.Lib/MO2Compiler.cs

@@ -101,6 +101,14 @@ namespace Wabbajack.Lib
             Utils.Log($"MO2 Folder: {MO2Folder}");
             Utils.Log($"Downloads Folder: {MO2DownloadsFolder}");
             Utils.Log($"Game Folder: {GamePath}");
+            
+            var watcher = new DiskSpaceWatcher(cancel, new []{MO2Folder, MO2DownloadsFolder, GamePath, AbsolutePath.EntryPoint}, (long)2 << 31,
+                drive =>
+                {
+                    Utils.Log($"Aborting due to low space on {drive.Name}");
+                    Abort();
+                });
+            var watcherTask = watcher.Start();
 
             if (cancel.IsCancellationRequested) return false;
             
@@ -297,6 +305,11 @@ namespace Wabbajack.Lib
             PrintNoMatches(noMatch);
             if (CheckForNoMatchExit(noMatch)) return false;
 
+            foreach (var ignored in results.OfType<IgnoredDirectly>())
+            {
+                Utils.Log($"Ignored {ignored.To} because {ignored.Reason}");
+            }
+
             InstallDirectives.SetTo(results.Where(i => !(i is IgnoredDirectly)));
 
             Info("Getting Nexus api_key, please click authorize if a browser window appears");
@@ -578,8 +591,6 @@ namespace Wabbajack.Lib
                 new IgnorePathContains(this,"temporary_logs"),
                 new IgnorePathContains(this, "GPUCache"),
                 new IgnorePathContains(this, "SSEEdit Cache"),
-                new IgnoreEndsWith(this, ".pyc"),
-                new IgnoreEndsWith(this, ".log"),
                 new IgnoreOtherProfiles(this),
                 new IgnoreDisabledMods(this),
                 new IncludeThisProfile(this),
@@ -594,6 +605,8 @@ namespace Wabbajack.Lib
                 new IncludeModIniData(this),
                 new DirectMatch(this),
                 new IncludeTaggedMods(this, Consts.WABBAJACK_INCLUDE),
+                new IgnoreEndsWith(this, ".pyc"),
+                new IgnoreEndsWith(this, ".log"),
                 new DeconstructBSAs(this), // Deconstruct BSAs before building patches so we don't generate massive patch files
                 new IncludePatches(this),
                 new IncludeDummyESPs(this),

Wabbajack.Lib/MO2Installer.cs

@@ -81,6 +81,14 @@ namespace Wabbajack.Lib
                 Utils.Log("Exiting because we couldn't find the game folder.");
                 return false;
             }
+            
+            var watcher = new DiskSpaceWatcher(cancel, new[]{OutputFolder, DownloadFolder, GameFolder.Value, AbsolutePath.EntryPoint}, (long)2 << 31,
+                drive =>
+                {
+                    Utils.Log($"Aborting due to low space on {drive.Name}");
+                    Abort();
+                });
+            var watcherTask = watcher.Start();
 
             if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Validating Game ESMs");

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -17,7 +17,7 @@
         <Version>3.1.0</Version>
       </PackageReference>
       <PackageReference Include="Fody">
-        <Version>6.2.0</Version>
+        <Version>6.2.1</Version>
       </PackageReference>
       <PackageReference Include="Genbox.AlphaFS">
         <Version>2.2.2.1</Version>
@@ -44,7 +44,7 @@
         <Version>11.4.17</Version>
       </PackageReference>
       <PackageReference Include="SharpCompress">
-        <Version>0.25.1</Version>
+        <Version>0.26.0</Version>
       </PackageReference>
       <PackageReference Include="System.Collections.Immutable">
         <Version>5.0.0-preview.6.20305.6</Version>

Wabbajack.Server.Test/Wabbajack.Server.Test.csproj

@@ -11,7 +11,7 @@
         <PackageReference Include="xunit" Version="2.4.1" />
         <PackageReference Include="xunit.runner.visualstudio" Version="2.4.2" />
         <PackageReference Include="coverlet.collector" Version="1.3.0" />
-        <PackageReference Include="XunitContext" Version="2.0.0" />
+        <PackageReference Include="XunitContext" Version="2.0.1" />
     </ItemGroup>
 
     <ItemGroup>

Wabbajack.Server/Wabbajack.Server.csproj

@@ -3,8 +3,8 @@
     <PropertyGroup>
         <OutputType>Exe</OutputType>
         <TargetFramework>netcoreapp3.1</TargetFramework>
-        <AssemblyVersion>2.1.3.3</AssemblyVersion>
+        <AssemblyVersion>2.1.3.4</AssemblyVersion>
-        <FileVersion>2.1.3.3</FileVersion>
+        <FileVersion>2.1.3.4</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Server</Description>
         <RuntimeIdentifier>win-x64</RuntimeIdentifier>

Wabbajack.Test/DownloaderTests.cs

@@ -490,6 +490,7 @@ namespace Wabbajack.Test
             Assert.Equal(entries, new List<string> {@"Data\TestCK.esp", @"Data\TestCK.ini"});
         }
         
+        /*
         [Fact]
         public async Task YoutubeDownloader()
         {
@@ -515,6 +516,7 @@ namespace Wabbajack.Test
             await converted.Download(new Archive(state: null!) { Name = "yt_test.zip"}, tempFile.Path);
             Assert.Equal(Hash.FromBase64("pD7UoVNY4o8="), await tempFile.Path.FileHashAsync());
         }
+        */
         
         
         /// <summary>

Wabbajack.Test/EndToEndTests.cs

@@ -62,7 +62,7 @@ namespace Wabbajack.Test
                 DownloadAndInstall(Game.Fallout4, 11925, "Anti-Tank Rifle"), 
                 DownloadAndInstall(Game.SkyrimSpecialEdition, 4783, "Frost Armor UNP"), 
                 DownloadAndInstall(Game.SkyrimSpecialEdition, 32359, "Frost Armor HDT"),
-                DownloadAndInstall(Game.SkyrimSpecialEdition, 31667, "Nemesis"),
+                DownloadAndInstall("https://github.com/ShikyoKira/Project-New-Reign---Nemesis-Main/releases/download/v0.84-beta/Nemesis.Unlimited.Behavior.Engine.v0.84-beta.rar", "Nemesis.Unlimited.Behavior.Engine.v0.84-beta.rar", "Nemesis"),
                 DownloadAndInstall(Game.Fallout4, 40136, "RAR test File"));
             
             // We're going to fully patch this mod from another source.
@@ -94,8 +94,10 @@ namespace Wabbajack.Test
 
         }
 
-        private async Task DownloadAndInstall(string url, string filename, string modName = null)
+        private async Task<(AbsolutePath Download, AbsolutePath ModFolder)> DownloadAndInstall(string url, string filename, string modName = null)
         {
+            if (modName != null)
+                await utils.AddMod(modName);
             var src = _downloadFolder.Combine(filename);
             if (!src.Exists)
             {
@@ -105,10 +107,13 @@ namespace Wabbajack.Test
 
             utils.DownloadsFolder.CreateDirectory();
 
-            await src.CopyToAsync(utils.DownloadsFolder.Combine(filename));
+            var destFile = utils.DownloadsFolder.Combine(filename);
+            await src.CopyToAsync(destFile);
 
             await using var dest = await FileExtractor.ExtractAll(Queue, src);
-            await dest.MoveAllTo(modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName));
+            var modFolder = modName == null ? utils.MO2Folder : utils.ModsFolder.Combine(modName);
+            await dest.MoveAllTo(modFolder);
+            return (destFile, modFolder);
         }
 
         private async Task<(AbsolutePath Download, AbsolutePath ModFolder)> DownloadAndInstall(Game game, int modId, string modName)

Wabbajack.Test/Wabbajack.Test.csproj

@@ -31,7 +31,7 @@
       <PackageReference Include="xunit" Version="2.4.1" />
       <PackageReference Include="xunit.runner.visualstudio" Version="2.4.2" />
       <PackageReference Include="coverlet.collector" Version="1.3.0" />
-      <PackageReference Include="XunitContext" Version="2.0.0" />
+      <PackageReference Include="XunitContext" Version="2.0.1" />
   </ItemGroup>
 
   <ItemGroup>

Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs

@@ -37,7 +37,7 @@ namespace Wabbajack.VirtualFileSystem
             return false;
         }
 
-        public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> OnlyFiles)
+        public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> OnlyFiles, bool throwOnError)
         {
             throw new Exception("BSAs can't contain archives");
         }

Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs

@@ -33,9 +33,9 @@ namespace Wabbajack.VirtualFileSystem
             return await FileExtractor.CanExtract(_path);
         }
 
-        public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> onlyFiles)
+        public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> onlyFiles, bool throwOnError)
         {
-            return FileExtractor.ExtractAll(queue, _path, onlyFiles);
+            return FileExtractor.ExtractAll(queue, _path, onlyFiles, throwOnError);
         }
 
         public async Task MoveTo(AbsolutePath path)

Wabbajack.VirtualFileSystem/FileExtractor.cs

@@ -27,7 +27,7 @@ namespace Wabbajack.VirtualFileSystem
             Definitions.FileType.RAR,
             Definitions.FileType._7Z);
         
-        public static async Task<ExtractedFiles> ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable<RelativePath> OnlyFiles = null)
+        public static async Task<ExtractedFiles> ExtractAll(WorkQueue queue, AbsolutePath source, IEnumerable<RelativePath> OnlyFiles = null, bool throwOnError = true)
         {
             try
             {
@@ -35,6 +35,8 @@ namespace Wabbajack.VirtualFileSystem
                 
                 if (source.Extension == Consts.OMOD)
                     return await ExtractAllWithOMOD(source);
+
+                Utils.Log($"Extracting {sig}");
                 
                 switch (sig)
                 {
@@ -53,6 +55,9 @@ namespace Wabbajack.VirtualFileSystem
             }
             catch (Exception ex)
             {
+                if (!throwOnError)
+                    return new ExtractedFiles(await TempFolder.Create());
+
                 Utils.ErrorThrow(ex, $"Error while extracting {source}");
                 throw new Exception();
             }
@@ -219,22 +224,24 @@ namespace Wabbajack.VirtualFileSystem
         /// <returns></returns>
         public static async Task<bool> CanExtract(AbsolutePath v)
         {
-            var ext = v.Extension;
+            var found = await archiveSigs.MatchesAsync(v);
-            if(ext != _exeExtension && !Consts.TestArchivesBeforeExtraction.Contains(ext))
+            switch (found)
-                return Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext);
-
-            var isArchive = await TestWith7z(v);
-
-            if (isArchive)
-                return true;
-
-            var process = new ProcessHelper
             {
-                Path = @"Extractors\innounp.exe".RelativeTo(AbsolutePath.EntryPoint),
+                case null:
-                Arguments = new object[] {"-t", v},
+                    return false;
-            };
+                case Definitions.FileType.EXE:
+                {
+                    var process = new ProcessHelper
+                    {
+                        Path = @"Extractors\innounp.exe".RelativeTo(AbsolutePath.EntryPoint),
+                        Arguments = new object[] {"-t", v},
+                    };
 
-            return await process.Start() == 0;
+                    return await process.Start() == 0;
+                }
+                default:
+                    return true;
+            }
         }
 
         public static async Task<bool> TestWith7z(AbsolutePath file)

Wabbajack.VirtualFileSystem/IExtractedFile.cs

@@ -16,7 +16,7 @@ namespace Wabbajack.VirtualFileSystem
 
         public Task<bool> CanExtract();
 
-        public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> Only = null);
+        public Task<ExtractedFiles> ExtractAll(WorkQueue queue, IEnumerable<RelativePath> Only = null, bool throwOnError = false);
 
         public Task MoveTo(AbsolutePath path);
 

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -233,7 +233,7 @@ namespace Wabbajack.VirtualFileSystem
             try
             {
 
-                await using var extracted = await extractedFile.ExtractAll(context.Queue);
+                await using var extracted = await extractedFile.ExtractAll(context.Queue, throwOnError:false);
 
                 var list = await extracted
                     .PMap(context.Queue,

Wabbajack/Wabbajack.csproj

@@ -58,12 +58,12 @@
     <PackageReference Include="CefSharp.Wpf" Version="83.4.20" />
     <PackageReference Include="DynamicData" Version="6.16.1" />
     <PackageReference Include="Extended.Wpf.Toolkit" Version="4.0.1" />
-    <PackageReference Include="Fody" Version="6.2.0">
+    <PackageReference Include="Fody" Version="6.2.1">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
-    <PackageReference Include="GitInfo" Version="2.0.29">
+    <PackageReference Include="GitInfo" Version="2.0.30">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>