Merge branch 'master' into code-cleanup

CHANGELOG.md

@@ -12,6 +12,7 @@ install thread.
 * Fixed Extraction so that zip files no longer cause WJ to CTD
 * Better path logging during install and compilation
 * Fix the "this was created with a newer version of Wabbajack" issue
+* If a downloaded file doesn't match the expected hash, try alternative download locations, if allowed
 
 
 #### Version - 2.2.2.0 - 8/31/2020

Wabbajack.Common/Paths/FileCompaction.cs

@@ -1,4 +1,5 @@
-using System.Threading.Tasks;
+using System.Linq;
+using System.Threading.Tasks;
 using Wabbajack.Common.IO;
 
 namespace Wabbajack.Common
@@ -60,7 +61,12 @@ namespace Wabbajack.Common
 
         public static async Task CompactFolder(this AbsolutePath folder, WorkQueue queue, Algorithm algorithm)
         {
-            await folder.EnumerateFiles(true)
+            var driveInfo = folder.DriveInfo().DiskSpaceInfo;
+            var clusterSize = driveInfo.SectorsPerCluster * driveInfo.BytesPerSector;
+
+            await folder
+                .EnumerateFiles(true)
+                .Where(f => f.Size > clusterSize)
                 .PMap(queue, async path =>
                 {
                     Utils.Status($"Compacting {path.FileName}");

Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs

@@ -137,15 +137,16 @@ namespace Wabbajack.Lib.Downloaders
 
             public override async Task<bool> Download(Archive a, AbsolutePath destination)
             {
-                using var stream = await ResolveDownloadStream(a);
-                if (stream == null) return false;
+                var (isValid, istream) = await ResolveDownloadStream(a, false);
+                if (!isValid) return false;
+                using var stream = istream!;
                 await using var fromStream = await stream.Content.ReadAsStreamAsync();
                 await using var toStream = await destination.Create();
                 await fromStream.CopyToAsync(toStream);
                 return true;
             }
 
-            private async Task<HttpResponseMessage?> ResolveDownloadStream(Archive a)
+            private async Task<(bool, HttpResponseMessage?)> ResolveDownloadStream(Archive a, bool quickMode)
             {
                 TOP:
                 string url;
@@ -168,7 +169,7 @@ namespace Wabbajack.Lib.Downloaders
                     if (csrfKey == null)
                     {
                         Utils.Log($"Returning null from IPS4 Downloader because no csrfKey was found");
-                        return null;
+                        return (false, null);
                     }
 
                     var sep = Site.EndsWith("?") ? "&" : "?";
@@ -199,10 +200,10 @@ namespace Wabbajack.Lib.Downloaders
                     if (a.Size != 0 && headerContentSize != 0 && a.Size != headerContentSize)
                     {
                         Utils.Log($"Bad Header Content sizes {a.Size} vs {headerContentSize}");
-                        return null;
+                        return (false, null);
                     }
 
-                    return streamResult;
+                    return (true, streamResult);
                 }
 
                 // Sometimes LL hands back a json object telling us to wait until a certain time
@@ -210,6 +211,7 @@ namespace Wabbajack.Lib.Downloaders
                 var secs = times.Download - times.CurrentTime;
                 for (int x = 0; x < secs; x++)
                 {
+                    if (quickMode) return (true, default);
                     Utils.Status($"Waiting for {secs} at the request of {Downloader.SiteName}", Percent.FactoryPutInRange(x, secs));
                     await Task.Delay(1000);
                 }
@@ -228,7 +230,8 @@ namespace Wabbajack.Lib.Downloaders
 
             public override async Task<bool> Verify(Archive a)
             {
-                var stream = await ResolveDownloadStream(a);
+                var (isValid, stream) = await ResolveDownloadStream(a, true);
+                if (!isValid) return false;
                 if (stream == null)
                     return false;
 

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -104,8 +104,9 @@ namespace Wabbajack.Lib.Downloaders
         {
             if (await Download(archive, destination))
             {
-                await destination.FileHashCachedAsync();
-                return DownloadResult.Success;
+                var downloadedHash = await destination.FileHashCachedAsync();
+                if (downloadedHash == archive.Hash || archive.Hash == default) 
+                    return DownloadResult.Success;
             }
 
             

Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs

@@ -24,8 +24,21 @@ namespace Wabbajack.VirtualFileSystem
             Definitions.FileType.RAR_OLD,
             Definitions.FileType.RAR_NEW,
             Definitions.FileType._7Z);
-        
+
         private static Extension OMODExtension = new Extension(".omod");
+        private static Extension BSAExtension = new Extension(".bsa");
+
+        public static readonly HashSet<Extension> ExtractableExtensions = new HashSet<Extension>
+        {
+            new Extension(".bsa"),
+            new Extension(".ba2"),
+            new Extension(".7z"),
+            new Extension(".7zip"),
+            new Extension(".rar"),
+            new Extension(".zip"),
+            OMODExtension
+        };
+        
         
         /// <summary>
         /// When true, will allow 7z to use multiple threads and cache more data in memory, potentially
@@ -64,11 +77,16 @@ namespace Wabbajack.VirtualFileSystem
                     }
                 }
 
-                case Definitions.FileType.TES3:
                 case Definitions.FileType.BSA:
                 case Definitions.FileType.BA2:
                     return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
-                
+
+                case Definitions.FileType.TES3:
+                    if (sFn.Name.FileName.Extension == BSAExtension) 
+                        return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
+                    else
+                        throw new Exception($"Invalid file format {sFn.Name}");
+
 
                 default:
                     throw new Exception($"Invalid file format {sFn.Name}");

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -175,16 +175,27 @@ namespace Wabbajack.VirtualFileSystem
         public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
             IPath relPath, int depth = 0)
         {
-            await using var stream = await extractedFile.GetStream();
-            var hash = await stream.xxHashAsync();
-            stream.Position = 0;
+            Hash hash = default;
+            if (extractedFile is NativeFileStreamFactory)
+            {
+                hash = await ((AbsolutePath)extractedFile.Name).FileHashCachedAsync();
+            } 
+            else
+            {
+                await using var hstream = await extractedFile.GetStream();
+                hash = await hstream.xxHashAsync();
+            }
 
+            if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
+            {
+                return vself;
+            }
+
+            
+            await using var stream = await extractedFile.GetStream();
             var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
             stream.Position = 0;
             
-            if (sig.HasValue && TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
-                return vself;
-            
             var self = new VirtualFile
             {
                 Context = context,
@@ -202,7 +213,7 @@ namespace Wabbajack.VirtualFileSystem
                 self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
 
             // Can't extract, so return
-            if (!sig.HasValue) return self;
+            if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension)) return self;
 
             try
             {