Merge pull request #399 from Noggog/corrupted-modlists-fix

Corrupted Modlists Fix

CHANGELOG.md

@@ -3,6 +3,7 @@
 * Progress ring displays when downloading modlist images
 * GUI releases memory of download modlists better when navigating around
 * Fixed phrasing after failed installations to say "failed".
+* Fixed download bug that was marking some modlists as corrupted if they were replacing older versions.
 
 #### Version - 1.0 beta 15 - 1/6/2020
 * Don't delete the download folder when deleting empty folders during an update

Compression.BSA.Test/BSATests.cs

@@ -110,7 +110,7 @@ namespace Compression.BSA.Test
                         Directory.CreateDirectory(Path.GetDirectoryName(absName));
 
 
-                    using (var fs = File.OpenWrite(absName))
+                    using (var fs = File.Open(absName, FileMode.Create))
                     {
                         file.CopyDataTo(fs);
                     }

Compression.BSA/BA2Builder.cs

@@ -51,7 +51,7 @@ namespace Compression.BSA
         public void Build(string filename)
         {
             SortEntries();
-            using (var fs = File.OpenWrite(filename))
+            using (var fs = File.Open(filename, FileMode.Create))
             using (var bw = new BinaryWriter(fs))
             {
                 bw.Write(Encoding.ASCII.GetBytes(_state.HeaderMagic));

Compression.BSA/BSABuilder.cs

@@ -90,9 +90,7 @@ namespace Compression.BSA
         public void Build(string outputName)
         {
             RegenFolderRecords();
-            if (File.Exists(outputName)) File.Delete(outputName);
-
-            using (var fs = File.OpenWrite(outputName))
+            using (var fs = File.Open(outputName, FileMode.Create))
             using (var wtr = new BinaryWriter(fs))
             {
                 wtr.Write(_fileId);

Wabbajack.Common/FileExtractor.cs

@@ -27,10 +27,7 @@ namespace Wabbajack.Common
 
         private static void ExtractResource(string from, string to)
         {
-            if (File.Exists(to))
-                File.Delete(to);
-
-            using (var ous = File.OpenWrite(to))
+            using (var ous = File.Open(to, System.IO.FileMode.Create))
             using (var ins = new GZipInputStream(Assembly.GetExecutingAssembly().GetManifestResourceStream(from)))
             {
                 ins.CopyTo(ous);
@@ -144,7 +141,7 @@ namespace Wabbajack.Common
                             if (!Directory.Exists(parent))
                                 Directory.CreateDirectory(parent);
 
-                            using (var fs = File.OpenWrite(outPath))
+                            using (var fs = File.Open(outPath, System.IO.FileMode.Create))
                             {
                                 f.CopyDataTo(fs);
                             }

Wabbajack.Common/Metrics.cs

@@ -9,6 +9,10 @@ namespace Wabbajack.Common
 {
     public class Metrics
     {
+        public const string Downloading = "downloading";
+        public const string BeginInstall = "begin_install";
+        public const string FinishInstall = "finish_install";
+
         static Metrics()
         {
             if (!Utils.HaveEncryptedJson(Consts.MetricsKeyHeader))

Wabbajack.Common/Utils.cs

@@ -417,7 +417,7 @@ namespace Wabbajack.Common
         /*
         public static void ToBSON<T>(this T obj, string filename)
         {
-            using (var fo = File.OpenWrite(filename))
+            using (var fo = File.Open(filename, System.IO.FileMode.Create))
             using (var br = new BsonDataWriter(fo))
             {
                 fo.SetLength(0);
@@ -518,7 +518,7 @@ namespace Wabbajack.Common
 
         public static void BZip2ExtractToFile(this Stream src, string dest)
         {
-            using (var os = File.OpenWrite(dest))
+            using (var os = File.Open(dest, System.IO.FileMode.Create))
             {
                 os.SetLength(0);
                 using (var bz = new BZip2InputStream(src))
@@ -828,7 +828,7 @@ namespace Wabbajack.Common
                 {
                     var tmpName = Path.Combine("patch_cache", Guid.NewGuid() + ".tmp");
 
-                    using (var f = File.OpenWrite(tmpName))
+                    using (var f = File.Open(tmpName, System.IO.FileMode.Create))
                     {
                         Status("Creating Patch");
                         BSDiff.Create(a, b, f);
@@ -947,7 +947,7 @@ namespace Wabbajack.Common
                     long size = 0;
                     byte[] buffer = new byte[1024 * 8];
                     random.NextBytes(buffer);
-                    using (var fs = File.OpenWrite(file))
+                    using (var fs = File.Open(file, System.IO.FileMode.Create))
                     {
                         while (DateTime.Now < startTime + new TimeSpan(0, 0, seconds))
                         {

Wabbajack.Lib/ACompiler.cs

@@ -179,7 +179,7 @@ namespace Wabbajack.Lib
                 }
             }
 
-            using (var fs = File.OpenWrite($"{ModList.Name}.md"))
+            using (var fs = File.Open($"{ModList.Name}.md", System.IO.FileMode.Create))
             {
                 fs.SetLength(0);
                 using (var reporter = new ReportBuilder(fs, ModListOutputFolder))

Wabbajack.Lib/AInstaller.cs

@@ -214,7 +214,7 @@ namespace Wabbajack.Lib
                     File.Delete(toFile);
 
                     // Patch it
-                    using (var outStream = File.OpenWrite(toFile))
+                    using (var outStream = File.Open(toFile, FileMode.Create))
                     {
                         BSDiff.Apply(oldData, () => new MemoryStream(patchData), outStream);
                     }

Wabbajack.Lib/Downloaders/AbstractDownloadState.cs

@@ -60,15 +60,15 @@ namespace Wabbajack.Lib.Downloaders
         /// Downloads this file to the given destination location
         /// </summary>
         /// <param name="destination"></param>
-        public abstract Task Download(Archive a, string destination);
+        public abstract Task<bool> Download(Archive a, string destination);
 
-        public async Task Download(string destination)
+        public async Task<bool> Download(string destination)
         {
             var path = Path.GetDirectoryName(destination);
             if (!string.IsNullOrEmpty(path) && !Directory.Exists(path))
                 Directory.CreateDirectory(path);
             
-            await Download(new Archive {Name = Path.GetFileName(destination)}, destination);
+            return await Download(new Archive {Name = Path.GetFileName(destination)}, destination);
         }
 
         /// <summary>

Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs

@@ -56,13 +56,14 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override async Task Download(Archive a, string destination)
+            public override async Task<bool> Download(Archive a, string destination)
             {
                 var stream = await ResolveDownloadStream();
-                using (var file = File.OpenWrite(destination))
+                using (var file = File.Open(destination, FileMode.Create))
                 {
                     stream.CopyTo(file);
                 }
+                return true;
             }
 
             private async Task<Stream> ResolveDownloadStream()

Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs

@@ -62,14 +62,15 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override async Task Download(Archive a, string destination)
+            public override async Task<bool> Download(Archive a, string destination)
             {
                 using(var src = File.OpenRead(SourcePath))
-                using (var dest = File.OpenWrite(destination))
+                using (var dest = File.Open(destination, System.IO.FileMode.Create))
                 {
                     var size = new FileInfo(SourcePath).Length;
                     src.CopyToWithStatus(size, dest, "Copying from Game folder");
                 }
+                return true;
             }
 
             public override async Task<bool> Verify(Archive a)

Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs

@@ -44,10 +44,10 @@ namespace Wabbajack.Lib.Downloaders
                 return whitelist.GoogleIDs.Contains(Id);
             }
 
-            public override async Task Download(Archive a, string destination)
+            public override async Task<bool> Download(Archive a, string destination)
             {
                 var state = await ToHttpState();
-                await state.Download(a, destination);
+                return await state.Download(a, destination);
             }
 
             private async Task<HTTPDownloader.State> ToHttpState()

Wabbajack.Lib/Downloaders/HTTPDownloader.cs

@@ -70,7 +70,7 @@ namespace Wabbajack.Lib.Downloaders
                 return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p));
             }
 
-            public override Task Download(Archive a, string destination)
+            public override Task<bool> Download(Archive a, string destination)
             {
                 return DoDownload(a, destination, true);
             }
@@ -84,7 +84,7 @@ namespace Wabbajack.Lib.Downloaders
                         Directory.CreateDirectory(parent.FullName);
                 }
 
-                using (var fs = download ? File.OpenWrite(destination) : null)
+                using (var fs = download ? File.Open(destination, FileMode.Create) : null)
                 {
                     var client = Client ?? new HttpClient();
                     client.DefaultRequestHeaders.Add("User-Agent", Consts.UserAgent);
@@ -103,11 +103,11 @@ namespace Wabbajack.Lib.Downloaders
 
                     Utils.Status($"Starting Download {a?.Name ?? Url}", 0);
                     var response = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead);
-                    TOP:
+TOP:
 
                     if (!response.IsSuccessStatusCode)
                         throw new HttpException((int)response.StatusCode, response.ReasonPhrase);
-                    
+
                     Stream stream;
                     try
                     {

Wabbajack.Lib/Downloaders/MEGADownloader.cs

@@ -26,7 +26,7 @@ namespace Wabbajack.Lib.Downloaders
 
         public class State : HTTPDownloader.State
         {
-            public override async Task Download(Archive a, string destination)
+            public override async Task<bool> Download(Archive a, string destination)
             {
                 var client = new MegaApiClient();
                 Utils.Status("Logging into MEGA (as anonymous)");
@@ -35,6 +35,7 @@ namespace Wabbajack.Lib.Downloaders
                 var node = client.GetNodeFromLink(fileLink);
                 Utils.Status($"Downloading MEGA file: {a.Name}");
                 client.DownloadFile(fileLink, destination);
+                return true;
             }
 
             public override async Task<bool> Verify(Archive a)

Wabbajack.Lib/Downloaders/ManualDownloader.cs

@@ -80,7 +80,7 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override async Task Download(Archive a, string destination)
+            public override async Task<bool> Download(Archive a, string destination)
             {
                 var downloader = (ManualDownloader)GetDownloader();
                 var absPath = Path.Combine(downloader._downloadfolder.Path, a.Name);
@@ -109,6 +109,7 @@ namespace Wabbajack.Lib.Downloaders
                         downloader._watcher.EnableRaisingEvents = false;
                     }
                 }
+                return true;
             }
 
             public override async Task<bool> Verify(Archive a)

Wabbajack.Lib/Downloaders/MediaFireDownloader.cs

@@ -31,10 +31,10 @@ namespace Wabbajack.Lib.Downloaders
                 return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p));
             }
 
-            public override async Task Download(Archive a, string destination)
+            public override async Task<bool> Download(Archive a, string destination)
             {
                 var result = await Resolve();
-                await result.Download(a, destination);
+                return await result.Download(a, destination);
             }
 
             public override async Task<bool> Verify(Archive a)

Wabbajack.Lib/Downloaders/ModDBDownloader.cs

@@ -46,23 +46,25 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override async Task Download(Archive a, string destination)
+            public override async Task<bool> Download(Archive a, string destination)
             {
                 var urls = await GetDownloadUrls();
                 Utils.Log($"Found {urls.Length} ModDB mirrors for {a.Name}");
-                foreach (var (url, idx) in urls.Zip(Enumerable.Range(0, urls.Length), (s, i) => (s, i))) {
+                foreach (var (url, idx) in urls.Zip(Enumerable.Range(0, urls.Length), (s, i) => (s, i)))
+                {
                     try
                     {
                         await new HTTPDownloader.State {Url = url}.Download(a, destination);
-                        break;
+                        return true;
                     }
-                    catch (Exception ex)
+                    catch (Exception)
                     {
                         if (idx == urls.Length - 1)
                             throw;
                         Utils.Log($"Download from {url} failed, trying next mirror");
                     }
                 }
+                return false;
             }
 
             private async Task<string[]> GetDownloadUrls()

Wabbajack.Lib/Downloaders/NexusDownloader.cs

@@ -136,7 +136,7 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override async Task Download(Archive a, string destination)
+            public override async Task<bool> Download(Archive a, string destination)
             {
                 string url;
                 try
@@ -147,16 +147,15 @@ namespace Wabbajack.Lib.Downloaders
                 catch (Exception ex)
                 {
                     Utils.Log($"{a.Name} - Error getting Nexus download URL - {ex.Message}");
-                    return;
+                    return false;
                 }
 
                 Utils.Log($"Downloading Nexus Archive - {a.Name} - {GameName} - {ModID} - {FileID}");
 
-                await new HTTPDownloader.State
+                return await new HTTPDownloader.State
                 {
                     Url = url
                 }.Download(a, destination);
-
             }
 
             public override async Task<bool> Verify(Archive a)

Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs

@@ -49,7 +49,7 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override async Task Download(Archive a, string destination)
+            public override async Task<bool> Download(Archive a, string destination)
             {
                 var currentLib = Item.Game.Universe;
 
@@ -77,6 +77,8 @@ namespace Wabbajack.Lib.Downloaders
 
                     Thread.Sleep(1000);
                 }
+
+                return true;
             }
 
             public override async Task<bool> Verify(Archive a)

Wabbajack.Lib/LibCefHelpers/Init.cs

@@ -23,7 +23,7 @@ namespace Wabbajack.Lib.LibCefHelpers
         {
             if (File.Exists("cefsharp.7z") && File.Exists("libcef.dll")) return;
 
-            using (var fs = File.OpenWrite("cefsharp.7z"))
+            using (var fs = File.Open("cefsharp.7z", System.IO.FileMode.Create))
             using (var rs = Assembly.GetExecutingAssembly().GetManifestResourceStream("Wabbajack.Lib.LibCefHelpers.cefsharp.7z"))
             {
                 rs.CopyTo(fs);

Wabbajack.Lib/MO2Installer.cs

@@ -44,7 +44,7 @@ namespace Wabbajack.Lib
         protected override async Task<bool> _Begin(CancellationToken cancel)
         {
             if (cancel.IsCancellationRequested) return false;
-            var metric = Metrics.Send("begin_install", ModList.Name);
+            var metric = Metrics.Send(Metrics.BeginInstall, ModList.Name);
 
             ConfigureProcessor(20, ConstructDynamicNumThreads(await RecommendQueueSize()));
             var game = ModList.GameType.MetaData();
@@ -147,7 +147,7 @@ namespace Wabbajack.Lib
             SetScreenSizeInPrefs();
 
             UpdateTracker.NextStep("Installation complete! You may exit the program.");
-            var metric2 = Metrics.Send("finish_install", ModList.Name);
+            var metric2 = Metrics.Send(Metrics.FinishInstall, ModList.Name);
 
             return true;
         }
@@ -295,7 +295,7 @@ namespace Wabbajack.Lib
             var patchData = LoadBytesFromPath(directive.SourceDataID);
             var toFile = Path.Combine(OutputFolder, directive.To);
             Status($"Patching {filename}");
-            using (var output = File.OpenWrite(toFile))
+            using (var output = File.Open(toFile, FileMode.Create))
             using (var input = File.OpenRead(gameFile))
             {
                 BSDiff.Apply(input, () => new MemoryStream(patchData), output);

Wabbajack.Lib/VortexInstaller.cs

@@ -39,7 +39,7 @@ namespace Wabbajack.Lib
         protected override async Task<bool> _Begin(CancellationToken cancel)
         {
             if (cancel.IsCancellationRequested) return false;
-            var metric = Metrics.Send("begin_install", ModList.Name);
+            var metric = Metrics.Send(Metrics.BeginInstall, ModList.Name);
             var result = await Utils.Log(new YesNoIntervention(
                 "Vortex Support is still experimental and may produce unexpected results. " +
                 "If anything fails please go to the special Vortex support channels on the Wabbajack Discord and contact @erri120#2285 " +
@@ -103,7 +103,7 @@ namespace Wabbajack.Lib
             await InstallSteamWorkshopItems();
 
             //InstallIncludedDownloadMetas();
-            var metric2 = Metrics.Send("finish_install", ModList.Name);
+            var metric2 = Metrics.Send(Metrics.FinishInstall, ModList.Name);
             UpdateTracker.NextStep("Installation complete! You may exit the program.");
             return true;
         }

Wabbajack.Lib/zEditIntegration.cs

@@ -175,7 +175,7 @@ namespace Wabbajack.Lib
 
                     var patch_data = installer.LoadBytesFromPath(m.PatchID);
 
-                    using (var fs = File.OpenWrite(Path.Combine(installer.OutputFolder, m.To))) 
+                    using (var fs = File.Open(Path.Combine(installer.OutputFolder, m.To), FileMode.Create)) 
                         BSDiff.Apply(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs);
                 });
         }

Wabbajack.VirtualFileSystem/Context.cs

@@ -148,7 +148,7 @@ namespace Wabbajack.VirtualFileSystem
 
         public async Task WriteToFile(string filename)
         {
-            using (var fs = File.OpenWrite(filename))
+            using (var fs = File.Open(filename, FileMode.Create))
             using (var bw = new BinaryWriter(fs, Encoding.UTF8, true))
             {
                 fs.SetLength(0);

Wabbajack/View Models/ModListMetadataVM.cs

@@ -31,7 +31,7 @@ namespace Wabbajack
         private readonly ObservableAsPropertyHelper<bool> _Exists;
         public bool Exists => _Exists.Value;
 
-        public string Location => Path.Combine(Consts.ModListDownloadFolder, Metadata.Links.MachineURL + ExtensionManager.Extension);
+        public string Location { get; }
 
         [Reactive]
         public double ProgressPercent { get; private set; }
@@ -52,6 +52,7 @@ namespace Wabbajack
         {
             _parent = parent;
             Metadata = metadata;
+            Location = Path.Combine(Consts.ModListDownloadFolder, Metadata.Links.MachineURL + ExtensionManager.Extension);
             IsBroken = metadata.ValidationSummary.HasFailures;
             OpenWebsiteCommand = ReactiveCommand.Create(() => Process.Start($"https://www.wabbajack.org/modlist/{Metadata.Links.MachineURL}"));
             ExecuteCommand = ReactiveCommand.CreateFromObservable<Unit, Unit>(
@@ -69,7 +70,12 @@ namespace Wabbajack
                     {
                         try
                         {
-                            await Download();
+                            var success = await Download();
+                            if (!success)
+                            {
+                                Error = ErrorResponse.Fail("Download was marked unsuccessful");
+                                return false;
+                            }
                         }
                         catch (Exception ex)
                         {
@@ -136,24 +142,34 @@ namespace Wabbajack
                 .ToGuiProperty(this, nameof(LoadingImage));
         }
 
-        private Task Download()
+        private async Task<bool> Download()
         {
             ProgressPercent = 0d;
-            var queue = new WorkQueue(1);
-            var sub = queue.Status.Select(i => i.ProgressPercent)
-                .Subscribe(percent => ProgressPercent = percent);
-            TaskCompletionSource<bool> tcs = new TaskCompletionSource<bool>();
-            var metric = Metrics.Send("downloading", Metadata.Title);
-            queue.QueueTask(async () =>
+            using (var queue = new WorkQueue(1))
+            using (queue.Status.Select(i => i.ProgressPercent)
+                .Subscribe(percent => ProgressPercent = percent))
             {
-                var downloader = DownloadDispatcher.ResolveArchive(Metadata.Links.Download);
-                await downloader.Download(new Archive{ Name = Metadata.Title, Size = Metadata.DownloadMetadata?.Size ?? 0}, Location);
-                Location.FileHashCached();
-                sub.Dispose();
-                tcs.SetResult(true);
-            });
+                var tcs = new TaskCompletionSource<bool>();
+                queue.QueueTask(async () =>
+                {
+                    try
+                    {
+                        var downloader = DownloadDispatcher.ResolveArchive(Metadata.Links.Download);
+                        var result = await downloader.Download(new Archive { Name = Metadata.Title, Size = Metadata.DownloadMetadata?.Size ?? 0 }, Location);
+                        // Want to rehash to current file, even if failed?
+                        Location.FileHashCached();
+                        tcs.SetResult(result);
+                    }
+                    catch (Exception ex)
+                    {
+                        tcs.SetException(ex);
+                    }
+                });
 
-            return tcs.Task;
+                await Metrics.Send(Metrics.Downloading, Metadata.Title);
+
+                return await tcs.Task;
+            }
         }
     }
 }