Merge branch 'master' of https://github.com/wabbajack-tools/wabbajack

Wabbajack.Common/Utils.cs

@@ -103,17 +103,25 @@ namespace Wabbajack.Common
             return sha.Hash.ToBase64();
         }
 
-        public static string FileHash(this string file)
+        public static string FileHash(this string file, bool nullOnIOError = false)
         {
-            var hash = new xxHashConfig();
-            hash.HashSizeInBits = 64;
-            hash.Seed = 0x42;
-            using (var fs = File.OpenRead(file))
+            try
             {
-                var config = new xxHashConfig();
-                config.HashSizeInBits = 64;
-                var value = xxHashFactory.Instance.Create(config).ComputeHash(fs);
-                return value.AsBase64String();
+                var hash = new xxHashConfig();
+                hash.HashSizeInBits = 64;
+                hash.Seed = 0x42;
+                using (var fs = File.OpenRead(file))
+                {
+                    var config = new xxHashConfig();
+                    config.HashSizeInBits = 64;
+                    var value = xxHashFactory.Instance.Create(config).ComputeHash(fs);
+                    return value.AsBase64String();
+                }
+            }
+            catch (IOException ex)
+            {
+                if (nullOnIOError) return null;
+                throw ex;
             }
         }
 

Wabbajack.Lib/Compiler.cs

@@ -352,7 +352,11 @@ namespace Wabbajack.Lib
             var metadata = new ModlistMetadata.DownloadMetadata
             {
                 Size = File.GetSize(ModListOutputFile),
-                Hash = ModListOutputFile.FileHash()
+                Hash = ModListOutputFile.FileHash(),
+                NumberOfArchives = ModList.Archives.Count,
+                SizeOfArchives = ModList.Archives.Sum(a => a.Size),
+                NumberOfInstalledFiles = ModList.Directives.Count,
+                SizeOfInstalledFiles = ModList.Directives.Sum(a => a.Size)
             };
             metadata.ToJSON(ModListOutputFile + ".meta.json");
 

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -18,8 +18,8 @@ namespace Wabbajack.Lib.Downloaders
             new ModDBDownloader(),
             new NexusDownloader(),
             new MediaFireDownloader(),
+            new HTTPDownloader(),
             new ManualDownloader(),
-            new HTTPDownloader()
         };
 
         private static readonly Dictionary<Type, IDownloader> IndexedDownloaders;

Wabbajack.Lib/Downloaders/ManualDownloader.cs

@@ -1,15 +1,67 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
 using System.Linq;
+using System.Reactive.Linq;
+using System.Reactive.Subjects;
+using System.Reactive.Threading.Tasks;
 using System.Text;
 using System.Threading.Tasks;
+using Syroot.Windows.IO;
 using Wabbajack.Common;
 using Wabbajack.Lib.Validation;
+using File = System.IO.File;
 
 namespace Wabbajack.Lib.Downloaders
 {
-    class ManualDownloader : IDownloader
+    public class ManualDownloader : IDownloader
     {
+        private FileSystemWatcher _watcher;
+        private Subject<FileEvent> _fileEvents = new Subject<FileEvent>();
+        private KnownFolder _downloadfolder;
+
+        class FileEvent
+        {
+            public string FullPath { get; set; }
+            public string Name { get; set; }
+            public long Size { get; set; }
+        }
+
+        public ManualDownloader()
+        {
+            _downloadfolder = new KnownFolder(KnownFolderType.DownloadsLocalized);
+            _watcher = new FileSystemWatcher(_downloadfolder.Path);
+            _watcher.Created += _watcher_Created;
+            _watcher.Changed += _watcher_Changed;
+        }
+
+        private void _watcher_Changed(object sender, FileSystemEventArgs e)
+        {
+            PublishEvent(e);
+        }
+        private void _watcher_Created(object sender, FileSystemEventArgs e)
+        {
+            PublishEvent(e);
+        }
+
+        private void PublishEvent(FileSystemEventArgs e)
+        {
+            try
+            {
+                _fileEvents.OnNext(new FileEvent
+                {
+                    Size = new FileInfo(e.FullPath).Length,
+                    Name = e.Name,
+                    FullPath = e.FullPath
+                });
+            }
+            catch (IOException)
+            {
+
+            }
+        }
+
         public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
         {
             var url = archive_ini?.General?.manualURL;
@@ -30,7 +82,33 @@ namespace Wabbajack.Lib.Downloaders
 
             public override void Download(Archive a, string destination)
             {
-                Utils.Log($"You must manually visit {Url} and download {a.Name} file by hand.");
+                var downloader = (ManualDownloader)GetDownloader();
+                var abs_path = Path.Combine(downloader._downloadfolder.Path, a.Name);
+                lock (downloader)
+                {
+                    try
+                    {
+                        Utils.Log($"You must manually visit {Url} and download {a.Name} file by hand.");
+                        Utils.Log($"Waiting for {a.Name}");
+                        downloader._watcher.EnableRaisingEvents = true;
+                        var watcher = downloader._fileEvents
+                            .Where(f => f.Size == a.Size)
+                            .Where(f => f.FullPath.FileHash(true) == a.Hash)
+                            .Buffer(new TimeSpan(0, 5, 0), 1)
+                            .Select(x => x.FirstOrDefault())
+                            .FirstOrDefaultAsync();
+                        Process.Start(Url);
+                        
+                        abs_path = watcher.Wait()?.FullPath;
+                        if (!File.Exists(abs_path))
+                            throw new InvalidDataException($"File not found after manual download operation");
+                        File.Move(abs_path, destination);
+                    }
+                    finally
+                    {
+                        downloader._watcher.EnableRaisingEvents = false;
+                    }
+                }
             }
 
             public override bool Verify()

Wabbajack.Lib/Installer.cs

@@ -229,19 +229,6 @@ namespace Wabbajack.Lib
             Info("Done! You may now exit the application!");
         }
 
-        private bool LocateGameFolder()
-        {
-            var fs = UIUtils.ShowFolderSelectionDialog("Please locate your game installation path");
-            if (fs != null)
-            {
-                GameFolder = fs;
-                return true;
-            }
-
-            return false;
-        }
-
-
         /// <summary>
         ///     We don't want to make the installer index all the archives, that's just a waste of time, so instead
         ///     we'll pass just enough information to VFS to let it know about the files we have.
@@ -410,13 +397,29 @@ namespace Wabbajack.Lib
 
             Status($"Copying files for {archive.Name}");
 
-            vfiles.DoIndexed((idx, file) =>
+            void CopyFile(string from, string to, bool use_move)
+            {
+                if (File.Exists(to))
+                    File.Delete(to);
+                if (use_move)
+                    File.Move(from, to);
+                else
+                    File.Copy(from, to);
+            }
+
+            vfiles.GroupBy(f => f.FromFile)
+                  .DoIndexed((idx, group) =>
             {
                 Utils.Status("Installing files", idx * 100 / vfiles.Count);
-                var dest = Path.Combine(Outputfolder, file.To);
-                if (File.Exists(dest))
-                    File.Delete(dest);
-                File.Copy(file.FromFile.StagedPath, dest);
+                var first_dest = Path.Combine(Outputfolder, group.First().To);
+                CopyFile(group.Key.StagedPath, first_dest, true);
+                
+                foreach (var copy in group.Skip(1))
+                {
+                    var next_dest = Path.Combine(Outputfolder, copy.To);
+                    CopyFile(first_dest, next_dest, false);
+                }
+
             });
 
             Status("Unstaging files");
@@ -457,17 +460,27 @@ namespace Wabbajack.Lib
 
             Info("Getting Nexus API Key, if a browser appears, please accept");
 
-            var dispatchers = ModList.Archives.Select(m => m.State.GetDownloader()).Distinct();
+            var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct();
 
             foreach (var dispatcher in dispatchers)
                 dispatcher.Prepare();
-
+            
             DownloadMissingArchives(missing);
         }
 
         private void DownloadMissingArchives(List<Archive> missing, bool download = true)
         {
-            missing.PMap(archive =>
+            if (download)
+            {
+                foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State)))
+                {
+                    var output_path = Path.Combine(DownloadFolder, a.Name);
+                    a.State.Download(a, output_path);
+                }
+            }
+
+            missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State))
+                   .PMap(archive =>
             {
                 Info($"Downloading {archive.Name}");
                 var output_path = Path.Combine(DownloadFolder, archive.Name);

Wabbajack.Lib/ModListRegistry/ModListMetadata.cs

@@ -63,6 +63,12 @@ namespace Wabbajack.Lib.ModListRegistry
         {
             public string Hash { get; set; }
             public long Size { get; set; }
+
+            public long NumberOfArchives { get; set; }
+            public long SizeOfArchives { get; set; }
+            public long NumberOfInstalledFiles { get; set; }
+            public long SizeOfInstalledFiles { get; set; }
+
         }
 
 
@@ -79,7 +85,7 @@ namespace Wabbajack.Lib.ModListRegistry
             if (!File.Exists(modlistPath)) return true;
             if (Links.DownloadMetadata?.Hash == null)
             {
-                return false;
+                return true;
             }
             return Links.DownloadMetadata.Hash != modlistPath.FileHash();
         }

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -54,6 +54,9 @@
   <ItemGroup>
     <Reference Include="PresentationCore" />
     <Reference Include="PresentationFramework" />
+    <Reference Include="Syroot.KnownFolders">
+      <HintPath>..\..\..\Users\tbald\.nuget\packages\syroot.windows.io.knownfolders\1.2.1\lib\net452\Syroot.KnownFolders.dll</HintPath>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="System.Design" />
@@ -186,6 +189,9 @@
     <PackageReference Include="SharpCompress">
       <Version>0.23.0</Version>
     </PackageReference>
+    <PackageReference Include="Syroot.Windows.IO.KnownFolders">
+      <Version>1.2.1</Version>
+    </PackageReference>
     <PackageReference Include="System.Reactive">
       <Version>4.2.0</Version>
     </PackageReference>

Wabbajack.Test/DownloaderTests.cs

@@ -147,6 +147,29 @@ namespace Wabbajack.Test
             Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
         }
 
+        [TestMethod]
+        public void ManualDownload()
+        {
+            var ini = @"[General]
+                        manualURL=http://build.wabbajack.org/WABBAJACK_TEST_FILE.zip";
+
+            var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+
+            Assert.IsNotNull(state);
+
+            var converted = state.ViaJSON();
+            Assert.IsTrue(converted.Verify());
+            var filename = Guid.NewGuid().ToString();
+
+            Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://build.wabbajack.org/" } }));
+
+            converted.Download(new Archive { Name = "WABBAJACK_TEST_FILE.zip", Size = 20, Hash = "eSIyd+KOG3s="}, filename);
+
+            Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
+
+            Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
+        }
+
         [TestMethod]
         public void MediaFireDownload()
         {

Wabbajack.Test/ModlistMetadataTests.cs

@@ -19,8 +19,6 @@ namespace Wabbajack.Test
             Assert.IsTrue(modlists.Count > 0);
         }
 
-        // Disabled until the list of modlists stabalizes a bit
-        /*
         [TestMethod]
         public void VerifyLogoURLs()
         {
@@ -30,11 +28,8 @@ namespace Wabbajack.Test
             {
                 var logo_state = DownloadDispatcher.ResolveArchive(modlist.ImageUri);
                 Assert.IsNotNull(logo_state);
-                //Assert.IsTrue(logo_state.Verify(), $"{modlist.ImageUri} is not valid");
-
-                //modlist.LoadLogo();
-               //Assert.IsNotNull(modlist.Logo);
+                Assert.IsTrue(logo_state.Verify(), $"{modlist.ImageUri} is not valid");
             }
-        }*/
+        }
     }
 }

Wabbajack.Test/SanityTests.cs

@@ -34,6 +34,29 @@ namespace Wabbajack.Test
             utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex");
         }
 
+        [TestMethod]
+        public void TestDuplicateFilesAreCopied()
+        {
+
+            var profile = utils.AddProfile();
+            var mod = utils.AddMod();
+            var test_pex = utils.AddModFile(mod, @"Data\scripts\test.pex", 10);
+
+            // Make a copy to make sure it gets picked up and moved around.
+            File.Copy(test_pex, test_pex + ".copy");
+
+            utils.Configure();
+
+            utils.AddManualDownload(
+                new Dictionary<string, byte[]> { { "/baz/biz.pex", File.ReadAllBytes(test_pex) } });
+
+            CompileAndInstall(profile);
+
+            utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex");
+            utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex.copy");
+        }
+
+
         [TestMethod]
         public void CleanedESMTest()
         {