IPS4 reworks

CHANGELOG.md

@@ -1,5 +1,10 @@
 ### Changelog
 
+#### Version - 2.3.0.0 - ???
+* Rewrote the file extraction routines. New code uses less memory, less disk space, and performs less thrashing on HDDs
+* Reworked IPS4 integration to reduce download failures
+
+
 #### Version - 2.2.2.0 - 8/31/2020
 * Route CDN requests through a reverse proxy to improve reliability
 

Compression.BSA/Compression.BSA.csproj

@@ -20,7 +20,7 @@
     </PropertyGroup>
     <ItemGroup>
       <PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
-      <PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.2.4-alpha" />
+      <PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.2.5" />
       <PackageReference Include="SharpZipLib" Version="1.2.0" />
       <PackageReference Include="System.Text.Encoding.CodePages" Version="5.0.0-preview.8.20407.11" />
     </ItemGroup>

Wabbajack.Common/Consts.cs

@@ -131,6 +131,8 @@ namespace Wabbajack.Common
         public static AbsolutePath SettingsFile => LocalAppDataPath.Combine("settings.json");
         public static RelativePath SettingsIni = (RelativePath)"settings.ini";
         public static byte SettingsVersion => 2;
+        public static bool IsServer = false;
+
         public static string CompressedBodyHeader = "x-compressed-body";
 
         public static AbsolutePath CefCacheLocation = @"CEF".RelativeTo(LocalAppDataPath);

Wabbajack.Lib/ClientAPI.cs

@@ -13,8 +13,9 @@ using System.Threading.Tasks;
 using Wabbajack.Common.Exceptions;
 using Wabbajack.Common.Serialization.Json;
 using Wabbajack.Lib.Downloaders;
+ using Wabbajack.Lib.LibCefHelpers;
 
-namespace Wabbajack.Lib
+ namespace Wabbajack.Lib
 {
     public static class BuildServerStatus
     {
@@ -254,5 +255,12 @@ namespace Wabbajack.Lib
                 return null;
             }
         }
+
+        public static async Task<Helpers.Cookie[]> GetAuthInfo<T>(string key)
+        {
+            var client = await GetClient();
+            return await client.GetJsonAsync<Helpers.Cookie[]>(
+                $"{Consts.WabbajackBuildServerUri}site-integration/auth-info/{key}");
+        }
     }
 }

Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs

@@ -86,9 +86,11 @@ namespace Wabbajack.Lib.Downloaders
             return cookies;
         }
         
-        public async Task<Wabbajack.Lib.Http.Client> GetAuthedClient()
+        public async Task<Http.Client> GetAuthedClient()
         {
             Helpers.Cookie[] cookies;
+            if (Consts.IsServer)
+            {
                 try
                 {
                     cookies = await Utils.FromEncryptedJson<Helpers.Cookie[]>(_encryptedKeyName);
@@ -101,6 +103,10 @@ namespace Wabbajack.Lib.Downloaders
                 return Helpers.GetClient(cookies, SiteURL.ToString());
             }
 
+            cookies = await ClientAPI.GetAuthInfo<Helpers.Cookie[]>(_encryptedKeyName);
+            return Helpers.GetClient(cookies, SiteURL.ToString());
+        }
+        
         public async Task Prepare()
         {
             if (_isPrepared) return;

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -56,8 +56,13 @@ namespace Wabbajack.Lib.Downloaders
                 if (state != null)
                     return state;
             }
-            return null;
             
+            var meta = string.Join("\n", new string[]
+            {
+                "[General]",
+                $"directURL={uri}"    
+            });
+            return (AbstractDownloadState)(await ResolveArchive(meta.LoadIniString()));
         }
 
         public static T GetInstance<T>() where T : IDownloader

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -17,7 +17,9 @@
         <Version>3.1.0</Version>
       </PackageReference>
       <PackageReference Include="Fody">
-        <Version>6.2.4</Version>
+        <Version>6.2.5</Version>
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       </PackageReference>
       <PackageReference Include="Genbox.AlphaFS">
         <Version>2.2.2.1</Version>

Wabbajack.Server/Program.cs

@@ -4,6 +4,7 @@ using System.Net;
 using System.Security.Cryptography.X509Certificates;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Hosting;
+using Wabbajack.Common;
 
 namespace Wabbajack.Server
 {
@@ -11,6 +12,7 @@ namespace Wabbajack.Server
     {
         public static void Main(string[] args)
         {
+            Consts.IsServer = true;
             bool testMode = args.Contains("TESTMODE");
             CreateHostBuilder(args, testMode).Build().Run();
         }

Wabbajack.Server/Wabbajack.Server.csproj

@@ -18,7 +18,7 @@
         <PackageReference Include="Dapper" Version="2.0.35" />
         <PackageReference Include="FluentFTP" Version="32.4.4" />
         <PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
-        <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.7" />
+        <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.8" />
         <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
         <PackageReference Include="Nettle" Version="1.3.0" />
         <PackageReference Include="Swashbuckle.AspNetCore" Version="5.5.1" />

Wabbajack/Wabbajack.csproj

@@ -56,9 +56,9 @@
 
   <ItemGroup>
     <PackageReference Include="CefSharp.Wpf" Version="84.4.10" />
-    <PackageReference Include="DynamicData" Version="6.16.8" />
+    <PackageReference Include="DynamicData" Version="6.16.9" />
     <PackageReference Include="Extended.Wpf.Toolkit" Version="4.0.1" />
-    <PackageReference Include="Fody" Version="6.2.4">
+    <PackageReference Include="Fody" Version="6.2.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>