Purge all non async IO routines from Paths.cs

Wabbajack.CLI/Verbs/Decrypt.cs

@@ -16,7 +16,7 @@ namespace Wabbajack.CLI.Verbs
 
         protected override async Task<ExitCode> Run()
         {
-            File.WriteAllBytes(Output, Utils.FromEncryptedData(Name));
+            File.WriteAllBytes(Output, await Utils.FromEncryptedData(Name));
             return 0;
         }
     }

Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Common
+{
+    public static class CircuitBreaker
+    {
+        public static TimeSpan DEFAULT_DELAY = TimeSpan.FromMilliseconds(100);
+        public static int DEFAULT_DELAY_MULTIPLIER = 2;
+        public static int DEFAULT_RETRIES = 5;
+
+        public static async ValueTask<TR> WithAutoRetry<TR, TE>(Func<ValueTask<TR>> f, TimeSpan? delay = null, int? multipler = null, int? maxRetries = null) where TE : Exception
+        {
+            int retries = 0;
+            delay ??= DEFAULT_DELAY;
+            multipler ??= DEFAULT_DELAY_MULTIPLIER;
+            maxRetries ??= DEFAULT_RETRIES;
+
+            TOP:
+            try
+            {
+                return await f();
+            }
+            catch (TE ex)
+            {
+                retries += 1;
+                if (retries > maxRetries)
+                    throw;
+                Utils.Log($"(Retry {retries} of {maxRetries}), got exception {ex.Message}, waiting {delay.Value.TotalMilliseconds}ms");
+                await Task.Delay(delay.Value);
+                delay = delay * multipler;
+                goto TOP;
+            }
+        }
+
+    }
+}

Wabbajack.Common/IAsyncEnumerableExtensions.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Common
+{
+    public static class IAsyncEnumerableExtensions
+    {
+        /// <summary>
+        /// Same as .Select but expects a function that returns an async result
+        /// </summary>
+        /// <param name="coll"></param>
+        /// <param name="mapFn"></param>
+        /// <typeparam name="TIn"></typeparam>
+        /// <typeparam name="TOut"></typeparam>
+        /// <returns></returns>
+        public static async IAsyncEnumerable<TOut> SelectAsync<TIn, TOut>(this IEnumerable<TIn> coll,
+            Func<TIn, ValueTask<TOut>> mapFn)
+        {
+            foreach (var itm in coll)
+            {
+                yield return await mapFn(itm);
+            }
+        }
+
+        public static async ValueTask<List<T>> ToList<T>(this IAsyncEnumerable<T> coll)
+        {
+            var list =new List<T>();
+            await foreach (var itm in coll)
+                list.Add(itm);
+            return list;
+        }
+        
+    }
+}

Wabbajack.Common/Metrics.cs

@@ -32,7 +32,7 @@ namespace Wabbajack.Common
             try
             {
                 client.DefaultRequestHeaders.Add(Consts.MetricsKeyHeader,
-                    Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader));
+                    await Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader));
                 await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}");
             }
             catch (Exception)

Wabbajack.Common/Paths.cs

@@ -87,19 +87,21 @@ namespace Wabbajack.Common
 
         public Extension Extension => Extension.FromPath(_path);
 
-        public FileStream OpenRead()
+        public ValueTask<FileStream> OpenRead()
         {
-            return File.OpenRead(_path);
+            return OpenShared();
         }
 
-        public FileStream Create()
+        public ValueTask<FileStream> Create()
         {
-            return File.Create(_path);
+            var path = _path;
+            return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () => File.Create(path));
         }
 
-        public FileStream OpenWrite()
+        public ValueTask<FileStream> OpenWrite()
         {
-            return File.OpenWrite(_path);
+            var path = _path;
+            return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () => File.OpenWrite(path));
         }
 
         public async Task WriteAllTextAsync(string text)
@@ -107,12 +109,6 @@ namespace Wabbajack.Common
             await using var fs = File.Create(_path);
             await fs.WriteAsync(Encoding.UTF8.GetBytes(text));
         }
-        
-        public void WriteAllText(string text)
-        {
-            using var fs = File.Create(_path);
-            fs.Write(Encoding.UTF8.GetBytes(text));
-        }
 
         public bool Exists => File.Exists(_path) || Directory.Exists(_path);
         public bool IsFile => File.Exists(_path);
@@ -175,24 +171,6 @@ namespace Wabbajack.Common
 
         public AbsolutePath Root => (AbsolutePath)Path.GetPathRoot(_path);
 
-        /// <summary>
-        ///     Moves this file to the specified location, will use Copy if required
-        /// </summary>
-        /// <param name="otherPath"></param>
-        /// <param name="overwrite">Replace the destination file if it exists</param>
-        public void MoveTo(AbsolutePath otherPath, bool overwrite = false)
-        {
-            if (Root != otherPath.Root)
-            {
-                if (otherPath.Exists && overwrite)
-                    otherPath.Delete();
-                
-                CopyTo(otherPath);
-                return;
-            }
-            File.Move(_path, otherPath._path, overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None);
-        }
-        
         /// <summary>
         ///     Moves this file to the specified location, will use Copy if required
         /// </summary>
@@ -286,7 +264,7 @@ namespace Wabbajack.Common
 
         public async Task<byte[]> ReadAllBytesAsync()
         {
-            await using var f = OpenRead();
+            await using var f = await OpenShared();
             return await f.ReadAllAsync();
         }
 
@@ -325,37 +303,19 @@ namespace Wabbajack.Common
             return File.ReadAllLines(_path);
         }
 
-        public void WriteAllBytes(byte[] data)
-        {
-            using var fs = Create();
-            fs.Write(data);
-        }
-
         public async Task WriteAllBytesAsync(byte[] data)
         {
-            await using var fs = Create();
+            await using var fs = await Create();
             await fs.WriteAsync(data);
         }
 
         public async Task WriteAllAsync(Stream data, bool disposeAfter = true)
         {
-            await using var fs = Create();
+            await using var fs = await Create();
             await data.CopyToAsync(fs);
             if (disposeAfter) await data.DisposeAsync();
         }
 
-        public void AppendAllText(string text)
-        {
-            File.AppendAllText(_path, text);
-        }
-
-        public void CopyTo(AbsolutePath dest)
-        {
-            File.Copy(_path, dest._path);
-        }
-        
-        
-
         [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Auto)]
         private static extern bool CreateHardLink(string lpFileName, string lpExistingFileName, IntPtr lpSecurityAttributes);
 
@@ -384,12 +344,6 @@ namespace Wabbajack.Common
             return (await ReadAllTextAsync()).Split(new[] {'\n', '\r'}, StringSplitOptions.RemoveEmptyEntries);
         }
 
-        public byte[] ReadAllBytes()
-        {
-            using var file = OpenShared();
-            return file.ReadAll();
-        }
-
         public static AbsolutePath GetCurrentDirectory()
         {
             return new AbsolutePath(Directory.GetCurrentDirectory());
@@ -397,8 +351,8 @@ namespace Wabbajack.Common
 
         public async Task CopyToAsync(AbsolutePath destFile)
         {
-            await using var src = OpenRead();
+            await using var src = await OpenRead();
-            await using var dest = destFile.Create();
+            await using var dest = await destFile.Create();
             await src.CopyToAsync(dest);
         }
 
@@ -413,11 +367,6 @@ namespace Wabbajack.Common
             await WriteAllTextAsync(string.Join("\r\n",strings));
         }
 
-        public void	 WriteAllLines(params string[] strings)
-        {
-            WriteAllText(string.Join("\n",strings));
-        }
-
         public int CompareTo(AbsolutePath other)
         {
             return string.Compare(_path, other._path, StringComparison.Ordinal);
@@ -428,14 +377,18 @@ namespace Wabbajack.Common
             return File.ReadAllText(_path);
         }
 
-        public FileStream OpenShared()
+        public ValueTask<FileStream> OpenShared()
         {
-            return File.Open(_path, FileMode.Open, FileAccess.Read, FileShare.Read);
+            var path = _path;
+            return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () =>
+                File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read));
         }
 
-        public FileStream WriteShared()
+        public ValueTask<FileStream> WriteShared()
         {
-            return File.Open(_path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite);
+            var path = _path;
+            return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () =>
+                File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite));
         }
 
         public async Task CopyDirectoryToAsync(AbsolutePath destination)

Wabbajack.Common/Utils.cs

@@ -165,7 +165,7 @@ namespace Wabbajack.Common
             if (LogFile == default) return;
             lock (_lock)
             {
-                LogFile.AppendAllText($"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n");
+                File.AppendAllText(LogFile.ToString(), $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n");
             }
         }
 
@@ -794,16 +794,16 @@ namespace Wabbajack.Common
             }
         }
 
-        public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch)
+        public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out AbsolutePath ePatch)
         {
             var patchName = Consts.PatchCacheFolder.Combine($"{foundHash.ToHex()}_{fileHash.ToHex()}.patch");
             if (patchName.Exists)
             {
-                ePatch = patchName.ReadAllBytes();
+                ePatch = patchName;
                 return true;
             }
 
-            ePatch = Array.Empty<byte>();
+            ePatch = default;
             return false;
         }
 
@@ -1039,9 +1039,9 @@ namespace Wabbajack.Common
             bytes.ToEcryptedData(key);
         }
 
-        public static T FromEncryptedJson<T>(string key)
+        public static async Task<T> FromEncryptedJson<T>(string key)
         {
-            var decoded = FromEncryptedData(key);
+            var decoded = await FromEncryptedData(key);
             return Encoding.UTF8.GetString(decoded).FromJsonString<T>();
         }
 
@@ -1053,9 +1053,9 @@ namespace Wabbajack.Common
             
             Consts.LocalAppDataPath.Combine(key).WriteAllBytes(encoded);
         }
-        public static byte[] FromEncryptedData(string key)
+        public static async Task<byte[]> FromEncryptedData(string key)
         {
-            var bytes = Consts.LocalAppDataPath.Combine(key).ReadAllBytes();
+            var bytes = await Consts.LocalAppDataPath.Combine(key).ReadAllBytesAsync();
             return ProtectedData.Unprotect(bytes, Encoding.UTF8.GetBytes(key), DataProtectionScope.LocalMachine);
         }
 

Wabbajack.Lib/AuthorApi/Client.cs

@@ -63,7 +63,7 @@ namespace Wabbajack.Lib.AuthorApi
                 {
                     progressFn("Hashing file parts", Percent.FactoryPutInRange(part.Index, parts.Length));
                     var buffer = new byte[part.Size];
-                    await using (var fs = path.OpenShared())
+                    await using (var fs = await path.OpenShared())
                     {
                         fs.Position = part.Offset;
                         await fs.ReadAsync(buffer);
@@ -91,7 +91,7 @@ namespace Wabbajack.Lib.AuthorApi
             {
                 progressFn("Uploading Part", Percent.FactoryPutInRange(part.Index, definition.Parts.Length));
                 var buffer = new byte[part.Size];
-                await using (var fs = path.OpenShared())
+                await using (var fs = await path.OpenShared())
                 {
                     fs.Position = part.Offset;
                     await fs.ReadAsync(buffer);

Wabbajack.Lib/ClientAPI.cs

@@ -43,11 +43,11 @@ namespace Wabbajack.Lib
     
     public class ClientAPI
     {
-        public static Common.Http.Client GetClient()
+        public static async Task<Common.Http.Client> GetClient()
         {
             var client = new Common.Http.Client();
             if (Utils.HaveEncryptedJson(Consts.MetricsKeyHeader)) 
-                client.Headers.Add((Consts.MetricsKeyHeader, Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)));
+                client.Headers.Add((Consts.MetricsKeyHeader, await Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)));
             return client;
         }
 
@@ -63,7 +63,7 @@ namespace Wabbajack.Lib
             
             RETRY:
             
-            var response = await GetClient()
+            var response = await (await GetClient())
                 .PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json"));
 
             if (response.IsSuccessStatusCode)
@@ -114,7 +114,7 @@ namespace Wabbajack.Lib
 
         public static async Task<NexusCacheStats> GetNexusCacheStats()
         {
-            return await GetClient()
+            return await (await GetClient())
                 .GetJsonAsync<NexusCacheStats>($"{Consts.WabbajackBuildServerUri}nexus_cache/stats");
         }
 

Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs

@@ -91,7 +91,7 @@ namespace Wabbajack.Lib.Downloaders
             Helpers.Cookie[] cookies;
             try
             {
-                cookies = Utils.FromEncryptedJson<Helpers.Cookie[]>(_encryptedKeyName);
+                cookies = await Utils.FromEncryptedJson<Helpers.Cookie[]>(_encryptedKeyName);
                 if (cookies != null)
                     return Helpers.GetClient(cookies, SiteURL.ToString());
             }

Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs

@@ -220,7 +220,7 @@ namespace Wabbajack.Lib.Downloaders
             {
                 var info = new CollectedBNetInfo();
 
-                var login_info = Utils.FromEncryptedJson<BethesdaNetData>(DataName);
+                var login_info = await Utils.FromEncryptedJson<BethesdaNetData>(DataName);
 
                 var client = new Common.Http.Client();
 

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -128,10 +128,10 @@ namespace Wabbajack.Lib.Downloaders
             response.Dispose();
 
             Utils.Log($"Applying patch to {archive.Name}");
-            await using(var src = result.NewFile.Path.OpenShared())
+            await using(var src = await result.NewFile.Path.OpenShared())
             await using (var final = destination.Create())
             {
-                Utils.ApplyPatch(src, () => tempFile.Path.OpenShared(), final);
+                Utils.ApplyPatch(src, () => tempFile.Path.OpenShared().Result, final);
             }
 
             var hash = await destination.FileHashCachedAsync();

Wabbajack.Lib/Downloaders/MEGADownloader.cs

@@ -153,7 +153,7 @@ namespace Wabbajack.Lib.Downloaders
                 else
                 {
                     Utils.Status("Logging into MEGA with saved credentials.");
-                    var infos = Utils.FromEncryptedJson<MEGAAuthInfos>(DataName);
+                    var infos = await Utils.FromEncryptedJson<MEGAAuthInfos>(DataName);
                     var authInfo = infos.ToAuthInfos();
                     await MegaApiClient.LoginAsync(authInfo);
                 }

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -58,7 +58,7 @@ namespace Wabbajack.Lib.NexusApi
 
                 try
                 {
-                    return Utils.FromEncryptedJson<string>("nexusapikey");
+                    return await Utils.FromEncryptedJson<string>("nexusapikey");
                 }
                 catch (Exception)
                 {

Wabbajack.Lib/zEditIntegration.cs

@@ -185,7 +185,7 @@ namespace Wabbajack.Lib
                     };
                 }));
 
-                var srcData = result.Sources.Select(f => _mo2Compiler.MO2Folder.Combine(f.RelativePath).ReadAllBytes())
+                var srcData = (await result.Sources.SelectAsync(async f => await _mo2Compiler.MO2Folder.Combine(f.RelativePath).ReadAllBytesAsync()).ToList())
                     .ConcatArrays();
 
                 var dstData = await source.AbsolutePath.ReadAllBytesAsync();
@@ -264,7 +264,8 @@ namespace Wabbajack.Lib
                 {
                     Utils.LogStatus($"Generating zEdit merge: {m.To}");
 
-                    var srcData = m.Sources.Select(s => installer.OutputFolder.Combine(s.RelativePath).ReadAllBytes())
+                    var srcData = (await m.Sources.SelectAsync(async s => await installer.OutputFolder.Combine(s.RelativePath).ReadAllBytesAsync())
+                        .ToList())
                         .ConcatArrays();
 
                     var patchData = await installer.LoadBytesFromPath(m.PatchID);