Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Purge all non async IO routines from Paths.cs

Browse Source
This commit is contained in:
Timothy Baldridge 2020-05-25 08:31:56 -06:00
parent a9eb43faf3
commit f9dc9148e7
14 changed files with 118 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.CLI/Verbs/Decrypt.cs
View File

@ -16,7 +16,7 @@ namespace Wabbajack.CLI.Verbs
protected override async Task<ExitCode> Run()
{
File.WriteAllBytes(Output, Utils.FromEncryptedData(Name));
File.WriteAllBytes(Output, await Utils.FromEncryptedData(Name));
return 0;
}
}

37
Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs Normal file
View File

@ -0,0 +1,37 @@
using System;
using System.Threading.Tasks;
namespace Wabbajack.Common
{
public static class CircuitBreaker
{
public static TimeSpan DEFAULT_DELAY = TimeSpan.FromMilliseconds(100);
public static int DEFAULT_DELAY_MULTIPLIER = 2;
public static int DEFAULT_RETRIES = 5;
public static async ValueTask<TR> WithAutoRetry<TR, TE>(Func<ValueTask<TR>> f, TimeSpan? delay = null, int? multipler = null, int? maxRetries = null) where TE : Exception
{
int retries = 0;
delay ??= DEFAULT_DELAY;
multipler ??= DEFAULT_DELAY_MULTIPLIER;
maxRetries ??= DEFAULT_RETRIES;
TOP:
try
{
return await f();
}
catch (TE ex)
{
retries += 1;
if (retries > maxRetries)
throw;
Utils.Log($"(Retry {retries} of {maxRetries}), got exception {ex.Message}, waiting {delay.Value.TotalMilliseconds}ms");
await Task.Delay(delay.Value);
delay = delay * multipler;
goto TOP;
}
}
}
}

35
Wabbajack.Common/IAsyncEnumerableExtensions.cs Normal file
View File

@ -0,0 +1,35 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace Wabbajack.Common
{
public static class IAsyncEnumerableExtensions
{
/// <summary>
/// Same as .Select but expects a function that returns an async result
/// </summary>
/// <param name="coll"></param>
/// <param name="mapFn"></param>
/// <typeparam name="TIn"></typeparam>
/// <typeparam name="TOut"></typeparam>
/// <returns></returns>
public static async IAsyncEnumerable<TOut> SelectAsync<TIn, TOut>(this IEnumerable<TIn> coll,
Func<TIn, ValueTask<TOut>> mapFn)
{
foreach (var itm in coll)
{
yield return await mapFn(itm);
}
}
public static async ValueTask<List<T>> ToList<T>(this IAsyncEnumerable<T> coll)
{
var list =new List<T>();
await foreach (var itm in coll)
list.Add(itm);
return list;
}
}
}

2
Wabbajack.Common/Metrics.cs
View File

@ -32,7 +32,7 @@ namespace Wabbajack.Common
try
{
client.DefaultRequestHeaders.Add(Consts.MetricsKeyHeader,
Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader));
await Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader));
await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}");
}
catch (Exception)

89
Wabbajack.Common/Paths.cs
View File

@ -87,19 +87,21 @@ namespace Wabbajack.Common
public Extension Extension => Extension.FromPath(_path);
public FileStream OpenRead()
public ValueTask<FileStream> OpenRead()
{
return File.OpenRead(_path);
return OpenShared();
}
public FileStream Create()
public ValueTask<FileStream> Create()
{
return File.Create(_path);
var path = _path;
return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () => File.Create(path));
}
public FileStream OpenWrite()
public ValueTask<FileStream> OpenWrite()
{
return File.OpenWrite(_path);
var path = _path;
return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () => File.OpenWrite(path));
}
public async Task WriteAllTextAsync(string text)
@ -107,12 +109,6 @@ namespace Wabbajack.Common
await using var fs = File.Create(_path);
await fs.WriteAsync(Encoding.UTF8.GetBytes(text));
}
public void WriteAllText(string text)
{
using var fs = File.Create(_path);
fs.Write(Encoding.UTF8.GetBytes(text));
}
public bool Exists => File.Exists(_path) || Directory.Exists(_path);
public bool IsFile => File.Exists(_path);
@ -175,24 +171,6 @@ namespace Wabbajack.Common
public AbsolutePath Root => (AbsolutePath)Path.GetPathRoot(_path);
/// <summary>
/// Moves this file to the specified location, will use Copy if required
/// </summary>
/// <param name="otherPath"></param>
/// <param name="overwrite">Replace the destination file if it exists</param>
public void MoveTo(AbsolutePath otherPath, bool overwrite = false)
{
if (Root != otherPath.Root)
{
if (otherPath.Exists && overwrite)
otherPath.Delete();
CopyTo(otherPath);
return;
}
File.Move(_path, otherPath._path, overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None);
}
/// <summary>
/// Moves this file to the specified location, will use Copy if required
/// </summary>
@ -286,7 +264,7 @@ namespace Wabbajack.Common
public async Task<byte[]> ReadAllBytesAsync()
{
await using var f = OpenRead();
await using var f = await OpenShared();
return await f.ReadAllAsync();
}
@ -325,37 +303,19 @@ namespace Wabbajack.Common
return File.ReadAllLines(_path);
}
public void WriteAllBytes(byte[] data)
{
using var fs = Create();
fs.Write(data);
}
public async Task WriteAllBytesAsync(byte[] data)
{
await using var fs = Create();
await using var fs = await Create();
await fs.WriteAsync(data);
}
public async Task WriteAllAsync(Stream data, bool disposeAfter = true)
{
await using var fs = Create();
await using var fs = await Create();
await data.CopyToAsync(fs);
if (disposeAfter) await data.DisposeAsync();
}
public void AppendAllText(string text)
{
File.AppendAllText(_path, text);
}
public void CopyTo(AbsolutePath dest)
{
File.Copy(_path, dest._path);
}
[DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Auto)]
private static extern bool CreateHardLink(string lpFileName, string lpExistingFileName, IntPtr lpSecurityAttributes);
@ -384,12 +344,6 @@ namespace Wabbajack.Common
return (await ReadAllTextAsync()).Split(new[] {'\n', '\r'}, StringSplitOptions.RemoveEmptyEntries);
}
public byte[] ReadAllBytes()
{
using var file = OpenShared();
return file.ReadAll();
}
public static AbsolutePath GetCurrentDirectory()
{
return new AbsolutePath(Directory.GetCurrentDirectory());
@ -397,8 +351,8 @@ namespace Wabbajack.Common
public async Task CopyToAsync(AbsolutePath destFile)
{
await using var src = OpenRead();
await using var dest = destFile.Create();
await using var src = await OpenRead();
await using var dest = await destFile.Create();
await src.CopyToAsync(dest);
}
@ -413,11 +367,6 @@ namespace Wabbajack.Common
await WriteAllTextAsync(string.Join("\r\n",strings));
}
public void WriteAllLines(params string[] strings)
{
WriteAllText(string.Join("\n",strings));
}
public int CompareTo(AbsolutePath other)
{
return string.Compare(_path, other._path, StringComparison.Ordinal);
@ -428,14 +377,18 @@ namespace Wabbajack.Common
return File.ReadAllText(_path);
}
public FileStream OpenShared()
public ValueTask<FileStream> OpenShared()
{
return File.Open(_path, FileMode.Open, FileAccess.Read, FileShare.Read);
var path = _path;
return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () =>
File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read));
}
public FileStream WriteShared()
public ValueTask<FileStream> WriteShared()
{
return File.Open(_path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite);
var path = _path;
return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () =>
File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite));
}
public async Task CopyDirectoryToAsync(AbsolutePath destination)

16
Wabbajack.Common/Utils.cs
View File

@ -165,7 +165,7 @@ namespace Wabbajack.Common
if (LogFile == default) return;
lock (_lock)
{
LogFile.AppendAllText($"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n");
File.AppendAllText(LogFile.ToString(), $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n");
}
}
@ -794,16 +794,16 @@ namespace Wabbajack.Common
}
}
public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch)
public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out AbsolutePath ePatch)
{
var patchName = Consts.PatchCacheFolder.Combine($"{foundHash.ToHex()}_{fileHash.ToHex()}.patch");
if (patchName.Exists)
{
ePatch = patchName.ReadAllBytes();
ePatch = patchName;
return true;
}
ePatch = Array.Empty<byte>();
ePatch = default;
return false;
}
@ -1039,9 +1039,9 @@ namespace Wabbajack.Common
bytes.ToEcryptedData(key);
}
public static T FromEncryptedJson<T>(string key)
public static async Task<T> FromEncryptedJson<T>(string key)
{
var decoded = FromEncryptedData(key);
var decoded = await FromEncryptedData(key);
return Encoding.UTF8.GetString(decoded).FromJsonString<T>();
}
@ -1053,9 +1053,9 @@ namespace Wabbajack.Common
Consts.LocalAppDataPath.Combine(key).WriteAllBytes(encoded);
}
public static byte[] FromEncryptedData(string key)
public static async Task<byte[]> FromEncryptedData(string key)
{
var bytes = Consts.LocalAppDataPath.Combine(key).ReadAllBytes();
var bytes = await Consts.LocalAppDataPath.Combine(key).ReadAllBytesAsync();
return ProtectedData.Unprotect(bytes, Encoding.UTF8.GetBytes(key), DataProtectionScope.LocalMachine);
}

4
Wabbajack.Lib/AuthorApi/Client.cs
View File

@ -63,7 +63,7 @@ namespace Wabbajack.Lib.AuthorApi
{
progressFn("Hashing file parts", Percent.FactoryPutInRange(part.Index, parts.Length));
var buffer = new byte[part.Size];
await using (var fs = path.OpenShared())
await using (var fs = await path.OpenShared())
{
fs.Position = part.Offset;
await fs.ReadAsync(buffer);
@ -91,7 +91,7 @@ namespace Wabbajack.Lib.AuthorApi
{
progressFn("Uploading Part", Percent.FactoryPutInRange(part.Index, definition.Parts.Length));
var buffer = new byte[part.Size];
await using (var fs = path.OpenShared())
await using (var fs = await path.OpenShared())
{
fs.Position = part.Offset;
await fs.ReadAsync(buffer);

8
Wabbajack.Lib/ClientAPI.cs
View File

@ -43,11 +43,11 @@ namespace Wabbajack.Lib
public class ClientAPI
{
public static Common.Http.Client GetClient()
public static async Task<Common.Http.Client> GetClient()
{
var client = new Common.Http.Client();
if (Utils.HaveEncryptedJson(Consts.MetricsKeyHeader))
client.Headers.Add((Consts.MetricsKeyHeader, Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)));
client.Headers.Add((Consts.MetricsKeyHeader, await Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)));
return client;
}
@ -63,7 +63,7 @@ namespace Wabbajack.Lib
RETRY:
var response = await GetClient()
var response = await (await GetClient())
.PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json"));
if (response.IsSuccessStatusCode)
@ -114,7 +114,7 @@ namespace Wabbajack.Lib
public static async Task<NexusCacheStats> GetNexusCacheStats()
{
return await GetClient()
return await (await GetClient())
.GetJsonAsync<NexusCacheStats>($"{Consts.WabbajackBuildServerUri}nexus_cache/stats");
}

2
Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs
View File

@ -91,7 +91,7 @@ namespace Wabbajack.Lib.Downloaders
Helpers.Cookie[] cookies;
try
{
cookies = Utils.FromEncryptedJson<Helpers.Cookie[]>(_encryptedKeyName);
cookies = await Utils.FromEncryptedJson<Helpers.Cookie[]>(_encryptedKeyName);
if (cookies != null)
return Helpers.GetClient(cookies, SiteURL.ToString());
}

2
Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs
View File

@ -220,7 +220,7 @@ namespace Wabbajack.Lib.Downloaders
{
var info = new CollectedBNetInfo();
var login_info = Utils.FromEncryptedJson<BethesdaNetData>(DataName);
var login_info = await Utils.FromEncryptedJson<BethesdaNetData>(DataName);
var client = new Common.Http.Client();

4
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -128,10 +128,10 @@ namespace Wabbajack.Lib.Downloaders
response.Dispose();
Utils.Log($"Applying patch to {archive.Name}");
await using(var src = result.NewFile.Path.OpenShared())
await using(var src = await result.NewFile.Path.OpenShared())
await using (var final = destination.Create())
{
Utils.ApplyPatch(src, () => tempFile.Path.OpenShared(), final);
Utils.ApplyPatch(src, () => tempFile.Path.OpenShared().Result, final);
}
var hash = await destination.FileHashCachedAsync();

2
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -153,7 +153,7 @@ namespace Wabbajack.Lib.Downloaders
else
{
Utils.Status("Logging into MEGA with saved credentials.");
var infos = Utils.FromEncryptedJson<MEGAAuthInfos>(DataName);
var infos = await Utils.FromEncryptedJson<MEGAAuthInfos>(DataName);
var authInfo = infos.ToAuthInfos();
await MegaApiClient.LoginAsync(authInfo);
}

2
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -58,7 +58,7 @@ namespace Wabbajack.Lib.NexusApi
try
{
return Utils.FromEncryptedJson<string>("nexusapikey");
return await Utils.FromEncryptedJson<string>("nexusapikey");
}
catch (Exception)
{

5
Wabbajack.Lib/zEditIntegration.cs
View File

@ -185,7 +185,7 @@ namespace Wabbajack.Lib
};
}));
var srcData = result.Sources.Select(f => _mo2Compiler.MO2Folder.Combine(f.RelativePath).ReadAllBytes())
var srcData = (await result.Sources.SelectAsync(async f => await _mo2Compiler.MO2Folder.Combine(f.RelativePath).ReadAllBytesAsync()).ToList())
.ConcatArrays();
var dstData = await source.AbsolutePath.ReadAllBytesAsync();
@ -264,7 +264,8 @@ namespace Wabbajack.Lib
{
Utils.LogStatus($"Generating zEdit merge: {m.To}");
var srcData = m.Sources.Select(s => installer.OutputFolder.Combine(s.RelativePath).ReadAllBytes())
var srcData = (await m.Sources.SelectAsync(async s => await installer.OutputFolder.Combine(s.RelativePath).ReadAllBytesAsync())
.ToList())
.ConcatArrays();
var patchData = await installer.LoadBytesFromPath(m.PatchID);