using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reactive.Linq;
using System.Runtime.InteropServices;
using System.Security;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using ICSharpCode.SharpZipLib.BZip2;
using IniParser;
using IniParser.Model.Configuration;
using IniParser.Parser;
using Microsoft.Win32;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
using Directory = System.IO.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Common
{
public static partial class Utils
{
public static bool IsMO2Running(string mo2Path)
{
Process[] processList = Process.GetProcesses();
return processList.Where(process => process.ProcessName == "ModOrganizer").Any(process => Path.GetDirectoryName(process.MainModule?.FileName) == mo2Path);
}
static Utils()
{
InitalizeLogging().Wait();
}
private static readonly string[] Suffix = {"B", "KB", "MB", "GB", "TB", "PB", "EB"}; // Longs run out around EB
public static void CopyToWithStatus(this Stream istream, long maxSize, Stream ostream, string status)
{
var buffer = new byte[1024 * 64];
if (maxSize == 0) maxSize = 1;
long totalRead = 0;
while (true)
{
var read = istream.Read(buffer, 0, buffer.Length);
if (read == 0) break;
totalRead += read;
ostream.Write(buffer, 0, read);
Status(status, Percent.FactoryPutInRange(totalRead, maxSize));
}
}
public static async Task CopyToWithStatusAsync(this Stream istream, long maxSize, Stream ostream, string status)
{
var buffer = new byte[1024 * 64];
if (maxSize == 0) maxSize = 1;
long totalRead = 0;
long remain = maxSize;
while (true)
{
var toRead = Math.Min(buffer.Length, remain);
var read = await istream.ReadAsync(buffer, 0, (int)toRead);
remain -= read;
if (read == 0) break;
totalRead += read;
await ostream.WriteAsync(buffer, 0, read);
Status(status, Percent.FactoryPutInRange(totalRead, maxSize));
}
await ostream.FlushAsync();
}
///
/// Returns a Base64 encoding of these bytes
///
///
///
public static string ToBase64(this byte[] data)
{
return Convert.ToBase64String(data);
}
public static string ToHex(this byte[] bytes)
{
var builder = new StringBuilder();
for (var i = 0; i < bytes.Length; i++) builder.Append(bytes[i].ToString("x2"));
return builder.ToString();
}
public static byte[] FromHex(this string hex)
{
return Enumerable.Range(0, hex.Length)
.Where(x => x % 2 == 0)
.Select(x => Convert.ToByte(hex.Substring(x, 2), 16))
.ToArray();
}
public static DateTime AsUnixTime(this long timestamp)
{
DateTime dtDateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
dtDateTime = dtDateTime.AddSeconds(timestamp);
return dtDateTime;
}
public static ulong AsUnixTime(this DateTime timestamp)
{
var diff = timestamp - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
return (ulong)diff.TotalSeconds;
}
///
/// Returns data from a base64 stream
///
///
///
public static byte[] FromBase64(this string data)
{
return Convert.FromBase64String(data);
}
///
/// Executes the action for every item in coll
///
///
///
///
public static void Do(this IEnumerable coll, Action f)
{
foreach (var i in coll) f(i);
}
///
/// Executes the action for every item in coll
///
///
///
///
public static async Task DoAsync(this IEnumerable coll, Func f)
{
foreach (var i in coll) await f(i);
}
public static void DoIndexed(this IEnumerable coll, Action f)
{
var idx = 0;
foreach (var i in coll)
{
f(idx, i);
idx += 1;
}
}
public static async Task DoIndexed(this IEnumerable coll, Func f)
{
var idx = 0;
foreach (var i in coll)
{
await f(idx, i);
idx += 1;
}
}
public static Task PDoIndexed(this IEnumerable coll, WorkQueue queue, Action f)
{
return coll.Zip(Enumerable.Range(0, int.MaxValue), (v, idx) => (v, idx))
.PMap(queue, vs=> f(vs.idx, vs.v));
}
private static IniDataParser IniParser()
{
var config = new IniParserConfiguration {AllowDuplicateKeys = true, AllowDuplicateSections = true};
var parser = new IniDataParser(config);
return parser;
}
///
/// Loads INI data from the given filename and returns a dynamic type that
/// can use . operators to navigate the INI.
///
///
///
public static dynamic LoadIniFile(this AbsolutePath file)
{
return new DynamicIniData(new FileIniDataParser(IniParser()).ReadFile((string)file));
}
///
/// Loads a INI from the given string
///
///
///
public static dynamic LoadIniString(this string file)
{
return new DynamicIniData(new FileIniDataParser(IniParser()).ReadData(new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(file)))));
}
public static bool FileExists(this string filename)
{
return File.Exists(filename);
}
public static string RelativeTo(this string file, string folder)
{
return file.Substring(folder.Length + 1);
}
///
/// Returns the string compressed via BZip2
///
///
///
public static byte[] BZip2String(this string data)
{
using (var os = new MemoryStream())
{
using (var bz = new BZip2OutputStream(os))
{
using (var bw = new BinaryWriter(bz))
{
bw.Write(data);
}
}
return os.ToArray();
}
}
public static void BZip2ExtractToFile(this Stream src, string dest)
{
using (var os = File.Open(dest, System.IO.FileMode.Create))
{
os.SetLength(0);
using (var bz = new BZip2InputStream(src))
bz.CopyTo(os);
}
}
///
/// Returns the string compressed via BZip2
///
///
///
public static string BZip2String(this byte[] data)
{
using (var s = new MemoryStream(data))
{
using (var bz = new BZip2InputStream(s))
{
using (var bw = new BinaryReader(bz))
{
return bw.ReadString();
}
}
}
}
///
/// A combination of .Select(func).Where(v => v != default). So select and filter default values.
///
///
///
///
///
///
public static IEnumerable Keep(this IEnumerable coll, Func func)
where TOut : struct, IComparable
{
return coll.Select(func).Where(v => v.CompareTo(default) != 0);
}
public static byte[] ReadAll(this Stream ins)
{
using (var ms = new MemoryStream())
{
ins.CopyTo(ms);
return ms.ToArray();
}
}
public static async Task ReadAllAsync(this Stream ins)
{
await using var ms = new MemoryStream();
await ins.CopyToAsync(ms);
return ms.ToArray();
}
public static async Task ReadAllTextAsync(this Stream ins)
{
await using var ms = new MemoryStream();
await ins.CopyToAsync(ms);
return Encoding.UTF8.GetString(ms.ToArray());
}
public static async Task PMap(this IEnumerable coll, WorkQueue queue, StatusUpdateTracker updateTracker,
Func f)
{
var cnt = 0;
var collist = coll.ToList();
return await collist.PMap(queue, itm =>
{
updateTracker.MakeUpdate(collist.Count, Interlocked.Increment(ref cnt));
return f(itm);
});
}
public static async Task PMap(this IEnumerable coll, WorkQueue queue, StatusUpdateTracker updateTracker,
Func> f)
{
var cnt = 0;
var collist = coll.ToList();
return await collist.PMap(queue, itm =>
{
updateTracker.MakeUpdate(collist.Count, Interlocked.Increment(ref cnt));
return f(itm);
});
}
public static async Task PMap(this IEnumerable coll, WorkQueue queue, StatusUpdateTracker updateTracker,
Func f)
{
var cnt = 0;
var collist = coll.ToList();
await collist.PMap(queue, async itm =>
{
updateTracker.MakeUpdate(collist.Count, Interlocked.Increment(ref cnt));
await f(itm);
});
}
public static async Task PMap(this IEnumerable coll, WorkQueue queue, StatusUpdateTracker updateTracker,
Action f)
{
var cnt = 0;
var collist = coll.ToList();
await collist.PMap(queue, itm =>
{
updateTracker.MakeUpdate(collist.Count, Interlocked.Increment(ref cnt));
f(itm);
return true;
});
}
public static async Task PMap(this IEnumerable coll, WorkQueue queue,
Func f)
{
var colllst = coll.ToList();
var remainingTasks = colllst.Count;
var tasks = colllst.Select(i =>
{
var tc = new TaskCompletionSource();
queue.QueueTask(async () =>
{
try
{
tc.SetResult(f(i));
}
catch (Exception ex)
{
tc.SetException(ex);
}
Interlocked.Decrement(ref remainingTasks);
});
return tc.Task;
}).ToList();
// To avoid thread starvation, we'll start to help out in the work queue
if (WorkQueue.WorkerThread)
{
while (true)
{
var (got, a) = await queue.Queue.TryTake(TimeSpan.FromMilliseconds(100), CancellationToken.None);
if (got)
{
await a();
}
else
{
break;
}
}
}
return await Task.WhenAll(tasks);
}
public static async Task
PMap(this IEnumerable coll, WorkQueue queue,
Func> f)
{
var colllst = coll.ToList();
var remainingTasks = colllst.Count;
var tasks = colllst.Select(i =>
{
var tc = new TaskCompletionSource();
queue.QueueTask(async () =>
{
try
{
tc.SetResult(await f(i));
}
catch (Exception ex)
{
tc.SetException(ex);
}
Interlocked.Decrement(ref remainingTasks);
});
return tc.Task;
}).ToList();
// To avoid thread starvation, we'll start to help out in the work queue
if (WorkQueue.WorkerThread)
{
while (remainingTasks > 0)
{
var (got, a) = await queue.Queue.TryTake(TimeSpan.FromMilliseconds(200), CancellationToken.None);
if (got)
{
await a();
}
}
}
return await Task.WhenAll(tasks);
}
public static async Task PMap(this IEnumerable coll, WorkQueue queue,
Func f)
{
var colllst = coll.ToList();
var remainingTasks = colllst.Count;
var tasks = colllst.Select(i =>
{
var tc = new TaskCompletionSource();
queue.QueueTask(async () =>
{
try
{
await f(i);
tc.SetResult(true);
}
catch (Exception ex)
{
tc.SetException(ex);
}
Interlocked.Decrement(ref remainingTasks);
});
return tc.Task;
}).ToList();
// To avoid thread starvation, we'll start to help out in the work queue
if (WorkQueue.WorkerThread)
{
while (remainingTasks > 0)
{
var (got, a) = await queue.Queue.TryTake(TimeSpan.FromMilliseconds(200), CancellationToken.None);
if (got)
{
await a();
}
}
}
await Task.WhenAll(tasks);
}
public static async Task PMap(this IEnumerable coll, WorkQueue queue, Action f)
{
await coll.PMap(queue, i =>
{
f(i);
return false;
});
}
public static void DoProgress(this IEnumerable coll, string msg, Action f)
{
var lst = coll.ToList();
lst.DoIndexed((idx, i) =>
{
Status(msg, Percent.FactoryPutInRange(idx, lst.Count));
f(i);
});
}
public static async Task DoProgress(this IEnumerable coll, string msg, Func f)
{
var lst = coll.ToList();
await lst.DoIndexed(async (idx, i) =>
{
Status(msg, Percent.FactoryPutInRange(idx, lst.Count));
await f(i);
});
}
public static void OnQueue(Action f)
{
new List().Do(_ => f());
}
public static async Task PostStream(this HttpClient client, string url, HttpContent content)
{
var result = await client.PostAsync(url, content);
return await result.Content.ReadAsStreamAsync();
}
public static IEnumerable DistinctBy(this IEnumerable vs, Func select)
{
var set = new HashSet();
foreach (var v in vs)
{
var key = select(v);
if (set.Contains(key)) continue;
set.Add(key);
yield return v;
}
}
public static T Last(this T[] a)
{
if (a == null || a.Length == 0)
throw new InvalidDataException("null or empty array");
return a[a.Length - 1];
}
[return: MaybeNull]
public static V GetOrDefault(this IDictionary dict, K key)
where K : notnull
{
if (dict.TryGetValue(key, out var v)) return v;
return default;
}
public static string ToFileSizeString(this long byteCount)
{
if (byteCount == 0)
return "0" + Suffix[0];
var bytes = Math.Abs(byteCount);
var place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
var num = Math.Round(bytes / Math.Pow(1024, place), 1);
return Math.Sign(byteCount) * num + Suffix[place];
}
public static string ToFileSizeString(this int byteCount)
{
return ToFileSizeString((long)byteCount);
}
public static IEnumerable ButLast(this IEnumerable coll)
{
var lst = coll.ToList();
return lst.Take(lst.Count() - 1);
}
public static byte[] ConcatArrays(this IEnumerable arrays)
{
var outarr = new byte[arrays.Sum(a => a.Length)];
int offset = 0;
foreach (var arr in arrays)
{
Array.Copy(arr, 0, outarr, offset, arr.Length);
offset += arr.Length;
}
return outarr;
}
///
/// Roundtrips the value through the JSON routines
///
public static T ViaJSON(this T tv)
{
var json = tv.ToJson();
return json.FromJsonString();
}
/*
public static void Error(string msg)
{
Log(msg);
throw new Exception(msg);
}*/
public static Stream GetEmbeddedResourceStream(string name)
{
return (from assembly in AppDomain.CurrentDomain.GetAssemblies()
where !assembly.IsDynamic
from rname in assembly.GetManifestResourceNames()
where rname == name
select assembly.GetManifestResourceStream(name)).First();
}
public static T FromYaml(this Stream s)
{
var d = new DeserializerBuilder()
.WithNamingConvention(PascalCaseNamingConvention.Instance)
.Build();
return d.Deserialize(new StreamReader(s));
}
public static T FromYaml(this string s)
{
var d = new DeserializerBuilder()
.WithNamingConvention(PascalCaseNamingConvention.Instance)
.Build();
return d.Deserialize(new StringReader(s));
}
public static T FromYaml(this AbsolutePath s)
{
var d = new DeserializerBuilder()
.WithNamingConvention(PascalCaseNamingConvention.Instance)
.Build();
return d.Deserialize(new StringReader((string)s));
}
public static void LogStatus(string s)
{
Status(s);
Log(s);
}
private static async Task TestDiskSpeedInner(WorkQueue queue, AbsolutePath path)
{
var seconds = 10;
var runTime = new TimeSpan(0, 0, seconds);
Log($"Running disk benchmark, this will take {seconds} seconds");
var results = Enumerable.Range(0, queue.DesiredNumWorkers)
.PMap(queue, async idx =>
{
var startTime = DateTime.Now;
var random = new Random();
var file = path.Combine($"size_test{idx}.bin");
long size = 0;
byte[] buffer = new byte[1024 * 8];
random.NextBytes(buffer);
await using (var fs = await file.Create())
{
while (DateTime.Now - startTime < runTime)
{
fs.Write(buffer, 0, buffer.Length);
// Flush to make sure large buffers don't cause the rate to be higher than it should
fs.Flush();
size += buffer.Length;
}
}
await file.DeleteAsync();
return size;
});
for (int x = 0; x < seconds; x++)
{
Log($"Running Disk benchmark {Percent.FactoryPutInRange(x, seconds)} complete");
await Task.Delay(TimeSpan.FromSeconds(1));
}
return (await results).Sum() / seconds;
}
public static async Task TestDiskSpeed(WorkQueue queue, AbsolutePath path)
{
var benchmarkFile = path.Combine("disk_benchmark.bin");
if (benchmarkFile.Exists)
{
try
{
return benchmarkFile.FromJson();
}
catch (Exception)
{
// ignored
}
}
var speed = await TestDiskSpeedInner(queue, path);
await speed.ToJsonAsync(benchmarkFile);
return speed;
}
/// https://stackoverflow.com/questions/422090/in-c-sharp-check-that-filename-is-possibly-valid-not-that-it-exists
public static IErrorResponse IsFilePathValid(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
return ErrorResponse.Fail("Path is empty.");
}
try
{
var fi = new System.IO.FileInfo(path);
}
catch (ArgumentException ex)
{
return ErrorResponse.Fail(ex.Message);
}
catch (PathTooLongException ex)
{
return ErrorResponse.Fail(ex.Message);
}
catch (NotSupportedException ex)
{
return ErrorResponse.Fail(ex.Message);
}
return ErrorResponse.Success;
}
public static IErrorResponse IsDirectoryPathValid(AbsolutePath path)
{
if (path == default)
{
return ErrorResponse.Fail("Path is empty");
}
try
{
var fi = new System.IO.DirectoryInfo((string)path);
}
catch (ArgumentException ex)
{
return ErrorResponse.Fail(ex.Message);
}
catch (PathTooLongException ex)
{
return ErrorResponse.Fail(ex.Message);
}
catch (NotSupportedException ex)
{
return ErrorResponse.Fail(ex.Message);
}
return ErrorResponse.Success;
}
///
/// Both AlphaFS and C#'s Directory.Delete sometimes fail when certain files are read-only
/// or have other weird attributes. This is the only 100% reliable way I've found to completely
/// delete a folder. If you don't like this code, it's unlikely to change without a ton of testing.
///
///
public static async Task DeleteDirectory(AbsolutePath path)
{
if (!path.Exists)
return;
var process = new ProcessHelper
{
Path = ((RelativePath)"cmd.exe").RelativeToSystemDirectory(),
Arguments = new object[] {"/c", "del", "/f", "/q", "/s", $"\"{(string)path}\"", "&&", "rmdir", "/q", "/s", $"\"{(string)path}\""},
};
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
{
Status($"Deleting: {p.Line}");
});
var exitCode = await process.Start();
await result;
}
public static bool IsUnderneathDirectory(string path, string dirPath)
{
return path.StartsWith(dirPath, StringComparison.OrdinalIgnoreCase);
}
///
/// Writes a file to JSON but in an encrypted format in the user's app local directory.
/// The data will be encrypted so that it can only be read by this machine and this user.
///
///
///
///
public static async ValueTask ToEcryptedJson(this T data, string key)
{
try
{
var bytes = Encoding.UTF8.GetBytes(data.ToJson());
await bytes.ToEcryptedData(key);
}
catch (Exception ex)
{
Log($"Error encrypting data {key} {ex}");
throw;
}
}
public static async Task FromEncryptedJson(string key)
{
var decoded = await FromEncryptedData(key);
return Encoding.UTF8.GetString(decoded).FromJsonString();
}
public static async ValueTask ToEcryptedData(this byte[] bytes, string key)
{
var encoded = ProtectedData.Protect(bytes, Encoding.UTF8.GetBytes(key), DataProtectionScope.LocalMachine);
Consts.LocalAppDataPath.CreateDirectory();
await Consts.LocalAppDataPath.Combine(key).WriteAllBytesAsync(encoded);
}
public static async Task FromEncryptedData(string key)
{
var bytes = await Consts.LocalAppDataPath.Combine(key).ReadAllBytesAsync();
return ProtectedData.Unprotect(bytes, Encoding.UTF8.GetBytes(key), DataProtectionScope.LocalMachine);
}
public static bool HaveEncryptedJson(string key)
{
return Consts.LocalAppDataPath.Combine(key).IsFile;
}
public static bool HaveRegKey()
{
return Registry.CurrentUser.OpenSubKey(@"Software\Wabbajack") != null;
}
public static bool HaveRegKeyMetricsKey()
{
if (HaveRegKey())
{
return Registry.CurrentUser.OpenSubKey(@"Software\Wabbajack")!.GetValueNames().Contains(Consts.MetricsKeyHeader);
}
return false;
}
public static IObservable HaveEncryptedJsonObservable(string key)
{
var path = Consts.LocalAppDataPath.Combine(key);
return WJFileWatcher.AppLocalEvents
.Where(t => (AbsolutePath)t.Item2.FullPath.ToLower() == path)
.Select(_ => path.Exists)
.StartWith(path.Exists)
.DistinctUntilChanged();
}
public static async ValueTask DeleteEncryptedJson(string key)
{
await Consts.LocalAppDataPath.Combine(key).DeleteAsync();
}
public static void StartProcessFromFile(string file)
{
Process.Start(new ProcessStartInfo("cmd.exe", $"/c {file}")
{
CreateNoWindow = true,
});
}
public static void OpenWebsite(Uri url)
{
Process.Start(new ProcessStartInfo("cmd.exe", $"/c start {url}")
{
CreateNoWindow = true,
});
}
public static bool IsInPath(this string path, string parent)
{
return path.ToLower().TrimEnd('\\').StartsWith(parent.ToLower().TrimEnd('\\') + "\\", StringComparison.OrdinalIgnoreCase);
}
public static async Task CopyToLimitAsync(this Stream frm, Stream tw, long limit)
{
var buff = new byte[1024];
var initalLimit = limit;
while (limit > 0)
{
var to_read = Math.Min(buff.Length, limit);
var read = await frm.ReadAsync(buff, 0, (int)to_read);
if (read == 0)
throw new Exception("End of stream before end of limit");
await tw.WriteAsync(buff, 0, read);
limit -= read;
}
tw.Flush();
}
public class NexusErrorResponse
{
public int code;
public string message = string.Empty;
}
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)]
public class MEMORYSTATUSEX
{
public uint dwLength;
public uint dwMemoryLoad;
public ulong ullTotalPhys;
public ulong ullAvailPhys;
public ulong ullTotalPageFile;
public ulong ullAvailPageFile;
public ulong ullTotalVirtual;
public ulong ullAvailVirtual;
public ulong ullAvailExtendedVirtual;
public MEMORYSTATUSEX()
{
dwLength = (uint)Marshal.SizeOf(typeof(MEMORYSTATUSEX));
}
}
[return: MarshalAs(UnmanagedType.Bool)]
[DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
static extern bool GlobalMemoryStatusEx([In, Out] MEMORYSTATUSEX lpBuffer);
public static MEMORYSTATUSEX GetMemoryStatus()
{
var mstat = new MEMORYSTATUSEX();
GlobalMemoryStatusEx(mstat);
return mstat;
}
public static string MakeRandomKey()
{
var random = new Random();
byte[] bytes = new byte[32];
random.NextBytes(bytes);
return bytes.ToHex();
}
public static byte[] RandomData(int size)
{
var random = new Random();
byte[] bytes = new byte[size];
random.NextBytes(bytes);
return bytes;
}
public static string ToNormalString(this SecureString value)
{
var valuePtr = IntPtr.Zero;
try
{
valuePtr = Marshal.SecureStringToGlobalAllocUnicode(value);
return Marshal.PtrToStringUni(valuePtr) ?? "";
}
finally
{
Marshal.ZeroFreeGlobalAllocUnicode(valuePtr);
}
}
public static IEnumerable> Partition(this IEnumerable coll, int size)
{
var lst = new List();
foreach (var itm in coll)
{
lst.Add(itm);
if (lst.Count != size) continue;
yield return lst;
lst = new List();
}
if (lst.Count > 0 && lst.Count != size)
yield return lst;
}
private static Random _random = new Random();
public static int NextRandom(int min, int max)
{
return _random.Next(min, max);
}
}
}