Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/extraction-fixes' into extraction-fixes

Browse Source
This commit is contained in:
Timothy Baldridge 2020-10-05 20:52:44 -06:00
commit 2cd7c1eb03
12 changed files with 276 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -15,6 +15,7 @@ install thread.
* Fixed Extraction so that zip files no longer cause WJ to CTD
* Better path logging during install and compilation
* Fix the "this was created with a newer version of Wabbajack" issue
* If a downloaded file doesn't match the expected hash, try alternative download locations, if allowed
#### Version - 2.2.2.0 - 8/31/2020

171
Wabbajack.Common/Logging.cs Normal file
View File

@ -0,0 +1,171 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Subjects;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Common.StatusFeed.Errors;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Common
{
public static class LoggingSettings
{
// False by default, so that library users do not have to swap it
public static bool LogToFile = false;
}
public static partial class Utils
{
public static AbsolutePath LogFile { get; private set; }
public static AbsolutePath LogFolder { get; private set; }
private static object _logLock = new object();
private static DateTime _startTime;
private static readonly Subject<IStatusMessage> LoggerSubj = new Subject<IStatusMessage>();
public static IObservable<IStatusMessage> LogMessages => LoggerSubj;
private static async Task InitalizeLogging()
{
_startTime = DateTime.Now;
if (LoggingSettings.LogToFile)
{
LogFolder = Consts.LogsFolder;
LogFile = Consts.LogFile;
Consts.LocalAppDataPath.CreateDirectory();
Consts.LogsFolder.CreateDirectory();
if (LogFile.Exists)
{
var newPath = Consts.LogsFolder.Combine(Consts.EntryPoint.FileNameWithoutExtension + LogFile.LastModified.ToString(" yyyy-MM-dd HH_mm_ss") + ".log");
await LogFile.MoveToAsync(newPath, true);
}
var logFiles = LogFolder.EnumerateFiles(false).ToList();
if (logFiles.Count >= Consts.MaxOldLogs)
{
Log($"Maximum amount of old logs reached ({logFiles.Count} >= {Consts.MaxOldLogs})");
var filesToDelete = logFiles
.Where(f => f.IsFile)
.OrderBy(f => f.LastModified)
.Take(logFiles.Count - Consts.MaxOldLogs)
.ToList();
Log($"Found {filesToDelete.Count} old log files to delete");
var success = 0;
var failed = 0;
filesToDelete.Do(f =>
{
try
{
f.Delete();
success++;
}
catch (Exception e)
{
failed++;
Log($"Could not delete log at {f}!\n{e}");
}
});
Log($"Deleted {success} log files, failed to delete {failed} logs");
}
}
}
public static void Log(string msg)
{
Log(new GenericInfo(msg));
}
public static T Log<T>(T msg) where T : IStatusMessage
{
LogStraightToFile(string.IsNullOrWhiteSpace(msg.ExtendedDescription) ? msg.ShortDescription : msg.ExtendedDescription);
LoggerSubj.OnNext(msg);
return msg;
}
public static void Error(string errMessage)
{
Log(errMessage);
}
public static void Error(Exception ex, string? extraMessage = null)
{
Log(new GenericException(ex, extraMessage));
}
public static void ErrorThrow(Exception ex, string? extraMessage = null)
{
Error(ex, extraMessage);
throw ex;
}
public static void Error(IException err)
{
LogStraightToFile($"{err.ShortDescription}\n{err.Exception.StackTrace}");
LoggerSubj.OnNext(err);
}
public static void ErrorThrow(IException err)
{
Error(err);
throw err.Exception;
}
public static void LogStraightToFile(string msg)
{
if (!LoggingSettings.LogToFile || LogFile == default) return;
lock (_logLock)
{
File.AppendAllText(LogFile.ToString(), $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n");
}
}
public static void Status(string msg, Percent progress, bool alsoLog = false)
{
WorkQueue.AsyncLocalCurrentQueue.Value?.Report(msg, progress);
if (alsoLog)
{
Utils.Log(msg);
}
}
public static void Status(string msg, bool alsoLog = false)
{
Status(msg, Percent.Zero, alsoLog: alsoLog);
}
public static void CatchAndLog(Action a)
{
try
{
a();
}
catch (Exception ex)
{
Utils.Error(ex);
}
}
public static async Task CatchAndLog(Func<Task> f)
{
try
{
await f();
}
catch (Exception ex)
{
Utils.Error(ex);
}
}
}
}

10
Wabbajack.Common/Paths/FileCompaction.cs
View File

@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common.IO;
namespace Wabbajack.Common
@ -60,7 +61,12 @@ namespace Wabbajack.Common
public static async Task CompactFolder(this AbsolutePath folder, WorkQueue queue, Algorithm algorithm)
{
await folder.EnumerateFiles(true)
var driveInfo = folder.DriveInfo().DiskSpaceInfo;
var clusterSize = driveInfo.SectorsPerCluster * driveInfo.BytesPerSector;
await folder
.EnumerateFiles(true)
.Where(f => f.Size > clusterSize)
.PMap(queue, async path =>
{
Utils.Status($"Compacting {path.FileName}");

175
Wabbajack.Common/Utils.cs
View File

@ -1,16 +1,12 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data.HashFunction.xxHash;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reactive.Concurrency;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Security;
using System.Security.Cryptography;
@ -23,9 +19,6 @@ using IniParser;
using IniParser.Model.Configuration;
using IniParser.Parser;
using Microsoft.Win32;
using Newtonsoft.Json;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Common.StatusFeed.Errors;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
using Directory = System.IO.Directory;
@ -43,167 +36,13 @@ namespace Wabbajack.Common
return processList.Where(process => process.ProcessName == "ModOrganizer").Any(process => Path.GetDirectoryName(process.MainModule?.FileName) == mo2Path);
}
public static AbsolutePath LogFile { get; }
public static AbsolutePath LogFolder { get; }
public enum FileEventType
{
Created,
Changed,
Deleted
}
static Utils()
{
LogFolder = Consts.LogsFolder;
LogFile = Consts.LogFile;
Consts.LocalAppDataPath.CreateDirectory();
Consts.LogsFolder.CreateDirectory();
_startTime = DateTime.Now;
if (LogFile.Exists)
{
var newPath = Consts.LogsFolder.Combine(Consts.EntryPoint.FileNameWithoutExtension + LogFile.LastModified.ToString(" yyyy-MM-dd HH_mm_ss") + ".log");
LogFile.MoveToAsync(newPath, true).Wait();
}
var logFiles = LogFolder.EnumerateFiles(false).ToList();
if (logFiles.Count >= Consts.MaxOldLogs)
{
Log($"Maximum amount of old logs reached ({logFiles.Count} >= {Consts.MaxOldLogs})");
var filesToDelete = logFiles
.Where(f => f.IsFile)
.OrderBy(f => f.LastModified)
.Take(logFiles.Count - Consts.MaxOldLogs)
.ToList();
Log($"Found {filesToDelete.Count} old log files to delete");
var success = 0;
var failed = 0;
filesToDelete.Do(f =>
{
try
{
f.Delete();
success++;
}
catch (Exception e)
{
failed++;
Log($"Could not delete log at {f}!\n{e}");
}
});
Log($"Deleted {success} log files, failed to delete {failed} logs");
}
var watcher = new FileSystemWatcher((string)Consts.LocalAppDataPath);
AppLocalEvents = Observable.Merge(Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Changed += h, h => watcher.Changed -= h).Select(e => (FileEventType.Changed, e.EventArgs)),
Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Created += h, h => watcher.Created -= h).Select(e => (FileEventType.Created, e.EventArgs)),
Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Deleted += h, h => watcher.Deleted -= h).Select(e => (FileEventType.Deleted, e.EventArgs)))
.ObserveOn(Scheduler.Default);
watcher.EnableRaisingEvents = true;
InitalizeLogging().Wait();
}
private static readonly Subject<IStatusMessage> LoggerSubj = new Subject<IStatusMessage>();
public static IObservable<IStatusMessage> LogMessages => LoggerSubj;
private static readonly string[] Suffix = {"B", "KB", "MB", "GB", "TB", "PB", "EB"}; // Longs run out around EB
private static object _lock = new object();
private static DateTime _startTime;
public static void Log(string msg)
{
Log(new GenericInfo(msg));
}
public static T Log<T>(T msg) where T : IStatusMessage
{
LogStraightToFile(string.IsNullOrWhiteSpace(msg.ExtendedDescription) ? msg.ShortDescription : msg.ExtendedDescription);
LoggerSubj.OnNext(msg);
return msg;
}
public static void Error(string errMessage)
{
Log(errMessage);
}
public static void Error(Exception ex, string? extraMessage = null)
{
Log(new GenericException(ex, extraMessage));
}
public static void ErrorThrow(Exception ex, string? extraMessage = null)
{
Error(ex, extraMessage);
throw ex;
}
public static void Error(IException err)
{
LogStraightToFile($"{err.ShortDescription}\n{err.Exception.StackTrace}");
LoggerSubj.OnNext(err);
}
public static void ErrorThrow(IException err)
{
Error(err);
throw err.Exception;
}
public static void LogStraightToFile(string msg)
{
if (LogFile == default) return;
lock (_lock)
{
File.AppendAllText(LogFile.ToString(), $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n");
}
}
public static void Status(string msg, Percent progress, bool alsoLog = false)
{
WorkQueue.AsyncLocalCurrentQueue.Value?.Report(msg, progress);
if (alsoLog)
{
Utils.Log(msg);
}
}
public static void Status(string msg, bool alsoLog = false)
{
Status(msg, Percent.Zero, alsoLog: alsoLog);
}
public static void CatchAndLog(Action a)
{
try
{
a();
}
catch (Exception ex)
{
Utils.Error(ex);
}
}
public static async Task CatchAndLog(Func<Task> f)
{
try
{
await f();
}
catch (Exception ex)
{
Utils.Error(ex);
}
}
public static void CopyToWithStatus(this Stream istream, long maxSize, Stream ostream, string status)
{
var buffer = new byte[1024 * 64];
@ -993,16 +832,14 @@ namespace Wabbajack.Common
return false;
}
public static IObservable<(FileEventType, FileSystemEventArgs)> AppLocalEvents { get; }
public static IObservable<bool> HaveEncryptedJsonObservable(string key)
{
var path = Consts.LocalAppDataPath.Combine(key);
return AppLocalEvents.Where(t => (AbsolutePath)t.Item2.FullPath.ToLower() == path)
.Select(_ => path.Exists)
.StartWith(path.Exists)
.DistinctUntilChanged();
return WJFileWatcher.AppLocalEvents
.Where(t => (AbsolutePath)t.Item2.FullPath.ToLower() == path)
.Select(_ => path.Exists)
.StartWith(path.Exists)
.DistinctUntilChanged();
}
public static async ValueTask DeleteEncryptedJson(string key)

32
Wabbajack.Common/WJFileWatcher.cs Normal file
View File

@ -0,0 +1,32 @@
using System;
using System.IO;
using System.Reactive.Concurrency;
using System.Reactive.Linq;
using System.Text;
namespace Wabbajack.Common
{
public static class WJFileWatcher
{
public enum FileEventType
{
Created,
Changed,
Deleted
}
public static IObservable<(FileEventType, FileSystemEventArgs)> AppLocalEvents { get; }
static WJFileWatcher()
{
var watcher = new FileSystemWatcher((string)Consts.LocalAppDataPath);
AppLocalEvents = Observable.Merge(
Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Changed += h, h => watcher.Changed -= h).Select(e => (FileEventType.Changed, e.EventArgs)),
Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Created += h, h => watcher.Created -= h).Select(e => (FileEventType.Created, e.EventArgs)),
Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Deleted += h, h => watcher.Deleted -= h).Select(e => (FileEventType.Deleted, e.EventArgs)))
.ObserveOn(Scheduler.Default);
watcher.EnableRaisingEvents = true;
}
}
}

17
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -137,15 +137,16 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, AbsolutePath destination)
{
using var stream = await ResolveDownloadStream(a);
if (stream == null) return false;
var (isValid, istream) = await ResolveDownloadStream(a, false);
if (!isValid) return false;
using var stream = istream!;
await using var fromStream = await stream.Content.ReadAsStreamAsync();
await using var toStream = await destination.Create();
await fromStream.CopyToAsync(toStream);
return true;
}
private async Task<HttpResponseMessage?> ResolveDownloadStream(Archive a)
private async Task<(bool, HttpResponseMessage?)> ResolveDownloadStream(Archive a, bool quickMode)
{
TOP:
string url;
@ -168,7 +169,7 @@ namespace Wabbajack.Lib.Downloaders
if (csrfKey == null)
{
Utils.Log($"Returning null from IPS4 Downloader because no csrfKey was found");
return null;
return (false, null);
}
var sep = Site.EndsWith("?") ? "&" : "?";
@ -199,10 +200,10 @@ namespace Wabbajack.Lib.Downloaders
if (a.Size != 0 && headerContentSize != 0 && a.Size != headerContentSize)
{
Utils.Log($"Bad Header Content sizes {a.Size} vs {headerContentSize}");
return null;
return (false, null);
}
return streamResult;
return (true, streamResult);
}
// Sometimes LL hands back a json object telling us to wait until a certain time
@ -210,6 +211,7 @@ namespace Wabbajack.Lib.Downloaders
var secs = times.Download - times.CurrentTime;
for (int x = 0; x < secs; x++)
{
if (quickMode) return (true, default);
Utils.Status($"Waiting for {secs} at the request of {Downloader.SiteName}", Percent.FactoryPutInRange(x, secs));
await Task.Delay(1000);
}
@ -228,7 +230,8 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Verify(Archive a)
{
var stream = await ResolveDownloadStream(a);
var (isValid, stream) = await ResolveDownloadStream(a, true);
if (!isValid) return false;
if (stream == null)
return false;

5
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -104,8 +104,9 @@ namespace Wabbajack.Lib.Downloaders
{
if (await Download(archive, destination))
{
await destination.FileHashCachedAsync();
return DownloadResult.Success;
var downloadedHash = await destination.FileHashCachedAsync();
if (downloadedHash == archive.Hash || archive.Hash == default)
return DownloadResult.Success;
}

1
Wabbajack.Server/Program.cs
View File

@ -12,6 +12,7 @@ namespace Wabbajack.Server
{
public static void Main(string[] args)
{
LoggingSettings.LogToFile = true;
Consts.IsServer = true;
bool testMode = args.Contains("TESTMODE");
CreateHostBuilder(args, testMode).Build().Run();

20
Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs
View File

@ -26,6 +26,19 @@ namespace Wabbajack.VirtualFileSystem
Definitions.FileType._7Z);
private static Extension OMODExtension = new Extension(".omod");
private static Extension BSAExtension = new Extension(".bsa");
public static readonly HashSet<Extension> ExtractableExtensions = new HashSet<Extension>
{
new Extension(".bsa"),
new Extension(".ba2"),
new Extension(".7z"),
new Extension(".7zip"),
new Extension(".rar"),
new Extension(".zip"),
OMODExtension
};
/// <summary>
/// When true, will allow 7z to use multiple threads and cache more data in memory, potentially
@ -64,11 +77,16 @@ namespace Wabbajack.VirtualFileSystem
}
}
case Definitions.FileType.TES3:
case Definitions.FileType.BSA:
case Definitions.FileType.BA2:
return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
case Definitions.FileType.TES3:
if (sFn.Name.FileName.Extension == BSAExtension)
return await GatheringExtractWithBSA(sFn, (Definitions.FileType)sig, shouldExtract, mapfn);
else
throw new Exception($"Invalid file format {sFn.Name}");
default:
throw new Exception($"Invalid file format {sFn.Name}");

25
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -175,16 +175,27 @@ namespace Wabbajack.VirtualFileSystem
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
IPath relPath, int depth = 0)
{
await using var stream = await extractedFile.GetStream();
var hash = await stream.xxHashAsync();
stream.Position = 0;
Hash hash = default;
if (extractedFile is NativeFileStreamFactory)
{
hash = await ((AbsolutePath)extractedFile.Name).FileHashCachedAsync();
}
else
{
await using var hstream = await extractedFile.GetStream();
hash = await hstream.xxHashAsync();
}
if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
{
return vself;
}
await using var stream = await extractedFile.GetStream();
var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
stream.Position = 0;
if (sig.HasValue && TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
return vself;
var self = new VirtualFile
{
Context = context,
@ -202,7 +213,7 @@ namespace Wabbajack.VirtualFileSystem
self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
// Can't extract, so return
if (!sig.HasValue) return self;
if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension)) return self;
try
{

2
Wabbajack/App.xaml.cs
View File

@ -11,6 +11,8 @@ namespace Wabbajack
{
public App()
{
LoggingSettings.LogToFile = true;
CLIOld.ParseOptions(Environment.GetCommandLineArgs());
if (CLIArguments.Help)
CLIOld.DisplayHelpText();

6
azure-pipelines.yml
View File

@ -65,12 +65,12 @@ steps:
- task: DownloadSecureFile@1
inputs:
secureFile: 'CertFile.p12'
secureFile: 'CodeCert2020.pfx'
- task: codesigning@2
displayName: "Sign main app"
condition: eq(variables['Build.SourceBranch'], 'refs/heads/master')
inputs:
secureFileId: 'CertFile.p12'
secureFileId: 'CodeCert2020.pfx'
signCertPassword: '$(CertPassword)'
files: '$(System.DefaultWorkingDirectory)/PublishApp/*abbajack*.exe'
hashingAlgorithm: 'SHA256'
@ -80,7 +80,7 @@ steps:
displayName: "Sign launcher"
condition: eq(variables['Build.SourceBranch'], 'refs/heads/master')
inputs:
secureFileId: 'CertFile.p12'
secureFileId: 'CodeCert2020.pfx'
signCertPassword: '$(CertPassword)'
files: '$(System.DefaultWorkingDirectory)/PublishLauncher/*abbajack*.exe'
hashingAlgorithm: 'SHA256'