Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix a bunch of issues with compilation, indexing and VFS loading performance

Browse Source
This commit is contained in:
Timothy Baldridge 2020-02-14 06:30:58 -07:00
parent f036420b42
commit b77db18bfa
9 changed files with 74 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Wabbajack.BuildServer/Controllers/IndexedFiles.cs
View File

@ -112,6 +112,9 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log("No valid INI parser for: \n" + iniString);
continue;
}
if (data is ManualDownloader.State)
continue;
var key = data.PrimaryKeyString;
var found = await Db.DownloadStates.AsQueryable().Where(f => f.Key == key).Take(1).ToListAsync();

3
Wabbajack.BuildServer/Models/Jobs/IndexJob.cs
View File

@ -24,6 +24,9 @@ namespace Wabbajack.BuildServer.Models.Jobs
public override bool UsesNexus { get => Archive.State is NexusDownloader.State; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
{
if (Archive.State is ManualDownloader.State)
return JobResult.Success();
var pk = new List<object>();
pk.Add(AbstractDownloadState.TypeToName[Archive.State.GetType()]);
pk.AddRange(Archive.State.PrimaryKey);

1
Wabbajack.Common/Consts.cs
View File

@ -102,5 +102,6 @@ namespace Wabbajack.Common
public const string MO2ModFolderName = "mods";
public static string PatchCacheFolder => Path.Combine(LocalAppDataPath, "patch_cache");
public static int MaxConnectionsPerServer = 4;
}
}

2
Wabbajack.Lib/ACompiler.cs
View File

@ -22,7 +22,7 @@ namespace Wabbajack.Lib
public bool ReadmeIsWebsite;
public string WabbajackVersion;
protected static string _vfsCacheName => Path.Combine(Consts.LocalAppDataPath, "vfs_compile_cache.bin");
protected string _vfsCacheName => Path.Combine(Consts.LocalAppDataPath, $"vfs_compile_cache_{ModListName.StringSHA256Hex()}.bin");
/// <summary>
/// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task
/// and the current progress.

12
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -121,7 +121,7 @@ namespace Wabbajack.Lib.FileUploader
public static HttpClient GetAuthorizedClient()
{
var handler = new HttpClientHandler {MaxConnectionsPerServer = MAX_CONNECTIONS};
var handler = new HttpClientHandler {MaxConnectionsPerServer = Consts.MaxConnectionsPerServer};
var client = new HttpClient(handler);
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
return client;
@ -144,7 +144,7 @@ namespace Wabbajack.Lib.FileUploader
return await RunJob("UpdateModLists");
}
public static async Task UploadPackagedInis(IEnumerable<IndexedArchive> archives)
public static async Task UploadPackagedInis(WorkQueue queue, IEnumerable<Archive> archives)
{
archives = archives.ToArray(); // defensive copy
Utils.Log($"Packaging {archives.Count()} inis");
@ -153,12 +153,12 @@ namespace Wabbajack.Lib.FileUploader
await using var ms = new MemoryStream();
using (var z = new ZipArchive(ms, ZipArchiveMode.Create, true))
{
foreach (var archive in archives)
foreach (var e in archives)
{
var state = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(archive.IniData));
var entry = z.CreateEntry(Path.GetFileName(archive.Name));
if (e.State == null) continue;
var entry = z.CreateEntry(Path.GetFileName(e.Name));
await using var os = entry.Open();
await os.WriteAsync(Encoding.UTF8.GetBytes(string.Join("\n", state.GetMetaIni())));
await os.WriteAsync(Encoding.UTF8.GetBytes(string.Join("\n", e.State.GetMetaIni())));
}
}

25
Wabbajack.Lib/MO2Compiler.cs
View File

@ -123,12 +123,13 @@ namespace Wabbajack.Lib
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p], Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath))));
}
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Cleaning output folder");
if (Directory.Exists(ModListOutputFolder))
Utils.DeleteDirectory(ModListOutputFolder);
/*
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Inferring metas for game file downloads");
await InferMetas();
@ -137,9 +138,11 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep("Reindexing downloads after meta inferring");
await VFS.AddRoot(MO2DownloadsFolder);
await VFS.WriteToFile(_vfsCacheName);
*/
if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Pre-validating Archives");
IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder)
.Where(f => File.Exists(f + Consts.MetaFileExtension))
@ -152,8 +155,7 @@ namespace Wabbajack.Lib
})
.ToList();
// Don't await this because we don't care if it fails.
var _ = AuthorAPI.UploadPackagedInis(IndexedArchives);
await CleanInvalidArchives();
@ -261,6 +263,11 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep("Gathering Archives");
await GatherArchives();
// Don't await this because we don't care if it fails.
Utils.Log("Finding States to package");
await AuthorAPI.UploadPackagedInis(Queue, SelectedArchives.ToArray());
UpdateTracker.NextStep("Including Archive Metadata");
await IncludeArchiveMetadata();
UpdateTracker.NextStep("Building Patches");
@ -348,8 +355,16 @@ namespace Wabbajack.Lib
var response =
await client.GetAsync(
$"http://build.wabbajack.org/indexed_files/{vf.Hash.FromBase64().ToHex()}/meta.ini");
if (!response.IsSuccessStatusCode) return;
if (!response.IsSuccessStatusCode)
{
File.WriteAllLines(vf.FullPath + Consts.MetaFileExtension, new []
{
"[General]",
"unknownArchive=true"
});
return;
}
var ini_data = await response.Content.ReadAsStringAsync();
Utils.Log($"Inferred .meta for {Path.GetFileName(vf.FullPath)}, writing to disk");

28
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -25,8 +25,8 @@ namespace Wabbajack.Lib.NexusApi
private static string _additionalEntropy = "vtP2HF6ezg";
private static object _diskLock = new object();
public HttpClient HttpClient { get; } = new HttpClient();
public HttpClient HttpClient { get; } = new HttpClient(new HttpClientHandler {MaxConnectionsPerServer = Consts.MaxConnectionsPerServer}) {Timeout = TimeSpan.FromMinutes(1), DefaultRequestHeaders = {ConnectionClose = true}};
#region Authentication
@ -227,16 +227,17 @@ namespace Wabbajack.Lib.NexusApi
TOP:
try
{
var response = await HttpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
using var response = await HttpClient.GetAsync(url, HttpCompletionOption.ResponseContentRead);
UpdateRemaining(response);
if (!response.IsSuccessStatusCode)
throw new HttpRequestException($"{response.StatusCode} - {response.ReasonPhrase}");
using (var stream = await response.Content.ReadAsStreamAsync())
{
return stream.FromJSON<T>();
Utils.Log($"Nexus call failed: {response.RequestMessage}");
throw new HttpRequestException($"{response.StatusCode} - {response.ReasonPhrase}");
}
await using var stream = await response.Content.ReadAsStreamAsync();
return stream.FromJSON<T>();
}
catch (TimeoutException)
{
@ -246,13 +247,18 @@ namespace Wabbajack.Lib.NexusApi
retries++;
goto TOP;
}
catch (Exception e)
{
Utils.Log(e.ToString());
throw;
}
}
private async Task<T> GetCached<T>(string url)
{
try
{
var builder = new UriBuilder(url) { Host = Consts.WabbajackCacheHostname, Port = Consts.WabbajackCachePort, Scheme = "http" };
var builder = new UriBuilder(url) { Host = Consts.WabbajackCacheHostname, Scheme = "https" };
return await Get<T>(builder.ToString());
}
catch (Exception)
@ -273,6 +279,10 @@ namespace Wabbajack.Lib.NexusApi
}
catch (HttpRequestException)
{
if (await IsPremium())
{
throw;
}
}
try

7
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -24,7 +24,7 @@ namespace Wabbajack.VirtualFileSystem
Utils.Log("Cleaning VFS, this may take a bit of time");
Utils.DeleteDirectory(_stagingFolder);
}
public const ulong FileVersion = 0x02;
public const ulong FileVersion = 0x03;
public const string Magic = "WABBAJACK VFS FILE";
private static readonly string _stagingFolder = "vfs_staging";
@ -420,13 +420,16 @@ namespace Wabbajack.VirtualFileSystem
public TemporaryDirectory(string name)
{
FullName = name;
if (!Directory.Exists(FullName))
Directory.CreateDirectory(FullName);
}
public string FullName { get; }
public void Dispose()
{
Utils.DeleteDirectory(FullName);
if (Directory.Exists(FullName))
Utils.DeleteDirectory(FullName);
}
}
}

16
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -112,6 +112,20 @@ namespace Wabbajack.VirtualFileSystem
return _thisAndAllChildren;
}
}
public T ThisAndAllChildrenReduced<T>(T acc, Func<T, VirtualFile, T> fn)
{
acc = fn(acc, this);
return this.Children.Aggregate(acc, (current, itm) => itm.ThisAndAllChildrenReduced<T>(current, fn));
}
public void ThisAndAllChildrenReduced(Action<VirtualFile> fn)
{
fn(this);
foreach (var itm in Children)
itm.ThisAndAllChildrenReduced(fn);
}
/// <summary>
@ -233,6 +247,7 @@ namespace Wabbajack.VirtualFileSystem
private void Write(BinaryWriter bw)
{
bw.Write(Name);
bw.Write(FullPath);
bw.Write(Hash);
bw.Write(Size);
bw.Write(LastModified);
@ -258,6 +273,7 @@ namespace Wabbajack.VirtualFileSystem
Context = context,
Parent = parent,
Name = br.ReadString(),
_fullPath = br.ReadString(),
Hash = br.ReadString(),
Size = br.ReadInt64(),
LastModified = br.ReadInt64(),