Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

* Disable server-side indexing of all mods from the Nexus

Browse Source 
* Accept download states from clients and index the mods we haven't seen
* Fixes for Skyrin VR USSEP patch
* Remember the download states that we index on the server
* Only print remaining nexus quotas when they change
This commit is contained in:
Timothy Baldridge 2020-02-13 05:29:59 -07:00
parent 05eb427ce5
commit f036420b42
11 changed files with 127 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CHANGELOG.md
View File

@ -1,5 +1,13 @@
### Changelog
#### Version - 0.9.19.0
* Disable server-side indexing of all mods from the Nexus
* Accept download states from clients and index the mods we haven't seen
* Fixes for Skyrin VR USSEP patch
* Remember the download states that we index on the server
* Only print remaining nexus quotas when they change
#### Version - 0.9.18.0 - 2/11/2020
* Auto update functionality added client-side.
* Slideshow now moves to next slide when users clicks, even if paused
* Installer now prints to log what modlist it is installing
@ -7,8 +15,8 @@
that mod (issue #465)
* Added support for non-premium Nexus downloads via manual downloading through the in-app browser.
* Downloads from Bethesda.NET are now supported. Can login via SkyrimSE or Fallout 4.
=======
* Manual URL downloads are streamlined
* AFKMods.com download support is improved
#### Version - 1.0 beta 17 - 1/22/2020
* Build server now indexes CDN files after they are uploaded

51
Wabbajack.BuildServer/Controllers/IndexedFiles.cs
View File

@ -1,6 +1,9 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using DynamicData;
using Microsoft.AspNetCore.Authorization;
@ -11,7 +14,10 @@ using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
using IndexedFile = Wabbajack.BuildServer.Models.IndexedFile;
@ -87,6 +93,51 @@ namespace Wabbajack.BuildServer.Controllers
return Ok(acc.ToList());
}
[HttpPost]
[Route("notify")]
public async Task<IActionResult> Notify()
{
Utils.Log("Starting ingestion of uploaded INIs");
var body = await Request.Body.ReadAllAsync();
await using var ms = new MemoryStream(body);
using var za = new ZipArchive(ms, ZipArchiveMode.Read);
int enqueued = 0;
foreach (var entry in za.Entries)
{
await using var ins = entry.Open();
var iniString = Encoding.UTF8.GetString(await ins.ReadAllAsync());
var data = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(iniString.LoadIniString()));
if (data == null)
{
Utils.Log("No valid INI parser for: \n" + iniString);
continue;
}
var key = data.PrimaryKeyString;
var found = await Db.DownloadStates.AsQueryable().Where(f => f.Key == key).Take(1).ToListAsync();
if (found.Count > 0)
continue;
await Db.Jobs.InsertOneAsync(new Job
{
Priority = Job.JobPriority.Low,
Payload = new IndexJob()
{
Archive = new Archive
{
Name = entry.Name,
State = data
}
}
});
enqueued += 1;
}
Utils.Log($"Enqueued {enqueued} out of {za.Entries.Count} entries from uploaded ini package");
return Ok(enqueued.ToString());
}
[HttpGet]
[Route("{xxHashAsBase64}")]
public async Task<IActionResult> GetFile(string xxHashAsBase64)

1
Wabbajack.BuildServer/JobManager.cs
View File

@ -82,7 +82,6 @@ namespace Wabbajack.BuildServer
await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30), Job.JobPriority.High);
await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2), Job.JobPriority.Low);
await ScheduledJob<EnqueueAllGameFiles>(TimeSpan.FromHours(24), Job.JobPriority.High);
await ScheduledJob<EnqueueRecentFiles>(TimeSpan.FromHours(6), Job.JobPriority.Low);
await ScheduledJob<IndexDynDOLOD>(TimeSpan.FromHours(1), Job.JobPriority.Normal);
await Task.Delay(10000);
}

10
Wabbajack.BuildServer/Models/Jobs/IndexJob.cs
View File

@ -32,7 +32,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
var found = await db.DownloadStates.AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync();
if (found.Count > 0)
return JobResult.Success();
string fileName = Archive.Name;
string folder = Guid.NewGuid().ToString();
Utils.Log($"Indexer is downloading {fileName}");
@ -46,6 +46,11 @@ namespace Wabbajack.BuildServer.Models.Jobs
var archive = vfs.Index.ByRootPath.First().Value;
await sql.MergeVirtualFile(archive);
await db.DownloadStates.InsertOneAsync(new DownloadState
{
Key = pk_str, Hash = archive.Hash, State = Archive.State, IsValid = true
});
var to_path = Path.Combine(settings.ArchiveDir,
$"{Path.GetFileName(fileName)}_{archive.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}");
@ -54,7 +59,10 @@ namespace Wabbajack.BuildServer.Models.Jobs
else
File.Move(downloadDest, to_path);
Utils.DeleteDirectory(Path.Combine(settings.DownloadDir, folder));
}
return JobResult.Success();
}

7
Wabbajack.BuildServer/Wabbajack.BuildServer.csproj
View File

@ -4,10 +4,11 @@
<TargetFramework>netcoreapp3.1</TargetFramework>
<UserSecretsId>aspnet-Wabbajack.BuildServer-6E798B30-DB04-4436-BE65-F043AF37B314</UserSecretsId>
<WebProject_DirectoryAccessLevelKey>0</WebProject_DirectoryAccessLevelKey>
<PublishReadyToRun>true</PublishReadyToRun>
<PublishSingleFile>true</PublishSingleFile>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<Configurations>Debug;Release</Configurations>
<Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<AssemblyName>Wabbajack</AssemblyName>
<RootNamespace>Wabbajack</RootNamespace>
</PropertyGroup>
<ItemGroup>

3
Wabbajack.Common/GameMetaData.cs
View File

@ -295,7 +295,8 @@ namespace Wabbajack.Common
RequiredFiles = new List<string>
{
"SkyrimVR.exe"
}
},
MainExecutable = "SkyrimVR.exe"
}
},
{

7
Wabbajack.Common/Utils.cs
View File

@ -601,6 +601,13 @@ namespace Wabbajack.Common
}
}
public static async Task<byte[]> ReadAllAsync(this Stream ins)
{
await using var ms = new MemoryStream();
await ins.CopyToAsync(ms);
return ms.ToArray();
}
public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
Func<TI, TR> f)
{

6
Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
View File

@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders
if (gameFile == null || gameFile == null)
return null;
var game = GameRegistry.GetByMO2ArchiveName(gameName);
var game = GameRegistry.GetByFuzzyName(gameName);
if (game == null) return null;
var path = game.GameLocation();
@ -34,10 +34,10 @@ namespace Wabbajack.Lib.Downloaders
return new State
{
Game = GameRegistry.GetByMO2ArchiveName(gameName).Game,
Game = game.Game,
GameFile = gameFile,
Hash = hash,
GameVersion = GameRegistry.GetByMO2ArchiveName(gameName).InstalledVersion
GameVersion = game.InstalledVersion
};
}

31
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -2,13 +2,16 @@
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Reactive.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
@ -140,5 +143,33 @@ namespace Wabbajack.Lib.FileUploader
{
return await RunJob("UpdateModLists");
}
public static async Task UploadPackagedInis(IEnumerable<IndexedArchive> archives)
{
archives = archives.ToArray(); // defensive copy
Utils.Log($"Packaging {archives.Count()} inis");
try
{
await using var ms = new MemoryStream();
using (var z = new ZipArchive(ms, ZipArchiveMode.Create, true))
{
foreach (var archive in archives)
{
var state = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(archive.IniData));
var entry = z.CreateEntry(Path.GetFileName(archive.Name));
await using var os = entry.Open();
await os.WriteAsync(Encoding.UTF8.GetBytes(string.Join("\n", state.GetMetaIni())));
}
}
var webClient = new WebClient();
await webClient.UploadDataTaskAsync($"https://{Consts.WabbajackCacheHostname}/indexed_files/notify",
"POST", ms.ToArray());
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
}
}
}
}

4
Wabbajack.Lib/MO2Compiler.cs
View File

@ -12,6 +12,7 @@ using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
@ -150,6 +151,9 @@ namespace Wabbajack.Lib
Meta = File.ReadAllText(f + Consts.MetaFileExtension)
})
.ToList();
// Don't await this because we don't care if it fails.
var _ = AuthorAPI.UploadPackagedInis(IndexedArchives);
await CleanInvalidArchives();

7
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -173,15 +173,20 @@ namespace Wabbajack.Lib.NexusApi
{
try
{
var oldDaily = _dailyRemaining;
var oldHourly = _hourlyRemaining;
var dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First());
var hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First());
Utils.Log($"Nexus requests remaining: {dailyRemaining} daily - {hourlyRemaining} hourly");
lock (RemainingLock)
{
_dailyRemaining = Math.Min(dailyRemaining, hourlyRemaining);
_hourlyRemaining = Math.Min(dailyRemaining, hourlyRemaining);
}
if (oldDaily != _dailyRemaining || oldHourly != _hourlyRemaining)
Utils.Log($"Nexus requests remaining: {dailyRemaining} daily - {hourlyRemaining} hourly");
this.RaisePropertyChanged(nameof(DailyRemaining));
this.RaisePropertyChanged(nameof(HourlyRemaining));
}