Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #954 from wabbajack-tools/cli-nice-to-haves

Browse Source 
Can now purge the cache, and extracs BSAs from the cli
This commit is contained in:
Timothy Baldridge 2020-07-10 18:35:50 -07:00 committed by GitHub
commit 91a7e665fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 272 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -23,7 +23,9 @@ namespace Wabbajack.CLI
typeof(BSADump),
typeof(MigrateGameFolderFiles),
typeof(HashFile),
typeof(InlinedFileReport)
typeof(InlinedFileReport),
typeof(ExtractBSA),
typeof(PurgeNexusCache)
};
}
}

17
Wabbajack.CLI/Program.cs
View File

@ -14,22 +14,7 @@ namespace Wabbajack.CLI
Utils.LogMessages.Subscribe(Console.WriteLine);
return Parser.Default.ParseArguments(args, OptionsDefinition.AllOptions)
.MapResult(
(Encrypt opts) => opts.Execute(),
(Decrypt opts) => opts.Execute(),
(Validate opts) => opts.Execute(),
(DownloadUrl opts) => opts.Execute(),
(UpdateModlists opts) => opts.Execute(),
(UpdateNexusCache opts) => opts.Execute(),
(ChangeDownload opts) => opts.Execute(),
(ServerLog opts) => opts.Execute(),
(MyFiles opts) => opts.Execute(),
(DeleteFile opts) => opts.Execute(),
(Changelog opts) => opts.Execute(),
(FindSimilar opts) => opts.Execute(),
(BSADump opts) => opts.Execute(),
(MigrateGameFolderFiles opts) => opts.Execute(),
(HashFile opts) => opts.Execute(),
(InlinedFileReport opts) => opts.Execute(),
(AVerb opts) => opts.Execute(),
errs => 1);
}
}

35
Wabbajack.CLI/Verbs/ExtractBSA.cs Normal file
View File

@ -0,0 +1,35 @@
using System;
using System.Threading.Tasks;
using CommandLine;
using Compression.BSA;
using Wabbajack.Common;
namespace Wabbajack.CLI.Verbs
{
[Verb("extract-bsa", HelpText = "Extracts a BSA/BA2 into a folder")]
public class ExtractBSA : AVerb
{
[Option('o', "output", Required = true, HelpText = @"Output folder to extract to")]
public string OutputFolder { get; set; } = "";
[IsFile(CustomMessage = "The input file %1 does not exist!")]
[Option('i', "input", Required = true, HelpText = @"BSA/BA2 to extract")]
public string InputFile { get; set; } = "";
protected override async Task<ExitCode> Run()
{
Console.WriteLine($"Extracting {InputFile} to {OutputFolder}");
var bsa = await BSADispatch.OpenRead((AbsolutePath)InputFile);
foreach (var file in bsa.Files)
{
Console.WriteLine($"Extracting {file.Path}");
var ofile = file.Path.RelativeTo((AbsolutePath)OutputFolder);
ofile.Parent.CreateDirectory();
await using var ostream = await ofile.Create();
await file.CopyDataTo(ostream);
}
return ExitCode.Ok;
}
}
}

19
Wabbajack.CLI/Verbs/PurgeNexusCache.cs Normal file
View File

@ -0,0 +1,19 @@
using System;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Lib.FileUploader;
namespace Wabbajack.CLI.Verbs
{
[Verb("purge-nexus-cache", HelpText = "Purge the Wabbajack Server's info about a given nexus Mod ID. Future requests for this mod will be grabbed from the Nexus")]
public class PurgeNexusCache : AVerb
{
[Option('i', "mod-id", Required = true, HelpText = @"Mod ID to purge")]
public long ModId { get; set; } = 0;
protected override async Task<ExitCode> Run()
{
Console.WriteLine(await AuthorAPI.PurgeNexusModInfo(ModId));
return ExitCode.Ok;
}
}
}

4
Wabbajack.CLI/Wabbajack.CLI.csproj
View File

@ -6,8 +6,8 @@
<AssemblyName>wabbajack-cli</AssemblyName>
<Company>Wabbajack</Company>
<Platforms>x64</Platforms>
<AssemblyVersion>2.1.0.2</AssemblyVersion>
<FileVersion>2.1.0.2</FileVersion>
<AssemblyVersion>2.1.0.1</AssemblyVersion>
<FileVersion>2.1.0.1</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>An automated ModList installer</Description>
<PublishReadyToRun>true</PublishReadyToRun>

4
Wabbajack.Lib/CompilationSteps/IncludePatches.cs
View File

@ -29,7 +29,7 @@ namespace Wabbajack.Lib.CompilationSteps
_indexedByName = _indexed.Values
.SelectMany(s => s)
.Where(f => f.IsNative)
.GroupBy(f => f.FullPath.FileName)
.GroupBy(f => f.Name.FileName)
.ToDictionary(f => f.Key, f => (IEnumerable<VirtualFile>)f);
_isGenericGame = _mo2Compiler.CompilingGame.IsGenericMO2Plugin;
@ -87,7 +87,7 @@ namespace Wabbajack.Lib.CompilationSteps
{
var dist = new Levenshtein();
found = arch.SelectMany(a => a.ThisAndAllChildren)
.OrderBy(a => dist.Distance(a.FullPath.FileName.ToString(), source.File.FullPath.FileName.ToString()))
.OrderBy(a => dist.Distance(a.Name.FileName.ToString(), source.File.Name.FileName.ToString()))
.Take(3)
.ToArray();
}

5
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -69,5 +69,10 @@ namespace Wabbajack.Lib.FileUploader
.DeleteStringAsync($"{Consts.WabbajackBuildServerUri}uploaded_files/{name}");
return result;
}
public static async Task<string> PurgeNexusModInfo(long modId)
{
return await (await GetAuthorizedClient()).GetStringAsync($"{Consts.WabbajackBuildServerUri}purge_nexus_cache/{modId}");
}
}
}

25
Wabbajack.Server.Test/NexusCacheTests.cs
View File

@ -5,6 +5,7 @@ using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
@ -58,6 +59,30 @@ namespace Wabbajack.BuildServer.Test
Assert.Single(modInfoResponse.files);
Assert.Equal("blerg", modInfoResponse.files.First().file_name);
}
[Fact]
public async Task TestCanPurgeModInfo()
{
var sqlService = Fixture.GetService<SqlService>();
var modId = long.MaxValue >> 3;
await sqlService.AddNexusModFiles(Game.SkyrimSpecialEdition, modId, DateTime.Now,
new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>
{
new NexusFileInfo
{
file_name = "blerg"
}
}});
var api = await NexusApiClient.Get();
var modInfoResponse = await api.GetModFiles(Game.SkyrimSpecialEdition, modId);
Assert.Single(modInfoResponse.files);
Assert.Equal("blerg", modInfoResponse.files.First().file_name);
await AuthorAPI.PurgeNexusModInfo(modId);
}
[Fact]
public async Task CanQueryAndFindNexusModfilesFast()

35
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -415,21 +415,32 @@ CREATE TABLE [dbo].[Metrics](
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[AccessLog](
[Id] [bigint] IDENTITY(1,1) NOT NULL,
[Timestamp] [datetime] NOT NULL,
[Action] [nvarchar](max) NOT NULL,
[Ip] [nvarchar](36) NOT NULL,
CONSTRAINT [PK_AccessLog] PRIMARY KEY CLUSTERED
(
[Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
[Id] [bigint] IDENTITY(1,1) NOT NULL,
[Timestamp] [datetime] NOT NULL,
[Action] [nvarchar](max) NOT NULL,
[Ip] [nvarchar](50) NOT NULL,
[MetricsKey] AS (json_value([Action],'$.Headers."x-metrics-key"[0]')),
CONSTRAINT [PK_AccessLog] PRIMARY KEY CLUSTERED
(
[Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
CREATE NONCLUSTERED INDEX [AccessLogByMetricsKey] ON [dbo].[AccessLog]
(
[MetricsKey] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: Index [AccessLogByIP] Script Date: 7/8/2020 10:14:01 PM ******/
CREATE NONCLUSTERED INDEX [AccessLogByIP] ON [dbo].[AccessLog]
(
[Ip] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: Table [dbo].[AuthoredFiles] Script Date: 5/9/2020 2:22:00 PM ******/
CREATE TABLE [dbo].[AuthoredFiles](

10
Wabbajack.Server/Controllers/NexusCache.cs
View File

@ -112,5 +112,15 @@ namespace Wabbajack.BuildServer.Controllers
Response.Headers.Add("x-cache-result", method);
return result;
}
[HttpGet]
[Authorize(Roles ="Author")]
[Route("/purge_nexus_cache/{ModId}")]
public async Task<IActionResult> PurgeNexusCache(long ModId)
{
_logger.LogInformation($"Purging nexus cache for {ModId}");
await _sql.PurgeNexusCache(ModId);
return Ok("Purged");
}
}
}

7
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -108,5 +108,12 @@ namespace Wabbajack.Server.DataLayer
new {Game = game.MetaData().NexusGameId, ModId = modId});
return result == null ? null : JsonConvert.DeserializeObject<NexusApiClient.GetModFilesResponse>(result);
}
public async Task PurgeNexusCache(long modId)
{
await using var conn = await Open();
await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId});
}
}
}

25
Wabbajack.Server/DataLayer/Patches.cs
View File

@ -200,5 +200,30 @@ namespace Wabbajack.Server.DataLayer
});
}
public async Task<HashSet<(Hash, Hash)>> AllPatchHashes()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, Hash)>(@"SELECT a1.Hash, a2.Hash
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a1 ON a1.Id = p.SrcId
LEFT JOIN dbo.ArchiveDownloads a2 on a2.Id = p.DestId")).ToHashSet();
}
public async Task DeletePatchesForHashPair((Hash, Hash) sqlFile)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"DELETE p
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a1 ON a1.Id = p.SrcId
LEFT JOIN dbo.ArchiveDownloads a2 on a2.Id = p.DestId
WHERE a1.Hash = @SrcHash
AND a2.Hash = @DestHash", new
{
SrcHash = sqlFile.Item1,
DestHash = sqlFile.Item2
});
}
}
}

2
Wabbajack.Server/Services/NexusKeyMaintainance.cs
View File

@ -13,7 +13,7 @@ namespace Wabbajack.Server.Services
{
private SqlService _sql;
public NexusKeyMaintainance(ILogger<NexusKeyMaintainance> logger, AppSettings settings, SqlService sql, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(1))
public NexusKeyMaintainance(ILogger<NexusKeyMaintainance> logger, AppSettings settings, SqlService sql, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(4))
{
_sql = sql;
}

25
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -1,5 +1,6 @@
using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
@ -129,11 +130,11 @@ namespace Wabbajack.Server.Services
{
_logger.LogInformation($"Cleaning patch {patch.Src.Archive.Hash} -> {patch.Dest.Archive.Hash}");
await _discordWebHook.Send(Channel.Ham,
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Removing patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString} due it no longer being required by curated lists"
$"Removing {patch.PatchSize.FileSizeToString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString} due it no longer being required by curated lists"
});
if (!await DeleteFromCDN(client, PatchName(patch)))
@ -145,8 +146,26 @@ namespace Wabbajack.Server.Services
var pendingPatch = await _sql.GetPendingPatch();
if (pendingPatch != default) break;
}
var files = await client.GetListingAsync($"{Consts.ArchiveUpdatesCDNFolder}\\");
_logger.LogInformation($"Found {files.Length} on the CDN");
var sqlFiles = await _sql.AllPatchHashes();
_logger.LogInformation($"Found {sqlFiles.Count} in SQL");
var hashPairs = files.Select(f => f.Name).Where(f => f.Contains("_")).Select(p =>
{
var lst = p.Split("_", StringSplitOptions.RemoveEmptyEntries).Select(Hash.FromHex).ToArray();
return (lst[0], lst[1]);
}).ToHashSet();
foreach (var sqlFile in sqlFiles.Where(s => !hashPairs.Contains(s)))
{
_logger.LogInformation($"Removing SQL File entry for {sqlFile.Item1} -> {sqlFile.Item2} it's not on the CDN");
await _sql.DeletePatchesForHashPair(sqlFile);
}
}
private async Task UploadToCDN(AbsolutePath patchFile, string patchName)

8
Wabbajack.Server/Startup.cs
View File

@ -36,10 +36,10 @@ namespace Wabbajack.Server
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddSwaggerGen(c =>
/*services.AddSwaggerGen(c =>
{
c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"});
});
});*/
services.AddAuthentication(options =>
{
@ -99,12 +99,14 @@ namespace Wabbajack.Server
provider.Mappings[".wabbajack"] = "application/zip";
app.UseStaticFiles();
/*
app.UseSwagger();
app.UseSwaggerUI(c =>
{
c.SwaggerEndpoint("/swagger/v1/swagger.json", "Wabbajack Build API");
c.RoutePrefix = string.Empty;
});
}); */
app.UseRouting();
app.UseAuthentication();

4
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -71,7 +71,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, new ExtractedDiskFile(f), f, 0);
return await VirtualFile.Analyze(this, null, new RootDiskFile(f), f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -103,7 +103,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, new ExtractedDiskFile(f), f, 0);
return await VirtualFile.Analyze(this, null, new RootDiskFile(f), f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());

4
Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs
View File

@ -8,7 +8,7 @@ namespace Wabbajack.VirtualFileSystem
{
public class ExtractedDiskFile : IExtractedFile
{
private AbsolutePath _path;
protected AbsolutePath _path;
public ExtractedDiskFile(AbsolutePath path)
{
@ -17,7 +17,7 @@ namespace Wabbajack.VirtualFileSystem
_path = path;
}
public async Task<Hash> HashAsync()
public virtual async Task<Hash> HashAsync()
{
return await _path.FileHashAsync();
}

56
Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs
View File

@ -1,4 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
@ -14,5 +17,58 @@ namespace Wabbajack.VirtualFileSystem
public Hash Hash { get; set; }
public long Size { get; set; }
public List<IndexedVirtualFile> Children { get; set; } = new List<IndexedVirtualFile>();
private void Write(BinaryWriter bw)
{
bw.Write(Name.ToString());
bw.Write((ulong)Hash);
bw.Write(Size);
bw.Write(Children.Count);
foreach (var file in Children)
file.Write(bw);
}
public void Write(Stream s)
{
using var bw = new BinaryWriter(s);
bw.Write(Size);
bw.Write(Children.Count);
foreach (var file in Children)
file.Write(bw);
}
private static IndexedVirtualFile Read(BinaryReader br)
{
var ivf = new IndexedVirtualFile
{
Name = (RelativePath)br.ReadString(),
Hash = Hash.FromULong(br.ReadUInt64()),
Size = br.ReadInt64(),
};
var lst = new List<IndexedVirtualFile>();
var count = br.ReadInt32();
for (int x = 0; x < count; x++)
{
lst.Add(Read(br));
}
return ivf;
}
public static IndexedVirtualFile Read(Stream s)
{
using var br = new BinaryReader(s);
var ivf = new IndexedVirtualFile
{
Size = br.ReadInt64(),
};
var lst = new List<IndexedVirtualFile>();
ivf.Children = lst;
var count = br.ReadInt32();
for (int x = 0; x < count; x++)
{
lst.Add(Read(br));
}
return ivf;
}
}
}

20
Wabbajack.VirtualFileSystem/RootDiskFile.cs Normal file
View File

@ -0,0 +1,20 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public class RootDiskFile : ExtractedDiskFile
{
public RootDiskFile(AbsolutePath path) : base(path)
{
}
public override async Task<Hash> HashAsync()
{
return await _path.FileHashCachedAsync();
}
}
}

11
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -19,7 +19,7 @@ namespace Wabbajack.VirtualFileSystem
static VirtualFile()
{
var options = new DbOptions().SetCreateIfMissing(true);
_vfsCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalVFSCache.rocksDb"));
_vfsCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalVFSCache2.rocksDb"));
}
private AbsolutePath _stagedPath;
@ -170,8 +170,10 @@ namespace Wabbajack.VirtualFileSystem
return false;
}
var data = new MemoryStream(result).FromJson<IndexedVirtualFile>();
var data = IndexedVirtualFile.Read(new MemoryStream(result));
found = ConvertFromIndexedFile(context, data, path, parent, extractedFile);
found.Name = path;
found.Hash = hash;
return true;
}
@ -195,7 +197,8 @@ namespace Wabbajack.VirtualFileSystem
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
{
var result = await TryGetContentsFromServer(hash);
// Disabled because it isn't enabled on the server
IndexedVirtualFile result = null; //await TryGetContentsFromServer(hash);
if (result != null)
{
@ -245,7 +248,7 @@ namespace Wabbajack.VirtualFileSystem
}
await using var ms = new MemoryStream();
self.ToIndexedVirtualFile().ToJson(ms);
self.ToIndexedVirtualFile().Write(ms);
_vfsCache.Put(self.Hash.ToArray(), ms.ToArray());
return self;