Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Index all archives from all modlists

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-10 06:16:41 -07:00
parent 51aa59ecee
commit a1258b5ea9
7 changed files with 106 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Compression.BSA/Compression.BSA.csproj
View File

@ -9,6 +9,7 @@
<PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
<PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.1.11" />
<PackageReference Include="SharpZipLib" Version="1.2.0" />
<PackageReference Include="System.Text.Encoding.CodePages" Version="4.7.0" />
</ItemGroup>
<ItemGroup>
<Folder Include="Properties\" />

8
Compression.BSA/Utils.cs
View File

@ -8,7 +8,13 @@ namespace Compression.BSA
{
internal static class Utils
{
private static readonly Encoding Windows1252 = Encoding.GetEncoding(1252);
private static readonly Encoding Windows1252;
static Utils()
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
Windows1252 = Encoding.GetEncoding(1252);
}
private static Encoding GetEncoding(VersionType version)
{

1
Wabbajack.BuildServer/JobManager.cs
View File

@ -73,6 +73,7 @@ namespace Wabbajack.BuildServer
await KillOrphanedJobs();
await ScheduledJob<GetNexusUpdatesJob>(TimeSpan.FromHours(2));
await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30));
await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2));
await Task.Delay(10000);
}
}

3
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
View File

@ -14,7 +14,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
{
typeof(IndexJob),
typeof(GetNexusUpdatesJob),
typeof(UpdateModLists)
typeof(UpdateModLists),
typeof(EnqueueAllArchives)
};
public static Dictionary<Type, string> TypeToName { get; set; }
public static Dictionary<string, Type> NameToType { get; set; }

87
Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs Normal file
View File

@ -0,0 +1,87 @@
using System;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using System.Linq;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class EnqueueAllArchives : AJobPayload
{
public override string Description => "Add missing modlist archives to indexer";
public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
{
Utils.Log("Starting modlist indexing");
var modlists = await ModlistMetadata.LoadFromGithub();
using (var queue = new WorkQueue())
{
foreach (var list in modlists)
{
try
{
await ValidateList(db, list, queue);
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
}
}
}
return JobResult.Success();
}
private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue)
{
var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
var modlist_path = Path.Combine(Consts.ModListDownloadFolder,
list.Links.MachineURL + ExtensionManager.Extension);
if (list.NeedsDownload(modlist_path))
{
if (File.Exists(modlist_path))
File.Delete(modlist_path);
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlist_path);
}
else
{
Utils.Log($"No changes detected from downloaded modlist");
}
Utils.Log($"Loading {modlist_path}");
var installer = AInstaller.LoadFromFile(modlist_path);
var archives = installer.Archives;
Utils.Log($"Found {archives.Count} archives in {installer.Name} to index");
var searching = archives.Select(a => a.Hash).Distinct().ToArray();
Utils.Log($"Looking for missing archives");
var knownArchives = (await db.IndexedFiles.AsQueryable().Where(a => searching.Contains(a.Hash))
.Select(d => d.Hash).ToListAsync()).ToDictionary(a => a);
Utils.Log($"Found {knownArchives.Count} pre-existing archives");
var missing = archives.Where(a => !knownArchives.ContainsKey(a.Hash)).ToList();
Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}});
Utils.Log($"Writing jobs to the DB");
await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false});
Utils.Log($"Done adding archives for {installer.Name}");
}
}
}

7
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -19,10 +19,15 @@ namespace Wabbajack.VirtualFileSystem
{
public class Context
{
static Context()
{
Utils.Log("Cleaning VFS, this may take a bit of time");
Utils.DeleteDirectory(_stagingFolder);
}
public const ulong FileVersion = 0x02;
public const string Magic = "WABBAJACK VFS FILE";
private readonly string _stagingFolder = "vfs_staging";
private static readonly string _stagingFolder = "vfs_staging";
public IndexRoot Index { get; private set; } = IndexRoot.Empty;
/// <summary>

2
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -139,7 +139,7 @@ namespace Wabbajack.VirtualFileSystem
var hash = abs_path.FileHash();
var fi = new FileInfo(abs_path);
if (FileExtractor.MightBeArchive(abs_path))
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(abs_path))
{
var result = await TryGetContentsFromServer(hash);