diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj index bf6f75fc..ff02e115 100644 --- a/Compression.BSA/Compression.BSA.csproj +++ b/Compression.BSA/Compression.BSA.csproj @@ -9,6 +9,7 @@ + diff --git a/Compression.BSA/Utils.cs b/Compression.BSA/Utils.cs index b4e36e99..3042f9dc 100644 --- a/Compression.BSA/Utils.cs +++ b/Compression.BSA/Utils.cs @@ -8,8 +8,14 @@ namespace Compression.BSA { internal static class Utils { - private static readonly Encoding Windows1252 = Encoding.GetEncoding(1252); + private static readonly Encoding Windows1252; + static Utils() + { + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + Windows1252 = Encoding.GetEncoding(1252); + } + private static Encoding GetEncoding(VersionType version) { if (version == VersionType.SSE) @@ -155,4 +161,4 @@ namespace Compression.BSA tw.Flush(); } } -} \ No newline at end of file +} diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index 6eedabb4..30fde1d4 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -73,6 +73,7 @@ namespace Wabbajack.BuildServer await KillOrphanedJobs(); await ScheduledJob(TimeSpan.FromHours(2)); await ScheduledJob(TimeSpan.FromMinutes(30)); + await ScheduledJob(TimeSpan.FromHours(2)); await Task.Delay(10000); } } diff --git a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs index a71369d1..51f99b41 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs @@ -14,7 +14,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue { typeof(IndexJob), typeof(GetNexusUpdatesJob), - typeof(UpdateModLists) + typeof(UpdateModLists), + typeof(EnqueueAllArchives) }; public static Dictionary TypeToName { get; set; } public static Dictionary NameToType { get; set; } diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs new file mode 100644 index 00000000..cc77cfb2 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs @@ -0,0 +1,87 @@ +using System; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using System.Linq; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.ModListRegistry; + +namespace Wabbajack.BuildServer.Models.Jobs +{ + public class EnqueueAllArchives : AJobPayload + { + public override string Description => "Add missing modlist archives to indexer"; + public override async Task Execute(DBContext db, AppSettings settings) + { + Utils.Log("Starting modlist indexing"); + var modlists = await ModlistMetadata.LoadFromGithub(); + + using (var queue = new WorkQueue()) + { + foreach (var list in modlists) + { + try + { + await ValidateList(db, list, queue); + } + catch (Exception ex) + { + Utils.Log(ex.ToString()); + } + } + } + + return JobResult.Success(); + } + + private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue) + { + var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL); + + var modlist_path = Path.Combine(Consts.ModListDownloadFolder, + list.Links.MachineURL + ExtensionManager.Extension); + + if (list.NeedsDownload(modlist_path)) + { + if (File.Exists(modlist_path)) + File.Delete(modlist_path); + + var state = DownloadDispatcher.ResolveArchive(list.Links.Download); + Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}"); + await state.Download(modlist_path); + } + else + { + Utils.Log($"No changes detected from downloaded modlist"); + } + + Utils.Log($"Loading {modlist_path}"); + + var installer = AInstaller.LoadFromFile(modlist_path); + + var archives = installer.Archives; + + Utils.Log($"Found {archives.Count} archives in {installer.Name} to index"); + var searching = archives.Select(a => a.Hash).Distinct().ToArray(); + + Utils.Log($"Looking for missing archives"); + var knownArchives = (await db.IndexedFiles.AsQueryable().Where(a => searching.Contains(a.Hash)) + .Select(d => d.Hash).ToListAsync()).ToDictionary(a => a); + + Utils.Log($"Found {knownArchives.Count} pre-existing archives"); + var missing = archives.Where(a => !knownArchives.ContainsKey(a.Hash)).ToList(); + + Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs"); + + var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}}); + + Utils.Log($"Writing jobs to the DB"); + await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false}); + Utils.Log($"Done adding archives for {installer.Name}"); + } + } +} diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index d8db2cae..04a043fd 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -19,10 +19,15 @@ namespace Wabbajack.VirtualFileSystem { public class Context { + static Context() + { + Utils.Log("Cleaning VFS, this may take a bit of time"); + Utils.DeleteDirectory(_stagingFolder); + } public const ulong FileVersion = 0x02; public const string Magic = "WABBAJACK VFS FILE"; - private readonly string _stagingFolder = "vfs_staging"; + private static readonly string _stagingFolder = "vfs_staging"; public IndexRoot Index { get; private set; } = IndexRoot.Empty; /// diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 47979868..ca6bad00 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -139,7 +139,7 @@ namespace Wabbajack.VirtualFileSystem var hash = abs_path.FileHash(); var fi = new FileInfo(abs_path); - if (FileExtractor.MightBeArchive(abs_path)) + if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(abs_path)) { var result = await TryGetContentsFromServer(hash);