Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Everything compiles

Browse Source
This commit is contained in:
Timothy Baldridge 2020-03-28 14:42:45 -06:00
parent c01ed4375c
commit b5006a0737
20 changed files with 89 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Wabbajack.BuildServer/AppSettings.cs
View File

@ -1,4 +1,5 @@
using Microsoft.Extensions.Configuration;
using Wabbajack.Common;
namespace Wabbajack.BuildServer
{
@ -9,8 +10,8 @@ namespace Wabbajack.BuildServer
config.Bind("WabbajackSettings", this);
}
public string DownloadDir { get; set; }
public string ArchiveDir { get; set; }
public AbsolutePath DownloadDir { get; set; }
public AbsolutePath ArchiveDir { get; set; }
public bool JobScheduler { get; set; }
public bool JobRunner { get; set; }

4
Wabbajack.BuildServer/Controllers/ModlistUpdater.cs
View File

@ -145,7 +145,7 @@ namespace Wabbajack.BuildServer.Controllers
if (startingHash == newArchive.Hash)
return NotFound("End hash same as old hash");
if (!AlphaFile.Exists(PatchArchive.CdnPath(startingHash, newArchive.Hash)))
if (!PatchArchive.CdnPath(startingHash, newArchive.Hash).Exists)
{
Db.Jobs.InsertOne(new Job
{
@ -162,7 +162,7 @@ namespace Wabbajack.BuildServer.Controllers
private async Task<Archive> FindAlternatives(NexusDownloader.State state, Hash srcHash)
{
var origSize = AlphaFile.GetSize(_settings.PathForArchive(srcHash));
var origSize = _settings.PathForArchive(srcHash).Size;
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var allMods = await api.GetModFiles(GameRegistry.GetByFuzzyName(state.GameName).Game,
int.Parse(state.ModID));

16
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -126,16 +126,16 @@ namespace Wabbajack.BuildServer.Controllers
if (!Key.All(a => HexChars.Contains(a)))
return BadRequest("NOT A VALID FILENAME");
var parts = Encoding.UTF8.GetString(Key.FromHex()).Split('|');
var final_name = $"{parts[0]}-{parts[1]}{parts[2]}";
var original_name = $"{parts[0]}{parts[2]}";
var finalName = $"{parts[0]}-{parts[1]}{parts[2]}";
var originalName = $"{parts[0]}{parts[2]}";
var final_path = Path.Combine("public", "files", final_name);
System.IO.File.Move(Path.Combine("public", "tmp_files", Key), final_path);
var hash = await final_path.FileHashAsync();
var finalPath = "public".RelativeTo(AbsolutePath.EntryPoint).Combine("files", finalName);
"public".RelativeTo(AbsolutePath.EntryPoint).MoveTo(finalPath);
var hash = await finalPath.FileHashAsync();
if (expectedHash != hash)
{
System.IO.File.Delete(final_path);
finalPath.Delete();
return BadRequest($"Bad Hash, Expected: {expectedHash} Got: {hash}");
}
@ -144,9 +144,9 @@ namespace Wabbajack.BuildServer.Controllers
{
Id = parts[1],
Hash = hash,
Name = original_name,
Name = originalName,
Uploader = user,
Size = new FileInfo(final_path).Length,
Size = finalPath.Size,
CDNName = "wabbajackpush"
};
await Db.UploadedFiles.InsertOneAsync(record);

17
Wabbajack.BuildServer/Extensions.cs
View File

@ -47,22 +47,19 @@ namespace Wabbajack.BuildServer
return authenticationBuilder.AddScheme<ApiKeyAuthenticationOptions, ApiKeyAuthenticationHandler>(ApiKeyAuthenticationOptions.DefaultScheme, options);
}
private static readonly ConcurrentDictionary<Hash, string> PathForArchiveHash = new ConcurrentDictionary<Hash, string>();
public static string PathForArchive(this AppSettings settings, Hash hash)
private static readonly ConcurrentDictionary<Hash, AbsolutePath> PathForArchiveHash = new ConcurrentDictionary<Hash, AbsolutePath>();
public static AbsolutePath PathForArchive(this AppSettings settings, Hash hash)
{
if (PathForArchiveHash.TryGetValue(hash, out string result))
if (PathForArchiveHash.TryGetValue(hash, out AbsolutePath result))
return result;
var hexHash = hash.ToHex();
var ends = "_" + hexHash + "_";
var file = Directory.EnumerateFiles(settings.ArchiveDir, DirectoryEnumerationOptions.Files,
new DirectoryEnumerationFilters
{
InclusionFilter = f => Path.GetFileNameWithoutExtension(f.FileName).EndsWith(ends)
}).FirstOrDefault();
if (file != null)
var file = settings.ArchiveDir.EnumerateFiles()
.FirstOrDefault(f => ((string)f.FileNameWithoutExtension).EndsWith(ends));
if (file != default)
PathForArchiveHash.TryAdd(hash, file);
return file;
}

15
Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs
View File

@ -2,6 +2,7 @@
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using System.Linq;
using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
@ -43,26 +44,24 @@ namespace Wabbajack.BuildServer.Models.Jobs
{
var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
var modlist_path = Path.Combine(Consts.ModListDownloadFolder,
list.Links.MachineURL + Consts.ModListExtension);
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlist_path))
if (list.NeedsDownload(modlistPath))
{
if (File.Exists(modlist_path))
File.Delete(modlist_path);
modlistPath.Delete();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlist_path);
await state.Download(modlistPath);
}
else
{
Utils.Log($"No changes detected from downloaded ModList");
}
Utils.Log($"Loading {modlist_path}");
Utils.Log($"Loading {modlistPath}");
var installer = AInstaller.LoadFromFile(modlist_path);
var installer = AInstaller.LoadFromFile(modlistPath);
var archives = installer.Archives;

8
Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs
View File

@ -23,12 +23,12 @@ namespace Wabbajack.BuildServer.Models.Jobs
Utils.Log($"Indexing game files");
var states = GameRegistry.Games.Values
.Where(game => game.GameLocation() != null && game.MainExecutable != null)
.SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", SearchOption.AllDirectories)
.SelectMany(game => game.GameLocation().Value.EnumerateFiles()
.Select(file => new GameFileSourceDownloader.State
{
Game = game.Game,
GameVersion = game.InstalledVersion,
GameFile = file.RelativeTo(game.GameLocation()),
GameFile = file.RelativeTo(game.GameLocation().Value),
}))
.ToList();
@ -46,7 +46,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
await states.PMap(queue, async state =>
{
var path = Path.Combine(state.Game.MetaData().GameLocation(), state.GameFile);
var path = state.Game.MetaData().GameLocation().Value.Combine(state.GameFile);
Utils.Log($"Hashing Game file {path}");
try
{
@ -60,7 +60,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
var with_hash = states.Where(state => state.Hash != null).ToList();
Utils.Log($"Inserting {with_hash.Count} jobs.");
var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = Path.GetFileName(state.GameFile), State = state}})
var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = state.GameFile.FileName.ToString(), State = state}})
.Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
.ToList();

23
Wabbajack.BuildServer/Models/Jobs/IndexJob.cs
View File

@ -4,6 +4,7 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
@ -39,13 +40,13 @@ namespace Wabbajack.BuildServer.Models.Jobs
string fileName = Archive.Name;
string folder = Guid.NewGuid().ToString();
Utils.Log($"Indexer is downloading {fileName}");
var downloadDest = Path.Combine(settings.DownloadDir, folder, fileName);
var downloadDest = settings.DownloadDir.Combine(folder, fileName);
await Archive.State.Download(downloadDest);
using (var queue = new WorkQueue())
{
var vfs = new Context(queue, true);
await vfs.AddRoot(Path.Combine(settings.DownloadDir, folder));
await vfs.AddRoot(settings.DownloadDir.Combine(folder));
var archive = vfs.Index.ByRootPath.First().Value;
await sql.MergeVirtualFile(archive);
@ -55,13 +56,14 @@ namespace Wabbajack.BuildServer.Models.Jobs
Key = pk_str, Hash = archive.Hash, State = Archive.State, IsValid = true
});
var to_path = Path.Combine(settings.ArchiveDir,
var to_path = settings.ArchiveDir.Combine(
$"{Path.GetFileName(fileName)}_{archive.Hash.ToHex()}_{Path.GetExtension(fileName)}");
if (File.Exists(to_path))
File.Delete(downloadDest);
if (to_path.Exists)
downloadDest.Delete();
else
File.Move(downloadDest, to_path);
Utils.DeleteDirectory(Path.Combine(settings.DownloadDir, folder));
downloadDest.MoveTo(to_path);
await settings.DownloadDir.Combine(folder).DeleteDirectory();
}
@ -70,9 +72,10 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
/*
private List<IndexedFile> ConvertArchive(List<IndexedFile> files, VirtualFile file, bool isTop = true)
{
var name = isTop ? Path.GetFileName(file.Name) : file.Name;
var name = isTop ? file.Name.FileName : file.Name;
var ifile = new IndexedFile
{
Hash = file.Hash,
@ -89,7 +92,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return new ChildFile
{
Hash = f.Hash,
Name = f.Name.ToLowerInvariant(),
Name = f.Name,
Extension = Path.GetExtension(f.Name.ToLowerInvariant())
};
}).ToList() : new List<ChildFile>()
@ -97,7 +100,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
ifile.IsArchive = ifile.Children.Count > 0;
files.Add(ifile);
return files;
}
}*/
}

14
Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs
View File

@ -18,8 +18,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
{
using (var queue = new WorkQueue())
{
var files = Directory.EnumerateFiles(settings.ArchiveDir)
.Where(f => !f.EndsWith(Consts.HashFileExtension))
var files = settings.ArchiveDir.EnumerateFiles()
.Where(f => f.Extension != Consts.HashFileExtension)
.ToList();
var total_count = files.Count;
int completed = 0;
@ -33,18 +33,18 @@ namespace Wabbajack.BuildServer.Models.Jobs
if (await sql.HaveIndexdFile(await file.FileHashCachedAsync()))
{
Utils.Log($"({completed}/{total_count}) Skipping {Path.GetFileName(file)}, it's already indexed");
Utils.Log($"({completed}/{total_count}) Skipping {file.FileName}, it's already indexed");
return;
}
var sub_folder = Guid.NewGuid().ToString();
string folder = Path.Combine(settings.DownloadDir, sub_folder);
var folder = settings.DownloadDir.Combine(sub_folder);
Utils.Log($"({completed}/{total_count}) Copying {file}");
Directory.CreateDirectory(folder);
folder.CreateDirectory();
Utils.Log($"({completed}/{total_count}) Copying {file}");
File.Copy(file, Path.Combine(folder, Path.GetFileName(file)));
file.CopyTo(folder.Combine(file.FileName));
Utils.Log($"({completed}/{total_count}) Analyzing {file}");
var vfs = new Context(queue, true);
@ -56,7 +56,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
await sql.MergeVirtualFile(root);
Utils.Log($"({completed}/{total_count}) Cleaning up {file}");
Utils.DeleteDirectory(folder);
await Utils.DeleteDirectory(folder);
}
catch (Exception ex)
{

13
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -48,16 +48,15 @@ namespace Wabbajack.BuildServer.Models.Jobs
private async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{
var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + Consts.ModListExtension);
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlist_path))
if (list.NeedsDownload(modlistPath))
{
if (File.Exists(modlist_path))
File.Delete(modlist_path);
modlistPath.Delete();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlist_path);
await state.Download(modlistPath);
}
else
{
@ -65,9 +64,9 @@ namespace Wabbajack.BuildServer.Models.Jobs
}
Utils.Log($"Loading {modlist_path}");
Utils.Log($"Loading {modlistPath}");
var installer = AInstaller.LoadFromFile(modlist_path);
var installer = AInstaller.LoadFromFile(modlistPath);
Utils.Log($"{installer.Archives.Count} archives to validate");

6
Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs
View File

@ -36,7 +36,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
{
try
{
await client.UploadAsync(stream, file.MungedName, progress: new Progress(file.MungedName));
await client.UploadAsync(stream, file.MungedName, progress: new Progress((RelativePath)file.MungedName));
}
catch (Exception ex)
{
@ -71,10 +71,10 @@ namespace Wabbajack.BuildServer.Models.Jobs
public class Progress : IProgress<FluentFTP.FtpProgress>
{
private string _name;
private RelativePath _name;
private DateTime LastUpdate = DateTime.UnixEpoch;
public Progress(string name)
public Progress(RelativePath name)
{
_name = name;
}

17
Wabbajack.BuildServer/Models/PatchArchive.cs
View File

@ -32,15 +32,14 @@ namespace Wabbajack.BuildServer.Models
Utils.Log($"Creating Patch ({Src} -> {DestPK})");
var cdnPath = CdnPath(Src, destHash);
if (File.Exists(cdnPath))
if (cdnPath.Exists)
return JobResult.Success();
Utils.Log($"Calculating Patch ({Src} -> {DestPK})");
await using var fs = File.Create(cdnPath);
await using (var srcStream = File.OpenRead(srcPath))
await using (var destStream = File.OpenRead(destPath))
await using (var sigStream = File.Create(cdnPath + ".octo_sig"))
await using var fs = cdnPath.Create();
await using (var srcStream = srcPath.OpenRead())
await using (var destStream = destPath.OpenRead())
await using (var sigStream = cdnPath.WithExtension(Consts.OctoSig).Create())
{
OctoDiff.Create(destStream, srcStream, sigStream, fs);
}
@ -56,7 +55,7 @@ namespace Wabbajack.BuildServer.Models
await client.ConnectAsync();
try
{
await client.UploadAsync(fs, $"updates/{Src.ToHex()}_{destHash.ToHex()}", progress: new UploadToCDN.Progress(cdnPath));
await client.UploadAsync(fs, $"updates/{Src.ToHex()}_{destHash.ToHex()}", progress: new UploadToCDN.Progress(cdnPath.FileName));
}
catch (Exception ex)
{
@ -72,9 +71,9 @@ namespace Wabbajack.BuildServer.Models
}
public static string CdnPath(Hash srcHash, Hash destHash)
public static AbsolutePath CdnPath(Hash srcHash, Hash destHash)
{
return $"updates/{srcHash.ToHex()}_{destHash.ToHex()}";
return $"updates/{srcHash.ToHex()}_{destHash.ToHex()}".RelativeTo(AbsolutePath.EntryPoint);
}
}
}

3
Wabbajack.BuildServer/Models/Sql/ArchiveContent.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using Wabbajack.Common;
namespace Wabbajack.BuildServer.Model.Models
{
@ -7,7 +8,7 @@ namespace Wabbajack.BuildServer.Model.Models
{
public long Parent { get; set; }
public long Child { get; set; }
public string Path { get; set; }
public RelativePath Path { get; set; }
public byte[] PathHash { get; set; }
}
}

4
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -69,7 +69,7 @@ namespace Wabbajack.BuildServer.Model.Models
{
Parent = (long)root.Hash,
Child = (long)child.Hash,
Path = child.Name
Path = (RelativePath)child.Name
});
}
@ -120,7 +120,7 @@ namespace Wabbajack.BuildServer.Model.Models
{
return children.Select(f => new IndexedVirtualFile
{
Name = f.Path,
Name = (RelativePath)f.Path,
Hash = Hash.FromLong(f.Hash),
Size = f.Size,
Children = Build(f.Hash)

6
Wabbajack.CLI/Verbs/ChangeDownload.cs
View File

@ -69,13 +69,13 @@ namespace Wabbajack.CLI.Verbs
Directory.CreateDirectory(Output);
}
if (!Modlist.EndsWith(Consts.ModListExtension) && !Modlist.EndsWith("modlist.txt"))
if (!Modlist.EndsWith((string)Consts.ModListExtension) && !Modlist.EndsWith("modlist.txt"))
return CLIUtils.Exit($"The file {Modlist} is not a valid modlist file!", -1);
if (Copy && Move)
return CLIUtils.Exit("You can't set both copy and move flags!", -1);
var isModlist = Modlist.EndsWith(Consts.ModListExtension);
var isModlist = Modlist.EndsWith((string)Consts.ModListExtension);
var list = new List<TransferFile>();
@ -85,7 +85,7 @@ namespace Wabbajack.CLI.Verbs
try
{
modlist = AInstaller.LoadFromFile(Modlist);
modlist = AInstaller.LoadFromFile((AbsolutePath)Modlist);
}
catch (Exception e)
{

2
Wabbajack.CLI/Verbs/DownloadUrl.cs
View File

@ -36,7 +36,7 @@ namespace Wabbajack.CLI.Verbs
new[] {state}
.PMap(queue, async s =>
{
await s.Download(new Archive {Name = Path.GetFileName(Output)}, Output);
await s.Download(new Archive {Name = Path.GetFileName(Output)}, (AbsolutePath)Output);
}).Wait();
File.WriteAllLines(Output + ".meta", state.GetMetaIni());

4
Wabbajack.CLI/Verbs/Validate.cs
View File

@ -33,14 +33,14 @@ namespace Wabbajack.CLI.Verbs
return CLIUtils.Exit($"The file {Input} does not exist!", -1);
if (!Input.EndsWith(Consts.ModListExtension))
if (!Input.EndsWith((string)Consts.ModListExtension))
return CLIUtils.Exit($"The file {Input} does not end with {Consts.ModListExtension}!", -1);
ModList modlist;
try
{
modlist = AInstaller.LoadFromFile(Input);
modlist = AInstaller.LoadFromFile((AbsolutePath)Input);
}
catch (Exception e)
{

4
Wabbajack.Common/Consts.cs
View File

@ -17,7 +17,7 @@ namespace Wabbajack.Common
public static RelativePath LOOTFolderFilesDir = (RelativePath)"LOOT Config Files";
public static RelativePath BSACreationDir = (RelativePath)"TEMP_BSA_FILES";
public static string ModListDownloadFolder = "downloaded_mod_lists";
public static AbsolutePath ModListDownloadFolder = "downloaded_mod_lists".RelativeTo(AbsolutePath.EntryPoint);
public static string MegaPrefix = "https://mega.nz/#!";
@ -124,6 +124,8 @@ namespace Wabbajack.Common
public static AbsolutePath SettingsFile => LocalAppDataPath.Combine("settings.json");
public static RelativePath SettingsIni = (RelativePath)"settings.ini";
public static byte SettingsVersion => 1;
public static Extension OctoSig = new Extension(".octo_sig");
public static RelativePath ModListTxt = (RelativePath)"modlist.txt";
public static RelativePath ModOrganizer2Exe = (RelativePath)"ModOrganizer.exe";
public static RelativePath ModOrganizer2Ini = (RelativePath)"ModOrganizer.ini";

5
Wabbajack.Common/Paths.cs
View File

@ -504,6 +504,11 @@ namespace Wabbajack.Common
{
return (RelativePath)Path.Combine(paths.Select(p => (string)p).Cons(_path).ToArray());
}
public RelativePath Combine(params string[] paths )
{
return (RelativePath)Path.Combine(paths.Cons(_path).ToArray());
}
public int CompareTo(RelativePath other)
{

4
Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
View File

@ -38,7 +38,7 @@ namespace Wabbajack.Lib.Downloaders
return new State
{
Game = game.Game,
GameFile = gameFile,
GameFile = (RelativePath)gameFile,
Hash = hash,
GameVersion = game.InstalledVersion
};
@ -54,7 +54,7 @@ namespace Wabbajack.Lib.Downloaders
[Key(0)]
public Game Game { get; set; }
[Key(1)]
public string GameFile { get; set; }
public RelativePath GameFile { get; set; }
[Key(2)]
public Hash Hash { get; set; }
[Key(3)]

2
Wabbajack/View Models/Gallery/ModListMetadataVM.cs
View File

@ -52,7 +52,7 @@ namespace Wabbajack
{
_parent = parent;
Metadata = metadata;
Location = Consts.ModListDownloadFolder.RelativeTo(AbsolutePath.EntryPoint).Combine(Metadata.Links.MachineURL + Consts.ModListExtension);
Location = Consts.ModListDownloadFolder.Combine(Metadata.Links.MachineURL + (string)Consts.ModListExtension);
IsBroken = metadata.ValidationSummary.HasFailures;
OpenWebsiteCommand = ReactiveCommand.Create(() => Utils.OpenWebsite(new Uri($"https://www.wabbajack.org/modlist/{Metadata.Links.MachineURL}")));
ExecuteCommand = ReactiveCommand.CreateFromObservable<Unit, Unit>(