Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #200 from wabbajack-tools/compilation-fixes

Browse Source 
Compilation fixes
This commit is contained in:
Timothy Baldridge 2019-11-22 05:26:41 -07:00 committed by GitHub
commit 58f8d916c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 141 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Compression.BSA.Test/BSATests.cs
View File

@ -64,8 +64,8 @@ namespace Compression.BSA.Test
using (var client = new NexusApiClient())
{
var results = client.GetModFiles(info.Item1, info.Item2);
var file = results.FirstOrDefault(f => f.is_primary) ??
results.OrderByDescending(f => f.uploaded_timestamp).First();
var file = results.files.FirstOrDefault(f => f.is_primary) ??
results.files.OrderByDescending(f => f.uploaded_timestamp).First();
var src = Path.Combine(_stagingFolder, file.file_name);
if (File.Exists(src)) return src;

4
Wabbajack.CacheServer/NexusCacheModule.cs
View File

@ -85,7 +85,9 @@ namespace Wabbajack.CacheServer
var client = new HttpClient();
var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 80};
client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]);
return client.GetStringSync(builder.Uri.ToString());
client.GetStringSync(builder.Uri.ToString());
if (!File.Exists(path))
throw new InvalidDataException("Invalid Data");
}
Utils.Log($"{DateTime.Now} - From Cached - {url}");

4
Wabbajack.Common/StatusFileStream.cs
View File

@ -5,9 +5,9 @@ namespace Wabbajack.Common
public class StatusFileStream : Stream
{
private string _message;
private FileStream _inner;
private Stream _inner;
public StatusFileStream(FileStream fs, string message)
public StatusFileStream(Stream fs, string message)
{
_inner = fs;
_message = message;

42
Wabbajack.Common/Utils.cs
View File

@ -177,10 +177,29 @@ namespace Wabbajack.Common
Status(status, (int) (totalRead * 100 / maxSize));
}
}
public static string SHA256(this byte[] data)
public static string xxHash(this byte[] data, bool nullOnIOError = false)
{
return new SHA256Managed().ComputeHash(data).ToBase64();
try
{
var hash = new xxHashConfig();
hash.HashSizeInBits = 64;
hash.Seed = 0x42;
using (var fs = new MemoryStream(data))
{
var config = new xxHashConfig();
config.HashSizeInBits = 64;
using (var f = new StatusFileStream(fs, $"Hashing memory stream"))
{
var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
return value.AsBase64String();
}
}
}
catch (IOException ex)
{
if (nullOnIOError) return null;
throw ex;
}
}
/// <summary>
@ -594,8 +613,8 @@ namespace Wabbajack.Common
public static void CreatePatch(byte[] a, byte[] b, Stream output)
{
var dataA = a.SHA256().FromBase64().ToHex();
var dataB = b.SHA256().FromBase64().ToHex();
var dataA = a.xxHash().FromBase64().ToHex();
var dataB = b.xxHash().FromBase64().ToHex();
var cacheFile = Path.Combine("patch_cache", $"{dataA}_{dataB}.patch");
if (!Directory.Exists("patch_cache"))
Directory.CreateDirectory("patch_cache");
@ -618,7 +637,7 @@ namespace Wabbajack.Common
BSDiff.Create(a, b, f);
}
File.Move(tmpName, cacheFile);
File.Move(tmpName, cacheFile, MoveOptions.ReplaceExisting);
continue;
}
@ -626,11 +645,18 @@ namespace Wabbajack.Common
}
}
public static void TryGetPatch(string foundHash, string fileHash, out byte[] ePatch)
public static bool TryGetPatch(string foundHash, string fileHash, out byte[] ePatch)
{
var patchName = Path.Combine("patch_cache",
$"{foundHash.FromBase64().ToHex()}_{fileHash.FromBase64().ToHex()}.patch");
ePatch = File.Exists(patchName) ? File.ReadAllBytes(patchName) : null;
if (File.Exists(patchName))
{
ePatch = File.ReadAllBytes(patchName);
return true;
}
ePatch = null;
return false;
}
public static void Warning(string s)

1
Wabbajack.Lib/CompilationSteps/IncludePatches.cs
View File

@ -38,6 +38,7 @@ namespace Wabbajack.Lib.CompilationSteps
}
var e = source.EvolveTo<PatchedFromArchive>();
e.FromHash = found.Hash;
e.ArchiveHashPath = found.MakeRelativePaths();
e.To = source.Path;
e.Hash = source.File.Hash;

3
Wabbajack.Lib/Data.cs
View File

@ -207,6 +207,9 @@ namespace Wabbajack.Lib
/// The file to apply to the source file to patch it
/// </summary>
public string PatchID;
[Exclude]
public string FromHash;
}
public class SourcePatch

4
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -102,13 +102,13 @@ namespace Wabbajack.Lib.Downloaders
{
var modfiles = new NexusApiClient().GetModFiles(GameRegistry.GetByMO2ArchiveName(GameName).Game, int.Parse(ModID));
var fileid = ulong.Parse(FileID);
var found = modfiles
var found = modfiles.files
.FirstOrDefault(file => file.file_id == fileid && file.category_name != null);
return found != null;
}
catch (Exception ex)
{
Utils.Log($"{ModName} - {GameName} - {ModID} - {FileID} - Error Getting Nexus Download URL - {ex.Message}");
Utils.Log($"{ModName} - {GameName} - {ModID} - {FileID} - Error Getting Nexus Download URL - {ex}");
return false;
}

69
Wabbajack.Lib/MO2Compiler.cs
View File

@ -78,25 +78,32 @@ namespace Wabbajack.Lib
VFS.IntegrateFromFile(_vfsCacheName);
UpdateTracker.NextStep($"Indexing {MO2Folder}");
VFS.AddRoot(MO2Folder);
var roots = new List<string>()
{
MO2Folder, GamePath, MO2DownloadsFolder
};
// TODO: make this generic so we can add more paths
UpdateTracker.NextStep("Writing VFS Cache");
var lootPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"LOOT");
IEnumerable<RawSourceFile> lootFiles = new List<RawSourceFile>();
if (Directory.Exists(lootPath))
{
roots.Add(lootPath);
}
UpdateTracker.NextStep("Indexing folders");
VFS.AddRoots(roots);
VFS.WriteToFile(_vfsCacheName);
UpdateTracker.NextStep($"Indexing {GamePath}");
VFS.AddRoot(GamePath);
UpdateTracker.NextStep("Writing VFS Cache");
VFS.WriteToFile(_vfsCacheName);
UpdateTracker.NextStep($"Indexing {MO2DownloadsFolder}");
VFS.AddRoot(MO2DownloadsFolder);
UpdateTracker.NextStep("Writing VFS Cache");
VFS.WriteToFile(_vfsCacheName);
if (Directory.Exists(lootPath))
{
lootFiles = Directory.EnumerateFiles(lootPath, "userlist.yaml", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{ Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)) });
}
UpdateTracker.NextStep("Cleaning output folder");
if (Directory.Exists(ModListOutputFolder))
@ -114,23 +121,8 @@ namespace Wabbajack.Lib
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{ Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
var lootPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"LOOT");
// TODO: make this generic so we can add more paths
IEnumerable<RawSourceFile> lootFiles = new List<RawSourceFile>();
if (Directory.Exists(lootPath))
{
Info($"Indexing {lootPath}");
VFS.AddRoot(lootPath);
VFS.WriteToFile(_vfsCacheName);
lootFiles = Directory.EnumerateFiles(lootPath, "userlist.yaml", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{ Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)) });
}
IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder)
.Where(f => File.Exists(f + ".meta"))
@ -297,6 +289,15 @@ namespace Wabbajack.Lib
private void BuildPatches()
{
Info("Gathering patch files");
InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.PatchID == null)
.Do(p =>
{
if (Utils.TryGetPatch(p.FromHash, p.Hash, out var bytes))
p.PatchID = IncludeFile(bytes);
});
var groups = InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => p.PatchID == null)
.GroupBy(p => p.ArchiveHashPath[0])
@ -326,7 +327,7 @@ namespace Wabbajack.Lib
using (var output = new MemoryStream())
{
var a = origin.ReadAll();
var b = LoadDataForTo(entry.To, absolutePaths).Result;
var b = LoadDataForTo(entry.To, absolutePaths);
Utils.CreatePatch(a, b, output);
entry.PatchID = IncludeFile(output.ToArray());
var fileSize = File.GetSize(Path.Combine(ModListOutputFolder, entry.PatchID));
@ -336,7 +337,7 @@ namespace Wabbajack.Lib
}
}
private async Task<byte[]> LoadDataForTo(string to, Dictionary<string, string> absolutePaths)
private byte[] LoadDataForTo(string to, Dictionary<string, string> absolutePaths)
{
if (absolutePaths.TryGetValue(to, out var absolute))
return File.ReadAllBytes(absolute);

4
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -289,10 +289,10 @@ namespace Wabbajack.Lib.NexusApi
public List<NexusFileInfo> files;
}
public IList<NexusFileInfo> GetModFiles(Game game, int modid)
public GetModFilesResponse GetModFiles(Game game, int modid)
{
var url = $"https://api.nexusmods.com/v1/games/{GameRegistry.Games[game].NexusName}/mods/{modid}/files.json";
return GetCached<GetModFilesResponse>(url).files;
return GetCached<GetModFilesResponse>(url);
}
public List<MD5Response> GetModInfoFromMD5(Game game, string md5Hash)

5
Wabbajack.Lib/Validation/ValidateModlist.cs
View File

@ -77,6 +77,11 @@ namespace Wabbajack.Lib.Validation
/// <returns></returns>
public Permissions FilePermissions(NexusDownloader.State mod)
{
if (mod.Author == null || mod.GameName == null || mod.ModID == null || mod.FileID == null)
{
Utils.Error($"Error: Null data for {mod.Author} {mod.GameName} {mod.ModID} {mod.FileID}");
}
var author_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Permissions;
var game_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Permissions;
var mod_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Mods.GetOrDefault(mod.ModID)

2
Wabbajack.Test/EndToEndTests.cs
View File

@ -103,7 +103,7 @@ namespace Wabbajack.Test
{
utils.AddMod(mod_name);
var client = new NexusApiClient();
var file = client.GetModFiles(game, modid).First(f => f.is_primary);
var file = client.GetModFiles(game, modid).files.First(f => f.is_primary);
var src = Path.Combine(DOWNLOAD_FOLDER, file.file_name);
var ini = string.Join("\n",

65
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -6,6 +6,7 @@ using System.Linq;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Common.CSP;
@ -54,7 +55,7 @@ namespace Wabbajack.VirtualFileSystem
var byPath = filtered.ToImmutableDictionary(f => f.Name);
var filesToIndex = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive).ToList();
var filesToIndex = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive).Distinct().ToList();
var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing {root}", filesToIndex.Count));
@ -81,6 +82,42 @@ namespace Wabbajack.VirtualFileSystem
return newIndex;
}
public IndexRoot AddRoots(List<string> roots)
{
if (!roots.All(p => Path.IsPathRooted(p)))
throw new InvalidDataException($"Paths are not absolute");
var filtered = Index.AllFiles.Where(file => File.Exists(file.Name)).ToList();
var byPath = filtered.ToImmutableDictionary(f => f.Name);
var filesToIndex = roots.SelectMany(root => Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive)).ToList();
var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing roots", filesToIndex.Count));
var allFiles = filesToIndex
.PMap(Queue, f =>
{
if (byPath.TryGetValue(f, out var found))
{
var fi = new FileInfo(f);
if (found.LastModified == fi.LastWriteTimeUtc.Ticks && found.Size == fi.Length)
return found;
}
return VirtualFile.Analyze(this, null, f, f);
});
var newIndex = IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
lock (this)
{
Index = newIndex;
}
return newIndex;
}
class Box<T>
{
public T Value { get; set; }
@ -338,26 +375,26 @@ namespace Wabbajack.VirtualFileSystem
public IndexRoot Integrate(List<VirtualFile> files)
{
Utils.Log($"Integrating");
Utils.Log($"Integrating {files.Count} files");
var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList();
var byFullPath = allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToImmutableDictionary(f => f.FullPath);
var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToImmutableDictionary(f => f.FullPath));
var byHash = allFiles.SelectMany(f => f.ThisAndAllChildren)
var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.Where(f => f.Hash != null)
.ToGroupedImmutableDictionary(f => f.Hash);
.ToGroupedImmutableDictionary(f => f.Hash));
var byName = allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToGroupedImmutableDictionary(f => f.Name);
var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
.ToGroupedImmutableDictionary(f => f.Name));
var byRootPath = allFiles.ToImmutableDictionary(f => f.Name);
var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name));
var result = new IndexRoot(allFiles,
byFullPath,
byHash,
byRootPath,
byName);
byFullPath.Result,
byHash.Result,
byRootPath.Result,
byName.Result);
Utils.Log($"Done integrating");
return result;
}
@ -383,4 +420,4 @@ namespace Wabbajack.VirtualFileSystem
Directory.Delete(FullName, true, true);
}
}
}
}