Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Don't re-hash files during compilation

Browse Source
This commit is contained in:
Timothy Baldridge 2020-07-10 16:59:39 -06:00
parent a3efdfbbe0
commit 009b59289c
6 changed files with 89 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Wabbajack.Lib/CompilationSteps/IncludePatches.cs
View File

@ -29,7 +29,7 @@ namespace Wabbajack.Lib.CompilationSteps
_indexedByName = _indexed.Values
.SelectMany(s => s)
.Where(f => f.IsNative)
.GroupBy(f => f.FullPath.FileName)
.GroupBy(f => f.Name.FileName)
.ToDictionary(f => f.Key, f => (IEnumerable<VirtualFile>)f);
_isGenericGame = _mo2Compiler.CompilingGame.IsGenericMO2Plugin;
@ -87,7 +87,7 @@ namespace Wabbajack.Lib.CompilationSteps
{
var dist = new Levenshtein();
found = arch.SelectMany(a => a.ThisAndAllChildren)
.OrderBy(a => dist.Distance(a.FullPath.FileName.ToString(), source.File.FullPath.FileName.ToString()))
.OrderBy(a => dist.Distance(a.Name.FileName.ToString(), source.File.Name.FileName.ToString()))
.Take(3)
.ToArray();
}

4
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -71,7 +71,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, new ExtractedDiskFile(f), f, 0);
return await VirtualFile.Analyze(this, null, new RootDiskFile(f), f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@ -103,7 +103,7 @@ namespace Wabbajack.VirtualFileSystem
return found;
}
return await VirtualFile.Analyze(this, null, new ExtractedDiskFile(f), f, 0);
return await VirtualFile.Analyze(this, null, new RootDiskFile(f), f, 0);
});
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());

4
Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs
View File

@ -8,7 +8,7 @@ namespace Wabbajack.VirtualFileSystem
{
public class ExtractedDiskFile : IExtractedFile
{
private AbsolutePath _path;
protected AbsolutePath _path;
public ExtractedDiskFile(AbsolutePath path)
{
@ -17,7 +17,7 @@ namespace Wabbajack.VirtualFileSystem
_path = path;
}
public async Task<Hash> HashAsync()
public virtual async Task<Hash> HashAsync()
{
return await _path.FileHashAsync();
}

56
Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs
View File

@ -1,4 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
@ -14,5 +17,58 @@ namespace Wabbajack.VirtualFileSystem
public Hash Hash { get; set; }
public long Size { get; set; }
public List<IndexedVirtualFile> Children { get; set; } = new List<IndexedVirtualFile>();
private void Write(BinaryWriter bw)
{
bw.Write(Name.ToString());
bw.Write((ulong)Hash);
bw.Write(Size);
bw.Write(Children.Count);
foreach (var file in Children)
file.Write(bw);
}
public void Write(Stream s)
{
using var bw = new BinaryWriter(s);
bw.Write(Size);
bw.Write(Children.Count);
foreach (var file in Children)
file.Write(bw);
}
private static IndexedVirtualFile Read(BinaryReader br)
{
var ivf = new IndexedVirtualFile
{
Name = (RelativePath)br.ReadString(),
Hash = Hash.FromULong(br.ReadUInt64()),
Size = br.ReadInt64(),
};
var lst = new List<IndexedVirtualFile>();
var count = br.ReadInt32();
for (int x = 0; x < count; x++)
{
lst.Add(Read(br));
}
return ivf;
}
public static IndexedVirtualFile Read(Stream s)
{
using var br = new BinaryReader(s);
var ivf = new IndexedVirtualFile
{
Size = br.ReadInt64(),
};
var lst = new List<IndexedVirtualFile>();
ivf.Children = lst;
var count = br.ReadInt32();
for (int x = 0; x < count; x++)
{
lst.Add(Read(br));
}
return ivf;
}
}
}

20
Wabbajack.VirtualFileSystem/RootDiskFile.cs Normal file
View File

@ -0,0 +1,20 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public class RootDiskFile : ExtractedDiskFile
{
public RootDiskFile(AbsolutePath path) : base(path)
{
}
public override async Task<Hash> HashAsync()
{
return await _path.FileHashCachedAsync();
}
}
}

11
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -19,7 +19,7 @@ namespace Wabbajack.VirtualFileSystem
static VirtualFile()
{
var options = new DbOptions().SetCreateIfMissing(true);
_vfsCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalVFSCache.rocksDb"));
_vfsCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalVFSCache2.rocksDb"));
}
private AbsolutePath _stagedPath;
@ -170,8 +170,10 @@ namespace Wabbajack.VirtualFileSystem
return false;
}
var data = new MemoryStream(result).FromJson<IndexedVirtualFile>();
var data = IndexedVirtualFile.Read(new MemoryStream(result));
found = ConvertFromIndexedFile(context, data, path, parent, extractedFile);
found.Name = path;
found.Hash = hash;
return true;
}
@ -195,7 +197,8 @@ namespace Wabbajack.VirtualFileSystem
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
{
var result = await TryGetContentsFromServer(hash);
// Disabled because it isn't enabled on the server
IndexedVirtualFile result = null; //await TryGetContentsFromServer(hash);
if (result != null)
{
@ -245,7 +248,7 @@ namespace Wabbajack.VirtualFileSystem
}
await using var ms = new MemoryStream();
self.ToIndexedVirtualFile().ToJson(ms);
self.ToIndexedVirtualFile().Write(ms);
_vfsCache.Put(self.Hash.ToArray(), ms.ToArray());
return self;