Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #969 from wabbajack-tools/small-fixes2

Browse Source 
Fixes for the DiskSlabAllocator.cs, ListValidator.cs, and the constan…
This commit is contained in:
Timothy Baldridge 2020-07-19 19:14:51 -07:00 committed by GitHub
commit 95e88f0690
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 63 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Compression.BSA.Test/BSATests.cs
View File

@ -119,7 +119,7 @@ namespace Compression.BSA.Test
TestContext.WriteLine($"Building {bsa}");
await using (var w = ViaJson(a.State).MakeBuilder(size))
await using (var w = await ViaJson(a.State).MakeBuilder(size))
{
var streams = await a.Files.PMap(Queue, async file =>
{

6
Compression.BSA/BA2Builder.cs
View File

@ -29,10 +29,10 @@ namespace Compression.BSA
private List<IFileBuilder> _entries = new List<IFileBuilder>();
private DiskSlabAllocator _slab;
public BA2Builder(BA2StateObject state, long size)
public static async Task<BA2Builder> Create(BA2StateObject state, long size)
{
_state = state;
_slab = new DiskSlabAllocator(size);
var self = new BA2Builder {_state = state, _slab = await DiskSlabAllocator.Create(size)};
return self;
}
public async ValueTask DisposeAsync()

4
Compression.BSA/BA2Reader.cs
View File

@ -144,9 +144,9 @@ namespace Compression.BSA
public EntryType Type { get; set; }
public string HeaderMagic { get; set; }
public uint Version { get; set; }
public override IBSABuilder MakeBuilder(long size)
public override async Task<IBSABuilder> MakeBuilder(long size)
{
return new BA2Builder(this, size);
return await BA2Builder.Create(this, size);
}
}

22
Compression.BSA/BSABuilder.cs
View File

@ -26,18 +26,24 @@ namespace Compression.BSA
internal uint _version;
internal DiskSlabAllocator _slab;
public BSABuilder(long size)
public static async Task<BSABuilder> Create(long size)
{
_fileId = Encoding.ASCII.GetBytes("BSA\0");
_offset = 0x24;
_slab = new DiskSlabAllocator(size);
var self = new BSABuilder
{
_fileId = Encoding.ASCII.GetBytes("BSA\0"),
_offset = 0x24,
_slab = await DiskSlabAllocator.Create(size)
};
return self;
}
public BSABuilder(BSAStateObject bsaStateObject, long size) : this(size)
public static async Task<BSABuilder> Create(BSAStateObject bsaStateObject, long size)
{
_version = bsaStateObject.Version;
_fileFlags = bsaStateObject.FileFlags;
_archiveFlags = bsaStateObject.ArchiveFlags;
var self = await Create(size);
self._version = bsaStateObject.Version;
self._fileFlags = bsaStateObject.FileFlags;
self._archiveFlags = bsaStateObject.ArchiveFlags;
return self;
}
public IEnumerable<FileEntry> Files => _files;

4
Compression.BSA/BSAReader.cs
View File

@ -179,9 +179,9 @@ namespace Compression.BSA
}
public override IBSABuilder MakeBuilder(long size)
public override async Task<IBSABuilder> MakeBuilder(long size)
{
return new BSABuilder(this, size);
return await BSABuilder.Create(this, size);
}
public string Magic { get; set; }

2
Compression.BSA/IBSAReader.cs
View File

@ -26,7 +26,7 @@ namespace Compression.BSA
public class ArchiveStateObject
{
public virtual IBSABuilder MakeBuilder(long size)
public virtual async Task<IBSABuilder> MakeBuilder(long size)
{
throw new NotImplementedException();
}

2
Compression.BSA/TES3Reader.cs
View File

@ -100,7 +100,7 @@ namespace Compression.BSA
public uint HashOffset { get; set; }
public uint VersionNumber { get; set; }
public override IBSABuilder MakeBuilder(long size)
public override async Task<IBSABuilder> MakeBuilder(long size)
{
return new TES3Builder(this);
}

37
Wabbajack.Common/Util/DiskSlabAllocator.cs
View File

@ -12,19 +12,32 @@ namespace Wabbajack.Common
/// </summary>
public class DiskSlabAllocator : IAsyncDisposable
{
private readonly TempFile _file;
private readonly MemoryMappedFile _mmap;
private TempFile? _file;
private MemoryMappedFile? _mmap;
private long _head = 0;
private readonly FileStream _fileStream;
private FileStream? _fileStream;
private List<IAsyncDisposable> _allocated = new List<IAsyncDisposable>();
private long _size;
public DiskSlabAllocator(long size)
private DiskSlabAllocator()
{
_file = new TempFile();
_fileStream = _file.File.Open(FileMode.Create, FileAccess.ReadWrite);
_size = size;
_mmap = MemoryMappedFile.CreateFromFile(_fileStream, null, size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false);
}
public static async Task<DiskSlabAllocator> Create(long size)
{
var file = new TempFile();
var fileStream = await file.Path.Create();
size = Math.Max(size, 1024);
var self = new DiskSlabAllocator
{
_file = file,
_size = size,
_fileStream = fileStream,
_mmap = MemoryMappedFile.CreateFromFile(fileStream, null, size, MemoryMappedFileAccess.ReadWrite,
HandleInheritability.None, false)
};
return self;
}
public Stream Allocate(long size)
@ -39,7 +52,7 @@ namespace Wabbajack.Common
var startAt = _head;
_head += size;
var stream = _mmap.CreateViewStream(startAt, size, MemoryMappedFileAccess.ReadWrite);
var stream = _mmap!.CreateViewStream(startAt, size, MemoryMappedFileAccess.ReadWrite);
_allocated.Add(stream);
return stream;
}
@ -49,9 +62,9 @@ namespace Wabbajack.Common
{
foreach (var allocated in _allocated)
await allocated.DisposeAsync();
_mmap.Dispose();
await _fileStream.DisposeAsync();
await _file.DisposeAsync();
_mmap!.Dispose();
await _fileStream!.DisposeAsync();
await _file!.DisposeAsync();
}
}
}

3
Wabbajack.Lib/MO2Installer.cs
View File

@ -261,11 +261,12 @@ namespace Wabbajack.Lib
foreach (var bsa in bsas)
{
Status($"Building {bsa.To}");
Info($"Building {bsa.To}");
var sourceDir = OutputFolder.Combine(Consts.BSACreationDir, bsa.TempID);
var bsaSize = bsa.FileStates.Select(state => sourceDir.Combine(state.Path).Size).Sum();
await using var a = bsa.State.MakeBuilder(bsaSize);
await using var a = await bsa.State.MakeBuilder(bsaSize);
var streams = await bsa.FileStates.PMap(Queue, async state =>
{
Status($"Adding {state.Path} to BSA");

1
Wabbajack.Server/Controllers/ListsStatus.cs
View File

@ -167,6 +167,7 @@ namespace Wabbajack.BuildServer.Controllers
if (string.IsNullOrWhiteSpace(itm.Archive.Name))
itm.Archive.Name = itm.Archive.State.PrimaryKeyString;
});
results.Archives = results.Archives.OrderBy(a => a.Name).ToList();
return results;
}

4
Wabbajack.Server/Controllers/ModUpgrade.cs
View File

@ -76,7 +76,9 @@ namespace Wabbajack.BuildServer.Controllers
}
catch (Exception ex)
{
// ignore
_logger.LogInformation(
$"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), due to upgrade failure");
return NotFound("File is Valid");
}
var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive);

10
Wabbajack.Server/DataLayer/Patches.cs
View File

@ -164,15 +164,7 @@ namespace Wabbajack.Server.DataLayer
@"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id
LEFT JOIN dbo.ModListArchives m ON m.PrimaryKeyString = a.PrimaryKeyString AND m.Hash = a.Hash
WHERE m.PrimaryKeyString is null
UNION
SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id
LEFT JOIN dbo.ModListArchives m ON m.PrimaryKeyString = a.PrimaryKeyString AND m.Hash = a.Hash
WHERE m.PrimaryKeyString is not null
AND (p.LastUsed < DATEADD(d, -7, getutcdate()) OR p.LastUsed is null and p.Finished < DATEADD(d, -7, getutcdate()))");
WHERE a.Hash not in (SELECT Hash FROM dbo.ModListArchives)");
return await AsPatches(patches);
}

2
Wabbajack.Server/Services/ListValidator.cs
View File

@ -210,6 +210,8 @@ namespace Wabbajack.Server.Services
return (archive, ArchiveStatus.Valid);
case ModDBDownloader.State _:
return (archive, ArchiveStatus.Valid);
case MediaFireDownloader.State _:
return (archive, ArchiveStatus.Valid);
default:
{
if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),

14
Wabbajack.Test/ModlistMetadataTests.cs
View File

@ -16,20 +16,6 @@ namespace Wabbajack.Test
var modlists = await ModlistMetadata.LoadFromGithub();
Assert.True(modlists.Count > 0);
}
[Fact]
public async Task VerifyLogoURLs()
{
var modlists = await ModlistMetadata.LoadFromGithub();
foreach (var modlist in modlists.Select(m => m.Links))
{
var logoState = DownloadDispatcher.ResolveArchive(modlist.ImageUri);
Assert.NotNull(logoState);
Assert.True(await logoState.Verify(new Archive(state: null){Size = 0}), $"{modlist.ImageUri} is not valid");
}
}
public ModlistMetadataTests(ITestOutputHelper output) : base(output)
{

10
Wabbajack.Test/SanityTests.cs
View File

@ -302,7 +302,7 @@ namespace Wabbajack.Test
Magic = "BSA\0", Version = 0x69, ArchiveFlags = 0x107, FileFlags = 0x0,
};
await using (var bsa = bsaState.MakeBuilder(1024 * 1024))
await using (var bsa = await bsaState.MakeBuilder(1024 * 1024))
{
await bsa.AddFile(new BSAFileStateObject
{
@ -344,7 +344,7 @@ namespace Wabbajack.Test
var tempFileData = utils.RandomData(1024);
await using (var bsa = bsaState.MakeBuilder(1024 * 1024))
await using (var bsa = await bsaState.MakeBuilder(1024 * 1024))
{
await bsa.AddFile(new BSAFileStateObject
{
@ -382,7 +382,7 @@ namespace Wabbajack.Test
Magic = "BSA\0", Version = 0x69, ArchiveFlags = 0x107, FileFlags = 0x0,
};
await using (var bsa = bsaState.MakeBuilder(1024 * 1024))
await using (var bsa = await bsaState.MakeBuilder(1024 * 1024))
{
await bsa.AddFile(new BSAFileStateObject
{
@ -418,7 +418,7 @@ namespace Wabbajack.Test
// Create the download
await using var tempFile = new TempFile();
await using (var bsa = bsaState.MakeBuilder(1024 * 1024))
await using (var bsa = await bsaState.MakeBuilder(1024 * 1024))
{
await bsa.AddFile(new BSAFileStateObject
{
@ -431,7 +431,7 @@ namespace Wabbajack.Test
// Create the result
await using (var bsa = bsaState.MakeBuilder(1024 * 1024))
await using (var bsa = await bsaState.MakeBuilder(1024 * 1024))
{
await bsa.AddFile(new BSAFileStateObject
{