Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix several more tests

Browse Source
This commit is contained in:
Timothy Baldridge 2020-04-02 21:57:59 -06:00
parent aaba7267bd
commit 8af841f3d6
35 changed files with 268 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Compression.BSA.Test/BSATests.cs
View File

@ -60,9 +60,9 @@ namespace Compression.BSA.Test
var state = new NexusDownloader.State
{
ModID = mod.ToString(),
ModID = mod,
Game = game,
FileID = file.file_id.ToString()
FileID = file.file_id
};
await state.Download(src);
return src;

64
Compression.BSA/BA2Builder.cs
View File

@ -4,6 +4,7 @@ using System.IO;
using System.IO.MemoryMappedFiles;
using System.Linq;
using System.Text;
using ICSharpCode.SharpZipLib.Zip.Compression;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using Wabbajack.Common;
@ -56,40 +57,38 @@ namespace Compression.BSA
public void Build(AbsolutePath filename)
{
SortEntries();
using (var fs = filename.Create())
using (var bw = new BinaryWriter(fs))
using var fs = filename.Create();
using var bw = new BinaryWriter(fs);
bw.Write(Encoding.ASCII.GetBytes(_state.HeaderMagic));
bw.Write(_state.Version);
bw.Write(Encoding.ASCII.GetBytes(Enum.GetName(typeof(EntryType), _state.Type)));
bw.Write((uint)_entries.Count);
var table_offset_loc = bw.BaseStream.Position;
bw.Write((ulong)0);
foreach (var entry in _entries)
{
bw.Write(Encoding.ASCII.GetBytes(_state.HeaderMagic));
bw.Write(_state.Version);
bw.Write(Encoding.ASCII.GetBytes(Enum.GetName(typeof(EntryType), _state.Type)));
bw.Write((uint)_entries.Count);
var table_offset_loc = bw.BaseStream.Position;
bw.Write((ulong)0);
entry.WriteHeader(bw);
}
foreach (var entry in _entries)
{
entry.WriteHeader(bw);
}
foreach (var entry in _entries)
{
entry.WriteData(bw);
}
foreach (var entry in _entries)
{
entry.WriteData(bw);
}
if (!_state.HasNameTable) return;
if (_state.HasNameTable)
{
var pos = bw.BaseStream.Position;
bw.BaseStream.Seek(table_offset_loc, SeekOrigin.Begin);
bw.Write((ulong) pos);
bw.BaseStream.Seek(pos, SeekOrigin.Begin);
var pos = bw.BaseStream.Position;
bw.BaseStream.Seek(table_offset_loc, SeekOrigin.Begin);
bw.Write((ulong)pos);
bw.BaseStream.Seek(pos, SeekOrigin.Begin);
foreach (var entry in _entries)
{
var bytes = Encoding.UTF7.GetBytes(entry.FullName);
bw.Write((ushort)bytes.Length);
bw.BaseStream.Write(bytes, 0, bytes.Length);
}
}
foreach (var entry in _entries)
{
var bytes = Encoding.UTF7.GetBytes(entry.FullName);
bw.Write((ushort)bytes.Length);
bw.BaseStream.Write(bytes, 0, bytes.Length);
}
}
@ -108,8 +107,8 @@ namespace Compression.BSA
{
var builder = new BA2DX10FileEntryBuilder {_state = state};
var header_size = DDS.HeaderSizeForFormat((DXGI_FORMAT) state.PixelFormat) + 4;
new BinaryReader(src).ReadBytes((int)header_size);
var headerSize = DDS.HeaderSizeForFormat((DXGI_FORMAT) state.PixelFormat) + 4;
new BinaryReader(src).ReadBytes((int)headerSize);
// This can't be parallel because it all runs off the same base IO stream.
builder._chunks = new List<ChunkBuilder>();
@ -169,8 +168,9 @@ namespace Compression.BSA
}
else
{
var deflater = new Deflater(Deflater.BEST_COMPRESSION);
using var ms = new MemoryStream();
using (var ds = new DeflaterOutputStream(ms))
using (var ds = new DeflaterOutputStream(ms, deflater))
{
ds.IsStreamOwner = false;
src.CopyToLimit(ds, (int)chunk.FullSz);

23
Wabbajack.BuildServer.Test/IndexedFilesTests.cs
View File

@ -49,34 +49,35 @@ namespace Wabbajack.BuildServer.Test
[Fact]
public async Task CanNotifyOfInis()
{
var files = await @"sql\NotifyStates".RelativeTo(AbsolutePath.EntryPoint)
.EnumerateFiles()
.Where(f => f.Extension == Consts.IniExtension)
.PMap(Queue, async ini => (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(ini.LoadIniFile())));
var archives = files.Select(f =>
var archive =
new Archive
{
State = f,
State = new NexusDownloader.State
{
Game = Game.SkyrimSpecialEdition,
ModID = long.MaxValue >> 3,
FileID = long.MaxValue >> 3,
},
Name = Guid.NewGuid().ToString()
});
Assert.True(await AuthorAPI.UploadPackagedInis(archives));
};
Assert.True(await AuthorAPI.UploadPackagedInis(new[] {archive}));
var SQL = Fixture.GetService<SqlService>();
var job = await SQL.GetJob();
Assert.NotNull(job);
Assert.IsType<IndexJob>(job.Payload);
var payload = (IndexJob)job.Payload;
Assert.IsType<NexusDownloader.State>(payload.Archive.State);
var casted = (NexusDownloader.State)payload.Archive.State;
Assert.Equal(Game.Skyrim, casted.Game);
Assert.Equal(Game.SkyrimSpecialEdition, casted.Game);
// Insert the record into SQL
await SQL.AddDownloadState(Hash.FromHex("00e8bbbf591f61a3"), casted);
// Enqueue the same file again
Assert.True(await AuthorAPI.UploadPackagedInis(archives));
Assert.True(await AuthorAPI.UploadPackagedInis(new[] {archive}));
// File is aleady indexed so nothing gets enqueued
Assert.Null(await SQL.GetJob());

3
Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj
View File

@ -49,6 +49,9 @@
<None Update="xunit.runner.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="sql\uploaded_files_ingest.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>

12
Wabbajack.BuildServer.Test/sql/wabbajack_db.sql
View File

@ -277,9 +277,9 @@ GO
CREATE TABLE [dbo].[Metrics](
[Id] [bigint] IDENTITY(1,1) NOT NULL,
[Timestamp] [datetime] NOT NULL,
[Action] [varchar](64) NOT NULL,
[Subject] [varchar](max) NOT NULL,
[MetricsKey] [varchar](64) NULL,
[Action] [nvarchar](64) NOT NULL,
[Subject] [nvarchar](max) NOT NULL,
[MetricsKey] [nvarchar](64) NULL,
[GroupingSubject] AS (substring([Subject],(0),case when patindex('%[0-9].%',[Subject])=(0) then len([Subject])+(1) else patindex('%[0-9].%',[Subject]) end)),
CONSTRAINT [PK_Metrics] PRIMARY KEY CLUSTERED
(
@ -326,9 +326,9 @@ GO
CREATE TABLE [dbo].[DownloadStates](
[Id] [binary](32) NOT NULL,
[Hash] [bigint] NOT NULL,
[PrimaryKey] [varchar](max) NOT NULL,
[IniState] [varchar](max) NOT NULL,
[JsonState] [varchar](max) NOT NULL,
[PrimaryKey] [nvarchar](max) NOT NULL,
[IniState] [nvarchar](max) NOT NULL,
[JsonState] [nvarchar](max) NOT NULL,
CONSTRAINT [PK_DownloadStates] PRIMARY KEY CLUSTERED
(
[Id] ASC

6
Wabbajack.BuildServer/Controllers/IndexedFiles.cs
View File

@ -60,7 +60,7 @@ namespace Wabbajack.BuildServer.Controllers
int loadCount = 0;
foreach (var file in fullPath.EnumerateFiles().Where(f => f.Extension == Consts.IniExtension))
{
var loaded = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile()));
var loaded = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(file.LoadIniFile(), true));
if (loaded == null)
{
Utils.Log($"Unsupported Ini {file}");
@ -88,7 +88,9 @@ namespace Wabbajack.BuildServer.Controllers
{
await using var ins = entry.Open();
var iniString = Encoding.UTF8.GetString(await ins.ReadAllAsync());
var data = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(iniString.LoadIniString()));
Utils.Log(iniString);
var data = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(iniString.LoadIniString(), true));
if (data == null)
{
Utils.Log("No valid INI parser for: \n" + iniString);

10
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -27,9 +27,9 @@ namespace Wabbajack.BuildServer.Controllers
[HttpGet]
[Route("status.json")]
public async Task<IList<ModlistSummary>> HandleGetLists()
public async Task<IEnumerable<ModlistSummary>> HandleGetLists()
{
return await Db.ModListStatus.AsQueryable().Select(m => m.Summary).ToListAsync();
return await SQL.GetModListSummaries();
}
private static readonly Func<object, string> HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@"
@ -53,7 +53,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("status/{Name}/broken.rss")]
public async Task<ContentResult> HandleGetRSSFeed(string Name)
{
var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus;
var lst = await SQL.GetDetailedModlistStatus(Name);
var response = HandleGetRssFeedTemplate(new
{
lst,
@ -91,7 +91,7 @@ namespace Wabbajack.BuildServer.Controllers
public async Task<ContentResult> HandleGetListHtml(string Name)
{
var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus;
var lst = await SQL.GetDetailedModlistStatus(Name);
var response = HandleGetListTemplate(new
{
lst,
@ -112,7 +112,7 @@ namespace Wabbajack.BuildServer.Controllers
public async Task<ContentResult> HandleGetListJson(string Name)
{
var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus;
var lst = await SQL.GetDetailedModlistStatus(Name);
lst.Archives.Do(a => a.Archive.Meta = null);
return new ContentResult
{

10
Wabbajack.BuildServer/Controllers/ModlistUpdater.cs
View File

@ -41,8 +41,8 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/delete_updates")]
public async Task<IActionResult> DeleteUpdates()
{
var lists = await Db.ModListStatus.AsQueryable().ToListAsync();
var archives = lists.SelectMany(list => list.DetailedStatus.Archives)
var lists = await SQL.GetDetailedModlistStatuses();
var archives = lists.SelectMany(list => list.Archives)
.Select(a => a.Archive.Hash.ToHex())
.ToHashSet();
@ -108,7 +108,7 @@ namespace Wabbajack.BuildServer.Controllers
.ToListAsync();
if (mod_files.SelectMany(f => f.Data.files)
.Any(f => f.category_name != null && f.file_id.ToString() == nexusState.FileID))
.Any(f => f.category_name != null && f.file_id == nexusState.FileID))
{
await Metric("not_required_upgrade", startingHash.ToString());
return BadRequest("Upgrade Not Required");
@ -164,7 +164,7 @@ namespace Wabbajack.BuildServer.Controllers
{
var origSize = _settings.PathForArchive(srcHash).Size;
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var allMods = await api.GetModFiles(state.Game, int.Parse(state.ModID));
var allMods = await api.GetModFiles(state.Game, state.ModID);
var archive = allMods.files.Where(m => !string.IsNullOrEmpty(m.category_name))
.OrderBy(s => Math.Abs((long)s.size - origSize))
.Select(s => new Archive {
@ -174,7 +174,7 @@ namespace Wabbajack.BuildServer.Controllers
{
Game = state.Game,
ModID = state.ModID,
FileID = s.file_id.ToString()
FileID = s.file_id
}}).FirstOrDefault();
if (archive == null)

6
Wabbajack.BuildServer/Controllers/NexusCache.cs
View File

@ -116,14 +116,14 @@ namespace Wabbajack.BuildServer.Controllers
foreach (var record in data.ModInfos)
{
await SQL.AddNexusModInfo(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId),
await SQL.AddNexusModInfo(GameRegistry.GetByFuzzyName(record.Game).Game, record.ModId,
record.LastCheckedUTC, record.Data);
totalRows += 1;
}
foreach (var record in data.FileInfos)
{
await SQL.AddNexusFileInfo(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId),
await SQL.AddNexusFileInfo(GameRegistry.GetByFuzzyName(record.Game).Game, record.ModId,
long.Parse(record.FileId),
record.LastCheckedUTC, record.Data);
totalRows += 1;
@ -131,7 +131,7 @@ namespace Wabbajack.BuildServer.Controllers
foreach (var record in data.ModFiles)
{
await SQL.AddNexusModFiles(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId),
await SQL.AddNexusModFiles(GameRegistry.GetByFuzzyName(record.Game).Game, record.ModId,
record.LastCheckedUTC, record.Data);
totalRows += 1;
}

2
Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs
View File

@ -53,7 +53,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
{
var state = new NexusDownloader.State
{
Game = tuple.Game, ModID = tuple.ModId.ToString(), FileID = tuple.File.file_id.ToString()
Game = tuple.Game, ModID = tuple.ModId, FileID = tuple.File.file_id
};
return new Archive {State = state, Name = tuple.File.file_name};
}).ToList();

2
Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs
View File

@ -56,7 +56,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
// Mod activity could hide files
var b = d.mod.LastestModActivity.AsUnixTime();
return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.ModId.ToString()};
return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.ModId};
});
var purged = await collected.PMap(queue, async t =>

9
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -178,8 +178,6 @@ namespace Wabbajack.BuildServer.Models.Jobs
return false;
var game = gameMeta.Game;
if (!int.TryParse(state.ModID, out var modID))
return false;
var modFiles = (await db.NexusModFiles.AsQueryable().Where(g => g.Game == gameMeta.NexusName && g.ModId == state.ModID).FirstOrDefaultAsync())?.Data;
@ -187,14 +185,11 @@ namespace Wabbajack.BuildServer.Models.Jobs
{
Utils.Log($"No Cache for {state.PrimaryKeyString} falling back to HTTP");
var nexusApi = await NexusApiClient.Get();
modFiles = await nexusApi.GetModFiles(game, modID);
modFiles = await nexusApi.GetModFiles(game, state.ModID);
}
if (!ulong.TryParse(state.FileID, out var fileID))
return false;
var found = modFiles.files
.FirstOrDefault(file => file.file_id == fileID && file.category_name != null);
.FirstOrDefault(file => file.file_id == state.FileID && file.category_name != null);
return found != null;
}
catch (Exception ex)

2
Wabbajack.BuildServer/Models/NexusCacheData.cs
View File

@ -13,7 +13,7 @@ namespace Wabbajack.BuildServer.Models
public string Game { get; set; }
[BsonIgnoreIfNull]
public string ModId { get; set; }
public long ModId { get; set; }
public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow;

134
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -1,5 +1,4 @@
using System;
using System.Collections;
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
@ -7,15 +6,13 @@ using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Microsoft.Extensions.Configuration;
using Microsoft.VisualBasic;
using Newtonsoft.Json;
using ReactiveUI;
using Wabbajack.BuildServer.GraphQL;
using Wabbajack.BuildServer.Model.Models.Results;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.VirtualFileSystem;
@ -197,7 +194,7 @@ namespace Wabbajack.BuildServer.Model.Models
await conn.ExecuteAsync(
@"INSERT INTO dbo.Jobs (Created, Priority, Payload, OnSuccess) VALUES (GETDATE(), @Priority, @Payload, @OnSuccess)",
new {
Priority = job.Priority,
job.Priority,
Payload = job.Payload.ToJSON(),
OnSuccess = job.OnSuccess?.ToJSON() ?? null});
}
@ -212,8 +209,8 @@ namespace Wabbajack.BuildServer.Model.Models
await using var conn = await Open();
await conn.ExecuteAsync(
@"UPDATE dbo.Jobs SET Finshed = GETDATE(), Success = @Success, ResultContent = @ResultContent WHERE Id = @Id",
new {
Id = job.Id,
new {
job.Id,
Success = job.Result.ResultType == JobResultType.Success,
ResultPayload = job.Result.ToJSON()
@ -259,7 +256,7 @@ namespace Wabbajack.BuildServer.Model.Models
public override AJobPayload Parse(object value)
{
return Utils.FromJSONString<AJobPayload>((string)value);
return ((string)value).FromJSONString<AJobPayload>();
}
}
@ -288,12 +285,12 @@ namespace Wabbajack.BuildServer.Model.Models
new
{
Id = uf.Id.ToString(),
Name = uf.Name,
Size = uf.Size,
uf.Name,
uf.Size,
UploadedBy = uf.Uploader,
Hash = (long)uf.Hash,
UploadDate = uf.UploadDate,
CDNName = uf.CDNName
uf.UploadDate,
uf.CDNName
});
}
@ -334,10 +331,115 @@ namespace Wabbajack.BuildServer.Model.Models
public async Task<bool> HaveIndexedArchivePrimaryKey(string key)
{
await using var conn = await Open();
var results = await conn.QueryAsync<string>(
"SELECT * FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey",
var results = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT PrimaryKey FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey",
new {PrimaryKey = key});
return results.Any();
return results != null;
}
public async Task AddNexusFileInfo(Game game, long modId, long fileId, DateTime lastCheckedUtc, NexusFileInfo data)
{
await using var conn = await Open();
await conn.ExecuteAsync("INSERT INTO dbo.NexusFileInfos (Game, ModId, FileId, LastChecked, Data) VALUES " +
"(@Game, @ModId, @FileId, @LastChecked, @Data)",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
FileId = fileId,
LastChecked = lastCheckedUtc,
Data = JsonConvert.SerializeObject(data)
});
}
public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"MERGE dbo.NexusModInfos AS Target
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
LastChecked = lastCheckedUtc,
Data = JsonConvert.SerializeObject(data)
});
}
public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, NexusApiClient.GetModFilesResponse data)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"MERGE dbo.NexusModFiles AS Target
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
LastChecked = lastCheckedUtc,
Data = JsonConvert.SerializeObject(data)
});
}
public async Task<ModInfo> GetNexusModInfoString(Game game, long modId)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId",
new {Game = game.MetaData().NexusGameId, ModId = modId});
return result == null ? null : JsonConvert.DeserializeObject<ModInfo>(result);
}
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(Game game, long modId)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId",
new {Game = game.MetaData().NexusGameId, ModId = modId});
return result == null ? null : JsonConvert.DeserializeObject<NexusApiClient.GetModFilesResponse>(result);
}
#region ModLists
public async Task<IEnumerable<ModlistSummary>> GetModListSummaries()
{
await using var conn = await Open();
var results = await conn.QueryAsync<string>("SELECT Summary from dbo.ModLists");
return results.Select(s => s.FromJSONString<ModlistSummary>()).ToList();
}
public async Task<DetailedStatus> GetDetailedModlistStatus(string machineUrl)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>("SELECT DetailedStatus from dbo.ModLists WHERE MachineURL = @MachineURL",
new
{
machineUrl
});
return result.FromJSONString<DetailedStatus>();
}
public async Task<List<DetailedStatus>> GetDetailedModlistStatuses()
{
await using var conn = await Open();
var results = await conn.QueryAsync<string>("SELECT DetailedStatus from dbo.ModLists");
return results.Select(s => s.FromJSONString<DetailedStatus>()).ToList();
}
#endregion
}
}

11
Wabbajack.Common/Util/DiskSlabAllocator.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.MemoryMappedFiles;
@ -14,11 +15,14 @@ namespace Wabbajack.Common
private readonly MemoryMappedFile _mmap;
private long _head = 0;
private readonly FileStream _fileStream;
private List<IDisposable> _allocated = new List<IDisposable>();
private long _size;
public DiskSlabAllocator(long size)
{
_file = new TempFile();
_fileStream = _file.File.Open(FileMode.Create, FileAccess.ReadWrite);
_size = size;
_mmap = MemoryMappedFile.CreateFromFile(_fileStream, null, size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false);
}
@ -26,14 +30,19 @@ namespace Wabbajack.Common
{
lock (this)
{
if (_head + size >= _size)
throw new InvalidDataException($"Size out of range. Declared {_size} used {_head + size}");
var startAt = _head;
_head += size;
return _mmap.CreateViewStream(startAt, size, MemoryMappedFileAccess.ReadWrite);
var stream = _mmap.CreateViewStream(startAt, size, MemoryMappedFileAccess.ReadWrite);
_allocated.Add(stream);
return stream;
}
}
public void Dispose()
{
_allocated.Do(s => s.Dispose());
_mmap?.Dispose();
_fileStream?.Dispose();
_file?.Dispose();

9
Wabbajack.Common/Util/TempFile.cs
View File

@ -16,10 +16,17 @@ namespace Wabbajack.Common
public bool DeleteAfter = true;
public TempFile(bool deleteAfter = true, bool createFolder = true)
: this(new FileInfo((string)((AbsolutePath)AlphaPath.GetTempPath()).Combine(AlphaPath.GetRandomFileName())))
: this(new FileInfo((string)GetTempFilePath()))
{
}
private static AbsolutePath GetTempFilePath()
{
var path = (@"temp\" + Guid.NewGuid()).RelativeTo(AbsolutePath.EntryPoint);
path.Parent.CreateDirectory();
return path;
}
public TempFile(FileInfo file, bool deleteAfter = true, bool createFolder = true)
{
this.File = file;

2
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -24,7 +24,7 @@ namespace Wabbajack.Lib.Downloaders
public override string SiteName { get; }
public override Uri SiteURL { get; }
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
Uri url = DownloaderUtils.GetDirectURL(archiveINI);

2
Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs
View File

@ -39,7 +39,7 @@ namespace Wabbajack.Lib.Downloaders
var result = await Utils.Log(new RequestBethesdaNetLogin()).Task;
}
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = (Uri)DownloaderUtils.GetDirectURL(archiveINI);
return StateFromUrl(url);

4
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -62,9 +62,9 @@ namespace Wabbajack.Lib.Downloaders
return inst;
}
public static async Task<AbstractDownloadState> ResolveArchive(dynamic ini)
public static async Task<AbstractDownloadState> ResolveArchive(dynamic ini, bool quickMode = false)
{
var states = await Task.WhenAll(Downloaders.Select(d => (Task<AbstractDownloadState>)d.GetDownloaderState(ini)));
var states = await Task.WhenAll(Downloaders.Select(d => (Task<AbstractDownloadState>)d.GetDownloaderState(ini, quickMode)));
return states.FirstOrDefault(result => result != null);
}

2
Wabbajack.Lib/Downloaders/DropboxDownloader.cs
View File

@ -7,7 +7,7 @@ namespace Wabbajack.Lib.Downloaders
{
public class DropboxDownloader : IDownloader, IUrlDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var urlstring = archiveINI?.General?.directURL;
return GetDownloaderState(urlstring);

2
Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
View File

@ -14,7 +14,7 @@ namespace Wabbajack.Lib.Downloaders
{
public class GameFileSourceDownloader : IDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var gameName = (string)archiveINI?.General?.gameName;
var gameFile = (string)archiveINI?.General?.gameFile;

2
Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs
View File

@ -10,7 +10,7 @@ namespace Wabbajack.Lib.Downloaders
{
public class GoogleDriveDownloader : IDownloader, IUrlDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url);

2
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -16,7 +16,7 @@ namespace Wabbajack.Lib.Downloaders
public class HTTPDownloader : IDownloader, IUrlDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url, archiveINI);

2
Wabbajack.Lib/Downloaders/IDownloader.cs
View File

@ -4,7 +4,7 @@ namespace Wabbajack.Lib.Downloaders
{
public interface IDownloader
{
Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI);
Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode = false);
/// <summary>
/// Called before any downloads are inacted by the installer;

2
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -7,7 +7,7 @@ namespace Wabbajack.Lib.Downloaders
{
public class MegaDownloader : IDownloader, IUrlDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url);

2
Wabbajack.Lib/Downloaders/ManualDownloader.cs
View File

@ -61,7 +61,7 @@ namespace Wabbajack.Lib.Downloaders
}
}
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = archiveINI?.General?.manualURL;
return url != null ? new State { Url = url} : null;

2
Wabbajack.Lib/Downloaders/MediaFireDownloader.cs
View File

@ -10,7 +10,7 @@ namespace Wabbajack.Lib.Downloaders
{
public class MediaFireDownloader : IUrlDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
Uri url = DownloaderUtils.GetDirectURL(archiveINI);
if (url == null || url.Host != "www.mediafire.com") return null;

2
Wabbajack.Lib/Downloaders/ModDBDownloader.cs
View File

@ -13,7 +13,7 @@ namespace Wabbajack.Lib.Downloaders
{
public class ModDBDownloader : IDownloader, IUrlDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url);

37
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -51,13 +51,23 @@ namespace Wabbajack.Lib.Downloaders
canExecute: IsLoggedIn.ObserveOnGuiThread());
}
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var general = archiveINI?.General;
if (general.modID != null && general.fileID != null && general.gameName != null)
{
var game = GameRegistry.GetByFuzzyName((string)general.gameName).Game;
if (quickMode)
{
return new State
{
Game = GameRegistry.GetByFuzzyName((string)general.gameName).Game,
ModID = long.Parse(general.modID),
FileID = long.Parse(general.fileID),
};
}
var client = await NexusApiClient.Get();
ModInfo info;
try
@ -79,8 +89,8 @@ namespace Wabbajack.Lib.Downloaders
IsNSFW = info.contains_adult_content,
Description = NexusApiUtils.FixupSummary(info.summary),
Game = GameRegistry.GetByFuzzyName((string)general.gameName).Game,
ModID = general.modID,
FileID = general.fileID
ModID = long.Parse(general.modID),
FileID = long.Parse(general.fileID)
};
}
@ -151,9 +161,9 @@ namespace Wabbajack.Lib.Downloaders
public Game Game { get; set; }
[Key(7)]
public string ModID { get; set; }
public long ModID { get; set; }
[Key(8)]
public string FileID { get; set; }
public long FileID { get; set; }
public async Task<bool> LoadMetaData()
{
@ -195,22 +205,11 @@ namespace Wabbajack.Lib.Downloaders
{
try
{
var gameMeta = Game.MetaData();
if (gameMeta == null)
return false;
var game = gameMeta.Game;
if (!int.TryParse(ModID, out var modID))
return false;
var client = await NexusApiClient.Get();
var modFiles = await client.GetModFiles(game, modID);
if (!ulong.TryParse(FileID, out var fileID))
return false;
var modFiles = await client.GetModFiles(Game, ModID);
var found = modFiles.files
.FirstOrDefault(file => file.file_id == fileID && file.category_name != null);
.FirstOrDefault(file => file.file_id == FileID && file.category_name != null);
return found != null;
}
catch (Exception ex)
@ -233,7 +232,7 @@ namespace Wabbajack.Lib.Downloaders
public override string[] GetMetaIni()
{
return new[] {"[General]", $"gameName={Game.MetaData().MO2Name}", $"modID={ModID}", $"fileID={FileID}"};
return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"modID={ModID}", $"fileID={FileID}"};
}
}
}

2
Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
View File

@ -16,7 +16,7 @@ namespace Wabbajack.Lib.Downloaders
{
private SteamWorkshopItem _item;
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var id = archiveINI?.General?.itemID;
var steamID = archiveINI?.General?.steamID;

2
Wabbajack.Lib/Downloaders/YouTubeDownloader.cs
View File

@ -20,7 +20,7 @@ namespace Wabbajack.Lib.Downloaders
{
public class YouTubeDownloader : IDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var directURL = (Uri)DownloaderUtils.GetDirectURL(archiveINI);
var state = (State)UriToState(directURL);

10
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -158,8 +158,14 @@ namespace Wabbajack.Lib.FileUploader
}
var client = new Common.Http.Client();
await client.PostAsync($"{Consts.WabbajackBuildServerUri}indexed_files/notify", new ByteArrayContent(ms.ToArray()));
return true;
var response = await client.PostAsync($"{Consts.WabbajackBuildServerUri}indexed_files/notify", new ByteArrayContent(ms.ToArray()));
if (response.IsSuccessStatusCode) return true;
Utils.Log("Error sending Inis");
Utils.Log(await response.Content.ReadAsStringAsync());
return false;
}
catch (Exception ex)
{

10
Wabbajack.Lib/NexusApi/Dtos.cs
View File

@ -15,20 +15,20 @@ namespace Wabbajack.Lib.NexusApi
public class NexusFileInfo
{
public ulong category_id { get; set; }
public long category_id { get; set; }
public string category_name { get; set; }
public string changelog_html { get; set; }
public string description { get; set; }
public string external_virus_scan_url { get; set; }
public ulong file_id { get; set; }
public long file_id { get; set; }
public string file_name { get; set; }
public bool is_primary { get; set; }
public string mod_version { get; set; }
public string name { get; set; }
public ulong size { get; set; }
public ulong size_kb { get; set; }
public long size { get; set; }
public long size_kb { get; set; }
public DateTime uploaded_time { get; set; }
public ulong uploaded_timestamp { get; set; }
public long uploaded_timestamp { get; set; }
public string version { get; set; }
}

4
Wabbajack.Test/ContentRightsManagementTests.cs
View File

@ -54,8 +54,8 @@ namespace Wabbajack.Test
{
Game = Game.Skyrim,
Author = "bill",
ModID = "42",
FileID = "33",
ModID = 42,
FileID = 33,
},
Hash = Hash.FromLong(42)
}

14
Wabbajack.Test/DownloaderTests.cs
View File

@ -514,14 +514,14 @@ namespace Wabbajack.Test
await inst.DownloadMissingArchives(archivesa, true);
await inst.DownloadMissingArchives(archivesb, true);
Assert.Equal(folder.EnumerateFiles().Select(f => f.FileName).OrderBy(a => a).ToArray(),
new RelativePath[]
Assert.Equal(new[]
{
(RelativePath)@"Download.esm",
(RelativePath)@"Download.esm.xxHash",
(RelativePath)@"Download_c4047f2251d8eead22df4b4888cc4b833ae7d9a6766ff29128e083d944f9ec4b_.esm",
(RelativePath)@"Download_c4047f2251d8eead22df4b4888cc4b833ae7d9a6766ff29128e083d944f9ec4b_.esm.xxHash"
}.OrderBy(a => a).ToArray());
(RelativePath)@"Download_ed33cbb256e5328361da8d9227df9cab1bb43a79a87dca2f223b2e2762ccaad1_.esm",
(RelativePath)@"Download_ed33cbb256e5328361da8d9227df9cab1bb43a79a87dca2f223b2e2762ccaad1_.esm.xxHash"
}.OrderBy(a => a).ToArray(),
folder.EnumerateFiles().Select(f => f.FileName).OrderBy(a => a).ToArray());
Consts.TestMode = true;
@ -549,8 +549,8 @@ namespace Wabbajack.Test
State = new NexusDownloader.State
{
Game = Game.SkyrimSpecialEdition,
ModID = "24808",
FileID = "123501"
ModID = 24808,
FileID = 123501
}
};
Assert.True(await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, dest));