Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP autoheal

Browse Source
This commit is contained in:
Timothy Baldridge 2020-05-18 21:46:33 -06:00
parent a03cda8a31
commit 2a8021c6f9
7 changed files with 314 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
Wabbajack.Server.Test/ABuildServerSystemTest.cs
View File

@ -1,12 +1,18 @@
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Reactive.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Wabbajack.Common;
using Wabbajack.Common.Http;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Server;
using Wabbajack.Server.DataLayer;
using Xunit;
@ -182,5 +188,79 @@ namespace Wabbajack.BuildServer.Test
_unsubErr.Dispose();
}
protected async Task<Uri> MakeModList()
{
var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
await test_archive_path.WriteAllBytesAsync(archive_data);
ModListData = new ModList();
ModListData.Archives.Add(
new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt")))
{
Hash = await test_archive_path.FileHashAsync(),
Name = "test_archive",
Size = test_archive_path.Size,
});
var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
await using (var fs = modListPath.Create())
{
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
var entry = za.CreateEntry("modlist");
await using var es = entry.Open();
ModListData.ToJson(es);
}
ModListMetaData = new List<ModlistMetadata>
{
new ModlistMetadata
{
Official = false,
Author = "Test Suite",
Description = "A test",
DownloadMetadata = new DownloadMetadata
{
Hash = await modListPath.FileHashAsync(),
Size = modListPath.Size
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "test_list",
Download = MakeURL("test_modlist.wabbajack")
}
},
new ModlistMetadata
{
Official = true,
Author = "Test Suite",
Description = "A list with a broken hash",
DownloadMetadata = new DownloadMetadata()
{
Hash = Hash.FromLong(42),
Size = 42
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "broken_list",
Download = MakeURL("test_modlist.wabbajack")
}
}
};
var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
ModListMetaData.ToJson(metadataPath);
return new Uri(MakeURL("test_mod_list_metadata.json"));
}
public ModList ModListData { get; set; }
public List<ModlistMetadata> ModListMetaData { get; set; }
}
}

72
Wabbajack.Server.Test/ModListValidationTests.cs
View File

@ -135,79 +135,7 @@ namespace Wabbajack.BuildServer.Test
}
private async Task<Uri> MakeModList()
{
var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
await test_archive_path.WriteAllBytesAsync(archive_data);
ModListData = new ModList();
ModListData.Archives.Add(
new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt")))
{
Hash = await test_archive_path.FileHashAsync(),
Name = "test_archive",
Size = test_archive_path.Size,
});
var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
await using (var fs = modListPath.Create())
{
using var za = new ZipArchive(fs, ZipArchiveMode.Create);
var entry = za.CreateEntry("modlist");
await using var es = entry.Open();
ModListData.ToJson(es);
}
ModListMetaData = new List<ModlistMetadata>
{
new ModlistMetadata
{
Official = false,
Author = "Test Suite",
Description = "A test",
DownloadMetadata = new DownloadMetadata
{
Hash = await modListPath.FileHashAsync(),
Size = modListPath.Size
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "test_list",
Download = MakeURL("test_modlist.wabbajack")
}
},
new ModlistMetadata
{
Official = true,
Author = "Test Suite",
Description = "A list with a broken hash",
DownloadMetadata = new DownloadMetadata()
{
Hash = Hash.FromLong(42),
Size = 42
},
Links = new ModlistMetadata.LinksObject
{
MachineURL = "broken_list",
Download = MakeURL("test_modlist.wabbajack")
}
}
};
var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
ModListMetaData.ToJson(metadataPath);
return new Uri(MakeURL("test_mod_list_metadata.json"));
}
public ModList ModListData { get; set; }
public List<ModlistMetadata> ModListMetaData { get; set; }
}
}

134
Wabbajack.Server.Test/ModlistUpdater.cs Normal file
View File

@ -0,0 +1,134 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.BuildServer;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class ModlistUpdater : ABuildServerSystemTest
{
public ModlistUpdater(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output,
fixture)
{
}
[Fact]
public async Task CanIndexAndUpdateFiles()
{
var validator = Fixture.GetService<ListValidator>();
var nonNexus = Fixture.GetService<NonNexusDownloadValidator>();
var modLists = await MakeModList();
Consts.ModlistMetadataURL = modLists.ToString();
var listDownloader = Fixture.GetService<ModListDownloader>();
var downloader = Fixture.GetService<ArchiveDownloader>();
var archiver = Fixture.GetService<ArchiveMaintainer>();
var sql = Fixture.GetService<SqlService>();
var modId = long.MaxValue >> 1;
var oldFileId = long.MaxValue >> 2;
var newFileId = (long.MaxValue >> 2) + 1;
var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!");
var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!");
var oldDataHash = oldFileData.xxHash();
var newDataHash = newFileData.xxHash();
Assert.Equal(2, await listDownloader.CheckForNewLists());
Assert.Equal(1, await downloader.Execute());
Assert.Equal(0, await nonNexus.Execute());
Assert.Equal(0, await validator.Execute());
Assert.True(archiver.HaveArchive(oldDataHash));
Assert.False(archiver.HaveArchive(newDataHash));
var status = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
Assert.Equal(0, status.ValidationSummary.Failed);
// Update the archive
await "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData);
// Nothing new to do
Assert.Equal(0, await listDownloader.CheckForNewLists());
Assert.Equal(0, await downloader.Execute());
// List now fails after we check the manual link
Assert.Equal(1, await nonNexus.Execute());
Assert.Equal(1, await validator.Execute());
/*
Assert.True(await sql.HaveIndexdFile(oldDataHash));
Assert.True(await sql.HaveIndexdFile(newDataHash));
var settings = Fixture.GetService<AppSettings>();
Assert.Equal($"Oldfile_{oldDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(oldDataHash));
Assert.Equal($"Newfile_{newDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(newDataHash));
Utils.Log($"Download Updating {oldDataHash} -> {newDataHash}");
await using var conn = await sql.Open();
await conn.ExecuteAsync("DELETE FROM dbo.DownloadStates WHERE Hash in (@OldHash, @NewHash);",
new {OldHash = (long)oldDataHash, NewHash = (long)newDataHash});
await sql.AddDownloadState(oldDataHash, new NexusDownloader.State
{
Game = Game.Oblivion,
ModID = modId,
FileID = oldFileId
});
await sql.AddDownloadState(newDataHash, new NexusDownloader.State
{
Game = Game.Oblivion,
ModID = modId,
FileID = newFileId
});
Assert.NotNull(await sql.GetNexusStateByHash(oldDataHash));
Assert.NotNull(await sql.GetNexusStateByHash(newDataHash));
// No nexus info, so no upgrade
var noUpgrade = await ClientAPI.GetModUpgrade(oldDataHash);
Assert.Null(noUpgrade);
// Add Nexus info
await sql.AddNexusModFiles(Game.Oblivion, modId, DateTime.Now,
new NexusApiClient.GetModFilesResponse
{
files = new List<NexusFileInfo>
{
new NexusFileInfo {category_name = "MAIN", file_id = newFileId, file_name = "New File"},
new NexusFileInfo {category_name = null, file_id = oldFileId, file_name = "Old File"}
}
});
var enqueuedUpgrade = await ClientAPI.GetModUpgrade(oldDataHash);
// Not Null because upgrade was enqueued
Assert.NotNull(enqueuedUpgrade);
await RunAllJobs();
Assert.True($"{oldDataHash.ToHex()}_{newDataHash.ToHex()}".RelativeTo(Fixture.ServerUpdatesFolder).IsFile);
*/
}
}
}

14
Wabbajack.Server/DTOs/Patch.cs Normal file
View File

@ -0,0 +1,14 @@
using System;
using Wabbajack.Common;
namespace Wabbajack.Server.DTOs
{
public class Patch
{
public ArchiveDownload Src { get; set; }
public ArchiveDownload Dest { get; set; }
public Hash PatchHash { get; set; }
public long PatchSize { get; set; }
public DateTime? Finished { get; set; }
}
}

20
Wabbajack.Server/DataLayer/ArchiveDownloads.cs
View File

@ -37,6 +37,26 @@ namespace Wabbajack.Server.DataLayer
return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads")).ToHashSet();
}
public async Task<ArchiveDownload> GetArchiveDownload(Guid id)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE Id = @id",
new {Id = id});
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task<ArchiveDownload> GetNextPendingDownload(bool ignoreNexus = false)
{
await using var conn = await Open();

65
Wabbajack.Server/DataLayer/Patches.cs Normal file
View File

@ -0,0 +1,65 @@
using System;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
/// <summary>
/// Adds a patch record
/// </summary>
/// <param name="patch"></param>
/// <returns></returns>
public async Task AddPatch(Patch patch)
{
await using var conn = await Open();
await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
new {SrcId = patch.Src.Id, DestId = patch.Dest.Id});
}
/// <summary>
/// Adds a patch record
/// </summary>
/// <param name="patch"></param>
/// <returns></returns>
public async Task FinializePatch(Patch patch)
{
await using var conn = await Open();
await conn.ExecuteAsync("UPDATE dbo.Patches SET PatchSize = @Size, PatchHash = @PatchHash, Finished = @Finished WHERE SrcId = @SrcId AND DestID = @DestId",
new
{
SrcId = patch.Src.Id,
DestId = patch.Dest.Id,
PatchHash = patch.PatchHash,
PatchSize = patch.PatchSize,
Finshed = patch.Finished
});
}
public async Task<Patch> FindPatch(Guid src, Guid dest)
{
await using var conn = await Open();
var patch = await conn.QueryFirstOrDefaultAsync<(Hash, long, DateTime?)>(
"SELECT PatchHash, PatchSize, Finished FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId",
new
{
SrcId = src,
DestId = dest
});
if (patch == default)
return default(Patch);
return new Patch {
Src = await GetArchiveDownload(src),
Dest = await GetArchiveDownload(dest),
PatchHash = patch.Item1,
PatchSize = patch.Item2,
Finished = patch.Item3
};
}
}
}

1
Wabbajack.Server/Services/ArchiveDownloader.cs
View File

@ -67,6 +67,7 @@ namespace Wabbajack.Server.Services
if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
{
_logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}