Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

LL autohealing

Browse Source
This commit is contained in:
Timothy Baldridge 2020-08-12 16:23:02 -06:00
parent bc94ec4321
commit 39078b21d9
11 changed files with 94 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Wabbajack.Common/OctoDiff.cs
View File

@ -7,14 +7,13 @@ namespace Wabbajack.Common
{
public class OctoDiff
{
private static ProgressReporter reporter = new ProgressReporter();
public static void Create(byte[] oldData, byte[] newData, Stream output)
{
using var signature = CreateSignature(oldData);
using var oldStream = new MemoryStream(oldData);
using var newStream = new MemoryStream(newData);
var db = new DeltaBuilder {ProgressReporter = reporter};
db.BuildDelta(newStream, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
var db = new DeltaBuilder {ProgressReporter = new ProgressReporter()};
db.BuildDelta(newStream, new SignatureReader(signature, new ProgressReporter()), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
}
private static Stream CreateSignature(byte[] oldData)
@ -40,7 +39,7 @@ namespace Wabbajack.Common
{
CreateSignature(oldData, signature);
var db = new DeltaBuilder {ProgressReporter = reporter ?? new ProgressReporter()};
db.BuildDelta(newData, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
db.BuildDelta(newData, new SignatureReader(signature, reporter ?? new ProgressReporter()), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output)));
}
public class ProgressReporter : IProgressReporter
@ -76,13 +75,13 @@ namespace Wabbajack.Common
{
using var deltaStream = openPatchStream();
var deltaApplier = new DeltaApplier();
deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, reporter), output);
deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, new ProgressReporter()), output);
}
public static void Apply(FileStream input, FileStream patchStream, FileStream output)
{
var deltaApplier = new DeltaApplier();
deltaApplier.Apply(input, new BinaryDeltaReader(patchStream, reporter), output);
deltaApplier.Apply(input, new BinaryDeltaReader(patchStream, new ProgressReporter()), output);
}
}
}

2
Wabbajack.Common/Util/TempFile.cs
View File

@ -37,7 +37,7 @@ namespace Wabbajack.Common
}
public async ValueTask DisposeAsync()
{
if (DeleteAfter)
if (DeleteAfter && Path.Exists)
{
await Path.DeleteAsync();
}

2
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -141,7 +141,7 @@ namespace Wabbajack.Lib.Downloaders
}
public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
return await ServerFindUpgrade(a);
}

60
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
@ -6,6 +7,8 @@ using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web;
using F23.StringSimilarity;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
@ -25,9 +28,13 @@ namespace Wabbajack.Lib.Downloaders
}
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
{
Uri url = DownloaderUtils.GetDirectURL(archiveINI);
return await GetDownloaderStateFromUrl(url, quickMode);
}
public async Task<AbstractDownloadState?> GetDownloaderStateFromUrl(Uri url, bool quickMode)
{
var absolute = true;
if (url == null || url.Host != SiteURL.Host) return null;
@ -81,7 +88,7 @@ namespace Wabbajack.Lib.Downloaders
FileName = file
};
}
public class State<TStateDownloader> : AbstractDownloadState, IMetaState
where TStateDownloader : IDownloader
{
@ -223,12 +230,59 @@ namespace Wabbajack.Lib.Downloaders
stream.Dispose();
return true;
}
public override IDownloader GetDownloader()
{
return DownloadDispatcher.GetInstance<TDownloader>();
}
public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
var files = await GetFilesInGroup();
var nl = new Levenshtein();
var newFile = files.OrderBy(f => nl.Distance(a.Name.ToLowerInvariant(), f.Name.ToLowerInvariant())).FirstOrDefault();
if (newFile == null) return default;
var existing = await downloadResolver(newFile);
if (existing != default) return (newFile, new TempFile());
var tmp = new TempFile();
await DownloadDispatcher.PrepareAll(new []{newFile.State});
if (await newFile.State.Download(newFile, tmp.Path))
{
newFile.Size = tmp.Path.Size;
newFile.Hash = await tmp.Path.FileHashAsync();
return (newFile, tmp);
}
await tmp.DisposeAsync();
return default;
}
public async Task<List<Archive>> GetFilesInGroup()
{
var others = await Downloader.AuthedClient.GetHtmlAsync($"{Site}/files/file/{FileName}?do=download");
var pairs = others.DocumentNode.SelectNodes("//a[@data-action='download']")
.Select(item => (item.GetAttributeValue("href", ""),
item.ParentNode.ParentNode.SelectNodes("//div//h4//span").First().InnerText));
List<Archive> archives = new List<Archive>();
foreach (var (url, name) in pairs)
{
var ini = new[] {"[General]", $"directURL={url}"};
var state = (AbstractDownloadState)(await DownloadDispatcher.ResolveArchive(
string.Join("\n", ini).LoadIniString(), false));
if (state == null) continue;
archives.Add(new Archive(state) {Name = name});
}
return archives;
}
public override string GetManifestURL(Archive a)
{
return IsAttachment ? FullURL : $"{Site}/files/file/{FileName}/?do=download&r={FileID}";

11
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -112,11 +112,9 @@ namespace Wabbajack.Lib.Downloaders
return false;
}
var upgrade = (IUpgradingState)archive.State;
Utils.Log($"Trying to find solution to broken download for {archive.Name}");
var result = await upgrade.FindUpgrade(archive);
var result = await FindUpgrade(archive);
if (result == default)
{
Utils.Log(
@ -153,7 +151,14 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
public static async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>>? downloadResolver = null)
{
downloadResolver ??= async a => default;
return await a.State.FindUpgrade(a, downloadResolver);
}
private static async Task<bool> DownloadFromMirror(Archive archive, AbsolutePath destination)
{
try

2
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -212,7 +212,7 @@ TOP:
}
public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
var tmpFile = new TempFile();

5
Wabbajack.Lib/Downloaders/IUpgradingState.cs
View File

@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Lib.Downloaders
@ -11,7 +12,7 @@ namespace Wabbajack.Lib.Downloaders
/// </summary>
/// <param name="a"></param>
/// <returns></returns>
public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a);
public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver);
Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState);
}

2
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -194,7 +194,7 @@ namespace Wabbajack.Lib.Downloaders
}
}
public override Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
public override Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
return ServerFindUpgrade(a);
}

5
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -237,8 +237,7 @@ namespace Wabbajack.Lib.Downloaders
return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"modID={ModID}", $"fileID={FileID}"};
}
public static Func<Archive, Task<AbsolutePath>> DownloadShortcut = async a => default;
public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
var client = await NexusApiClient.Get();
@ -267,7 +266,7 @@ namespace Wabbajack.Lib.Downloaders
Name = newFile.file_name,
};
var fastPath = await DownloadShortcut(newArchive);
var fastPath = await downloadResolver(newArchive);
if (fastPath != default)
{
newArchive.Size = fastPath.Size;

4
Wabbajack.Server/Services/ListValidator.cs
View File

@ -190,7 +190,7 @@ namespace Wabbajack.Server.Services
var upgradeTime = DateTime.UtcNow;
_logger.LogInformation($"Validator Finding Upgrade for {archive.Hash} {archive.State.PrimaryKeyString}");
NexusDownloader.State.DownloadShortcut = async findIt =>
Func<Archive, Task<AbsolutePath>> resolver = async findIt =>
{
_logger.LogInformation($"Quick find for {findIt.State.PrimaryKeyString}");
var foundArchive = await _sql.GetArchiveDownload(findIt.State.PrimaryKeyString);
@ -203,7 +203,7 @@ namespace Wabbajack.Server.Services
return _archives.TryGetPath(foundArchive.Archive.Hash, out var path) ? path : default;
};
var upgrade = await (archive.State as IUpgradingState)?.FindUpgrade(archive);
var upgrade = await DownloadDispatcher.FindUpgrade(archive, resolver);
if (upgrade == default)

13
Wabbajack.Test/DownloaderTests.cs
View File

@ -308,6 +308,18 @@ namespace Wabbajack.Test
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
var files = await ((LoversLabDownloader.State)converted).GetFilesInGroup();
Assert.NotEmpty(files);
Assert.Equal("WABBAJACK_TEST_FILE.zip", files.First().Name);
((LoversLabDownloader.State)converted).FileID = "42";
var upgrade = await DownloadDispatcher.FindUpgrade(new Archive(converted) {Name = "WABBAJACK_TEST_FILE.zip"});
Assert.True(upgrade != default);
}
[Fact]
@ -336,6 +348,7 @@ namespace Wabbajack.Test
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
[Fact]