Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Few small fixes and optimizations

Browse Source
This commit is contained in:
Timothy Baldridge 2021-05-13 13:41:33 -06:00
parent 564b8331fc
commit e6a5e46af0
4 changed files with 56 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -39,7 +39,8 @@ namespace Wabbajack.CLI
typeof(AllKnownDownloadStates),
typeof(VerifyAllDownloads),
typeof(HashBenchmark),
typeof(StressTestURL)
typeof(StressTestURL),
typeof(MirrorFolder)
};
}
}

46
Wabbajack.CLI/Verbs/MirrorFolder.cs Normal file
View File

@ -0,0 +1,46 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
namespace Wabbajack.CLI.Verbs
{
[Verb("mirror-folder", HelpText = "Copies the files from one folder to the other, skipping file of the same size and copies in parallel")]
public class MirrorFolder : AVerb
{
[IsDirectory(CustomMessage = "Downloads folder at %1 does not exist!")]
[Option('f', "from", HelpText = "From folder", Required = true)]
public string _FromFolder { get; set; } = "";
public AbsolutePath FromFolder => (AbsolutePath)_FromFolder;
[IsDirectory(CustomMessage = "Downloads folder at %1 does not exist!")]
[Option('t', "to", HelpText = "To folder", Required = true)]
public string _ToFolder { get; set; } = "";
public AbsolutePath ToFolder => (AbsolutePath)_ToFolder;
protected override async Task<ExitCode> Run()
{
var queue = new WorkQueue();
var src = FromFolder.EnumerateFiles().Where(f => f.IsFile).ToList();
Console.WriteLine($"Found {src.Count} files");
int idx = 0;
await src.PMap(queue, async f =>
{
var thisidx = Interlocked.Increment(ref idx);
var dest = f.RelativeTo(FromFolder).RelativeTo(ToFolder);
if (dest.IsFile && f.Size == dest.Size) return;
Console.WriteLine($"({thisidx}/{src.Count}) Copying {f.RelativeTo(FromFolder)} - {f.Size.ToFileSizeString()}");
await f.CopyToAsync(dest);
});
return ExitCode.Ok;
}
}
}

1
Wabbajack.Common/Hash.cs
View File

@ -289,6 +289,7 @@ namespace Wabbajack.Common
try
{
await using var fs = await file.OpenRead();
await using var bs = new BufferedStream(fs, 1024 * 1024 * 10);
var config = new xxHashConfig { HashSizeInBits = 64 };
await using var hs = new StatusFileStream(fs, $"Hashing {file}");
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(hs);

8
Wabbajack.Server/Services/NonNexusDownloadValidator.cs
View File

@ -27,13 +27,16 @@ namespace Wabbajack.Server.Services
{
var archives = await _sql.GetNonNexusModlistArchives();
_logger.Log(LogLevel.Information, $"Validating {archives.Count} non-Nexus archives");
using var queue = new WorkQueue();
using var queue = new WorkQueue(10);
await DownloadDispatcher.PrepareAll(archives.Select(a => a.State));
var random = new Random();
var results = await archives.PMap(queue, async archive =>
{
try
{
await Task.Delay(random.Next(1000, 5000));
var token = new CancellationTokenSource();
token.CancelAfter(TimeSpan.FromMinutes(10));
@ -43,6 +46,9 @@ namespace Wabbajack.Server.Services
{
//case WabbajackCDNDownloader.State _:
//case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021
case GameFileSourceDownloader.State _:
isValid = true;
break;
case ManualDownloader.State _:
case ModDBDownloader.State _:
case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"):