Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes for inis, resuming downloads and caching services

Browse Source
This commit is contained in:
halgari 2019-12-17 16:17:44 -07:00
parent 9bae8a9c22
commit 91e27fc3d7
9 changed files with 338 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Wabbajack.CacheServer/ListValidationService.cs
View File

@ -148,6 +148,9 @@ namespace Wabbajack.CacheServer
if (list.NeedsDownload(modlist_path))
{
if (File.Exists(modlist_path))
File.Delete(modlist_path);
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
await state.Download(modlist_path);

2
Wabbajack.Common/Utils.cs
View File

@ -338,6 +338,8 @@ namespace Wabbajack.Common
public static void ToJSON<T>(this T obj, string filename)
{
if (File.Exists(filename))
File.Delete(filename);
File.WriteAllText(filename,
JsonConvert.SerializeObject(obj, Formatting.Indented,
new JsonSerializerSettings {TypeNameHandling = TypeNameHandling.Auto}));

143
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -4,8 +4,12 @@ using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Reflection.Emit;
using System.Threading.Tasks;
using Windows.Networking.BackgroundTransfer;
using Ceras;
using SharpCompress.Common;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
using File = Alphaleonis.Win32.Filesystem.File;
@ -71,68 +75,101 @@ namespace Wabbajack.Lib.Downloaders
public async Task<bool> DoDownload(Archive a, string destination, bool download)
{
var client = Client ?? new HttpClient();
client.DefaultRequestHeaders.Add("User-Agent", Consts.UserAgent);
using (var fs = download ? File.OpenWrite(destination) : null)
{
var client = Client ?? new HttpClient();
client.DefaultRequestHeaders.Add("User-Agent", Consts.UserAgent);
if (Headers != null)
foreach (var header in Headers)
if (Headers != null)
foreach (var header in Headers)
{
var idx = header.IndexOf(':');
var k = header.Substring(0, idx);
var v = header.Substring(idx + 1);
client.DefaultRequestHeaders.Add(k, v);
}
long totalRead = 0;
var bufferSize = 1024 * 32;
Utils.Status($"Starting Download {a?.Name ?? Url}", 0);
var response = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead);
TOP:
Stream stream;
try
{
var idx = header.IndexOf(':');
var k = header.Substring(0, idx);
var v = header.Substring(idx + 1);
client.DefaultRequestHeaders.Add(k, v);
stream = await response.Content.ReadAsStreamAsync();
}
catch (Exception ex)
{
Utils.Error(ex, $"While downloading {Url}");
return false;
}
long totalRead = 0;
var bufferSize = 1024 * 32;
if (!download)
return true;
Utils.Status($"Starting Download {a?.Name ?? Url}", 0);
var responseTask = client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead);
responseTask.Wait();
var response = await responseTask;
var headerVar = a.Size == 0 ? "1" : a.Size.ToString();
if (response.Content.Headers.Contains("Content-Length"))
headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null;
Stream stream;
try
{
stream = await response.Content.ReadAsStreamAsync();
}
catch (Exception ex)
{
Utils.Error(ex, $"While downloading {Url}");
return false;
}
var contentSize = headerVar != null ? long.Parse(headerVar) : 1;
FileInfo fileInfo = new FileInfo(destination);
if (!fileInfo.Directory.Exists)
{
Directory.CreateDirectory(fileInfo.Directory.FullName);
}
using (var webs = stream)
{
var buffer = new byte[bufferSize];
int read_this_cycle = 0;
while (true)
{
int read = 0;
try
{
read = await webs.ReadAsync(buffer, 0, bufferSize);
}
catch (Exception ex)
{
if (read_this_cycle == 0)
throw ex;
if (totalRead < contentSize)
{
if (supportsResume)
{
Utils.Log(
$"Abort during download, trying to resume {Url} from {totalRead.ToFileSizeString()}");
var msg = new HttpRequestMessage(HttpMethod.Get, Url);
msg.Headers.Range = new RangeHeaderValue(totalRead, null);
response = await client.SendAsync(msg,
HttpCompletionOption.ResponseHeadersRead);
goto TOP;
}
}
throw ex;
}
read_this_cycle += read;
if (read == 0) break;
Utils.Status($"Downloading {a.Name}", (int)(totalRead * 100 / contentSize));
fs.Write(buffer, 0, read);
totalRead += read;
}
}
if (!download)
return true;
var headerVar = a.Size == 0 ? "1" : a.Size.ToString();
if (response.Content.Headers.Contains("Content-Length"))
headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
var contentSize = headerVar != null ? long.Parse(headerVar) : 1;
FileInfo fileInfo = new FileInfo(destination);
if (!fileInfo.Directory.Exists)
{
Directory.CreateDirectory(fileInfo.Directory.FullName);
}
using (var webs = stream)
using (var fs = File.OpenWrite(destination))
{
var buffer = new byte[bufferSize];
while (true)
{
var read = webs.Read(buffer, 0, bufferSize);
if (read == 0) break;
Utils.Status($"Downloading {a.Name}", (int)(totalRead * 100 / contentSize));
fs.Write(buffer, 0, read);
totalRead += read;
}
}
return true;
}
public override async Task<bool> Verify()

2
Wabbajack.Lib/MO2Installer.cs
View File

@ -284,7 +284,7 @@ namespace Wabbajack.Lib
private void SetScreenSizeInPrefs()
{
var config = new IniParserConfiguration {AllowDuplicateKeys = true};
var config = new IniParserConfiguration {AllowDuplicateKeys = true, AllowDuplicateSections = true};
foreach (var file in Directory.EnumerateFiles(Path.Combine(OutputFolder, "profiles"), "*refs.ini",
DirectoryEnumerationOptions.Recursive))
{

49
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -24,6 +24,7 @@ using Xilium.CefGlue.Common.Handlers;
using Xilium.CefGlue.WPF;
using static Wabbajack.Lib.NexusApi.NexusApiUtils;
using System.Threading;
using Newtonsoft.Json;
namespace Wabbajack.Lib.NexusApi
{
@ -32,6 +33,8 @@ namespace Wabbajack.Lib.NexusApi
private static readonly string API_KEY_CACHE_FILE = "nexus.key_cache";
private static string _additionalEntropy = "vtP2HF6ezg";
private static object _diskLock = new object();
public HttpClient HttpClient { get; } = new HttpClient();
#region Authentication
@ -255,18 +258,31 @@ namespace Wabbajack.Lib.NexusApi
if (UseLocalCache)
{
if (!Directory.Exists(LocalCacheDir))
Directory.CreateDirectory(LocalCacheDir);
var cache_file = Path.Combine(LocalCacheDir, code);
if (File.Exists(cache_file))
lock (_diskLock)
{
return cache_file.FromJSON<T>();
if (!Directory.Exists(LocalCacheDir))
Directory.CreateDirectory(LocalCacheDir);
if (File.Exists(cache_file))
{
return cache_file.FromJSON<T>();
}
}
var result = await Get<T>(url);
if (result != null)
if (result == null)
return result;
lock (_diskLock)
{
result.ToJSON(cache_file);
}
return result;
}
@ -399,6 +415,27 @@ namespace Wabbajack.Lib.NexusApi
public async Task ClearUpdatedModsInCache()
{
if (!UseLocalCache) return;
using (var queue = new WorkQueue())
{
var invalid_json = (await Directory.EnumerateFiles(LocalCacheDir, "*.json")
.PMap(queue, f =>
{
var s = JsonSerializer.Create();
try
{
using (var tr = File.OpenText(f))
s.Deserialize(new JsonTextReader(tr));
return null;
}
catch (JsonReaderException)
{
return f;
}
})).Where(f => f != null).ToList();
Utils.Log($"Found {invalid_json.Count} bad json files");
foreach (var file in invalid_json)
File.Delete(file);
}
var gameTasks = GameRegistry.Games.Values
.Where(game => game.NexusName != null)

91
Wabbajack.Test/RestartingDownloadsTests.cs Normal file
View File

@ -0,0 +1,91 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Reactive.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.Test
{
public class CrappyRandomServer : SimpleHTTPServer
{
private Random _random;
private readonly byte[] _data;
public byte[] Data => _data;
public CrappyRandomServer() : base("")
{
_random = new Random();
_data = new byte[_random.Next(1024 * 1024, 1024 * 2048)];
_random.NextBytes(_data);
}
protected override void Process(HttpListenerContext context)
{
context.Response.ContentType = "application/octet-stream";
context.Response.ContentLength64 = _data.Length;
context.Response.AddHeader("Accept-Ranges", "bytes");
context.Response.StatusCode = (int)HttpStatusCode.OK;
var range = context.Request.Headers.Get("Range");
int start = 0;
if (range != null)
{
var match = new Regex("(?<=bytes=)[0-9]+(?=\\-)").Match(range);
if (match != null)
{
start = int.Parse(match.ToString());
}
}
var end = Math.Min(start + _random.Next(1024 * 32, 1024 * 64), _data.Length);
context.Response.OutputStream.Write(_data, start, end - start);
context.Response.OutputStream.Flush();
Thread.Sleep(500);
context.Response.Abort();
}
}
[TestClass]
public class RestartingDownloadsTests
{
public TestContext TestContext { get; set; }
[TestInitialize]
public void Setup()
{
Utils.LogMessages.OfType<IInfo>().Subscribe(onNext: msg => TestContext.WriteLine(msg.ShortDescription));
Utils.LogMessages.OfType<IUserIntervention>().Subscribe(msg =>
TestContext.WriteLine("ERROR: User intervetion required: " + msg.ShortDescription));
}
[TestMethod]
public async Task DownloadResume()
{
using (var server = new CrappyRandomServer())
{
var downloader = DownloadDispatcher.GetInstance<HTTPDownloader>();
var state = new HTTPDownloader.State {Url = $"http://localhost:{server.Port}/foo"};
await state.Download("test.resume_file");
CollectionAssert.AreEqual(server.Data, File.ReadAllBytes("test.resume_file"));
if (File.Exists("test.resume_file"))
File.Delete("test.resume_file");
}
}
}
}

7
Wabbajack.Test/SanityTests.cs
View File

@ -142,11 +142,16 @@ namespace Wabbajack.Test
File.WriteAllLines(Path.Combine(utils.MO2Folder, "profiles", profile, "somegameprefs.ini"),
new List<string>
{
// Beth inis are messy, let's make ours just as messy to catch some parse failures
"[Display]",
"foo=4",
"[Display]",
"STestFile=f",
"STestFile=",
"iSize H=3",
"iSize W=-200"
"iSize W=-200",
"[Display]",
"foo=4"
});
var modlist = await CompileAndInstall(profile);

100
Wabbajack.Test/SimpleHTTPServer.cs Normal file
View File

@ -0,0 +1,100 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Test
{
// MIT License - Copyright (c) 2016 Can Güney Aksakalli
// https://aksakalli.github.io/2014/02/24/simple-http-server-with-csparp.html
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net.Sockets;
using System.Net;
using System.IO;
using System.Threading;
using System.Diagnostics;
public abstract class SimpleHTTPServer : IDisposable
{
private Thread _serverThread;
private HttpListener _listener;
private int _port;
public int Port
{
get { return _port; }
}
/// <summary>
/// Construct server with given port.
/// </summary>
/// <param name="path">Directory path to serve.</param>
/// <param name="port">Port of the server.</param>
public SimpleHTTPServer(string path, int port)
{
this.Initialize(path, port);
}
/// <summary>
/// Construct server with suitable port.
/// </summary>
/// <param name="path">Directory path to serve.</param>
public SimpleHTTPServer(string path)
{
//get an empty port
TcpListener l = new TcpListener(IPAddress.Loopback, 0);
l.Start();
int port = ((IPEndPoint)l.LocalEndpoint).Port;
l.Stop();
Initialize(path, port);
}
/// <summary>
/// Stop server and dispose all functions.
/// </summary>
public void Stop()
{
_serverThread.Abort();
_listener.Stop();
}
private void Listen()
{
_listener = new HttpListener();
_listener.Prefixes.Add("http://localhost:" + _port.ToString() + "/");
_listener.Start();
while (true)
{
try
{
HttpListenerContext context = _listener.GetContext();
Process(context);
}
catch (Exception ex)
{
}
}
}
protected abstract void Process(HttpListenerContext context);
private void Initialize(string path, int port)
{
_port = port;
_serverThread = new Thread(this.Listen);
_serverThread.Start();
}
public void Dispose()
{
//Stop();
}
}
}

2
Wabbajack.Test/Wabbajack.Test.csproj
View File

@ -108,6 +108,8 @@
<Compile Include="FilePickerTests.cs" />
<Compile Include="MiscTests.cs" />
<Compile Include="ModlistMetadataTests.cs" />
<Compile Include="RestartingDownloadsTests.cs" />
<Compile Include="SimpleHTTPServer.cs" />
<Compile Include="TestUtils.cs" />
<Compile Include="SanityTests.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />