Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix a ton of bugs with the Author file uploader, and a crash on the build server

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-29 16:41:53 -07:00
parent 7c567da334
commit 08a3bc6f23
5 changed files with 106 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -16,15 +16,20 @@ namespace Wabbajack.BuildServer.Model.Models
public class SqlService public class SqlService
{ {
private IConfiguration _configuration; private IConfiguration _configuration;
private IDbConnection _conn; private AppSettings _settings;
public SqlService(AppSettings configuration) public SqlService(AppSettings settings)
{ {
_conn = new SqlConnection(configuration.SqlConnection); _settings = settings;
_conn.Open();
} }
public IDbConnection Connection { get => _conn; } private async Task<SqlConnection> Open()
{
var conn = new SqlConnection(_settings.SqlConnection);
await conn.OpenAsync();
return conn;
}
public async Task MergeVirtualFile(VirtualFile vfile) public async Task MergeVirtualFile(VirtualFile vfile)
{ {
@ -36,7 +41,8 @@ namespace Wabbajack.BuildServer.Model.Models
files = files.DistinctBy(f => f.Hash).ToList(); files = files.DistinctBy(f => f.Hash).ToList();
contents = contents.DistinctBy(c => (c.Parent, c.Path)).ToList(); contents = contents.DistinctBy(c => (c.Parent, c.Path)).ToList();
await Connection.ExecuteAsync("dbo.MergeIndexedFiles", new {Files = files.ToDataTable(), Contents = contents.ToDataTable()}, await using var conn = await Open();
await conn.ExecuteAsync("dbo.MergeIndexedFiles", new {Files = files.ToDataTable(), Contents = contents.ToDataTable()},
commandType: CommandType.StoredProcedure); commandType: CommandType.StoredProcedure);
} }
@ -72,7 +78,8 @@ namespace Wabbajack.BuildServer.Model.Models
public async Task<bool> HaveIndexdFile(string hash) public async Task<bool> HaveIndexdFile(string hash)
{ {
var row = await Connection.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash", await using var conn = await Open();
var row = await conn.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash",
new {Hash = BitConverter.ToInt64(hash.FromBase64())}); new {Hash = BitConverter.ToInt64(hash.FromBase64())});
return row.Any(); return row.Any();
} }
@ -97,8 +104,8 @@ namespace Wabbajack.BuildServer.Model.Models
/// <returns></returns> /// <returns></returns>
public async Task<IndexedVirtualFile> AllArchiveContents(long hash) public async Task<IndexedVirtualFile> AllArchiveContents(long hash)
{ {
await using var conn = await Open();
var files = await Connection.QueryAsync<ArchiveContentsResult>(@" var files = await conn.QueryAsync<ArchiveContentsResult>(@"
SELECT 0 as Parent, i.Hash, i.Size, null as Path FROM IndexedFile WHERE Hash = @Hash SELECT 0 as Parent, i.Hash, i.Size, null as Path FROM IndexedFile WHERE Hash = @Hash
UNION ALL UNION ALL
SELECT a.Parent, i.Hash, i.Size, a.Path FROM AllArchiveContent a SELECT a.Parent, i.Hash, i.Size, a.Path FROM AllArchiveContent a

73
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -3,6 +3,7 @@ using System.IO;
using System.Linq; using System.Linq;
using System.Net.Http; using System.Net.Http;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Wabbajack.Common; using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File; using File = Alphaleonis.Win32.Filesystem.File;
@ -12,10 +13,10 @@ namespace Wabbajack.Lib.FileUploader
{ {
public class AuthorAPI public class AuthorAPI
{ {
public static IObservable<bool> HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key"); public static IObservable<bool> HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key.txt");
public static IObservable<string> AuthorAPIKey => HaveAuthorAPIKey.Where(h => h) public static IObservable<string> AuthorAPIKey => HaveAuthorAPIKey.Where(h => h)
.Select(_ => File.ReadAllText(Path.Combine(Consts.LocalAppDataPath, "author-api-key"))); .Select(_ => File.ReadAllText(Path.Combine(Consts.LocalAppDataPath, "author-api-key.txt")));
public static string GetAPIKey() public static string GetAPIKey()
@ -26,13 +27,15 @@ namespace Wabbajack.Lib.FileUploader
public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file"); public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file");
public static long BLOCK_SIZE = (long)1024 * 1024 * 8; public static long BLOCK_SIZE = (long)1024 * 1024 * 2;
public static Task<string> UploadFile(WorkQueue queue, string filename) public static Task<string> UploadFile(WorkQueue queue, string filename, Action<double> progressFn)
{ {
var tcs = new TaskCompletionSource<string>(); var tcs = new TaskCompletionSource<string>();
queue.QueueTask(async () => Task.Run(async () =>
{ {
var client = new HttpClient(); var client = new HttpClient();
var fsize = new FileInfo(filename).Length; var fsize = new FileInfo(filename).Length;
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey()); client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
@ -44,41 +47,43 @@ namespace Wabbajack.Lib.FileUploader
} }
var key = await response.Content.ReadAsStringAsync(); var key = await response.Content.ReadAsStringAsync();
long sent = 0;
using (var iqueue = new WorkQueue(8)) using (var iqueue = new WorkQueue(8))
{ {
iqueue.Report("Starting Upload", 1);
await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE))
.PMap(iqueue, async block_idx =>
{
var block_offset = block_idx * BLOCK_SIZE;
var block_size = block_offset + BLOCK_SIZE > fsize
? fsize - block_offset
: BLOCK_SIZE;
Interlocked.Add(ref sent, block_size);
progressFn((double)sent / fsize);
await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE)) using (var fs = File.OpenRead(filename))
.PMap(iqueue, async block_idx =>
{ {
var block_offset = block_idx * BLOCK_SIZE; fs.Position = block_offset;
var block_size = block_offset + BLOCK_SIZE > fsize var data = new byte[block_size];
? fsize - block_offset await fs.ReadAsync(data, 0, data.Length);
: BLOCK_SIZE;
using (var fs = File.OpenRead(filename)) response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{ {
fs.Position = block_offset; tcs.SetResult("FAILED");
var data = new byte[block_size]; return;
await fs.ReadAsync(data, 0, data.Length);
response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != block_offset + data.Length)
{
tcs.SetResult("Sync Error");
return;
}
} }
});
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != block_offset + data.Length)
{
tcs.SetResult("Sync Error");
return;
}
}
});
} }
response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent("")); response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
@ -87,6 +92,8 @@ namespace Wabbajack.Lib.FileUploader
else else
tcs.SetResult("FAILED"); tcs.SetResult("FAILED");
progressFn(0.0);
}); });
return tcs.Task; return tcs.Task;
} }

4
Wabbajack/Util/FilePickerVM.cs
View File

@ -245,7 +245,7 @@ namespace Wabbajack
.ToGuiProperty(this, nameof(ErrorTooltip)); .ToGuiProperty(this, nameof(ErrorTooltip));
} }
public ICommand ConstructTypicalPickerCommand() public ICommand ConstructTypicalPickerCommand(IObservable<bool> canExecute = null)
{ {
return ReactiveCommand.Create( return ReactiveCommand.Create(
execute: () => execute: () =>
@ -280,7 +280,7 @@ namespace Wabbajack
} }
if (dlg.ShowDialog() != CommonFileDialogResult.Ok) return; if (dlg.ShowDialog() != CommonFileDialogResult.Ok) return;
TargetPath = dlg.FileName; TargetPath = dlg.FileName;
}); }, canExecute: canExecute);
} }
} }
} }

55
Wabbajack/View Models/Settings/AuthorFilesVM.cs
View File

@ -6,6 +6,8 @@ using System.Reactive.Subjects;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Windows; using System.Windows;
using System.Windows.Documents;
using System.Windows.Input;
using Alphaleonis.Win32.Filesystem; using Alphaleonis.Win32.Filesystem;
using Microsoft.WindowsAPICodePack.Shell.PropertySystem; using Microsoft.WindowsAPICodePack.Shell.PropertySystem;
using ReactiveUI; using ReactiveUI;
@ -20,39 +22,52 @@ namespace Wabbajack
{ {
public class AuthorFilesVM : BackNavigatingVM public class AuthorFilesVM : BackNavigatingVM
{ {
public Visibility IsVisible { get; } private readonly ObservableAsPropertyHelper<Visibility> _isVisible;
public Visibility IsVisible => _isVisible.Value;
[Reactive]
public string SelectedFile { get; set; }
public IReactiveCommand SelectFile { get; } private readonly ObservableAsPropertyHelper<string> _selectedFile;
public ICommand SelectFile { get; }
public ICommand HyperlinkCommand { get; }
public IReactiveCommand Upload { get; } public IReactiveCommand Upload { get; }
[Reactive] [Reactive] public double UploadProgress { get; set; }
public double UploadProgress { get; set; } [Reactive] public string FinalUrl { get; set; }
private WorkQueue Queue = new WorkQueue(1); private WorkQueue Queue = new WorkQueue(1);
public FilePickerVM Picker { get;}
private Subject<bool> _isUploading = new Subject<bool>();
private IObservable<bool> IsUploading { get; }
public AuthorFilesVM(SettingsVM vm) : base(vm.MWVM) public AuthorFilesVM(SettingsVM vm) : base(vm.MWVM)
{ {
var sub = new Subject<double>(); IsUploading = _isUploading;
Queue.Status.Select(s => (double)s.ProgressPercent).Subscribe(v => Picker = new FilePickerVM(this);
{
UploadProgress = v;
});
IsVisible = AuthorAPI.HasAPIKey ? Visibility.Visible : Visibility.Collapsed;
SelectFile = ReactiveCommand.Create(() => _isVisible = AuthorAPI.HaveAuthorAPIKey.Select(h => h ? Visibility.Visible : Visibility.Collapsed)
{ .ToProperty(this, x => x.IsVisible);
var fod = UIUtils.OpenFileDialog("*|*");
if (fod != null) SelectFile = Picker.ConstructTypicalPickerCommand(IsUploading.StartWith(false).Select(u => !u));
SelectedFile = fod;
}); HyperlinkCommand = ReactiveCommand.Create(() => Clipboard.SetText(FinalUrl));
Upload = ReactiveCommand.Create(async () => Upload = ReactiveCommand.Create(async () =>
{ {
SelectedFile = await AuthorAPI.UploadFile(Queue, SelectedFile); _isUploading.OnNext(true);
}); try
{
FinalUrl = await AuthorAPI.UploadFile(Queue, Picker.TargetPath, progress => UploadProgress = progress );
}
finally
{
_isUploading.OnNext(false);
}
}, IsUploading.StartWith(false).Select(u => !u)
.CombineLatest(Picker.WhenAnyValue(t => t.TargetPath).Select(f => f != null),
(a, b) => a && b));
} }
} }
} }

12
Wabbajack/Views/Settings/AuthorFilesView.xaml
View File

@ -24,6 +24,7 @@
<RowDefinition></RowDefinition> <RowDefinition></RowDefinition>
<RowDefinition></RowDefinition> <RowDefinition></RowDefinition>
<RowDefinition></RowDefinition> <RowDefinition></RowDefinition>
<RowDefinition></RowDefinition>
</Grid.RowDefinitions> </Grid.RowDefinitions>
<Grid.ColumnDefinitions> <Grid.ColumnDefinitions>
<ColumnDefinition Width="300"></ColumnDefinition> <ColumnDefinition Width="300"></ColumnDefinition>
@ -36,10 +37,15 @@
FontSize="20" FontSize="20"
FontWeight="Bold" FontWeight="Bold"
Text="File Uploader" /> Text="File Uploader" />
<TextBlock Margin="5" Grid.Row="1" Grid.ColumnSpan="2" Text="{Binding AuthorFile.SelectedFile}"></TextBlock> <TextBlock Margin="5" Grid.Row="1" Grid.ColumnSpan="2" Text="{Binding AuthorFile.Picker.TargetPath}"></TextBlock>
<ProgressBar Margin="5" Grid.Row="2" Grid.ColumnSpan="2" Value="{Binding AuthorFile.UploadProgress, Mode=OneWay}" Minimum="0" Maximum="1"></ProgressBar> <ProgressBar Margin="5" Grid.Row="2" Grid.ColumnSpan="2" Value="{Binding AuthorFile.UploadProgress, Mode=OneWay}" Minimum="0" Maximum="1"></ProgressBar>
<Button Margin="5" Grid.Row="3" Grid.Column="0" Command="{Binding AuthorFile.SelectFile, Mode=OneTime}">Select</Button> <TextBlock Margin="5" Grid.Row="3" Grid.ColumnSpan="2">
<Button Margin="5" Grid.Row="3" Grid.Column="1" Command="{Binding AuthorFile.Upload}">Upload</Button> <Hyperlink Command="{Binding AuthorFile.HyperlinkCommand}">
<TextBlock Text="{Binding AuthorFile.FinalUrl}"></TextBlock>
</Hyperlink>
</TextBlock>
<Button Margin="5" Grid.Row="4" Grid.Column="0" Command="{Binding AuthorFile.SelectFile, Mode=OneTime}">Select</Button>
<Button Margin="5" Grid.Row="4" Grid.Column="1" Command="{Binding AuthorFile.Upload}">Upload</Button>
</Grid> </Grid>
</Border> </Border>
</rxui:ReactiveUserControl> </rxui:ReactiveUserControl>