Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix a ton of bugs with the Author file uploader, and a crash on the build server

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-29 16:41:53 -07:00
parent 7c567da334
commit 08a3bc6f23
5 changed files with 106 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -16,15 +16,20 @@ namespace Wabbajack.BuildServer.Model.Models
public class SqlService
{
private IConfiguration _configuration;
private IDbConnection _conn;
private AppSettings _settings;
public SqlService(AppSettings configuration)
public SqlService(AppSettings settings)
{
_conn = new SqlConnection(configuration.SqlConnection);
_conn.Open();
_settings = settings;
}
public IDbConnection Connection { get => _conn; }
private async Task<SqlConnection> Open()
{
var conn = new SqlConnection(_settings.SqlConnection);
await conn.OpenAsync();
return conn;
}
public async Task MergeVirtualFile(VirtualFile vfile)
{
@ -36,7 +41,8 @@ namespace Wabbajack.BuildServer.Model.Models
files = files.DistinctBy(f => f.Hash).ToList();
contents = contents.DistinctBy(c => (c.Parent, c.Path)).ToList();
await Connection.ExecuteAsync("dbo.MergeIndexedFiles", new {Files = files.ToDataTable(), Contents = contents.ToDataTable()},
await using var conn = await Open();
await conn.ExecuteAsync("dbo.MergeIndexedFiles", new {Files = files.ToDataTable(), Contents = contents.ToDataTable()},
commandType: CommandType.StoredProcedure);
}
@ -72,7 +78,8 @@ namespace Wabbajack.BuildServer.Model.Models
public async Task<bool> HaveIndexdFile(string hash)
{
var row = await Connection.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash",
await using var conn = await Open();
var row = await conn.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash",
new {Hash = BitConverter.ToInt64(hash.FromBase64())});
return row.Any();
}
@ -97,8 +104,8 @@ namespace Wabbajack.BuildServer.Model.Models
/// <returns></returns>
public async Task<IndexedVirtualFile> AllArchiveContents(long hash)
{
var files = await Connection.QueryAsync<ArchiveContentsResult>(@"
await using var conn = await Open();
var files = await conn.QueryAsync<ArchiveContentsResult>(@"
SELECT 0 as Parent, i.Hash, i.Size, null as Path FROM IndexedFile WHERE Hash = @Hash
UNION ALL
SELECT a.Parent, i.Hash, i.Size, a.Path FROM AllArchiveContent a

75
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -3,6 +3,7 @@ using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File;
@ -12,10 +13,10 @@ namespace Wabbajack.Lib.FileUploader
{
public class AuthorAPI
{
public static IObservable<bool> HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key");
public static IObservable<bool> HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key.txt");
public static IObservable<string> AuthorAPIKey => HaveAuthorAPIKey.Where(h => h)
.Select(_ => File.ReadAllText(Path.Combine(Consts.LocalAppDataPath, "author-api-key")));
.Select(_ => File.ReadAllText(Path.Combine(Consts.LocalAppDataPath, "author-api-key.txt")));
public static string GetAPIKey()
@ -26,12 +27,14 @@ namespace Wabbajack.Lib.FileUploader
public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file");
public static long BLOCK_SIZE = (long)1024 * 1024 * 8;
public static Task<string> UploadFile(WorkQueue queue, string filename)
public static long BLOCK_SIZE = (long)1024 * 1024 * 2;
public static Task<string> UploadFile(WorkQueue queue, string filename, Action<double> progressFn)
{
var tcs = new TaskCompletionSource<string>();
queue.QueueTask(async () =>
Task.Run(async () =>
{
var client = new HttpClient();
var fsize = new FileInfo(filename).Length;
@ -44,41 +47,43 @@ namespace Wabbajack.Lib.FileUploader
}
var key = await response.Content.ReadAsStringAsync();
long sent = 0;
using (var iqueue = new WorkQueue(8))
{
await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE))
.PMap(iqueue, async block_idx =>
iqueue.Report("Starting Upload", 1);
await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE))
.PMap(iqueue, async block_idx =>
{
var block_offset = block_idx * BLOCK_SIZE;
var block_size = block_offset + BLOCK_SIZE > fsize
? fsize - block_offset
: BLOCK_SIZE;
Interlocked.Add(ref sent, block_size);
progressFn((double)sent / fsize);
using (var fs = File.OpenRead(filename))
{
var block_offset = block_idx * BLOCK_SIZE;
var block_size = block_offset + BLOCK_SIZE > fsize
? fsize - block_offset
: BLOCK_SIZE;
fs.Position = block_offset;
var data = new byte[block_size];
await fs.ReadAsync(data, 0, data.Length);
using (var fs = File.OpenRead(filename))
response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
fs.Position = block_offset;
var data = new byte[block_size];
await fs.ReadAsync(data, 0, data.Length);
response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != block_offset + data.Length)
{
tcs.SetResult("Sync Error");
return;
}
tcs.SetResult("FAILED");
return;
}
});
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != block_offset + data.Length)
{
tcs.SetResult("Sync Error");
return;
}
}
});
}
response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
@ -87,6 +92,8 @@ namespace Wabbajack.Lib.FileUploader
else
tcs.SetResult("FAILED");
progressFn(0.0);
});
return tcs.Task;
}

4
Wabbajack/Util/FilePickerVM.cs
View File

@ -245,7 +245,7 @@ namespace Wabbajack
.ToGuiProperty(this, nameof(ErrorTooltip));
}
public ICommand ConstructTypicalPickerCommand()
public ICommand ConstructTypicalPickerCommand(IObservable<bool> canExecute = null)
{
return ReactiveCommand.Create(
execute: () =>
@ -280,7 +280,7 @@ namespace Wabbajack
}
if (dlg.ShowDialog() != CommonFileDialogResult.Ok) return;
TargetPath = dlg.FileName;
});
}, canExecute: canExecute);
}
}
}

61
Wabbajack/View Models/Settings/AuthorFilesVM.cs
View File

@ -6,6 +6,8 @@ using System.Reactive.Subjects;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Documents;
using System.Windows.Input;
using Alphaleonis.Win32.Filesystem;
using Microsoft.WindowsAPICodePack.Shell.PropertySystem;
using ReactiveUI;
@ -20,39 +22,52 @@ namespace Wabbajack
{
public class AuthorFilesVM : BackNavigatingVM
{
public Visibility IsVisible { get; }
private readonly ObservableAsPropertyHelper<Visibility> _isVisible;
public Visibility IsVisible => _isVisible.Value;
[Reactive]
public string SelectedFile { get; set; }
public IReactiveCommand SelectFile { get; }
private readonly ObservableAsPropertyHelper<string> _selectedFile;
public ICommand SelectFile { get; }
public ICommand HyperlinkCommand { get; }
public IReactiveCommand Upload { get; }
[Reactive]
public double UploadProgress { get; set; }
[Reactive] public double UploadProgress { get; set; }
[Reactive] public string FinalUrl { get; set; }
private WorkQueue Queue = new WorkQueue(1);
public FilePickerVM Picker { get;}
private Subject<bool> _isUploading = new Subject<bool>();
private IObservable<bool> IsUploading { get; }
public AuthorFilesVM(SettingsVM vm) : base(vm.MWVM)
{
var sub = new Subject<double>();
Queue.Status.Select(s => (double)s.ProgressPercent).Subscribe(v =>
{
UploadProgress = v;
});
IsVisible = AuthorAPI.HasAPIKey ? Visibility.Visible : Visibility.Collapsed;
SelectFile = ReactiveCommand.Create(() =>
{
var fod = UIUtils.OpenFileDialog("*|*");
if (fod != null)
SelectedFile = fod;
});
IsUploading = _isUploading;
Picker = new FilePickerVM(this);
_isVisible = AuthorAPI.HaveAuthorAPIKey.Select(h => h ? Visibility.Visible : Visibility.Collapsed)
.ToProperty(this, x => x.IsVisible);
SelectFile = Picker.ConstructTypicalPickerCommand(IsUploading.StartWith(false).Select(u => !u));
HyperlinkCommand = ReactiveCommand.Create(() => Clipboard.SetText(FinalUrl));
Upload = ReactiveCommand.Create(async () =>
{
SelectedFile = await AuthorAPI.UploadFile(Queue, SelectedFile);
});
_isUploading.OnNext(true);
try
{
FinalUrl = await AuthorAPI.UploadFile(Queue, Picker.TargetPath, progress => UploadProgress = progress );
}
finally
{
_isUploading.OnNext(false);
}
}, IsUploading.StartWith(false).Select(u => !u)
.CombineLatest(Picker.WhenAnyValue(t => t.TargetPath).Select(f => f != null),
(a, b) => a && b));
}
}
}

12
Wabbajack/Views/Settings/AuthorFilesView.xaml
View File

@ -24,6 +24,7 @@
<RowDefinition></RowDefinition>
<RowDefinition></RowDefinition>
<RowDefinition></RowDefinition>
<RowDefinition></RowDefinition>
</Grid.RowDefinitions>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="300"></ColumnDefinition>
@ -36,10 +37,15 @@
FontSize="20"
FontWeight="Bold"
Text="File Uploader" />
<TextBlock Margin="5" Grid.Row="1" Grid.ColumnSpan="2" Text="{Binding AuthorFile.SelectedFile}"></TextBlock>
<TextBlock Margin="5" Grid.Row="1" Grid.ColumnSpan="2" Text="{Binding AuthorFile.Picker.TargetPath}"></TextBlock>
<ProgressBar Margin="5" Grid.Row="2" Grid.ColumnSpan="2" Value="{Binding AuthorFile.UploadProgress, Mode=OneWay}" Minimum="0" Maximum="1"></ProgressBar>
<Button Margin="5" Grid.Row="3" Grid.Column="0" Command="{Binding AuthorFile.SelectFile, Mode=OneTime}">Select</Button>
<Button Margin="5" Grid.Row="3" Grid.Column="1" Command="{Binding AuthorFile.Upload}">Upload</Button>
<TextBlock Margin="5" Grid.Row="3" Grid.ColumnSpan="2">
<Hyperlink Command="{Binding AuthorFile.HyperlinkCommand}">
<TextBlock Text="{Binding AuthorFile.FinalUrl}"></TextBlock>
</Hyperlink>
</TextBlock>
<Button Margin="5" Grid.Row="4" Grid.Column="0" Command="{Binding AuthorFile.SelectFile, Mode=OneTime}">Select</Button>
<Button Margin="5" Grid.Row="4" Grid.Column="1" Command="{Binding AuthorFile.Upload}">Upload</Button>
</Grid>
</Border>
</rxui:ReactiveUserControl>