New Features from External Contributors [Merged to Internal Test Branch so Tests can run!] (#2325)

* added enderalse GOGID

* Fix readme opening twice when loading last modlist

* Edit Wabbajack CLI button text

* Cancel running downloads when shutting down application

* Add resume support for IHttpDownloader

* Add resume support for manual downloads

* Update CHANGELOG.md

* Improve game selection to only show games with results combined with the amount of lists

* Undo accidental removal of loading settings

* Add more tooltips and improve existing ones

* Update CHANGELOG.md

* Main test external pull readme fix (#2335)

* Fix SelectedGameType crashing Wabbajack when no settings are present yet, fix readme being clickable when not specified resulting in crash

* Add readme fix to CHANGELOG, fix typo

* Add readme button fix to changelog

---------

Co-authored-by: UrbanCMC <UrbanCMC@web.de>
Co-authored-by: Angad <angadmisra28@gmail.com>
Co-authored-by: trawzified <55751269+tr4wzified@users.noreply.github.com>
Co-authored-by: Timothy Baldridge <tbaldridge@gmail.com>
This commit is contained in:
Luca 2023-05-07 22:32:18 +02:00 committed by GitHub
parent b66632f930
commit a87f8dac7f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 452 additions and 263 deletions

View File

@ -1,8 +1,20 @@
### Changelog ### Changelog
#### Version - TBD
* Fixed Readme opening twice
* Updated Text in the UI to better describe current app behavior
* Added support for resumable downloads after closing the app during downloads (not available for MEGA downloads)
* More and improved existing tooltips
* Fixed being able to click the readme button when there is no readme
* Game Selector Improvements:
* Only games that have modlists are shown now
* Amount of lists for a game is shown
* Now able to filter for game in combination with filtering on only installed modlists
* Game support:
* Added Enderal GOG support (compatibility with existing lists unclear)
#### Version - 3.0.6.2 - 1/28/2023 #### Version - 3.0.6.2 - 1/28/2023
* Add fallback for DDS compression when installing older lists. This should keep older DDS files from not being compressed without any mipmaps at all. * Add fallback for DDS compression when installing older lists. This should keep older DDS files from not being compressed without any mipmaps at all.
*
#### Version - 3.0.6.1 - 1/28/2023 #### Version - 3.0.6.1 - 1/28/2023
* Game support: * Game support:

View File

@ -5,8 +5,7 @@
xmlns:local="clr-namespace:Wabbajack" xmlns:local="clr-namespace:Wabbajack"
xmlns:controls="clr-namespace:Wabbajack.Extensions" xmlns:controls="clr-namespace:Wabbajack.Extensions"
ShutdownMode="OnExplicitShutdown" ShutdownMode="OnExplicitShutdown"
Startup="OnStartup" Startup="OnStartup">
Exit="OnExit">
<Application.Resources> <Application.Resources>
<ResourceDictionary> <ResourceDictionary>
<ResourceDictionary.MergedDictionaries> <ResourceDictionary.MergedDictionaries>

View File

@ -194,14 +194,5 @@ namespace Wabbajack
return services; return services;
} }
private void OnExit(object sender, ExitEventArgs e)
{
using (_host)
{
_host.StopAsync();
}
base.OnExit(e);
}
} }
} }

View File

@ -6,6 +6,7 @@ using System.Collections.ObjectModel;
using System.Linq; using System.Linq;
using System.Reactive.Disposables; using System.Reactive.Disposables;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Windows.Input; using System.Windows.Input;
using DynamicData; using DynamicData;
@ -25,9 +26,10 @@ namespace Wabbajack
public class ModListGalleryVM : BackNavigatingVM public class ModListGalleryVM : BackNavigatingVM
{ {
public MainWindowVM MWVM { get; } public MainWindowVM MWVM { get; }
private readonly SourceCache<ModListMetadataVM, string> _modLists = new(x => x.Metadata.NamespacedName); private readonly SourceCache<ModListMetadataVM, string> _modLists = new(x => x.Metadata.NamespacedName);
public ReadOnlyObservableCollection<ModListMetadataVM> _filteredModLists; public ReadOnlyObservableCollection<ModListMetadataVM> _filteredModLists;
public ReadOnlyObservableCollection<ModListMetadataVM> ModLists => _filteredModLists; public ReadOnlyObservableCollection<ModListMetadataVM> ModLists => _filteredModLists;
private const string ALL_GAME_TYPE = "All"; private const string ALL_GAME_TYPE = "All";
@ -44,19 +46,45 @@ namespace Wabbajack
[Reactive] public string GameType { get; set; } [Reactive] public string GameType { get; set; }
public List<string> GameTypeEntries => GetGameTypeEntries(); public class GameTypeEntry
{
public GameTypeEntry(string humanFriendlyName, int amount)
{
HumanFriendlyName = humanFriendlyName;
Amount = amount;
FormattedName = $"{HumanFriendlyName} ({Amount})";
}
public string HumanFriendlyName { get; set; }
public int Amount { get; set; }
public string FormattedName { get; set; }
}
[Reactive] public List<GameTypeEntry> GameTypeEntries { get; set; }
private bool _filteringOnGame;
private GameTypeEntry _selectedGameTypeEntry = null;
public GameTypeEntry SelectedGameTypeEntry
{
get => _selectedGameTypeEntry;
set
{
RaiseAndSetIfChanged(ref _selectedGameTypeEntry, value == null ? GameTypeEntries?.FirstOrDefault(gte => gte.HumanFriendlyName == ALL_GAME_TYPE) : value);
GameType = _selectedGameTypeEntry?.HumanFriendlyName;
}
}
private readonly Client _wjClient; private readonly Client _wjClient;
private readonly ILogger<ModListGalleryVM> _logger; private readonly ILogger<ModListGalleryVM> _logger;
private readonly GameLocator _locator; private readonly GameLocator _locator;
private readonly ModListDownloadMaintainer _maintainer; private readonly ModListDownloadMaintainer _maintainer;
private readonly SettingsManager _settingsManager; private readonly SettingsManager _settingsManager;
private readonly CancellationToken _cancellationToken;
private FiltersSettings settings { get; set; } = new(); private FiltersSettings settings { get; set; } = new();
public ICommand ClearFiltersCommand { get; set; } public ICommand ClearFiltersCommand { get; set; }
public ModListGalleryVM(ILogger<ModListGalleryVM> logger, Client wjClient, public ModListGalleryVM(ILogger<ModListGalleryVM> logger, Client wjClient, GameLocator locator,
GameLocator locator, SettingsManager settingsManager, ModListDownloadMaintainer maintainer) SettingsManager settingsManager, ModListDownloadMaintainer maintainer, CancellationToken cancellationToken)
: base(logger) : base(logger)
{ {
_wjClient = wjClient; _wjClient = wjClient;
@ -64,7 +92,8 @@ namespace Wabbajack
_locator = locator; _locator = locator;
_maintainer = maintainer; _maintainer = maintainer;
_settingsManager = settingsManager; _settingsManager = settingsManager;
_cancellationToken = cancellationToken;
ClearFiltersCommand = ReactiveCommand.Create( ClearFiltersCommand = ReactiveCommand.Create(
() => () =>
{ {
@ -72,9 +101,9 @@ namespace Wabbajack
ShowNSFW = false; ShowNSFW = false;
ShowUnofficialLists = false; ShowUnofficialLists = false;
Search = string.Empty; Search = string.Empty;
GameType = ALL_GAME_TYPE; SelectedGameTypeEntry = GameTypeEntries.FirstOrDefault();
}); });
BackCommand = ReactiveCommand.Create( BackCommand = ReactiveCommand.Create(
() => () =>
{ {
@ -83,13 +112,13 @@ namespace Wabbajack
this.WhenActivated(disposables => this.WhenActivated(disposables =>
{ {
LoadModLists().FireAndForget(); LoadModLists().FireAndForget();
LoadSettings().FireAndForget(); LoadSettings().FireAndForget();
Disposable.Create(() => SaveSettings().FireAndForget()) Disposable.Create(() => SaveSettings().FireAndForget())
.DisposeWith(disposables); .DisposeWith(disposables);
var searchTextPredicates = this.ObservableForProperty(vm => vm.Search) var searchTextPredicates = this.ObservableForProperty(vm => vm.Search)
.Select(change => change.Value) .Select(change => change.Value)
.StartWith("") .StartWith("")
@ -99,7 +128,7 @@ namespace Wabbajack
return item => item.Metadata.Title.ContainsCaseInsensitive(txt) || return item => item.Metadata.Title.ContainsCaseInsensitive(txt) ||
item.Metadata.Description.ContainsCaseInsensitive(txt); item.Metadata.Description.ContainsCaseInsensitive(txt);
}); });
var onlyInstalledGamesFilter = this.ObservableForProperty(vm => vm.OnlyInstalled) var onlyInstalledGamesFilter = this.ObservableForProperty(vm => vm.OnlyInstalled)
.Select(v => v.Value) .Select(v => v.Value)
.Select<bool, Func<ModListMetadataVM, bool>>(onlyInstalled => .Select<bool, Func<ModListMetadataVM, bool>>(onlyInstalled =>
@ -108,7 +137,7 @@ namespace Wabbajack
return item => _locator.IsInstalled(item.Metadata.Game); return item => _locator.IsInstalled(item.Metadata.Game);
}) })
.StartWith(_ => true); .StartWith(_ => true);
var showUnofficial = this.ObservableForProperty(vm => vm.ShowUnofficialLists) var showUnofficial = this.ObservableForProperty(vm => vm.ShowUnofficialLists)
.Select(v => v.Value) .Select(v => v.Value)
.StartWith(false) .StartWith(false)
@ -117,21 +146,22 @@ namespace Wabbajack
if (unoffical) return x => true; if (unoffical) return x => true;
return x => x.Metadata.Official; return x => x.Metadata.Official;
}); });
var showNSFWFilter = this.ObservableForProperty(vm => vm.ShowNSFW) var showNSFWFilter = this.ObservableForProperty(vm => vm.ShowNSFW)
.Select(v => v.Value) .Select(v => v.Value)
.Select<bool, Func<ModListMetadataVM, bool>>(showNsfw => { return item => item.Metadata.NSFW == showNsfw; }) .Select<bool, Func<ModListMetadataVM, bool>>(showNsfw => { return item => item.Metadata.NSFW == showNsfw; })
.StartWith(item => item.Metadata.NSFW == false); .StartWith(item => item.Metadata.NSFW == false);
var gameFilter = this.ObservableForProperty(vm => vm.GameType) var gameFilter = this.ObservableForProperty(vm => vm.GameType)
.Select(v => v.Value) .Select(v => v.Value)
.Select<string, Func<ModListMetadataVM, bool>>(selected => .Select<string, Func<ModListMetadataVM, bool>>(selected =>
{ {
_filteringOnGame = true;
if (selected is null or ALL_GAME_TYPE) return _ => true; if (selected is null or ALL_GAME_TYPE) return _ => true;
return item => item.Metadata.Game.MetaData().HumanFriendlyGameName == selected; return item => item.Metadata.Game.MetaData().HumanFriendlyGameName == selected;
}) })
.StartWith(_ => true); .StartWith(_ => true);
_modLists.Connect() _modLists.Connect()
.ObserveOn(RxApp.MainThreadScheduler) .ObserveOn(RxApp.MainThreadScheduler)
.Filter(searchTextPredicates) .Filter(searchTextPredicates)
@ -140,7 +170,18 @@ namespace Wabbajack
.Filter(showNSFWFilter) .Filter(showNSFWFilter)
.Filter(gameFilter) .Filter(gameFilter)
.Bind(out _filteredModLists) .Bind(out _filteredModLists)
.Subscribe() .Subscribe((_) =>
{
if (!_filteringOnGame)
{
var previousGameType = GameType;
SelectedGameTypeEntry = null;
GameTypeEntries = new(GetGameTypeEntries());
var nextEntry = GameTypeEntries.FirstOrDefault(gte => previousGameType == gte.HumanFriendlyName);
SelectedGameTypeEntry = nextEntry != default ? nextEntry : GameTypeEntries.FirstOrDefault(gte => GameType == ALL_GAME_TYPE);
}
_filteringOnGame = false;
})
.DisposeWith(disposables); .DisposeWith(disposables);
}); });
} }
@ -174,10 +215,10 @@ namespace Wabbajack
private async Task LoadSettings() private async Task LoadSettings()
{ {
using var ll = LoadingLock.WithLoading(); using var ll = LoadingLock.WithLoading();
RxApp.MainThreadScheduler.Schedule(await _settingsManager.Load<FilterSettings>("modlist_gallery"), RxApp.MainThreadScheduler.Schedule(await _settingsManager.Load<FilterSettings>("modlist_gallery"),
(_, s) => (_, s) =>
{ {
GameType = s.GameType; SelectedGameTypeEntry = GameTypeEntries?.FirstOrDefault(gte => gte.HumanFriendlyName.Equals(s.GameType));
ShowNSFW = s.ShowNSFW; ShowNSFW = s.ShowNSFW;
ShowUnofficialLists = s.ShowUnofficialLists; ShowUnofficialLists = s.ShowUnofficialLists;
Search = s.Search; Search = s.Search;
@ -196,7 +237,7 @@ namespace Wabbajack
{ {
e.Clear(); e.Clear();
e.AddOrUpdate(modLists.Select(m => e.AddOrUpdate(modLists.Select(m =>
new ModListMetadataVM(_logger, this, m, _maintainer, _wjClient))); new ModListMetadataVM(_logger, this, m, _maintainer, _wjClient, _cancellationToken)));
}); });
} }
catch (Exception ex) catch (Exception ex)
@ -207,13 +248,14 @@ namespace Wabbajack
ll.Succeed(); ll.Succeed();
} }
private List<string> GetGameTypeEntries() private List<GameTypeEntry> GetGameTypeEntries()
{ {
List<string> gameEntries = new List<string> {ALL_GAME_TYPE}; return ModLists.Select(fm => fm.Metadata)
gameEntries.AddRange(GameRegistry.Games.Values.Select(gameType => gameType.HumanFriendlyGameName)); .GroupBy(m => m.Game)
gameEntries.Sort(); .Select(g => new GameTypeEntry(g.Key.MetaData().HumanFriendlyGameName, g.Count()))
return gameEntries; .OrderBy(gte => gte.HumanFriendlyName)
.Prepend(new GameTypeEntry(ALL_GAME_TYPE, ModLists.Count))
.ToList();
} }
} }
} }

View File

@ -4,6 +4,7 @@ using System.Linq;
using System.Reactive; using System.Reactive;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Reactive.Subjects; using System.Reactive.Subjects;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Windows.Input; using System.Windows.Input;
using System.Windows.Media.Imaging; using System.Windows.Media.Imaging;
@ -94,15 +95,17 @@ namespace Wabbajack
private readonly ILogger _logger; private readonly ILogger _logger;
private readonly ModListDownloadMaintainer _maintainer; private readonly ModListDownloadMaintainer _maintainer;
private readonly Client _wjClient; private readonly Client _wjClient;
private readonly CancellationToken _cancellationToken;
public ModListMetadataVM(ILogger logger, ModListGalleryVM parent, ModlistMetadata metadata, public ModListMetadataVM(ILogger logger, ModListGalleryVM parent, ModlistMetadata metadata,
ModListDownloadMaintainer maintainer, Client wjClient) ModListDownloadMaintainer maintainer, Client wjClient, CancellationToken cancellationToken)
{ {
_logger = logger; _logger = logger;
_parent = parent; _parent = parent;
_maintainer = maintainer; _maintainer = maintainer;
Metadata = metadata; Metadata = metadata;
_wjClient = wjClient; _wjClient = wjClient;
_cancellationToken = cancellationToken;
Location = LauncherUpdater.CommonFolder.Value.Combine("downloaded_mod_lists", Metadata.NamespacedName).WithExtension(Ext.Wabbajack); Location = LauncherUpdater.CommonFolder.Value.Combine("downloaded_mod_lists", Metadata.NamespacedName).WithExtension(Ext.Wabbajack);
ModListTagList = new List<ModListTag>(); ModListTagList = new List<ModListTag>();
@ -184,7 +187,7 @@ namespace Wabbajack
Status = ModListStatus.Downloading; Status = ModListStatus.Downloading;
using var ll = LoadingLock.WithLoading(); using var ll = LoadingLock.WithLoading();
var (progress, task) = _maintainer.DownloadModlist(Metadata); var (progress, task) = _maintainer.DownloadModlist(Metadata, _cancellationToken);
var dispose = progress var dispose = progress
.BindToStrict(this, vm => vm.ProgressPercent); .BindToStrict(this, vm => vm.ProgressPercent);
try try

View File

@ -119,6 +119,7 @@ public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM
private readonly HttpClient _client; private readonly HttpClient _client;
private readonly DownloadDispatcher _downloadDispatcher; private readonly DownloadDispatcher _downloadDispatcher;
private readonly IEnumerable<INeedsLogin> _logins; private readonly IEnumerable<INeedsLogin> _logins;
private readonly CancellationToken _cancellationToken;
public ReadOnlyObservableCollection<CPUDisplayVM> StatusList => _resourceMonitor.Tasks; public ReadOnlyObservableCollection<CPUDisplayVM> StatusList => _resourceMonitor.Tasks;
[Reactive] [Reactive]
@ -146,7 +147,8 @@ public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM
public InstallerVM(ILogger<InstallerVM> logger, DTOSerializer dtos, SettingsManager settingsManager, IServiceProvider serviceProvider, public InstallerVM(ILogger<InstallerVM> logger, DTOSerializer dtos, SettingsManager settingsManager, IServiceProvider serviceProvider,
SystemParametersConstructor parametersConstructor, IGameLocator gameLocator, LogStream loggerProvider, ResourceMonitor resourceMonitor, SystemParametersConstructor parametersConstructor, IGameLocator gameLocator, LogStream loggerProvider, ResourceMonitor resourceMonitor,
Wabbajack.Services.OSIntegrated.Configuration configuration, HttpClient client, DownloadDispatcher dispatcher, IEnumerable<INeedsLogin> logins) : base(logger) Wabbajack.Services.OSIntegrated.Configuration configuration, HttpClient client, DownloadDispatcher dispatcher, IEnumerable<INeedsLogin> logins,
CancellationToken cancellationToken) : base(logger)
{ {
_logger = logger; _logger = logger;
_configuration = configuration; _configuration = configuration;
@ -160,18 +162,19 @@ public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM
_client = client; _client = client;
_downloadDispatcher = dispatcher; _downloadDispatcher = dispatcher;
_logins = logins; _logins = logins;
_cancellationToken = cancellationToken;
Installer = new MO2InstallerVM(this); Installer = new MO2InstallerVM(this);
BackCommand = ReactiveCommand.Create(() => NavigateToGlobal.Send(NavigateToGlobal.ScreenType.ModeSelectionView)); BackCommand = ReactiveCommand.Create(() => NavigateToGlobal.Send(NavigateToGlobal.ScreenType.ModeSelectionView));
BeginCommand = ReactiveCommand.Create(() => BeginInstall().FireAndForget()); BeginCommand = ReactiveCommand.Create(() => BeginInstall().FireAndForget());
OpenReadmeCommand = ReactiveCommand.Create(() => OpenReadmeCommand = ReactiveCommand.Create(() =>
{ {
UIUtils.OpenWebsite(new Uri(ModList!.Readme)); UIUtils.OpenWebsite(new Uri(ModList!.Readme));
}, LoadingLock.IsNotLoadingObservable); }, this.WhenAnyValue(vm => vm.LoadingLock.IsNotLoading, vm => vm.ModList.Readme, (isNotLoading, readme) => isNotLoading && !string.IsNullOrWhiteSpace(readme)));
VisitModListWebsiteCommand = ReactiveCommand.Create(() => VisitModListWebsiteCommand = ReactiveCommand.Create(() =>
{ {
UIUtils.OpenWebsite(ModList!.Website); UIUtils.OpenWebsite(ModList!.Website);
@ -312,7 +315,9 @@ public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM
{ {
var lst = await _settingsManager.Load<AbsolutePath>(LastLoadedModlist); var lst = await _settingsManager.Load<AbsolutePath>(LastLoadedModlist);
if (lst.FileExists()) if (lst.FileExists())
await LoadModlist(lst, null); {
ModListLocation.TargetPath = lst;
}
} }
private async Task LoadModlist(AbsolutePath path, ModlistMetadata? metadata) private async Task LoadModlist(AbsolutePath path, ModlistMetadata? metadata)
@ -437,8 +442,8 @@ public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM
TaskBarUpdate.Send(update.StatusText, TaskbarItemProgressState.Indeterminate, TaskBarUpdate.Send(update.StatusText, TaskbarItemProgressState.Indeterminate,
update.StepsProgress.Value); update.StepsProgress.Value);
}; };
if (!await installer.Begin(CancellationToken.None)) if (!await installer.Begin(_cancellationToken))
{ {
TaskBarUpdate.Send($"Error during install of {ModList.Name}", TaskbarItemProgressState.Error); TaskBarUpdate.Send($"Error during install of {ModList.Name}", TaskbarItemProgressState.Error);
InstallState = InstallState.Failure; InstallState = InstallState.Failure;

View File

@ -243,6 +243,20 @@ namespace Wabbajack
return false; return false;
} }
public void CancelRunningTasks(TimeSpan timeout)
{
var endTime = DateTime.Now.Add(timeout);
var cancellationTokenSource = _serviceProvider.GetRequiredService<CancellationTokenSource>();
cancellationTokenSource.Cancel();
bool IsInstalling() => Installer.InstallState is InstallState.Installing;
while (DateTime.Now < endTime && IsInstalling())
{
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
/* /*
public void NavigateTo(ViewModel vm) public void NavigateTo(ViewModel vm)
{ {

View File

@ -224,7 +224,7 @@
Margin="30,2" Margin="30,2"
FontSize="20" FontSize="20"
Style="{StaticResource LargeButtonStyle}" Style="{StaticResource LargeButtonStyle}"
ToolTip="Open the Discord for this Modlist"> ToolTip="Open the Discord for the Modlist">
<Grid> <Grid>
<Grid.ColumnDefinitions> <Grid.ColumnDefinitions>
<ColumnDefinition Width="30" /> <ColumnDefinition Width="30" />
@ -290,7 +290,7 @@
Margin="30,2" Margin="30,2"
FontSize="20" FontSize="20"
Style="{StaticResource LargeButtonStyle}" Style="{StaticResource LargeButtonStyle}"
ToolTip="Open an explicit listing of all actions this modlist will take"> ToolTip="Open an explicit listing of all archives this modlist contains">
<Grid> <Grid>
<Grid.ColumnDefinitions> <Grid.ColumnDefinitions>
<ColumnDefinition Width="30" /> <ColumnDefinition Width="30" />

View File

@ -25,23 +25,27 @@
VerticalAlignment="Center" VerticalAlignment="Center"
FontSize="14" FontSize="14"
Text="Modlist Installation Location" Text="Modlist Installation Location"
TextAlignment="Center" /> TextAlignment="Center"
ToolTip="The directory where the modlist will be installed" />
<local:FilePicker Grid.Row="0" Grid.Column="2" <local:FilePicker Grid.Row="0" Grid.Column="2"
Height="30" Height="30"
VerticalAlignment="Center" VerticalAlignment="Center"
FontSize="14" FontSize="14"
PickerVM="{Binding Location}" /> PickerVM="{Binding Location}"
ToolTip="The directory where the modlist will be installed" />
<TextBlock Grid.Row="1" Grid.Column="0" <TextBlock Grid.Row="1" Grid.Column="0"
HorizontalAlignment="Right" HorizontalAlignment="Right"
VerticalAlignment="Center" VerticalAlignment="Center"
FontSize="14" FontSize="14"
Text="Resource Download Location" Text="Resource Download Location"
TextAlignment="Center" /> TextAlignment="Center"
ToolTip="The directory where modlist archives will be downloaded to" />
<local:FilePicker Grid.Row="1" Grid.Column="2" <local:FilePicker Grid.Row="1" Grid.Column="2"
Height="30" Height="30"
VerticalAlignment="Center" VerticalAlignment="Center"
FontSize="14" FontSize="14"
PickerVM="{Binding DownloadLocation}" /> PickerVM="{Binding DownloadLocation}"
ToolTip="The directory where modlist archives will be downloaded to" />
<CheckBox Grid.Row="2" Grid.Column="2" <CheckBox Grid.Row="2" Grid.Column="2"
HorizontalAlignment="Right" HorizontalAlignment="Right"
Content="Overwrite Installation" Content="Overwrite Installation"

View File

@ -40,7 +40,7 @@
</Grid.ColumnDefinitions> </Grid.ColumnDefinitions>
<TextBlock Grid.Column="0" FontSize="16" Margin="0, 0, 8, 0" Name="AppName"></TextBlock> <TextBlock Grid.Column="0" FontSize="16" Margin="0, 0, 8, 0" Name="AppName"></TextBlock>
<TextBlock Grid.Column="1" FontSize="16" Margin="5, 0" Name="ResourceUsage" HorizontalAlignment="Right" VerticalAlignment="Center"></TextBlock> <TextBlock Grid.Column="1" FontSize="16" Margin="5, 0" Name="ResourceUsage" HorizontalAlignment="Right" VerticalAlignment="Center"></TextBlock>
<Button Grid.Column="2" Name="SettingsButton"> <Button Grid.Column="2" Name="SettingsButton" ToolTip="Open Wabbajack settings">
<icon:Material Kind="Cog"></icon:Material> <icon:Material Kind="Cog"></icon:Material>
</Button> </Button>
</Grid> </Grid>

View File

@ -3,7 +3,7 @@ using System.Collections.ObjectModel;
using System.ComponentModel; using System.ComponentModel;
using System.Linq; using System.Linq;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Threading.Tasks; using System.Threading;
using System.Windows; using System.Windows;
using System.Windows.Input; using System.Windows.Input;
using DynamicData; using DynamicData;
@ -55,11 +55,8 @@ namespace Wabbajack
Closed += (s, e) => Closed += (s, e) =>
{ {
Task.Run(async () => _mwvm.CancelRunningTasks(TimeSpan.FromSeconds(10));
{ Application.Current.Shutdown();
await Task.Delay(5000);
Environment.Exit(0);
});
}; };
MessageBus.Current.Listen<TaskBarUpdate>() MessageBus.Current.Listen<TaskBarUpdate>()

View File

@ -91,45 +91,52 @@
<Label <Label
Margin="0,0,0,0" Margin="0,0,0,0"
VerticalAlignment="Center" VerticalAlignment="Center"
Content="Game" /> Content="Game"
ToolTip="Select a game" />
<ComboBox <ComboBox
Width="150" Width="150"
Margin="0,0,10,0" Margin="0,0,10,0"
VerticalAlignment="Center" VerticalAlignment="Center"
Foreground="{StaticResource ForegroundBrush}" Foreground="{StaticResource ForegroundBrush}"
IsEnabled="{Binding OnlyInstalled, Converter={StaticResource InverseBooleanConverter}}" ItemsSource="{Binding GameTypeEntries, Mode=TwoWay}"
ItemsSource="{Binding Path=GameTypeEntries}" SelectedItem="{Binding SelectedGameTypeEntry, Mode=TwoWay}"
SelectedItem="{Binding GameType, Mode=TwoWay}" DisplayMemberPath="FormattedName"
IsSynchronizedWithCurrentItem="True"
ToolTip="Select a game" /> ToolTip="Select a game" />
<TextBlock <TextBlock
Margin="0,0,5,0" Margin="0,0,5,0"
VerticalAlignment="Center" VerticalAlignment="Center"
Foreground="{StaticResource ForegroundBrush}" Foreground="{StaticResource ForegroundBrush}"
Text="Search" /> Text="Search"
ToolTip="Search for a game" />
<TextBox <TextBox
x:Name="SearchBox" x:Name="SearchBox"
Width="95" Width="95"
VerticalContentAlignment="Center" /> VerticalContentAlignment="Center"
ToolTip="Only show Not Safe For Work (NSFW) modlists" />
<CheckBox <CheckBox
x:Name="ShowNSFW" x:Name="ShowNSFW"
Margin="10,0,10,0" Margin="10,0,10,0"
VerticalAlignment="Center" VerticalAlignment="Center"
Content="Show NSFW" Content="Show NSFW"
Foreground="{StaticResource ForegroundBrush}" /> Foreground="{StaticResource ForegroundBrush}"
ToolTip="Only show Not Safe For Work (NSFW) modlists" />
<CheckBox <CheckBox
x:Name="ShowUnofficialLists" x:Name="ShowUnofficialLists"
Margin="10,0,10,0" Margin="10,0,10,0"
VerticalAlignment="Center" VerticalAlignment="Center"
Content="Show Unofficial Lists" Content="Show Unofficial Lists"
Foreground="{StaticResource ForegroundBrush}" /> Foreground="{StaticResource ForegroundBrush}"
ToolTip="Show modlists from external repositories"/>
<CheckBox <CheckBox
x:Name="OnlyInstalledCheckbox" x:Name="OnlyInstalledCheckbox"
Margin="10,0,10,0" Margin="10,0,10,0"
VerticalAlignment="Center" VerticalAlignment="Center"
Content="Only Installed" Content="Only Installed"
Foreground="{StaticResource ForegroundBrush}" /> Foreground="{StaticResource ForegroundBrush}"
ToolTip="Only show modlists for games you have installed"/>
<Button <Button
x:Name="ClearFiltersButton" x:Name="ClearFiltersButton"
Margin="0,0,10,0" Margin="0,0,10,0"

View File

@ -367,7 +367,8 @@
Height="40" Height="40"
Margin="5,0" Margin="5,0"
VerticalAlignment="Center" VerticalAlignment="Center"
Style="{StaticResource IconBareButtonStyle}"> Style="{StaticResource IconBareButtonStyle}"
ToolTip="View modlist website in browser">
<iconPacks:Material <iconPacks:Material
Width="20" Width="20"
Height="20" Height="20"
@ -379,7 +380,8 @@
Height="40" Height="40"
Margin="5,0" Margin="5,0"
VerticalAlignment="Center" VerticalAlignment="Center"
Style="{StaticResource IconBareButtonStyle}"> Style="{StaticResource IconBareButtonStyle}"
ToolTip="View modlist archives in browser">
<iconPacks:Material <iconPacks:Material
Width="20" Width="20"
Height="20" Height="20"
@ -390,7 +392,8 @@
Width="40" Width="40"
Height="40" Height="40"
Margin="5,0" Margin="5,0"
VerticalAlignment="Center"> VerticalAlignment="Center"
ToolTip="Download modlist">
<StackPanel x:Name="IconContainer"> <StackPanel x:Name="IconContainer">
<iconPacks:Material <iconPacks:Material
x:Name="ErrorIcon" x:Name="ErrorIcon"

View File

@ -82,7 +82,7 @@
<Button Grid.Row="3" <Button Grid.Row="3"
Name="OpenTerminal" Name="OpenTerminal"
Margin="0,5,0,0" Margin="0,5,0,0"
Content="Open Terminal and Close WJ" /> Content="Launch Wabbajack CLI" />
</Grid> </Grid>
</Grid> </Grid>
</Border> </Border>

View File

@ -21,6 +21,7 @@ public static class Ext
public static Extension Md = new(".md"); public static Extension Md = new(".md");
public static Extension MetaData = new(".metadata"); public static Extension MetaData = new(".metadata");
public static Extension CompilerSettings = new(".compiler_settings"); public static Extension CompilerSettings = new(".compiler_settings");
public static Extension DownloadPackage = new(".download_package");
public static Extension Temp = new(".temp"); public static Extension Temp = new(".temp");
public static Extension ModlistMetadataExtension = new(".modlist_metadata"); public static Extension ModlistMetadataExtension = new(".modlist_metadata");
public static Extension Txt = new(".txt"); public static Extension Txt = new(".txt");

View File

@ -182,6 +182,7 @@ public static class GameRegistry
MO2Name = "Enderal Special Edition", MO2Name = "Enderal Special Edition",
MO2ArchiveName = "enderalse", MO2ArchiveName = "enderalse",
SteamIDs = new[] {976620}, SteamIDs = new[] {976620},
GOGIDs = new [] {1708684988},
RequiredFiles = new[] RequiredFiles = new[]
{ {
"SkyrimSE.exe".ToRelativePath() "SkyrimSE.exe".ToRelativePath()

View File

@ -43,6 +43,11 @@ public class DownloadDispatcher
public async Task<Hash> Download(Archive a, AbsolutePath dest, CancellationToken token, bool? proxy = null) public async Task<Hash> Download(Archive a, AbsolutePath dest, CancellationToken token, bool? proxy = null)
{ {
if (token.IsCancellationRequested)
{
return new Hash();
}
using var downloadScope = _logger.BeginScope("Downloading {Name}", a.Name); using var downloadScope = _logger.BeginScope("Downloading {Name}", a.Name);
using var job = await _limiter.Begin("Downloading " + a.Name, a.Size, token); using var job = await _limiter.Begin("Downloading " + a.Name, a.Size, token);
return await Download(a, dest, job, token, proxy); return await Download(a, dest, job, token, proxy);
@ -74,33 +79,40 @@ public class DownloadDispatcher
public async Task<Hash> Download(Archive a, AbsolutePath dest, Job<DownloadDispatcher> job, CancellationToken token, bool? useProxy = null) public async Task<Hash> Download(Archive a, AbsolutePath dest, Job<DownloadDispatcher> job, CancellationToken token, bool? useProxy = null)
{ {
if (!dest.Parent.DirectoryExists()) try
dest.Parent.CreateDirectory();
var downloader = Downloader(a);
if ((useProxy ?? _useProxyCache) && downloader is IProxyable p)
{ {
var uri = p.UnParse(a.State); if (!dest.Parent.DirectoryExists())
var newUri = await _wjClient.MakeProxyUrl(a, uri); dest.Parent.CreateDirectory();
if (newUri != null)
{
a = new Archive
{
Name = a.Name,
Size = a.Size,
Hash = a.Hash,
State = new DTOs.DownloadStates.Http()
{
Url = newUri
}
};
downloader = Downloader(a);
_logger.LogInformation("Downloading Proxy ({Hash}) {Uri}", (await uri.ToString().Hash()).ToHex(), uri);
}
}
var hash = await downloader.Download(a, dest, job, token); var downloader = Downloader(a);
return hash; if ((useProxy ?? _useProxyCache) && downloader is IProxyable p)
{
var uri = p.UnParse(a.State);
var newUri = await _wjClient.MakeProxyUrl(a, uri);
if (newUri != null)
{
a = new Archive
{
Name = a.Name,
Size = a.Size,
Hash = a.Hash,
State = new DTOs.DownloadStates.Http()
{
Url = newUri
}
};
downloader = Downloader(a);
_logger.LogInformation("Downloading Proxy ({Hash}) {Uri}", (await uri.ToString().Hash()).ToHex(), uri);
}
}
var hash = await downloader.Download(a, dest, job, token);
return hash;
}
catch (TaskCanceledException)
{
return new Hash();
}
} }
public Task<IDownloadState?> ResolveArchive(IReadOnlyDictionary<string, string> ini) public Task<IDownloadState?> ResolveArchive(IReadOnlyDictionary<string, string> ini)
@ -285,24 +297,6 @@ public class DownloadDispatcher
throw new NotImplementedException(); throw new NotImplementedException();
} }
public async Task<Archive?> FindUpgrade(Archive archive, TemporaryFileManager fileManager, CancellationToken token)
{
try
{
var downloader = Downloader(archive);
if (downloader is not IUpgradingDownloader ud) return null;
using var job = await _limiter.Begin($"Finding upgrade for {archive.Name} - {archive.State.PrimaryKeyString}", 0,
token);
return await ud.TryGetUpgrade(archive, job, fileManager, token);
}
catch (Exception ex)
{
_logger.LogCritical(ex, "While finding upgrade for {PrimaryKeyString}", archive.State.PrimaryKeyString);
return null;
}
}
public Task<IEnumerable<IDownloader>> AllDownloaders(IEnumerable<IDownloadState> downloadStates) public Task<IEnumerable<IDownloader>> AllDownloaders(IEnumerable<IDownloadState> downloadStates)
{ {
return Task.FromResult(downloadStates.Select(d => Downloader(new Archive {State = d})).Distinct()); return Task.FromResult(downloadStates.Select(d => Downloader(new Archive {State = d})).Distinct());

View File

@ -21,7 +21,7 @@ using Wabbajack.RateLimiter;
namespace Wabbajack.Downloaders.Http; namespace Wabbajack.Downloaders.Http;
public class HttpDownloader : ADownloader<DTOs.DownloadStates.Http>, IUrlDownloader, IUpgradingDownloader, IChunkedSeekableStreamDownloader public class HttpDownloader : ADownloader<DTOs.DownloadStates.Http>, IUrlDownloader, IChunkedSeekableStreamDownloader
{ {
private readonly HttpClient _client; private readonly HttpClient _client;
private readonly IHttpDownloader _downloader; private readonly IHttpDownloader _downloader;
@ -34,23 +34,6 @@ public class HttpDownloader : ADownloader<DTOs.DownloadStates.Http>, IUrlDownloa
_downloader = downloader; _downloader = downloader;
} }
public async Task<Archive?> TryGetUpgrade(Archive archive, IJob job, TemporaryFileManager temporaryFileManager,
CancellationToken token)
{
var state = (DTOs.DownloadStates.Http) archive.State;
await using var file = temporaryFileManager.CreateFile();
var newHash = await Download(archive, file.Path, job, token);
return new Archive
{
Hash = newHash,
Size = file.Path.Size(),
State = archive.State,
Name = archive.Name
};
}
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData) public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{ {
if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri)) if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri))

View File

@ -26,7 +26,7 @@ using Wabbajack.RateLimiter;
namespace Wabbajack.Downloaders.IPS4OAuth2Downloader; namespace Wabbajack.Downloaders.IPS4OAuth2Downloader;
public class AIPS4OAuth2Downloader<TDownloader, TLogin, TState> : ADownloader<TState>, IUpgradingDownloader public class AIPS4OAuth2Downloader<TDownloader, TLogin, TState> : ADownloader<TState>
where TLogin : OAuth2LoginState, new() where TLogin : OAuth2LoginState, new()
where TState : IPS4OAuth2, new() where TState : IPS4OAuth2, new()
{ {
@ -58,41 +58,6 @@ public class AIPS4OAuth2Downloader<TDownloader, TLogin, TState> : ADownloader<TS
public override Priority Priority => Priority.Normal; public override Priority Priority => Priority.Normal;
public async Task<Archive?> TryGetUpgrade(Archive archive, IJob job, TemporaryFileManager temporaryFileManager,
CancellationToken token)
{
var state = (TState) archive.State;
if (state.IsAttachment) return default;
var files = (await GetDownloads(state.IPS4Mod, token)).Files;
var nl = new Levenshtein();
foreach (var newFile in files.Where(f => f.Url != null)
.OrderBy(f => nl.Distance(archive.Name.ToLowerInvariant(), f.Name!.ToLowerInvariant())))
{
var newArchive = new Archive
{
State = new TState
{
IPS4Mod = state.IPS4Mod,
IPS4File = newFile.Name!
}
};
var tmp = temporaryFileManager.CreateFile();
var newHash = await Download(newArchive, (TState) newArchive.State, tmp.Path, job, token);
if (newHash != default)
{
newArchive.Size = tmp.Path.Size();
newArchive.Hash = newHash;
return newArchive;
}
await tmp.DisposeAsync();
}
return default;
}
public async ValueTask<HttpRequestMessage> MakeMessage(HttpMethod method, Uri url, bool useOAuth2 = true) public async ValueTask<HttpRequestMessage> MakeMessage(HttpMethod method, Uri url, bool useOAuth2 = true)
{ {
var msg = new HttpRequestMessage(method, url); var msg = new HttpRequestMessage(method, url);

View File

@ -1,13 +0,0 @@
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.DTOs;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
namespace Wabbajack.Downloaders.Interfaces;
public interface IUpgradingDownloader
{
public Task<Archive?> TryGetUpgrade(Archive archive, IJob job, TemporaryFileManager temporaryFileManager,
CancellationToken token);
}

View File

@ -6,6 +6,7 @@ using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.Interventions; using Wabbajack.DTOs.Interventions;
using Wabbajack.DTOs.Validation; using Wabbajack.DTOs.Validation;
using Wabbajack.Hashing.xxHash64; using Wabbajack.Hashing.xxHash64;
using Wabbajack.Networking.Http.Interfaces;
using Wabbajack.Paths; using Wabbajack.Paths;
using Wabbajack.Paths.IO; using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter; using Wabbajack.RateLimiter;
@ -16,45 +17,25 @@ public class ManualDownloader : ADownloader<DTOs.DownloadStates.Manual>, IProxya
{ {
private readonly ILogger<ManualDownloader> _logger; private readonly ILogger<ManualDownloader> _logger;
private readonly IUserInterventionHandler _interventionHandler; private readonly IUserInterventionHandler _interventionHandler;
private readonly HttpClient _client; private readonly IHttpDownloader _downloader;
public ManualDownloader(ILogger<ManualDownloader> logger, IUserInterventionHandler interventionHandler, HttpClient client) public ManualDownloader(ILogger<ManualDownloader> logger, IUserInterventionHandler interventionHandler, IHttpDownloader downloader)
{ {
_logger = logger; _logger = logger;
_interventionHandler = interventionHandler; _interventionHandler = interventionHandler;
_client = client; _downloader = downloader;
} }
public override async Task<Hash> Download(Archive archive, DTOs.DownloadStates.Manual state, AbsolutePath destination, IJob job, CancellationToken token) public override async Task<Hash> Download(Archive archive, DTOs.DownloadStates.Manual state, AbsolutePath destination, IJob job, CancellationToken token)
{ {
_logger.LogInformation("Starting manual download of {Url}", state.Url); _logger.LogInformation("Starting manual download of {Url}", state.Url);
if (state.Url.Host == "mega.nz") var intervention = new ManualDownload(archive);
{ _interventionHandler.Raise(intervention);
var intervention = new ManualBlobDownload(archive, destination); var browserState = await intervention.Task;
_interventionHandler.Raise(intervention);
await intervention.Task;
if (!destination.FileExists())
throw new Exception("File does not exist after download");
_logger.LogInformation("Hashing manually downloaded Mega file {File}", destination.FileName);
return await destination.Hash(token);
}
else
{
var intervention = new ManualDownload(archive);
_interventionHandler.Raise(intervention);
var browserState = await intervention.Task;
var msg = browserState.ToHttpRequestMessage(); var msg = browserState.ToHttpRequestMessage();
return await _downloader.Download(msg, destination, job, token);
using var response = await _client.SendAsync(msg, token);
if (!response.IsSuccessStatusCode)
throw new HttpRequestException(response.ReasonPhrase, null, statusCode: response.StatusCode);
await using var strm = await response.Content.ReadAsStreamAsync(token);
await using var os = destination.Open(FileMode.Create, FileAccess.Write, FileShare.None);
return await strm.HashingCopy(os, token, job);
}
} }

View File

@ -131,6 +131,11 @@ public class NexusDownloader : ADownloader<Nexus>, IUrlDownloader
state.FileID); state.FileID);
foreach (var link in urls.info) foreach (var link in urls.info)
{ {
if (token.IsCancellationRequested)
{
return new Hash();
}
try try
{ {
var message = new HttpRequestMessage(HttpMethod.Get, link.URI); var message = new HttpRequestMessage(HttpMethod.Get, link.URI);

View File

@ -228,7 +228,7 @@ public abstract class AInstaller<T>
.ToDictionary(a => a.Key); .ToDictionary(a => a.Key);
if (grouped.Count == 0) return; if (grouped.Count == 0) return;
if (token.IsCancellationRequested) return;
await _vfs.Extract(grouped.Keys.ToHashSet(), async (vf, sf) => await _vfs.Extract(grouped.Keys.ToHashSet(), async (vf, sf) =>
{ {
@ -237,6 +237,7 @@ public abstract class AInstaller<T>
token); token);
foreach (var directive in directives) foreach (var directive in directives)
{ {
if (token.IsCancellationRequested) return;
var file = directive.Directive; var file = directive.Directive;
UpdateProgress(file.Size); UpdateProgress(file.Size);
var destPath = file.To.RelativeTo(_configuration.Install); var destPath = file.To.RelativeTo(_configuration.Install);
@ -363,9 +364,10 @@ public abstract class AInstaller<T>
{ {
_logger.LogInformation("Downloading {Archive}", archive.Name); _logger.LogInformation("Downloading {Archive}", archive.Name);
var outputPath = _configuration.Downloads.Combine(archive.Name); var outputPath = _configuration.Downloads.Combine(archive.Name);
var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage);
if (download) if (download)
if (outputPath.FileExists()) if (outputPath.FileExists() && !downloadPackagePath.FileExists())
{ {
var origName = Path.GetFileNameWithoutExtension(archive.Name); var origName = Path.GetFileNameWithoutExtension(archive.Name);
var ext = Path.GetExtension(archive.Name); var ext = Path.GetExtension(archive.Name);
@ -396,6 +398,10 @@ public abstract class AInstaller<T>
var (result, hash) = var (result, hash) =
await _downloadDispatcher.DownloadWithPossibleUpgrade(archive, destination.Value, token); await _downloadDispatcher.DownloadWithPossibleUpgrade(archive, destination.Value, token);
if (token.IsCancellationRequested)
{
return false;
}
if (hash != archive.Hash) if (hash != archive.Hash)
{ {

View File

@ -109,12 +109,16 @@ public class StandardInstaller : AInstaller<StandardInstaller>
_configuration.Downloads.CreateDirectory(); _configuration.Downloads.CreateDirectory();
await OptimizeModlist(token); await OptimizeModlist(token);
if (token.IsCancellationRequested) return false;
await HashArchives(token); await HashArchives(token);
if (token.IsCancellationRequested) return false;
await DownloadArchives(token); await DownloadArchives(token);
if (token.IsCancellationRequested) return false;
await HashArchives(token); await HashArchives(token);
if (token.IsCancellationRequested) return false;
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
if (missing.Count > 0) if (missing.Count > 0)
@ -127,21 +131,27 @@ public class StandardInstaller : AInstaller<StandardInstaller>
} }
await ExtractModlist(token); await ExtractModlist(token);
if (token.IsCancellationRequested) return false;
await PrimeVFS(); await PrimeVFS();
await BuildFolderStructure(); await BuildFolderStructure();
await InstallArchives(token); await InstallArchives(token);
if (token.IsCancellationRequested) return false;
await InstallIncludedFiles(token); await InstallIncludedFiles(token);
if (token.IsCancellationRequested) return false;
await WriteMetaFiles(token); await WriteMetaFiles(token);
if (token.IsCancellationRequested) return false;
await BuildBSAs(token); await BuildBSAs(token);
if (token.IsCancellationRequested) return false;
// TODO: Port this // TODO: Port this
await GenerateZEditMerges(token); await GenerateZEditMerges(token);
if (token.IsCancellationRequested) return false;
await ForcePortable(); await ForcePortable();
await RemapMO2File(); await RemapMO2File();

View File

@ -1,4 +1,7 @@
using System.Linq;
using System.Net;
using System.Net.Http; using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.Json; using System.Text.Json;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -25,5 +28,16 @@ public static class Extensions
} }
return (await JsonSerializer.DeserializeAsync<T>(await result.Content.ReadAsStreamAsync(token.Value)))!; return (await JsonSerializer.DeserializeAsync<T>(await result.Content.ReadAsStreamAsync(token.Value)))!;
} }
public static WebHeaderCollection ToWebHeaderCollection(this HttpRequestHeaders headers)
{
var headerCollection = new WebHeaderCollection();
foreach (var header in headers.Where(header => !WebHeaderCollection.IsRestricted(header.Key)))
{
header.Value.ToList().ForEach(value => headerCollection.Add(header.Key, value));
}
return headerCollection;
}
} }

View File

@ -0,0 +1,144 @@
using System;
using System.ComponentModel;
using System.IO;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Downloader;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
namespace Wabbajack.Networking.Http;
internal class ResumableDownloader
{
private readonly IJob _job;
private readonly HttpRequestMessage _msg;
private readonly AbsolutePath _outputPath;
private readonly AbsolutePath _packagePath;
private CancellationToken _token;
private Exception? _error;
public ResumableDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job)
{
_job = job;
_msg = msg;
_outputPath = outputPath;
_packagePath = outputPath.WithExtension(Extension.FromPath(".download_package"));
}
public async Task<Hash> Download(CancellationToken token)
{
_token = token;
var downloader = new DownloadService(CreateConfiguration(_msg));
downloader.DownloadStarted += OnDownloadStarted;
downloader.DownloadProgressChanged += OnDownloadProgressChanged;
downloader.DownloadFileCompleted += OnDownloadFileCompleted;
// Attempt to resume previous download
var downloadPackage = LoadPackage();
if (downloadPackage != null)
{
// Resume with different Uri in case old one is no longer valid
downloadPackage.Address = _msg.RequestUri!.AbsoluteUri;
await downloader.DownloadFileTaskAsync(downloadPackage, token);
}
else
{
_outputPath.Delete();
await downloader.DownloadFileTaskAsync(_msg.RequestUri!.AbsoluteUri, _outputPath.ToString(), token);
}
// Save progress if download isn't completed yet
if (downloader.Status is DownloadStatus.Stopped or DownloadStatus.Failed)
{
SavePackage(downloader.Package);
if (_error != null && _error.GetType() != typeof(TaskCanceledException))
{
throw _error;
}
return new Hash();
}
if (downloader.Status == DownloadStatus.Completed)
{
DeletePackage();
}
if (!_outputPath.FileExists())
{
return new Hash();
}
return await _outputPath.Open(FileMode.Open).Hash(token);
}
private DownloadConfiguration CreateConfiguration(HttpRequestMessage message)
{
var configuration = new DownloadConfiguration
{
RequestConfiguration = new RequestConfiguration
{
Headers = message.Headers.ToWebHeaderCollection(),
ProtocolVersion = message.Version,
UserAgent = message.Headers.UserAgent.ToString()
}
};
return configuration;
}
private void OnDownloadFileCompleted(object? sender, AsyncCompletedEventArgs e)
{
_error = e.Error;
}
private async void OnDownloadProgressChanged(object? sender, DownloadProgressChangedEventArgs e)
{
var processedSize = e.ProgressedByteSize;
if (_job.Current == 0)
{
// Set current to total in case this download resumes from a previous one
processedSize = e.ReceivedBytesSize;
}
await _job.Report(processedSize, _token);
}
private void OnDownloadStarted(object? sender, DownloadStartedEventArgs e)
{
_job.ResetProgress();
_job.Size = e.TotalBytesToReceive;
// Get rid of package, since we can't use it to resume anymore
DeletePackage();
}
private void DeletePackage()
{
_packagePath.Delete();
}
private DownloadPackage? LoadPackage()
{
if (!_packagePath.FileExists())
{
return null;
}
var packageJson = _packagePath.ReadAllText();
return JsonSerializer.Deserialize<DownloadPackage>(packageJson);
}
private void SavePackage(DownloadPackage package)
{
var packageJson = JsonSerializer.Serialize(package);
_packagePath.WriteAllText(packageJson);
}
}

View File

@ -29,31 +29,34 @@ public class SingleThreadedDownloader : IHttpDownloader
public async Task<Hash> Download(HttpRequestMessage message, AbsolutePath outputPath, IJob job, public async Task<Hash> Download(HttpRequestMessage message, AbsolutePath outputPath, IJob job,
CancellationToken token) CancellationToken token)
{ {
using var response = await _client.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, token); var downloader = new ResumableDownloader(message, outputPath, job);
if (!response.IsSuccessStatusCode) return await downloader.Download(token);
throw new HttpException(response);
if (job.Size == 0)
job.Size = response.Content.Headers.ContentLength ?? 0;
/* Need to make this mulitthreaded to be much use // using var response = await _client.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, token);
if ((response.Content.Headers.ContentLength ?? 0) != 0 && // if (!response.IsSuccessStatusCode)
response.Headers.AcceptRanges.FirstOrDefault() == "bytes") // throw new HttpException(response);
{ //
return await ResettingDownloader(response, message, outputPath, job, token); // if (job.Size == 0)
} // job.Size = response.Content.Headers.ContentLength ?? 0;
*/ //
// /* Need to make this mulitthreaded to be much use
await using var stream = await response.Content.ReadAsStreamAsync(token); // if ((response.Content.Headers.ContentLength ?? 0) != 0 &&
await using var outputStream = outputPath.Open(FileMode.Create, FileAccess.Write); // response.Headers.AcceptRanges.FirstOrDefault() == "bytes")
return await stream.HashingCopy(outputStream, token, job); // {
// return await ResettingDownloader(response, message, outputPath, job, token);
// }
// */
//
// await using var stream = await response.Content.ReadAsStreamAsync(token);
// await using var outputStream = outputPath.Open(FileMode.Create, FileAccess.Write);
// return await stream.HashingCopy(outputStream, token, job);
} }
private const int CHUNK_SIZE = 1024 * 1024 * 8; private const int CHUNK_SIZE = 1024 * 1024 * 8;
private async Task<Hash> ResettingDownloader(HttpResponseMessage response, HttpRequestMessage message, AbsolutePath outputPath, IJob job, CancellationToken token) private async Task<Hash> ResettingDownloader(HttpResponseMessage response, HttpRequestMessage message, AbsolutePath outputPath, IJob job, CancellationToken token)
{ {
using var rented = MemoryPool<byte>.Shared.Rent(CHUNK_SIZE); using var rented = MemoryPool<byte>.Shared.Rent(CHUNK_SIZE);
var buffer = rented.Memory; var buffer = rented.Memory;
@ -65,7 +68,7 @@ public class SingleThreadedDownloader : IHttpDownloader
var inputStream = await response.Content.ReadAsStreamAsync(token); var inputStream = await response.Content.ReadAsStreamAsync(token);
await using var outputStream = outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); await using var outputStream = outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None);
long writePosition = 0; long writePosition = 0;
while (running && !token.IsCancellationRequested) while (running && !token.IsCancellationRequested)
{ {
var totalRead = 0; var totalRead = 0;
@ -112,7 +115,7 @@ public class SingleThreadedDownloader : IHttpDownloader
{ {
writePosition += totalRead; writePosition += totalRead;
if (job != null) if (job != null)
await job.Report(totalRead, token); await job.Report(totalRead, token);
message = CloneMessage(message); message = CloneMessage(message);
message.Headers.Range = new RangeHeaderValue(writePosition, writePosition + CHUNK_SIZE); message.Headers.Range = new RangeHeaderValue(writePosition, writePosition + CHUNK_SIZE);

View File

@ -8,6 +8,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Downloader" Version="3.0.4" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.2-mauipre.1.22102.15" /> <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.2-mauipre.1.22102.15" />
</ItemGroup> </ItemGroup>

View File

@ -9,6 +9,7 @@ public interface IJob
public long? Size { get; set; } public long? Size { get; set; }
public long Current { get; } public long Current { get; }
public string Description { get; } public string Description { get; }
public ValueTask Report(int processedSize, CancellationToken token); public ValueTask Report(long processedSize, CancellationToken token);
public void ReportNoWait(int processedSize); public void ReportNoWait(int processedSize);
public void ResetProgress();
} }

View File

@ -23,9 +23,9 @@ public class Job<T> : IJob, IDisposable
public long Current { get; internal set; } public long Current { get; internal set; }
public long? Size { get; set; } public long? Size { get; set; }
public async ValueTask Report(int processedSize, CancellationToken token) public async ValueTask Report(long processedSize, CancellationToken token)
{ {
await Resource.Report(this, processedSize, token); await Resource.Report(this, (int)Math.Min(processedSize, int.MaxValue), token);
Current += processedSize; Current += processedSize;
OnUpdate?.Invoke(this, (Percent.FactoryPutInRange(Current, Size ?? 1), Current)); OnUpdate?.Invoke(this, (Percent.FactoryPutInRange(Current, Size ?? 1), Current));
} }
@ -36,5 +36,11 @@ public class Job<T> : IJob, IDisposable
OnUpdate?.Invoke(this, (Percent.FactoryPutInRange(Current, Size ?? 1), Current)); OnUpdate?.Invoke(this, (Percent.FactoryPutInRange(Current, Size ?? 1), Current));
} }
public void ResetProgress()
{
Current = 0;
OnUpdate?.Invoke(this, (Percent.FactoryPutInRange(Current, Size ?? 1), Current));
}
public event EventHandler<(Percent Progress, long Processed)> OnUpdate; public event EventHandler<(Percent Progress, long Processed)> OnUpdate;
} }

View File

@ -19,7 +19,7 @@ public class Resource<T> : IResource<T>
private long _totalUsed; private long _totalUsed;
public IEnumerable<IJob> Jobs => _tasks.Values; public IEnumerable<IJob> Jobs => _tasks.Values;
public Resource(string? humanName = null, int? maxTasks = null, long maxThroughput = long.MaxValue) public Resource(string? humanName = null, int? maxTasks = null, long maxThroughput = long.MaxValue, CancellationToken? token = null)
{ {
Name = humanName ?? "<unknown>"; Name = humanName ?? "<unknown>";
MaxTasks = maxTasks ?? Environment.ProcessorCount; MaxTasks = maxTasks ?? Environment.ProcessorCount;
@ -27,11 +27,11 @@ public class Resource<T> : IResource<T>
_semaphore = new SemaphoreSlim(MaxTasks); _semaphore = new SemaphoreSlim(MaxTasks);
_channel = Channel.CreateBounded<PendingReport>(10); _channel = Channel.CreateBounded<PendingReport>(10);
_tasks = new ConcurrentDictionary<ulong, Job<T>>(); _tasks = new ConcurrentDictionary<ulong, Job<T>>();
var tsk = StartTask(CancellationToken.None); var tsk = StartTask(token ?? CancellationToken.None);
} }
public Resource(string humanName, Func<Task<(int MaxTasks, long MaxThroughput)>> settingGetter) public Resource(string humanName, Func<Task<(int MaxTasks, long MaxThroughput)>> settingGetter, CancellationToken? token = null)
{ {
Name = humanName; Name = humanName;
_tasks = new ConcurrentDictionary<ulong, Job<T>>(); _tasks = new ConcurrentDictionary<ulong, Job<T>>();
@ -43,9 +43,9 @@ public class Resource<T> : IResource<T>
MaxThroughput = maxThroughput; MaxThroughput = maxThroughput;
_semaphore = new SemaphoreSlim(MaxTasks); _semaphore = new SemaphoreSlim(MaxTasks);
_channel = Channel.CreateBounded<PendingReport>(10); _channel = Channel.CreateBounded<PendingReport>(10);
await StartTask(CancellationToken.None); await StartTask(token ?? CancellationToken.None);
}); }, token ?? CancellationToken.None);
} }
public int MaxTasks { get; set; } public int MaxTasks { get; set; }

View File

@ -4,6 +4,7 @@ using System.Diagnostics;
using System.Linq; using System.Linq;
using System.Net.Http; using System.Net.Http;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
@ -45,6 +46,10 @@ public static class ServiceExtensions
public static IServiceCollection AddOSIntegrated(this IServiceCollection service, public static IServiceCollection AddOSIntegrated(this IServiceCollection service,
Action<OSIntegratedOptions>? cfn = null) Action<OSIntegratedOptions>? cfn = null)
{ {
// Register app-wide cancellation token source to allow clean termination
service.AddSingleton(new CancellationTokenSource());
service.AddTransient(typeof(CancellationToken), s => s.GetRequiredService<CancellationTokenSource>().Token);
var options = new OSIntegratedOptions(); var options = new OSIntegratedOptions();
cfn?.Invoke(options); cfn?.Invoke(options);
@ -117,25 +122,25 @@ public static class ServiceExtensions
// Resources // Resources
service.AddAllSingleton<IResource, IResource<DownloadDispatcher>>(s => service.AddAllSingleton<IResource, IResource<DownloadDispatcher>>(s =>
new Resource<DownloadDispatcher>("Downloads", GetSettings(s, "Downloads"))); new Resource<DownloadDispatcher>("Downloads", GetSettings(s, "Downloads"), s.GetRequiredService<CancellationToken>()));
service.AddAllSingleton<IResource, IResource<HttpClient>>(s => new Resource<HttpClient>("Web Requests", GetSettings(s, "Web Requests"))); service.AddAllSingleton<IResource, IResource<HttpClient>>(s => new Resource<HttpClient>("Web Requests", GetSettings(s, "Web Requests"), s.GetRequiredService<CancellationToken>()));
service.AddAllSingleton<IResource, IResource<Context>>(s => new Resource<Context>("VFS", GetSettings(s, "VFS"))); service.AddAllSingleton<IResource, IResource<Context>>(s => new Resource<Context>("VFS", GetSettings(s, "VFS"), s.GetRequiredService<CancellationToken>()));
service.AddAllSingleton<IResource, IResource<FileHashCache>>(s => service.AddAllSingleton<IResource, IResource<FileHashCache>>(s =>
new Resource<FileHashCache>("File Hashing", GetSettings(s, "File Hashing"))); new Resource<FileHashCache>("File Hashing", GetSettings(s, "File Hashing"), s.GetRequiredService<CancellationToken>()));
service.AddAllSingleton<IResource, IResource<Client>>(s => service.AddAllSingleton<IResource, IResource<Client>>(s =>
new Resource<Client>("Wabbajack Client", GetSettings(s, "Wabbajack Client"))); new Resource<Client>("Wabbajack Client", GetSettings(s, "Wabbajack Client"), s.GetRequiredService<CancellationToken>()));
service.AddAllSingleton<IResource, IResource<FileExtractor.FileExtractor>>(s => service.AddAllSingleton<IResource, IResource<FileExtractor.FileExtractor>>(s =>
new Resource<FileExtractor.FileExtractor>("File Extractor", GetSettings(s, "File Extractor"))); new Resource<FileExtractor.FileExtractor>("File Extractor", GetSettings(s, "File Extractor"), s.GetRequiredService<CancellationToken>()));
service.AddAllSingleton<IResource, IResource<ACompiler>>(s => service.AddAllSingleton<IResource, IResource<ACompiler>>(s =>
new Resource<ACompiler>("Compiler", GetSettings(s, "Compiler"))); new Resource<ACompiler>("Compiler", GetSettings(s, "Compiler"), s.GetRequiredService<CancellationToken>()));
service.AddAllSingleton<IResource, IResource<IInstaller>>(s => service.AddAllSingleton<IResource, IResource<IInstaller>>(s =>
new Resource<IInstaller>("Installer", GetSettings(s, "Installer"))); new Resource<IInstaller>("Installer", GetSettings(s, "Installer"), s.GetRequiredService<CancellationToken>()));
service.AddAllSingleton<IResource, IResource<IUserInterventionHandler>>(s => service.AddAllSingleton<IResource, IResource<IUserInterventionHandler>>(s =>
new Resource<IUserInterventionHandler>("User Intervention", 1)); new Resource<IUserInterventionHandler>("User Intervention", 1, token: s.GetRequiredService<CancellationToken>()));
service.AddSingleton<LoggingRateLimiterReporter>(); service.AddSingleton<LoggingRateLimiterReporter>();

View File

@ -54,12 +54,10 @@ public class ModListDownloadMaintainer
return await _hashCache.FileHashCachedAsync(path, token.Value) == metadata.DownloadMetadata!.Hash; return await _hashCache.FileHashCachedAsync(path, token.Value) == metadata.DownloadMetadata!.Hash;
} }
public (IObservable<Percent> Progress, Task Task) DownloadModlist(ModlistMetadata metadata, CancellationToken? token = null) public (IObservable<Percent> Progress, Task Task) DownloadModlist(ModlistMetadata metadata, CancellationToken token)
{ {
var path = ModListPath(metadata); var path = ModListPath(metadata);
token ??= CancellationToken.None;
var progress = new Subject<Percent>(); var progress = new Subject<Percent>();
progress.OnNext(Percent.Zero); progress.OnNext(Percent.Zero);
@ -69,7 +67,7 @@ public class ModListDownloadMaintainer
{ {
Interlocked.Increment(ref _downloadingCount); Interlocked.Increment(ref _downloadingCount);
using var job = await _rateLimiter.Begin($"Downloading {metadata.Title}", metadata.DownloadMetadata!.Size, using var job = await _rateLimiter.Begin($"Downloading {metadata.Title}", metadata.DownloadMetadata!.Size,
token.Value); token);
job.OnUpdate += (_, pr) => { progress.OnNext(pr.Progress); }; job.OnUpdate += (_, pr) => { progress.OnNext(pr.Progress); };
@ -78,17 +76,21 @@ public class ModListDownloadMaintainer
State = _dispatcher.Parse(new Uri(metadata.Links.Download))!, State = _dispatcher.Parse(new Uri(metadata.Links.Download))!,
Size = metadata.DownloadMetadata.Size, Size = metadata.DownloadMetadata.Size,
Hash = metadata.DownloadMetadata.Hash Hash = metadata.DownloadMetadata.Hash
}, path, job, token.Value); }, path, job, token);
if (token.IsCancellationRequested)
{
return;
}
await _hashCache.FileHashWriteCache(path, hash); await _hashCache.FileHashWriteCache(path, hash);
await path.WithExtension(Ext.MetaData).WriteAllTextAsync(JsonSerializer.Serialize(metadata)); await path.WithExtension(Ext.MetaData).WriteAllTextAsync(JsonSerializer.Serialize(metadata), token);
} }
finally finally
{ {
progress.OnCompleted(); progress.OnCompleted();
Interlocked.Decrement(ref _downloadingCount); Interlocked.Decrement(ref _downloadingCount);
} }
}); }, token);
return (progress, tsk); return (progress, tsk);
} }

View File

@ -117,6 +117,8 @@ public class Context
async Task HandleFile(VirtualFile file, IExtractedFile sfn) async Task HandleFile(VirtualFile file, IExtractedFile sfn)
{ {
if (token.IsCancellationRequested) return;
if (filesByParent.ContainsKey(file)) if (filesByParent.ContainsKey(file))
sfn.CanMove = false; sfn.CanMove = false;
@ -138,6 +140,7 @@ public class Context
} }
catch (Exception ex) catch (Exception ex)
{ {
if (token.IsCancellationRequested) return;
await using var stream = await sfn.GetStream(); await using var stream = await sfn.GetStream();
var hash = await stream.HashingCopy(Stream.Null, token); var hash = await stream.HashingCopy(Stream.Null, token);
if (hash != file.Hash) if (hash != file.Hash)