Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1657 from wabbajack-tools/compiler-screens

Browse Source 
Compiler screens
This commit is contained in:
Timothy Baldridge 2021-10-16 15:38:21 -06:00 committed by GitHub
commit ead1bd48ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
83 changed files with 1315 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Wabbajack.App/App.axaml.cs
View File

@ -6,6 +6,7 @@ using Avalonia.Markup.Xaml;
using CefNet;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using ReactiveUI;
using Splat;
using Wabbajack.App.Controls;
@ -36,6 +37,10 @@ namespace Wabbajack.App
public override void OnFrameworkInitializationCompleted()
{
var host = Host.CreateDefaultBuilder(Array.Empty<string>())
.ConfigureLogging(c =>
{
c.ClearProviders();
})
.ConfigureServices((host, services) =>
{
services.AddAppServices();

1
Wabbajack.App/Controls/BrowseItemViewModel.cs
View File

@ -17,6 +17,7 @@ using Wabbajack.App.Messages;
using Wabbajack.App.ViewModels;
using Wabbajack.Common;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Installer;

22
Wabbajack.App/Controls/LogView.axaml Normal file
View File

@ -0,0 +1,22 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:controls="clr-namespace:Wabbajack.App.Controls"
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
x:Class="Wabbajack.App.Controls.LogView">
<ScrollViewer ScrollChanged="ScrollViewer_OnScrollChanged" x:Name="ScrollViewer">
<ItemsControl x:Name="Messages">
<ItemsControl.ItemsPanel>
<ItemsPanelTemplate>
<StackPanel></StackPanel>
</ItemsPanelTemplate>
</ItemsControl.ItemsPanel>
<ItemsControl.ItemTemplate>
<DataTemplate>
<controls:LogViewItem></controls:LogViewItem>
</DataTemplate>
</ItemsControl.ItemTemplate>
</ItemsControl>
</ScrollViewer>
</UserControl>

27
Wabbajack.App/Controls/LogView.axaml.cs Normal file
View File

@ -0,0 +1,27 @@
using Avalonia.Controls;
using Avalonia.Controls.Mixins;
using Avalonia.ReactiveUI;
using Microsoft.Extensions.DependencyInjection;
using ReactiveUI;
using Wabbajack.App.Utilities;
namespace Wabbajack.App.Controls;
public partial class LogView : ReactiveUserControl<LogViewModel>
{
public LogView()
{
DataContext = App.Services.GetService<LogViewModel>()!;
InitializeComponent();
this.WhenActivated(disposables =>
{
this.OneWayBind(ViewModel, vm => vm.Messages, view => view.Messages.Items)
.DisposeWith(disposables);
});
}
private void ScrollViewer_OnScrollChanged(object? sender, ScrollChangedEventArgs e)
{
ScrollViewer.ScrollToEnd();
}
}

8
Wabbajack.App/Controls/LogViewItem.axaml Normal file
View File

@ -0,0 +1,8 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
x:Class="Wabbajack.App.Controls.LogViewItem">
<TextBlock x:Name="Message" FontSize="10"></TextBlock>
</UserControl>

20
Wabbajack.App/Controls/LogViewItem.axaml.cs Normal file
View File

@ -0,0 +1,20 @@
using Avalonia.Controls.Mixins;
using Avalonia.ReactiveUI;
using ReactiveUI;
using Wabbajack.App.Utilities;
namespace Wabbajack.App.Controls;
public partial class LogViewItem : ReactiveUserControl<LoggerProvider.ILogMessage>, IActivatableView
{
public LogViewItem()
{
InitializeComponent();
this.WhenActivated(disposables =>
{
this.OneWayBind(ViewModel, vm => vm.ShortMessage, view => view.Message.Text)
.DisposeWith(disposables);
});
}
}

37
Wabbajack.App/Controls/LogViewModel.cs Normal file
View File

@ -0,0 +1,37 @@
using System;
using System.Collections.ObjectModel;
using Avalonia.Controls.Mixins;
using DynamicData;
using Microsoft.Extensions.Logging;
using ReactiveUI;
using Wabbajack.App.Utilities;
using Wabbajack.App.ViewModels;
namespace Wabbajack.App.Controls;
public class LogViewModel : ViewModelBase, IActivatableViewModel
{
private readonly LoggerProvider _provider;
private readonly SourceCache<LoggerProvider.ILogMessage, long> _messages;
public readonly ReadOnlyObservableCollection<LoggerProvider.ILogMessage> _messagesFiltered;
public ReadOnlyObservableCollection<LoggerProvider.ILogMessage> Messages => _messagesFiltered;
public LogViewModel(LoggerProvider provider)
{
_messages = new SourceCache<LoggerProvider.ILogMessage, long>(m => m.MessageId);
//_messages.LimitSizeTo(100);
Activator = new ViewModelActivator();
_provider = provider;
_messages.Connect()
.Bind(out _messagesFiltered)
.Subscribe();
_provider.Messages
.Subscribe(m => _messages.AddOrUpdate(m));
}
}

14
Wabbajack.App/Controls/RemovableListItem.axaml Normal file
View File

@ -0,0 +1,14 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:i="clr-namespace:Material.Icons.Avalonia;assembly=Material.Icons.Avalonia"
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
x:Class="Wabbajack.App.Controls.RemovableListItem">
<StackPanel Orientation="Horizontal">
<Button x:Name="DeleteButton">
<i:MaterialIcon Kind="MinusCircle"></i:MaterialIcon>
</Button>
<TextBlock x:Name="Text" VerticalAlignment="Center"></TextBlock>
</StackPanel>
</UserControl>

24
Wabbajack.App/Controls/RemovableListItem.axaml.cs Normal file
View File

@ -0,0 +1,24 @@
using Avalonia.Controls.Mixins;
using Avalonia.ReactiveUI;
using ReactiveUI;
using Wabbajack.App.ViewModels;
namespace Wabbajack.App.Controls;
public partial class RemovableListItem : ReactiveUserControl<RemovableItemViewModel>, IActivatableView
{
public RemovableListItem()
{
InitializeComponent();
this.WhenActivated(disposables =>
{
this.OneWayBind(ViewModel, vm => vm.Text, view => view.Text.Text)
.DisposeWith(disposables);
this.BindCommand(ViewModel, vm => vm.DeleteCommand, view => view.DeleteButton)
.DisposeWith(disposables);
});
}
}

22
Wabbajack.App/Controls/RemovableListItemViewModel.cs Normal file
View File

@ -0,0 +1,22 @@
using System;
using System.Reactive;
using ReactiveUI;
using ReactiveUI.Fody.Helpers;
using Wabbajack.App.ViewModels;
namespace Wabbajack.App.Controls;
public class RemovableItemViewModel : ViewModelBase
{
[Reactive]
public string Text { get; set; }
[Reactive]
public ReactiveCommand<Unit, Unit> DeleteCommand { get; set; }
public RemovableItemViewModel()
{
Activator = new ViewModelActivator();
}
}

8
Wabbajack.App/Messages/StartCompilation.cs Normal file
View File

@ -0,0 +1,8 @@
using Wabbajack.Compiler;
namespace Wabbajack.App.Messages;
public record StartCompilation(CompilerSettings Settings)
{
}

62
Wabbajack.App/Models/SettingsManager.cs Normal file
View File

@ -0,0 +1,62 @@
using System;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using JetBrains.Annotations;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
namespace Wabbajack.App.Models;
public class SettingsManager
{
private readonly Configuration _configuration;
private readonly DTOSerializer _dtos;
private readonly ILogger<SettingsManager> _logger;
public SettingsManager(ILogger<SettingsManager> logger, Configuration configuration, DTOSerializer dtos)
{
_logger = logger;
_dtos = dtos;
_configuration = configuration;
_configuration.SavedSettingsLocation.CreateDirectory();
}
private AbsolutePath GetPath(string key) => _configuration.SavedSettingsLocation.Combine(key).WithExtension(Ext.Json);
public async Task Save<T>(string key, T value)
{
var tmp = GetPath(key).WithExtension(Ext.Temp);
await using (var s = tmp.Open(FileMode.Create, FileAccess.Write))
{
await JsonSerializer.SerializeAsync(s, value, _dtos.Options);
}
await tmp.MoveToAsync(GetPath(key), true, CancellationToken.None);
}
public async Task<T> Load<T>(string key)
where T : new()
{
var path = GetPath(key);
try
{
if (path.FileExists())
{
await using (var s = path.Open(FileMode.Open))
{
return (await JsonSerializer.DeserializeAsync<T>(s, _dtos.Options))!;
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Loading settings {Key}", key);
}
return new T();
}
}

4
Wabbajack.App/Screens/BrowseViewModel.cs
View File

@ -24,6 +24,7 @@ using Wabbajack.Networking.WabbajackClientApi;
using DynamicData.Binding;
using Microsoft.Extensions.DependencyInjection;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Installer;
using Wabbajack.Paths;
@ -45,8 +46,7 @@ namespace Wabbajack.App.Screens
private readonly IResource<DownloadDispatcher> _dispatcherLimiter;
private SourceCache<BrowseItemViewModel, string> _modLists = new(x => x.MachineURL);
public readonly ReadOnlyObservableCollection<BrowseItemViewModel> _filteredModLists;
public ReadOnlyObservableCollection<BrowseItemViewModel> ModLists => _filteredModLists;

15
Wabbajack.App/Screens/CompilationView.axaml Normal file
View File

@ -0,0 +1,15 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
xmlns:i="clr-namespace:Material.Icons.Avalonia;assembly=Material.Icons.Avalonia"
xmlns:controls="clr-namespace:Wabbajack.App.Controls"
x:Class="Wabbajack.App.Screens.CompilationView">
<Grid RowDefinitions="40, 5, 5, *, 40">
<TextBlock Grid.Row="0" x:Name="StatusText" FontSize="20" FontWeight="Bold">[20/30] Installing Files</TextBlock>
<ProgressBar Grid.Row="1" x:Name="StepsProgress" Maximum="1000" Value="40"></ProgressBar>
<ProgressBar Grid.Row="2" x:Name="StepProgress" Maximum="10000" Value="30"></ProgressBar>
<controls:LogView Grid.Row="3" x:Name="LogView"></controls:LogView>
</Grid>
</UserControl>

26
Wabbajack.App/Screens/CompilationView.axaml.cs Normal file
View File

@ -0,0 +1,26 @@
using Avalonia.Controls.Mixins;
using ReactiveUI;
using Wabbajack.App.ViewModels;
using Wabbajack.App.Views;
namespace Wabbajack.App.Screens;
public partial class CompilationView : ScreenBase<CompilationViewModel>
{
public CompilationView()
{
InitializeComponent();
this.WhenActivated(disposables =>
{
this.OneWayBind(ViewModel, vm => vm.StatusText, view => view.StatusText.Text)
.DisposeWith(disposables);
this.OneWayBind(ViewModel, vm => vm.StepsProgress, view => view.StepsProgress.Value, p => p.Value * 1000)
.DisposeWith(disposables);
this.OneWayBind(ViewModel, vm => vm.StepProgress, view => view.StepProgress.Value, p => p.Value * 10000)
.DisposeWith(disposables);
});
}
}

68
Wabbajack.App/Screens/CompilationViewModel.cs Normal file
View File

@ -0,0 +1,68 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Avalonia.Threading;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using ReactiveUI;
using ReactiveUI.Fody.Helpers;
using Wabbajack.App.Messages;
using Wabbajack.App.ViewModels;
using Wabbajack.Common;
using Wabbajack.Compiler;
using Wabbajack.RateLimiter;
namespace Wabbajack.App.Screens;
public class CompilationViewModel : ViewModelBase, IReceiverMarker, IReceiver<StartCompilation>
{
private readonly IServiceProvider _provider;
private ACompiler _compiler;
private readonly ILogger<CompilationViewModel> _logger;
[Reactive] public string StatusText { get; set; } = "";
[Reactive] public Percent StepsProgress { get; set; } = Percent.Zero;
[Reactive] public Percent StepProgress { get; set; } = Percent.Zero;
public CompilationViewModel(ILogger<CompilationViewModel> logger, IServiceProvider provider)
{
_logger = logger;
_provider = provider;
Activator = new ViewModelActivator();
}
public void Receive(StartCompilation val)
{
if (val.Settings is MO2CompilerSettings mo2)
{
var compiler = _provider.GetService<MO2Compiler>()!;
compiler.Settings = mo2;
_compiler = compiler;
_compiler.OnStatusUpdate += (sender, update) =>
{
Dispatcher.UIThread.InvokeAsync(() =>
{
StatusText = update.StatusText;
StepsProgress = update.StepsProgress;
StepProgress = update.StepProgress;
});
};
}
Compile().FireAndForget();
}
public async Task Compile()
{
try
{
await _compiler.Begin(CancellationToken.None);
}
catch (Exception ex)
{
_logger.LogError(ex, "During Compilation: {Message}", ex.Message);
StatusText = $"ERRORED: {ex.Message}";
}
}
}

63
Wabbajack.App/Screens/CompilerConfigurationView.axaml Normal file
View File

@ -0,0 +1,63 @@
<UserControl xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
xmlns:i="clr-namespace:Material.Icons.Avalonia;assembly=Material.Icons.Avalonia"
xmlns:controls="clr-namespace:Wabbajack.App.Controls"
x:Class="Wabbajack.App.Screens.CompilerConfigurationView">
<Grid RowDefinitions="40, *, 40">
<TextBlock Grid.Row="0" x:Name="StatusText" FontSize="20" FontWeight="Bold">Compiler Configuration</TextBlock>
<Grid Grid.Row="1" ColumnDefinitions="Auto, *" RowDefinitions="Auto, Auto, Auto, Auto, Auto, Auto, Auto" Margin="4">
<Label Grid.Column="0" Grid.Row="0" HorizontalAlignment="Right">Title:</Label>
<TextBox Grid.Column="1" Grid.Row="0" x:Name="Title"></TextBox>
<Label Grid.Column="0" Grid.Row="1" HorizontalAlignment="Right">Settings File:</Label>
<controls:FileSelectionBox Grid.Column="1" Grid.Row="1" x:Name="SettingsFile"
AllowedExtensions=".txt|.json">
</controls:FileSelectionBox>
<Label Grid.Column="0" Grid.Row="2" HorizontalAlignment="Right">Source:</Label>
<controls:FileSelectionBox Grid.Column="1" Grid.Row="2" x:Name="Source" SelectFolder="True"></controls:FileSelectionBox>
<Label Grid.Column="0" Grid.Row="3" HorizontalAlignment="Right">Downloads Folder:</Label>
<controls:FileSelectionBox Grid.Column="1" Grid.Row="3" x:Name="DownloadsFolder" SelectFolder="True"></controls:FileSelectionBox>
<Label Grid.Column="0" Grid.Row="4" HorizontalAlignment="Right">Base Game:</Label>
<ComboBox Grid.Column="1" Grid.Row="4" x:Name="BaseGame">
<ComboBox.ItemTemplate>
<DataTemplate>
<TextBlock Text="{Binding Path=HumanFriendlyGameName}"></TextBlock>
</DataTemplate>
</ComboBox.ItemTemplate>
</ComboBox>
<Label Grid.Column="0" Grid.Row="5" HorizontalAlignment="Right">Output Folder:</Label>
<controls:FileSelectionBox Grid.Column="1" Grid.Row="5" x:Name="OutputFolder" SelectFolder="True"></controls:FileSelectionBox>
<Label Grid.Column="0" Grid.Row="6" HorizontalAlignment="Right" VerticalAlignment="Top">Always Enabled:</Label>
<StackPanel Grid.Column="1" Grid.Row="6" Orientation="Vertical">
<Button x:Name="AddAlwaysEnabled">
<i:MaterialIcon Kind="AddCircle"></i:MaterialIcon>
</Button>
<ItemsControl x:Name="AlwaysEnabledList">
<ItemsControl.ItemsPanel>
<ItemsPanelTemplate>
<StackPanel Orientation="Vertical"></StackPanel>
</ItemsPanelTemplate>
</ItemsControl.ItemsPanel>
<ItemsControl.ItemTemplate>
<DataTemplate>
<controls:RemovableListItem></controls:RemovableListItem>
</DataTemplate>
</ItemsControl.ItemTemplate>
</ItemsControl>
</StackPanel>
</Grid>
<Grid ColumnDefinitions="*, Auto, Auto" Grid.Row="2">
<Button Grid.Column="1" x:Name="InferSettings" Click="InferSettings_OnClick">
<TextBlock>Infer Settings</TextBlock>
</Button>
<Button Grid.Column="2" x:Name="StartCompilation">
<TextBlock>Start Compilation</TextBlock>
</Button>
</Grid>
</Grid>
</UserControl>

90
Wabbajack.App/Screens/CompilerConfigurationView.axaml.cs Normal file
View File

@ -0,0 +1,90 @@
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Disposables;
using System.Threading.Tasks;
using Avalonia;
using Avalonia.Controls;
using Avalonia.Interactivity;
using Avalonia.Threading;
using ReactiveUI;
using Wabbajack.App.Controls;
using Wabbajack.App.Views;
using Wabbajack.Common;
using Wabbajack.Paths;
namespace Wabbajack.App.Screens
{
public partial class CompilerConfigurationView : ScreenBase<CompilerConfigurationViewModel>
{
public CompilerConfigurationView()
{
InitializeComponent();
AddAlwaysEnabled.Command = ReactiveCommand.Create(() => AddAlwaysEnabled_Command().FireAndForget());
this.WhenActivated(disposables =>
{
this.Bind(ViewModel, vm => vm.SettingsFile, view => view.SettingsFile.SelectedPath)
.DisposeWith(disposables);
this.Bind(ViewModel, vm => vm.Title, view => view.Title.Text)
.DisposeWith(disposables);
this.Bind(ViewModel, vm => vm.SettingsFile, view => view.SettingsFile.SelectedPath)
.DisposeWith(disposables);
this.Bind(ViewModel, vm => vm.Source, view => view.Source.SelectedPath)
.DisposeWith(disposables);
this.Bind(ViewModel, vm => vm.Downloads, view => view.DownloadsFolder.SelectedPath)
.DisposeWith(disposables);
this.Bind(ViewModel, vm => vm.OutputFolder, view => view.OutputFolder.SelectedPath)
.DisposeWith(disposables);
this.OneWayBind(ViewModel, vm => vm.AllGames, view => view.BaseGame.Items)
.DisposeWith(disposables);
this.Bind(ViewModel, vm => vm.BaseGame, view => view.BaseGame.SelectedItem)
.DisposeWith(disposables);
this.BindCommand(ViewModel, vm => vm.StartCompilation, view => view.StartCompilation)
.DisposeWith(disposables);
this.OneWayBind(ViewModel, vm => vm.AlwaysEnabled, view => view.AlwaysEnabledList.Items,
d => d!.Select(itm => new RemovableItemViewModel()
{
Text = itm.ToString(),
DeleteCommand = ReactiveCommand.Create(() => { ViewModel?.RemoveAlwaysExcluded(itm); })
}))
.DisposeWith(disposables);
});
}
private async Task AddAlwaysEnabled_Command()
{
var dialog = new OpenFolderDialog()
{
Title = "Select a folder",
};
var result = await dialog.ShowAsync(App.MainWindow);
if (!string.IsNullOrWhiteSpace(result))
ViewModel!.AddAlwaysExcluded(result.ToAbsolutePath());
}
private void InferSettings_OnClick(object? sender, RoutedEventArgs e)
{
Dispatcher.UIThread.InvokeAsync(async () =>
{
var dialog = new OpenFileDialog()
{
Title = "Select a modlist.txt file",
Filters = new List<FileDialogFilter> { new() {Extensions = new List<string> {"txt"}, Name = "modlist.txt"}},
AllowMultiple = false
};
var result = await dialog.ShowAsync(App.MainWindow);
if (result is { Length: > 0 })
await ViewModel!.InferSettingsFromModlistTxt(result.First().ToAbsolutePath());
});
}
}
}

228
Wabbajack.App/Screens/CompilerConfigurationViewModel.cs Normal file
View File

@ -0,0 +1,228 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reactive;
using System.Text.Json;
using System.Threading.Tasks;
using Avalonia.Controls.Mixins;
using ReactiveUI;
using ReactiveUI.Fody.Helpers;
using Wabbajack.App.Messages;
using Wabbajack.App.Models;
using Wabbajack.App.ViewModels;
using Wabbajack.Common;
using Wabbajack.Compiler;
using Wabbajack.DTOs;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Installer;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Consts = Wabbajack.Compiler.Consts;
namespace Wabbajack.App.Screens;
public class CompilerConfigurationViewModel : ViewModelBase, IReceiverMarker
{
private readonly DTOSerializer _dtos;
private readonly SettingsManager _settingsManager;
[Reactive]
public string Title { get; set; }
[Reactive]
public AbsolutePath SettingsFile { get; set; }
[Reactive]
public AbsolutePath Downloads { get; set; }
[Reactive]
public GameMetaData BaseGame { get; set; }
[Reactive]
public AbsolutePath Source { get; set; }
[Reactive]
public AbsolutePath GamePath { get; set; }
[Reactive]
public string SelectedProfile { get; set; }
[Reactive]
public AbsolutePath OutputFolder { get; set; }
[Reactive]
public IEnumerable<GameMetaData> AllGames { get; set; }
[Reactive]
public ReactiveCommand<Unit, Unit> StartCompilation { get; set; }
[Reactive]
public IEnumerable<RelativePath> AlwaysEnabled { get; set; } = Array.Empty<RelativePath>();
public AbsolutePath SettingsOutputLocation => Source.Combine(Title).WithExtension(IsMO2Compilation ? Ext.MO2CompilerSettings : Ext.CompilerSettings);
[Reactive]
public bool IsMO2Compilation { get; set; }
public CompilerConfigurationViewModel(DTOSerializer dtos, SettingsManager settingsManager)
{
_settingsManager = settingsManager;
_dtos = dtos;
Activator = new ViewModelActivator();
AllGames = GameRegistry.Games.Values.ToArray();
StartCompilation = ReactiveCommand.Create(() => BeginCompilation().FireAndForget());
OutputFolder = KnownFolders.EntryPoint;
this.WhenActivated(disposables =>
{
LoadLastCompilation().FireAndForget();
this.WhenAnyValue(v => v.SettingsFile)
.Subscribe( location =>
{
LoadNewSettingsFile(location).FireAndForget();
})
.DisposeWith(disposables);
});
}
private async Task LoadNewSettingsFile(AbsolutePath location)
{
if (location == default) return;
if (location.FileExists()) await LoadSettings(location);
}
private async Task LoadLastCompilation()
{
var location = await _settingsManager.Load<AbsolutePath>("last_compilation");
SettingsFile = location;
}
private async Task BeginCompilation()
{
var settings = GetSettings();
await SaveSettingsFile();
await _settingsManager.Save("last_compilation", SettingsOutputLocation);
MessageBus.Instance.Send(new StartCompilation(settings));
MessageBus.Instance.Send(new NavigateTo(typeof(CompilationViewModel)));
}
private CompilerSettings GetSettings()
{
return new MO2CompilerSettings
{
Downloads = Downloads,
Source = Source,
Game = BaseGame.Game,
Profile = SelectedProfile,
UseGamePaths = true,
OutputFile = OutputFolder.Combine(SelectedProfile).WithExtension(Ext.Wabbajack),
AlwaysEnabled = AlwaysEnabled.ToArray()
};
}
public bool AddAlwaysExcluded(AbsolutePath path)
{
if (!path.InFolder(Source)) return false;
var relative = path.RelativeTo(Source);
AlwaysEnabled = AlwaysEnabled.Append(relative).Distinct().ToArray();
return true;
}
public void RemoveAlwaysExcluded(RelativePath path)
{
AlwaysEnabled = AlwaysEnabled.Where(p => p != path).ToArray();
}
public async Task InferSettingsFromModlistTxt(AbsolutePath settingsFile)
{
if (settingsFile.FileName == "modlist.txt".ToRelativePath() && settingsFile.Depth > 3)
{
var mo2Folder = settingsFile.Parent.Parent.Parent;
var mo2Ini = mo2Folder.Combine(Consts.MO2IniName);
if (mo2Ini.FileExists())
{
var iniData = mo2Ini.LoadIniFile();
var general = iniData["General"];
BaseGame = GameRegistry.GetByFuzzyName(general["gameName"].FromMO2Ini());
Source = mo2Folder;
SelectedProfile = general["selected_profile"].FromMO2Ini();
GamePath = general["gamePath"].FromMO2Ini().ToAbsolutePath();
Title = SelectedProfile;
var settings = iniData["Settings"];
Downloads = settings["download_directory"].FromMO2Ini().ToAbsolutePath();
IsMO2Compilation = true;
// Find Always Enabled mods
foreach (var modFolder in mo2Folder.Combine("mods").EnumerateDirectories())
{
var iniFile = modFolder.Combine("meta.ini");
if (!iniFile.FileExists()) continue;
var data = iniFile.LoadIniFile();
var generalModData = data["General"];
if ((generalModData["notes"]?.Contains("WABBAJACK_ALWAYS_ENABLE") ?? false) ||
(generalModData["comments"]?.Contains("WABBAJACK_ALWAYS_ENABLE") ?? false))
{
AlwaysEnabled = AlwaysEnabled.Append(modFolder.RelativeTo(mo2Folder)).ToArray();
}
}
if (mo2Folder.Depth > 1)
OutputFolder = mo2Folder.Parent;
await SaveSettingsFile();
SettingsFile = SettingsOutputLocation;
}
}
}
private async Task SaveSettingsFile()
{
await using var st = SettingsOutputLocation.Open(FileMode.Create, FileAccess.Write, FileShare.None);
if (IsMO2Compilation)
await JsonSerializer.SerializeAsync(st, (MO2CompilerSettings)GetSettings(), _dtos.Options);
else
await JsonSerializer.SerializeAsync(st, GetSettings(), _dtos.Options);
}
private async Task LoadSettings(AbsolutePath path)
{
CompilerSettings s;
if (path.Extension == Ext.MO2CompilerSettings)
{
var mo2 = await LoadSettingsFile<MO2CompilerSettings>(path);
AlwaysEnabled = mo2.AlwaysEnabled;
SelectedProfile = mo2.Profile;
s = mo2;
}
else
{
throw new NotImplementedException();
}
Source = s.Source;
Downloads = s.Downloads;
OutputFolder = s.OutputFile.Depth > 1 ? s.OutputFile.Parent : s.OutputFile;
BaseGame = s.Game.MetaData();
}
private async Task<T> LoadSettingsFile<T>(AbsolutePath path)
{
await using var st = path.Open(FileMode.Open);
return (await JsonSerializer.DeserializeAsync<T>(st, _dtos.Options))!;
}
}

1
Wabbajack.App/Screens/StandardInstallationViewModel.cs
View File

@ -19,6 +19,7 @@ using Wabbajack.App.Screens;
using Wabbajack.App.Utilities;
using Wabbajack.App.ViewModels.SubViewModels;
using Wabbajack.Common;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.JsonConverters;

12
Wabbajack.App/ServiceExtensions.cs
View File

@ -6,6 +6,7 @@ using System.Threading.Tasks;
using Avalonia.Threading;
using CefNet;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Wabbajack.App.Controls;
using Wabbajack.App.Interfaces;
using Wabbajack.App.Messages;
@ -31,19 +32,24 @@ namespace Wabbajack.App
{
public static IServiceCollection AddAppServices(this IServiceCollection services)
{
services.AddAllSingleton<ILoggerProvider, LoggerProvider>();
services.AddSingleton<MessageBus>();
services.AddSingleton<MainWindow>();
services.AddSingleton<BrowseViewModel>();
services.AddTransient<BrowseItemViewModel>();
services.AddTransient<LogViewModel>();
services.AddDTOConverters();
services.AddDTOSerializer();
services.AddSingleton<ModeSelectionViewModel>();
services.AddTransient<FileSelectionBoxViewModel>();
services.AddSingleton<IScreenView, ModeSelectionView>();
services.AddSingleton<IScreenView, InstallConfigurationView>();
services.AddSingleton<IScreenView, CompilerConfigurationView>();
services.AddSingleton<IScreenView, StandardInstallationView>();
services.AddSingleton<IScreenView, CompilationView>();
services.AddSingleton<IScreenView, SettingsView>();
services.AddSingleton<IScreenView, BrowseView>();
services.AddSingleton<IScreenView, LauncherView>();
@ -53,11 +59,13 @@ namespace Wabbajack.App
services.AddAllSingleton<IReceiverMarker, StandardInstallationViewModel>();
services.AddAllSingleton<IReceiverMarker, InstallConfigurationViewModel>();
services.AddAllSingleton<IReceiverMarker, CompilerConfigurationViewModel>();
services.AddAllSingleton<IReceiverMarker, MainWindowViewModel>();
services.AddAllSingleton<IReceiverMarker, SettingsViewModel>();
services.AddAllSingleton<IReceiverMarker, NexusLoginViewModel>();
services.AddAllSingleton<IReceiverMarker, LoversLabOAuthLoginViewModel>();
services.AddAllSingleton<IReceiverMarker, VectorPlexusOAuthLoginViewModel>();
services.AddAllSingleton<IReceiverMarker, CompilationViewModel>();
services.AddAllSingleton<IReceiverMarker, LauncherViewModel>();
// Services
@ -83,6 +91,8 @@ namespace Wabbajack.App
ModListsDownloadLocation = KnownFolders.EntryPoint.Combine("downloaded_mod_lists"),
SavedSettingsLocation = KnownFolders.WabbajackAppLocal.Combine("saved_settings")
});
services.AddSingleton<SettingsManager>();
services.AddSingleton(s =>
{

72
Wabbajack.App/Utilities/LoggerProvider.cs Normal file
View File

@ -0,0 +1,72 @@
using System;
using System.Collections.Immutable;
using System.Reactive.Disposables;
using System.Reactive.Subjects;
using System.Threading;
using Microsoft.Extensions.Logging;
namespace Wabbajack.App.Utilities;
public class LoggerProvider : ILoggerProvider
{
private Subject<ILogMessage> _messages = new();
public IObservable<ILogMessage> Messages => _messages;
private long _messageID = 0;
public long NextMessageId()
{
return Interlocked.Increment(ref _messageID);
}
public void Dispose()
{
_messages.Dispose();
}
public ILogger CreateLogger(string categoryName)
{
return new Logger(this, categoryName);
}
public class Logger : ILogger
{
private readonly LoggerProvider _provider;
private ImmutableList<object> Scopes = ImmutableList<object>.Empty;
private readonly string _categoryName;
public Logger(LoggerProvider provider, string categoryName)
{
_categoryName = categoryName;
_provider = provider;
}
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
{
_provider._messages.OnNext(new LogMessage<TState>(_provider.NextMessageId(), logLevel, eventId, state, exception, formatter));
}
public bool IsEnabled(LogLevel logLevel)
{
return true;
}
public IDisposable BeginScope<TState>(TState state)
{
Scopes = Scopes.Add(state);
return Disposable.Create(() => Scopes = Scopes.Remove(state));
}
}
public interface ILogMessage
{
long MessageId { get; }
string ShortMessage { get; }
}
record LogMessage<TState>(long MessageId, LogLevel LogLevel, EventId EventId, TState State, Exception? Exception, Func<TState, Exception?, string> Formatter) : ILogMessage
{
public string ShortMessage => Formatter(State, Exception);
}
}

1
Wabbajack.App/ViewModels/MainWindowViewModel.cs
View File

@ -78,6 +78,7 @@ namespace Wabbajack.App.ViewModels
.DisposeWith(disposables);
});
CurrentScreen = (Control)_screens.First(s => s.ViewModelType == typeof(ModeSelectionViewModel));
LoadFirstScreen().FireAndForget();

1
Wabbajack.App/Views/MainWindow.axaml
View File

@ -38,6 +38,7 @@
<Button Grid.Column="0" x:Name="BackButton">
<i:MaterialIcon Kind="ArrowBack"> </i:MaterialIcon>
</Button>
<TextBlock Grid.Column="1" HorizontalAlignment="Right" VerticalAlignment="Center" x:Name="ResourceStatus"></TextBlock>
<Button Grid.Column="2" x:Name="SettingsButton">

2
Wabbajack.App/Views/ModeSelectionView.axaml
View File

@ -17,7 +17,7 @@
<Grid Grid.Row="1" ColumnDefinitions="*, *, *, *" HorizontalAlignment="Center">
<controls:LargeIconButton Grid.Column="0" Text="Browse" Icon="CloudDownload" x:Name="BrowseButton"></controls:LargeIconButton>
<controls:LargeIconButton Grid.Column="1" Text="Install" Icon="HarddiskPlus" x:Name="InstallButton"></controls:LargeIconButton>
<controls:LargeIconButton Grid.Column="2" Text="Compile" Icon="DatabaseImport"></controls:LargeIconButton>
<controls:LargeIconButton Grid.Column="2" Text="Compile" Icon="DatabaseImport" x:Name="CompileButton"></controls:LargeIconButton>
<controls:LargeIconButton Grid.Column="3" Text="Play" Icon="TelevisionPlay"></controls:LargeIconButton>
</Grid>
</Grid>

5
Wabbajack.App/Views/ModeSelectionView.axaml.cs
View File

@ -24,6 +24,11 @@ namespace Wabbajack.App.Views
{
MessageBus.Instance.Send(new NavigateTo(typeof(BrowseViewModel)));
}).DisposeWith(disposables);
CompileButton.Button.Command = ReactiveCommand.Create(() =>
{
MessageBus.Instance.Send(new NavigateTo(typeof(CompilerConfigurationViewModel)));
}).DisposeWith(disposables);
});
}

10
Wabbajack.App/Wabbajack.App.csproj
View File

@ -12,16 +12,16 @@
<AvaloniaResource Include="Assets\Wabbajack.axaml" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Avalonia" Version="0.10.7" />
<PackageReference Include="Avalonia.Desktop" Version="0.10.7" />
<PackageReference Include="Avalonia.Diagnostics" Version="0.10.7" />
<PackageReference Include="Avalonia.ReactiveUI" Version="0.10.7" />
<PackageReference Include="Avalonia" Version="0.10.8" />
<PackageReference Include="Avalonia.Desktop" Version="0.10.8" />
<PackageReference Include="Avalonia.Diagnostics" Version="0.10.8" />
<PackageReference Include="Avalonia.ReactiveUI" Version="0.10.8" />
<PackageReference Include="CefNet.Avalonia" Version="94.0.21246.700" />
<PackageReference Include="DynamicData" Version="7.3.1" />
<PackageReference Include="Fizzler.Systems.HtmlAgilityPack" Version="1.2.1" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.37" />
<PackageReference Include="Material.Icons.Avalonia" Version="1.0.2" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="ReactiveUI.Fody" Version="16.2.6" />
<PackageReference Include="ReactiveUI.Validation" Version="2.2.1" />
<PackageReference Include="XamlNameReferenceGenerator" Version="1.3.4" />

12
Wabbajack.CLI/Wabbajack.CLI.csproj
View File

@ -13,12 +13,12 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="System.CommandLine" Version="2.0.0-beta1.21504.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="System.CommandLine" Version="2.0.0-beta1.21514.1" />
</ItemGroup>
<ItemGroup>

3
Wabbajack.Common/Ext.cs
View File

@ -20,5 +20,8 @@ namespace Wabbajack.Common
public static Extension Json = new(".json");
public static Extension Md = new(".md");
public static Extension MetaData = new(".metadata");
public static Extension CompilerSettings = new(".compiler_settings");
public static Extension MO2CompilerSettings = new (".mo2_compiler_settings");
public static Extension Temp = new(".temp");
}
}

17
Wabbajack.Common/HttpExtensions.cs
View File

@ -1,6 +1,12 @@
using System.Linq;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.DTOs.Logins;
using Wabbajack.Networking.Http;
using Wabbajack.RateLimiter;
namespace Wabbajack.Common
{
@ -18,5 +24,16 @@ namespace Wabbajack.Common
return msg;
}
public static async Task<TValue?> GetFromJsonAsync<TValue>(this HttpClient client, IResource<HttpClient> limiter, HttpRequestMessage msg,
JsonSerializerOptions? options, CancellationToken cancellationToken = default)
{
using var job = await limiter.Begin($"HTTP Get JSON {msg.RequestUri}", 0, cancellationToken);
using var response = await client.SendAsync(msg, cancellationToken);
if (!response.IsSuccessStatusCode)
throw new HttpException(response);
await job.Report((int)response.Content.Headers.ContentLength!, cancellationToken);
return await response.Content.ReadFromJsonAsync<TValue>(options, cancellationToken);
}
}
}

3
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -9,6 +9,7 @@
<ItemGroup>
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
<ProjectReference Include="..\Wabbajack.Networking.Http\Wabbajack.Networking.Http.csproj" />
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
</ItemGroup>
@ -28,7 +29,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="System.Reactive" Version="5.0.0" />
</ItemGroup>

2
Wabbajack.Compiler.Test/Wabbajack.Compiler.Test.csproj
View File

@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
<PackageReference Include="xunit" Version="2.4.2-pre.12" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">

113
Wabbajack.Compiler/ACompiler.cs
View File

@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Compiler.CompilationSteps;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.DownloadStates;
@ -20,6 +21,7 @@ using Wabbajack.Installer;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
using Wabbajack.VFS;
namespace Wabbajack.Compiler
@ -37,7 +39,7 @@ namespace Wabbajack.Compiler
private readonly FileHashCache _hashCache;
protected readonly Context _vfs;
private readonly TemporaryFileManager _manager;
public readonly CompilerSettings _settings;
public CompilerSettings _settings;
private readonly AbsolutePath _stagingFolder;
public readonly ParallelOptions _parallelOptions;
@ -48,6 +50,16 @@ namespace Wabbajack.Compiler
public readonly IGameLocator _locator;
private readonly DTOSerializer _dtos;
public readonly IBinaryPatchCache _patchCache;
private long _maxStepProgress = 0;
private int _currentStep = 0;
private string _statusText;
private long _currentStepProgress;
private readonly Stopwatch _updateStopWatch = new();
protected long MaxSteps { get; set; }
public event EventHandler<StatusUpdate> OnStatusUpdate;
public ACompiler(ILogger logger, FileExtractor.FileExtractor extractor, FileHashCache hashCache, Context vfs, TemporaryFileManager manager, CompilerSettings settings,
ParallelOptions parallelOptions, DownloadDispatcher dispatcher, Client wjClient, IGameLocator locator, DTOSerializer dtos,
@ -69,10 +81,49 @@ namespace Wabbajack.Compiler
_patchOptions = new();
_sourceFileLinks = new();
_patchCache = patchCache;
_updateStopWatch = new();
}
public void NextStep(string statusText, long maxStepProgress = 1)
{
_updateStopWatch.Restart();
_maxStepProgress = maxStepProgress;
_currentStep += 1;
_statusText = statusText;
_logger.LogInformation("Compiler Step: {Step}", statusText);
public CompilerSettings Settings { get; set; }
if (OnStatusUpdate != null)
{
OnStatusUpdate(this, new StatusUpdate($"[{_currentStep}/{MaxSteps}] " + statusText, Percent.FactoryPutInRange(_currentStep, MaxSteps),
Percent.Zero));
}
}
public void UpdateProgress(long stepProgress)
{
Interlocked.Add(ref _currentStepProgress, stepProgress);
lock (_updateStopWatch)
{
if (_updateStopWatch.ElapsedMilliseconds < 100) return;
_updateStopWatch.Restart();
}
if (OnStatusUpdate != null)
{
OnStatusUpdate(this, new StatusUpdate(_statusText, Percent.FactoryPutInRange(_currentStep, MaxSteps),
Percent.FactoryPutInRange(_currentStepProgress, _maxStepProgress)));
}
}
public abstract Task<bool> Begin(CancellationToken token);
public CompilerSettings Settings
{
get => _settings;
set => _settings = value;
}
public Dictionary<Game, HashSet<Hash>> GameHashes { get; set; } = new Dictionary<Game, HashSet<Hash>>();
public Dictionary<Hash, Game[]> GamesWithHashes { get; set; } = new Dictionary<Hash, Game[]>();
@ -144,8 +195,10 @@ namespace Wabbajack.Compiler
public async Task<bool> GatherMetaData()
{
_logger.LogInformation("Getting meta data for {count} archives", SelectedArchives.Count);
NextStep("Gathering Metadata", SelectedArchives.Count);
await SelectedArchives.PDo(_parallelOptions, async a =>
{
UpdateProgress(1);
await _dispatcher.FillInMetadata(a);
});
@ -155,6 +208,7 @@ namespace Wabbajack.Compiler
protected async Task IndexGameFileHashes()
{
NextStep("Indexing Game Files", 1);
if (_settings.UseGamePaths)
{
//taking the games in Settings.IncludedGames + currently compiling game so you can eg
@ -189,7 +243,8 @@ namespace Wabbajack.Compiler
return new IndexedArchive(
_vfs.Index.ByRootPath[path.Combine(state.GameFile)])
{
Name = state.GameFile.ToString().Replace("/", "_").Replace("\\", "_")
Name = state.GameFile.ToString().Replace("/", "_").Replace("\\", "_"),
State = state
};
}));
}
@ -207,6 +262,7 @@ namespace Wabbajack.Compiler
protected async Task CleanInvalidArchivesAndFillState()
{
NextStep("Cleaning Invalid Archives", 1);
var remove = await IndexedArchives.PMap(_parallelOptions, async a =>
{
try
@ -240,6 +296,7 @@ namespace Wabbajack.Compiler
protected async Task InferMetas(CancellationToken token)
{
async Task<bool> HasInvalidMeta(AbsolutePath filename)
{
var metaName = filename.WithExtension(Ext.Meta);
@ -267,6 +324,7 @@ namespace Wabbajack.Compiler
.Where(f => f.FileExists())
.ToList();
NextStep("InferMetas", toFind.Count);
if (toFind.Count == 0)
{
return;
@ -274,8 +332,9 @@ namespace Wabbajack.Compiler
_logger.LogInformation("Attempting to infer {count} metas from the server.", toFind.Count);
await toFind.PDo(_parallelOptions, async f =>
await toFind.PDoAll(async f =>
{
UpdateProgress(1);
var vf = _vfs.Index.ByRootPath[f];
var archives = await _wjClient.GetArchivesForHash(vf.Hash);
@ -307,6 +366,7 @@ namespace Wabbajack.Compiler
protected async Task ExportModList(CancellationToken token)
{
NextStep("Exporting Modlist");
_logger.LogInformation("Exporting ModList to {location}", _settings.OutputFile);
// Modify readme and ModList image to relative paths if they exist
@ -374,8 +434,6 @@ namespace Wabbajack.Compiler
/// </summary>
protected async Task BuildPatches(CancellationToken token)
{
_logger.LogInformation("Gathering patch files");
var toBuild = InstallDirectives.OfType<PatchedFromArchive>()
.Where(p => _patchOptions.GetValueOrDefault(p, Array.Empty<VirtualFile>()).Length > 0)
.SelectMany(p => _patchOptions[p].Select(c => new PatchedFromArchive
@ -387,6 +445,7 @@ namespace Wabbajack.Compiler
}))
.ToArray();
NextStep("Generating Patches", toBuild.Length);
if (toBuild.Length == 0)
{
return;
@ -398,6 +457,7 @@ namespace Wabbajack.Compiler
await _vfs.Extract( indexed.Keys.ToHashSet(),
async (vf, sf) =>
{
UpdateProgress(1);
// For each, extract the destination
var matches = indexed[vf];
foreach (var match in matches)
@ -476,6 +536,7 @@ namespace Wabbajack.Compiler
public async Task GenerateManifest()
{
NextStep("Generating Manifest");
var manifest = new Manifest(ModList);
await using var of = _settings.OutputFile.Open(FileMode.Create, FileAccess.Write);
await _dtos.Serialize(manifest, of);
@ -483,6 +544,7 @@ namespace Wabbajack.Compiler
public async Task GatherArchives()
{
NextStep("Gathering Archives");
_logger.LogInformation("Building a list of archives based on the files required");
var hashes = InstallDirectives.OfType<FromArchive>()
@ -494,7 +556,11 @@ namespace Wabbajack.Compiler
.ToDictionary(f => f.Key, f => f.First());
SelectedArchives.Clear();
SelectedArchives.AddRange(await hashes.PMap(_parallelOptions, hash => ResolveArchive(hash, archives)).ToList());
SelectedArchives.AddRange(await hashes.PMap(_parallelOptions, hash =>
{
UpdateProgress(1);
return ResolveArchive(hash, archives);
}).ToList());
}
public async Task<Archive> ResolveArchive(Hash hash, IDictionary<Hash, IndexedArchive> archives)
@ -509,7 +575,7 @@ namespace Wabbajack.Compiler
public async Task<Archive?> ResolveArchive(IndexedArchive archive)
{
if (archive.IniData == null)
if (archive.State == null && archive.IniData == null)
{
_logger.LogWarning(
"No download metadata found for {archive}, please use MO2 to query info or add a .meta file and try again.",
@ -517,31 +583,38 @@ namespace Wabbajack.Compiler
return null;
}
var state = await _dispatcher.ResolveArchive(archive.IniData!["General"].ToDictionary(d => d.KeyName, d => d.Value));
if (state == null)
IDownloadState? state;
if (archive.State == null)
{
_logger.LogWarning("{archive} could not be handled by any of the downloaders", archive.Name);
return null;
state = await _dispatcher.ResolveArchive(archive.IniData!["General"]
.ToDictionary(d => d.KeyName, d => d.Value));
if (state == null)
{
_logger.LogWarning("{archive} could not be handled by any of the downloaders", archive.Name);
return null;
}
}
else
{
state = archive.State;
}
var result = new Archive
{
State = state,
State = state!,
Name = archive.Name ?? "",
Hash = archive.File.Hash,
Size = archive.File.Size
};
var downloader = _dispatcher.Downloader(result);
await downloader.Prepare();
var token = new CancellationTokenSource();
token.CancelAfter(_settings.MaxVerificationTime);
if (!await _dispatcher.Verify(result, token.Token))
{
_logger.LogWarning(
"Unable to resolve link for {archive}. If this is hosted on the Nexus the file may have been removed.", archive);
"Unable to resolve link for {Archive}. If this is hosted on the Nexus the file may have been removed.", result.State!.PrimaryKeyString);
}
result.Meta = "[General]\n" + string.Join("\n", _dispatcher.MetaIni(result));
@ -581,11 +654,13 @@ namespace Wabbajack.Compiler
.GroupBy(f => _sourceFileLinks[f].File)
.ToDictionary(k => k.Key);
NextStep("Inlining Files");
if (grouped.Count == 0) return;
await _vfs.Extract(grouped.Keys.ToHashSet(), async (vf, sfn) =>
{
UpdateProgress(1);
await using var stream = await sfn.GetStream();
var id = await IncludeFile(stream);
var id = await IncludeFile(stream, token);
foreach (var file in grouped[vf])
{
file.SourceDataID = id;

11
Wabbajack.Compiler/CompilationSteps/IgnoreDisabledMods.cs
View File

@ -16,16 +16,17 @@ namespace Wabbajack.Compiler.CompilationSteps
public IgnoreDisabledMods(ACompiler compiler) : base(compiler)
{
_mo2Compiler = (MO2Compiler) compiler;
var alwaysEnabled = _mo2Compiler.ModInis.Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_ENABLE)).Select(f => f.Key).Distinct();
var alwaysDisabled = _mo2Compiler.ModInis
.Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_DISABLE)).Select(f => f.Key).Distinct();
//var alwaysEnabled = _mo2Compiler.ModInis.Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_ENABLE)).Select(f => f.Key).Distinct();
// TODO: Re-enable this
//var alwaysDisabled = _mo2Compiler.ModInis
// .Where(f => HasFlagInNotes(f.Value, Consts.WABBAJACK_ALWAYS_DISABLE)).Select(f => f.Key).Distinct();
_allEnabledMods = _mo2Compiler._settings.SelectedProfiles
.SelectMany(p => _mo2Compiler._settings.Source.Combine("profiles", p, "modlist.txt").ReadAllLines())
.Where(line => line.StartsWith("+") || line.EndsWith("_separator"))
.Select(line => line[1..].ToRelativePath().RelativeTo(_mo2Compiler.MO2ModsFolder))
.Concat(alwaysEnabled)
.Except(alwaysDisabled)
.Concat(_mo2Compiler.Mo2Settings.AlwaysEnabled.Select(r => r.RelativeTo(_mo2Compiler.Settings.Source)))
//.Except(alwaysDisabled)
.ToList();
}

6
Wabbajack.Compiler/Consts.cs
View File

@ -35,7 +35,7 @@ namespace Wabbajack.Compiler
public static readonly HashSet<Extension> SupportedBSAs = new[] {".bsa", ".ba2"}
.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml"}.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> ConfigFileExtensions = new[]{".json", ".ini", ".yml", ".xml", ".yaml", ".compiler_settings", ".mo2_compiler_settings"}.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> ESPFileExtensions = new []{ ".esp", ".esm", ".esl"}.Select(s => new Extension(s)).ToHashSet();
public static HashSet<Extension> AssetFileExtensions = new[] {".dds", ".tga", ".nif", ".psc", ".pex"}.Select(s => new Extension(s)).ToHashSet();
@ -50,7 +50,7 @@ namespace Wabbajack.Compiler
public static string DOWNLOAD_PATH_MAGIC_BACK = "{--||DOWNLOAD_PATH_MAGIC_BACK||--}";
public static string DOWNLOAD_PATH_MAGIC_DOUBLE_BACK = "{--||DOWNLOAD_PATH_MAGIC_DOUBLE_BACK||--}";
public static string DOWNLOAD_PATH_MAGIC_FORWARD = "{--||DOWNLOAD_PATH_MAGIC_FORWARD||--}";
public static RelativePath MO2IniName => "ModOrganizer.ini".ToRelativePath();
public static object CompilerSettings => "compiler_settings.json".ToRelativePath();
}
}

71
Wabbajack.Compiler/MO2Compiler.cs
View File

@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Compiler.CompilationSteps;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.JsonConverters;
@ -21,22 +22,22 @@ namespace Wabbajack.Compiler
{
public class MO2Compiler : ACompiler
{
private readonly MO2CompilerSettings _mo2Settings;
public MO2CompilerSettings Mo2Settings => (MO2CompilerSettings)Settings;
public MO2Compiler(ILogger<MO2Compiler> logger, FileExtractor.FileExtractor extractor, FileHashCache hashCache, Context vfs,
TemporaryFileManager manager, MO2CompilerSettings settings, ParallelOptions parallelOptions, DownloadDispatcher dispatcher,
Client wjClient, IGameLocator locator, DTOSerializer dtos, IBinaryPatchCache patchCache) :
base(logger, extractor, hashCache, vfs, manager, settings, parallelOptions, dispatcher, wjClient, locator, dtos, patchCache)
{
_mo2Settings = settings;
MaxSteps = 14;
}
public AbsolutePath MO2ModsFolder => _settings.Source.Combine(Consts.MO2ModFolderName);
public AbsolutePath MO2ModsFolder => Settings.Source.Combine(Consts.MO2ModFolderName);
public IniData MO2Ini { get; }
public AbsolutePath MO2ProfileDir => _settings.Source.Combine(Consts.MO2Profiles, _mo2Settings.Profile);
public AbsolutePath MO2ProfileDir => Settings.Source.Combine(Consts.MO2Profiles, Mo2Settings.Profile);
public ConcurrentBag<Directive> ExtraFiles { get; private set; } = new();
public Dictionary<AbsolutePath, IniData> ModInis { get; set; } = new();
@ -45,21 +46,23 @@ namespace Wabbajack.Compiler
return mo2Folder.Combine("downloads");
}
public async Task<bool> Begin(CancellationToken token)
public override async Task<bool> Begin(CancellationToken token)
{
await _wjClient.SendMetric("begin_compiling", _mo2Settings.Profile);
await _wjClient.SendMetric("begin_compiling", Mo2Settings.Profile);
var roots = new List<AbsolutePath> {_settings.Source, _settings.Downloads};
roots.AddRange(_settings.OtherGames.Append(_settings.Game).Select(g => _locator.GameLocation(g)));
var roots = new List<AbsolutePath> {Settings.Source, Settings.Downloads};
roots.AddRange(Settings.OtherGames.Append(Settings.Game).Select(g => _locator.GameLocation(g)));
await _vfs.AddRoots(roots, token);
NextStep("Add Roots", 1);
await _vfs.AddRoots(roots, token); // Step 1
await InferMetas(token);
await InferMetas(token); // Step 2
await _vfs.AddRoot(_settings.Downloads, token);
NextStep("Add Download Roots", 1);
await _vfs.AddRoot(Settings.Downloads, token); // Step 3
// Find all Downloads
IndexedArchives = await _settings.Downloads.EnumerateFiles()
IndexedArchives = await Settings.Downloads.EnumerateFiles()
.Where(f => f.WithExtension(Ext.Meta).FileExists())
.PMap(_parallelOptions,
async f => new IndexedArchive(_vfs.Index.ByRootPath[f])
@ -68,8 +71,7 @@ namespace Wabbajack.Compiler
IniData = f.WithExtension(Ext.Meta).LoadIniFile(),
Meta = await f.WithExtension(Ext.Meta).ReadAllTextAsync()
}).ToList();
await IndexGameFileHashes();
IndexedArchives = IndexedArchives.DistinctBy(a => a.File.AbsoluteName).ToList();
@ -77,9 +79,9 @@ namespace Wabbajack.Compiler
await CleanInvalidArchivesAndFillState();
var mo2Files = _settings.Source.EnumerateFiles()
var mo2Files = Settings.Source.EnumerateFiles()
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(_vfs.Index.ByRootPath[p], p.RelativeTo(_settings.Source)));
.Select(p => new RawSourceFile(_vfs.Index.ByRootPath[p], p.RelativeTo(Settings.Source)));
// If Game Folder Files exists, ignore the game folder
IndexedFiles = IndexedArchives.SelectMany(f => f.File.ThisAndAllChildren)
@ -101,7 +103,7 @@ namespace Wabbajack.Compiler
return false;
}
ModInis = _settings.Source.Combine(Consts.MO2ModFolderName)
ModInis = Settings.Source.Combine(Consts.MO2ModFolderName)
.EnumerateDirectories()
.Select(f =>
{
@ -117,10 +119,15 @@ namespace Wabbajack.Compiler
var stack = MakeStack();
var results = await AllFiles.PMap(_parallelOptions, f => RunStack(stack, f)).ToList();
NextStep("Running Compilation Stack", AllFiles.Count);
var results = await AllFiles.PMap(_parallelOptions, f =>
{
UpdateProgress(1);
return RunStack(stack, f);
}).ToList();
NextStep("Updating Extra files");
// Add the extra files that were generated by the stack
results = results.Concat(ExtraFiles).ToList();
var noMatch = results.OfType<NoMatch>().ToArray();
@ -147,18 +154,18 @@ namespace Wabbajack.Compiler
ModList = new ModList
{
GameType = _settings.Game,
GameType = Settings.Game,
WabbajackVersion = Consts.CurrentMinimumWabbajackVersion,
Archives = SelectedArchives.ToArray(),
Directives = InstallDirectives.ToArray(),
Name = _settings.ModListName,
Author = _settings.ModListAuthor,
Description = _settings.ModListDescription,
Readme = _settings.ModlistReadme,
Name = Settings.ModListName,
Author = Settings.ModListAuthor,
Description = Settings.ModListDescription,
Readme = Settings.ModlistReadme,
Image = ModListImage != default ? ModListImage.FileName : default,
Website = _settings.ModListWebsite,
Version = _settings.ModlistVersion,
IsNSFW = _settings.ModlistIsNSFW
Website = Settings.ModListWebsite,
Version = Settings.ModlistVersion,
IsNSFW = Settings.ModlistIsNSFW
};
await InlineFiles(token);
@ -176,9 +183,11 @@ namespace Wabbajack.Compiler
private async Task RunValidation(ModList modList)
{
NextStep("Validating Archives", modList.Archives.Length);
var allowList = await _wjClient.LoadDownloadAllowList();
foreach (var archive in modList.Archives)
{
UpdateProgress(1);
if (!_dispatcher.IsAllowed(archive, allowList))
{
_logger.LogCritical("Archive {name}, {primaryKeyString} is not allowed", archive.Name,
@ -242,7 +251,7 @@ namespace Wabbajack.Compiler
new IncludeRegex(this, "^[^\\\\]*\\.bat$"),
new IncludeModIniData(this),
new DirectMatch(this),
new IncludeTaggedFiles(this, _settings.Include),
new IncludeTaggedFiles(this, Settings.Include),
// TODO: rework tagged files
// new IncludeTaggedFolders(this, Consts.WABBAJACK_INCLUDE),
new IgnoreExtension(this, Ext.Pyc),
@ -272,10 +281,14 @@ namespace Wabbajack.Compiler
// TODO
//new zEditIntegration.IncludeZEditPatches(this),
new IncludeTaggedFiles(this, _settings.NoMatchInclude),
new IncludeTaggedFiles(this, Settings.NoMatchInclude),
new IncludeRegex(this, ".*\\.txt"),
new IgnorePathContains(this,@"\Edit Scripts\Export\"),
new IgnoreExtension(this, new Extension(".CACHE")),
// Misc
new IncludeRegex(this, "modlist-image\\.png"),
new DropAll(this)
};

1
Wabbajack.Compiler/MO2CompilerSettings.cs
View File

@ -5,5 +5,6 @@ namespace Wabbajack.Compiler
public class MO2CompilerSettings : CompilerSettings
{
public string Profile { get; set; }
public RelativePath[] AlwaysEnabled { get; set; }
}
}

4
Wabbajack.Compression.BSA.Test/Wabbajack.Compression.BSA.Test.csproj
View File

@ -7,10 +7,10 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
<PackageReference Include="xunit" Version="2.4.2-pre.12" />
<PackageReference Include="Xunit.DependencyInjection" Version="7.6.0" />
<PackageReference Include="Xunit.DependencyInjection" Version="7.7.0" />
<PackageReference Include="Xunit.DependencyInjection.Logging" Version="7.5.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

2
Wabbajack.Compression.BSA/Wabbajack.Compression.BSA.csproj
View File

@ -14,7 +14,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.2.12" />
<PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.2.13" />
<PackageReference Include="SharpZipLib" Version="1.3.3" />
</ItemGroup>

2
Wabbajack.DTOs.Test/Wabbajack.DTOs.Test.csproj
View File

@ -10,7 +10,7 @@
<PackageReference Include="FsCheck.Xunit" Version="3.0.0-beta1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
<PackageReference Include="xunit" Version="2.4.2-pre.12" />
<PackageReference Include="Xunit.DependencyInjection" Version="7.6.0" />
<PackageReference Include="Xunit.DependencyInjection" Version="7.7.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>

2
Wabbajack.DTOs/Wabbajack.DTOs.csproj
View File

@ -8,7 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
<ItemGroup>

2
Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj
View File

@ -7,7 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
<PackageReference Include="xunit" Version="2.4.2-pre.12" />
<PackageReference Include="Xunit.DependencyInjection.Logging" Version="7.5.1" />

1
Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs
View File

@ -22,6 +22,7 @@ namespace Wabbajack.Downloaders
.AddNexusDownloader()
.AddIPS4OAuth2Downloaders()
.AddWabbajackCDNDownloader()
.AddGameFileDownloader()
.AddSingleton<DownloadDispatcher>();
}
}

3
Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj
View File

@ -8,6 +8,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Downloaders.GameFile\Wabbajack.Downloaders.GameFile.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.GoogleDrive\Wabbajack.Downloaders.GoogleDrive.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.Http\Wabbajack.Downloaders.Http.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
@ -21,7 +22,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
</Project>

68
Wabbajack.Downloaders.GameFile/GameFileDownloader.cs Normal file
View File

@ -0,0 +1,68 @@
using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.Validation;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
using Wabbajack.VFS;
namespace Wabbajack.Downloaders.GameFile;
public class GameFileDownloader : ADownloader<GameFileSource>
{
private readonly IGameLocator _locator;
private readonly FileHashCache _hashCache;
public GameFileDownloader(IGameLocator locator, FileHashCache hashCache)
{
_locator = locator;
_hashCache = hashCache;
}
public override Task<Hash> Download(Archive archive, GameFileSource state, AbsolutePath destination, IJob job, CancellationToken token)
{
throw new NotImplementedException();
}
public override Task<bool> Prepare()
{
return Task.FromResult(true);
}
public override bool IsAllowed(ServerAllowList allowList, IDownloadState state)
{
return true;
}
public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
{
if (!iniData.TryGetValue("gameName", out var gameName) || !iniData.TryGetValue("gameFile", out var gameFile) ||
!GameRegistry.TryGetByFuzzyName(gameName, out var game)) return null;
return new GameFileSource
{
Game = game.Game,
GameFile = gameFile.ToRelativePath()
};
}
public override Priority Priority => Priority.Normal;
public override async Task<bool> Verify(Archive archive, GameFileSource archiveState, IJob job, CancellationToken token)
{
var fp = archiveState.GameFile.RelativeTo(_locator.GameLocation(archiveState.Game));
if (!fp.FileExists()) return false;
return await _hashCache.FileHashCachedAsync(fp, token) == archive.Hash;
}
public override IEnumerable<string> MetaIni(Archive a, GameFileSource state)
{
return new[]
{
$"gameName={state.Game}",
$"gameFile={state.GameFile}"
};
}
}

12
Wabbajack.Installer/GameLocator.cs → Wabbajack.Downloaders.GameFile/GameLocator.cs
View File

@ -9,7 +9,7 @@ using Microsoft.Extensions.Logging;
using Wabbajack.DTOs;
using Wabbajack.Paths;
namespace Wabbajack.Installer
namespace Wabbajack.Downloaders.GameFile
{
public class GameLocator : IGameLocator
{
@ -28,17 +28,17 @@ namespace Wabbajack.Installer
if (OperatingSystem.IsWindows())
{
_origin = new OriginHandler(true, false, logger);
_gog = new GOGHandler(logger);
_egs = new EGSHandler(logger);
}
_gog = new GOGHandler(logger);
_egs = new EGSHandler(logger);
_locationCache = new Dictionary<Game, AbsolutePath>();
_steam.FindAllGames();
_origin?.FindAllGames();
_gog.FindAllGames();
_egs.FindAllGames();
_gog?.FindAllGames();
_egs?.FindAllGames();
}
public AbsolutePath GameLocation(Game game)

2
Wabbajack.Installer/IGameLocator.cs → Wabbajack.Downloaders.GameFile/IGameLocator.cs
View File

@ -1,7 +1,7 @@
using Wabbajack.DTOs;
using Wabbajack.Paths;
namespace Wabbajack.Installer
namespace Wabbajack.Downloaders.GameFile
{
public interface IGameLocator
{

21
Wabbajack.Downloaders.GameFile/ServiceExtensions.cs Normal file
View File

@ -0,0 +1,21 @@
using Microsoft.Extensions.DependencyInjection;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs;
namespace Wabbajack.Downloaders
{
public static class ServiceExtensions
{
public static IServiceCollection AddGameFileDownloader(this IServiceCollection services)
{
return services.AddAllSingleton<IDownloader, IDownloader<DTOs.DownloadStates.GameFileSource>,
GameFileDownloader>();
}
public static IServiceCollection AddStandardGameLocator(this IServiceCollection services)
{
return services.AddAllSingleton<IGameLocator, GameLocator>();
}
}
}

27
Wabbajack.Downloaders.GameFile/Wabbajack.Downloaders.GameFile.csproj Normal file
View File

@ -0,0 +1,27 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.VFS\Wabbajack.VFS.csproj" />
</ItemGroup>
<ItemGroup>
<Reference Include="Microsoft.Extensions.Logging.Abstractions, Version=6.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60">
<HintPath>..\..\..\Program Files\dotnet\packs\Microsoft.AspNetCore.App.Ref\6.0.0-preview.6.21355.2\ref\net6.0\Microsoft.Extensions.Logging.Abstractions.dll</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="GameFinder.StoreHandlers.EGS" Version="1.7.1" />
<PackageReference Include="GameFinder.StoreHandlers.GOG" Version="1.7.1" />
<PackageReference Include="GameFinder.StoreHandlers.Origin" Version="1.7.1" />
<PackageReference Include="GameFinder.StoreHandlers.Steam" Version="1.7.1" />
</ItemGroup>
</Project>

2
Wabbajack.Downloaders.GoogleDrive/Wabbajack.Downloaders.GoogleDrive.csproj
View File

@ -8,7 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
<ItemGroup>

2
Wabbajack.Downloaders.Http/Wabbajack.Downloaders.Http.csproj
View File

@ -16,7 +16,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
</Project>

2
Wabbajack.Downloaders.IPS4OAuth2Downloader/Wabbajack.Downloaders.IPS4OAuth2Downloader.csproj
View File

@ -14,7 +14,7 @@
<ItemGroup>
<PackageReference Include="F23.StringSimilarity" Version="4.1.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
</Project>

4
Wabbajack.Downloaders.MediaFire/Wabbajack.Downloaders.MediaFire.csproj
View File

@ -7,8 +7,8 @@
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.37" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
<ItemGroup>

24
Wabbajack.Downloaders.Mega/MegaDownloader.cs
View File

@ -71,16 +71,24 @@ namespace Wabbajack.Downloaders.ModDB
{
if (!_apiClient.IsLoggedIn)
await _apiClient.LoginAsync();
try
for (var times = 0; times < 5; times ++)
{
var node = await _apiClient.GetNodeFromLinkAsync(archiveState.Url);
return node != null;
}
catch (Exception)
{
return false;
try
{
var node = await _apiClient.GetNodeFromLinkAsync(archiveState.Url);
if (node != null)
return true;
}
catch (Exception)
{
return false;
}
await Task.Delay(TimeSpan.FromMilliseconds(500), token);
}
return false;
}
public override IEnumerable<string> MetaIni(Archive a, Mega state)

2
Wabbajack.Downloaders.Mega/Wabbajack.Downloaders.Mega.csproj
View File

@ -12,7 +12,7 @@
<ItemGroup>
<PackageReference Include="MegaApiClient" Version="1.9.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
</Project>

2
Wabbajack.Downloaders.ModDB/Wabbajack.Downloaders.ModDB.csproj
View File

@ -13,7 +13,7 @@
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.37" />
<PackageReference Include="MegaApiClient" Version="1.9.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
</Project>

2
Wabbajack.Downloaders.Nexus/Wabbajack.Downloaders.Nexus.csproj
View File

@ -18,7 +18,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Xunit.DependencyInjection" Version="7.6.0" />
<PackageReference Include="Xunit.DependencyInjection" Version="7.7.0" />
<PackageReference Include="Xunit.DependencyInjection.Logging" Version="7.5.1" />
</ItemGroup>

2
Wabbajack.Downloaders.WabbajackCDN/Wabbajack.Downloaders.WabbajackCDN.csproj
View File

@ -15,7 +15,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.Toolkit.HighPerformance" Version="7.1.0" />
</ItemGroup>

6
Wabbajack.FileExtractor.Test/Wabbajack.FileExtractor.Test.csproj
View File

@ -7,11 +7,11 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
<PackageReference Include="xunit" Version="2.4.2-pre.12" />
<PackageReference Include="Xunit.DependencyInjection" Version="7.6.0" />
<PackageReference Include="Xunit.DependencyInjection" Version="7.7.0" />
<PackageReference Include="Xunit.DependencyInjection.Logging" Version="7.5.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

2
Wabbajack.FileExtractor/Wabbajack.FileExtractor.csproj
View File

@ -29,7 +29,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="OMODFramework" Version="3.0.1" />
</ItemGroup>

2
Wabbajack.Installer.Test/Wabbajack.Installer.Test.csproj
View File

@ -7,7 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
<PackageReference Include="xunit" Version="2.4.2-pre.12" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">

8
Wabbajack.Installer/AInstaller.cs
View File

@ -10,6 +10,7 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.DownloadStates;
@ -256,15 +257,18 @@ namespace Wabbajack.Installer
await Task.WhenAll(dispatchers.Select(d => d.Prepare()));
_logger.LogInformation("Downloading validation data");
var validationData = await _wjClient.LoadDownloadAllowList();
_logger.LogInformation("Validating Archives");
foreach (var archive in missing.Where(archive =>
!_downloadDispatcher.Downloader(archive).IsAllowed(validationData, archive.State)))
{
_logger.LogCritical("File {primaryKeyString} failed validation", archive.State.PrimaryKeyString);
return;
}
_logger.LogInformation("Downloading missing archives");
await DownloadMissingArchives(missing, token);
}
@ -437,7 +441,7 @@ namespace Wabbajack.Installer
var existingfiles = _configuration.Install.EnumerateFiles().ToHashSet();
NextStep("Optimizing Modlist: Removing redundant directives", indexed.Count);
await indexed.Values.PMap<Directive, Directive?>(_parallelOptions, async d =>
await indexed.Values.PMapAll<Directive, Directive?>(async d =>
{
UpdateProgress(1);
// Bit backwards, but we want to return null for

59
Wabbajack.Installer/IniExtensions.cs
View File

@ -1,6 +1,10 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using IniParser;
using IniParser.Exceptions;
using IniParser.Model;
using IniParser.Model.Configuration;
using IniParser.Parser;
@ -39,5 +43,60 @@ namespace Wabbajack.Installer
return new FileIniDataParser(IniParser()).ReadData(
new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(file))));
}
public static string FromMO2Ini(this string s)
{
if (string.IsNullOrEmpty(s))
{
return s;
}
if (s.StartsWith("@ByteArray(") && s.EndsWith(")"))
{
return UnescapeUTF8(s.Substring("@ByteArray(".Length, s.Length - "@ByteArray(".Length - ")".Length));
}
return UnescapeString(s);
}
private static string UnescapeString(string s)
{
if (s.Trim().StartsWith("\"") || s.Contains("\\\\"))
return Regex.Unescape(s.Trim('"'));
return s;
}
private static string UnescapeUTF8(string s)
{
var acc = new List<byte>();
for (var i = 0; i < s.Length; i++)
{
var c = s[i];
switch (c)
{
case '\\':
i++;
var nc = s[i];
switch (nc)
{
case '\\':
acc.Add((byte)'\\');
break;
case 'x':
var chrs = s[i + 1] + s[i + 2].ToString();
i += 2;
acc.Add(Convert.ToByte(chrs, 16));
break;
default:
throw new ParsingException($"Not a valid escape characer {nc}");
}
break;
default:
acc.Add((byte)c);
break;
}
}
return Encoding.UTF8.GetString(acc.ToArray());
}
}
}

1
Wabbajack.Installer/StandardInstaller.cs
View File

@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Compression.BSA;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.DownloadStates;

1
Wabbajack.Installer/Wabbajack.Installer.csproj
View File

@ -9,6 +9,7 @@
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Downloaders.Dispatcher\Wabbajack.Downloaders.Dispatcher.csproj" />
<ProjectReference Include="..\Wabbajack.Downloaders.GameFile\Wabbajack.Downloaders.GameFile.csproj" />
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
<ProjectReference Include="..\Wabbajack.FileExtractor\Wabbajack.FileExtractor.csproj" />
<ProjectReference Include="..\Wabbajack.Networking.WabbajackClientApi\Wabbajack.Networking.WabbajackClientApi.csproj" />

2
Wabbajack.Networking.Discord/Wabbajack.Networking.Discord.csproj
View File

@ -12,7 +12,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
</Project>

2
Wabbajack.Networking.GitHub/Wabbajack.Networking.GitHub.csproj
View File

@ -13,7 +13,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Octokit" Version="0.50.0" />
</ItemGroup>

2
Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj
View File

@ -8,7 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
<ItemGroup>

2
Wabbajack.Networking.NexusApi.Test/Wabbajack.Networking.NexusApi.Test.csproj
View File

@ -9,7 +9,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
<PackageReference Include="xunit" Version="2.4.2-pre.12" />
<PackageReference Include="Xunit.DependencyInjection" Version="7.6.0" />
<PackageReference Include="Xunit.DependencyInjection" Version="7.7.0" />
<PackageReference Include="Xunit.DependencyInjection.Logging" Version="7.5.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

4
Wabbajack.Networking.NexusApi/Wabbajack.Networking.NexusApi.csproj
View File

@ -8,8 +8,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
<ItemGroup>

34
Wabbajack.Networking.WabbajackClientApi/Client.cs
View File

@ -11,12 +11,15 @@ using Wabbajack.Common;
using Wabbajack.DTOs;
using Wabbajack.DTOs.CDN;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.DTOs.Logins;
using Wabbajack.DTOs.ModListValidation;
using Wabbajack.DTOs.ServerResponses;
using Wabbajack.DTOs.Validation;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Networking.Http.Interfaces;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
@ -28,27 +31,35 @@ namespace Wabbajack.Networking.WabbajackClientApi
private readonly HttpClient _client;
private readonly Configuration _configuration;
private readonly ITokenProvider<WabbajackApiState> _token;
private readonly ILogger<Client> _logger;
private readonly DTOSerializer _dtos;
private readonly ParallelOptions _parallelOptions;
private readonly IResource<HttpClient> _limiter;
private readonly Configuration _configuration;
public Client(ILogger<Client> logger, HttpClient client, Configuration configuration, DTOSerializer dtos, ParallelOptions parallelOptions)
public Client(ILogger<Client> logger, HttpClient client, ITokenProvider<WabbajackApiState> token, DTOSerializer dtos, IResource<HttpClient> limiter, Configuration configuration)
{
_configuration = configuration;
_token = token;
_client = client;
_logger = logger;
_logger.LogInformation("File hash check (-42) {key}", _configuration.MetricsKey);
_dtos = dtos;
_parallelOptions = parallelOptions;
_limiter = limiter;
}
private async ValueTask<HttpRequestMessage> MakeMessage(HttpMethod method, Uri uri)
{
var msg = new HttpRequestMessage(method, uri);
var key = (await _token.Get())!;
msg.Headers.Add(_configuration.MetricsKeyHeader, key.MetricsKey);
return msg;
}
public async Task SendMetric(string action, string subject)
{
var msg = new HttpRequestMessage(HttpMethod.Get,
$"{_configuration.BuildServerUrl}metrics/{action}/{subject}");
msg.Headers.Add(_configuration.MetricsKeyHeader, _configuration.MetricsKey);
var msg = await MakeMessage(HttpMethod.Get, new Uri($"{_configuration.BuildServerUrl}metrics/{action}/{subject}"));
await _client.SendAsync(msg);
}
@ -78,13 +89,14 @@ namespace Wabbajack.Networking.WabbajackClientApi
public async Task<Archive[]> GetGameArchives(Game game, string version)
{
var url = $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}.json";
return await _client.GetFromJsonAsync<Archive[]>(url) ?? Array.Empty<Archive>();
return await _client.GetFromJsonAsync<Archive[]>(url, _dtos.Options) ?? Array.Empty<Archive>();
}
public async Task<Archive[]> GetArchivesForHash(Hash hash)
{
return await _client.GetFromJsonAsync<Archive[]>(
$"{_configuration.BuildServerUrl}mod_files/by_hash/{hash.ToHex()}", _dtos.Options) ?? Array.Empty<Archive>();
var msg = await MakeMessage(HttpMethod.Get,
new Uri($"{_configuration.BuildServerUrl}mod_files/by_hash/{hash.ToHex()}"));
return await _client.GetFromJsonAsync<Archive[]>(_limiter, msg, _dtos.Options) ?? Array.Empty<Archive>();
}
public async Task<Uri?> GetMirrorUrl(Hash archiveHash)
@ -175,7 +187,7 @@ namespace Wabbajack.Networking.WabbajackClientApi
"https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/unlisted_modlists.json"})
.Take(includeUnlisted ? 3 : 2);
return await lists.PMap(_parallelOptions, async url => await _client.GetFromJsonAsync<ModlistMetadata[]>(url, _dtos.Options)!)
return await lists.PMapAll(async url => await _client.GetFromJsonAsync<ModlistMetadata[]>(_limiter, new HttpRequestMessage(HttpMethod.Get, url), _dtos.Options)!)
.SelectMany(x => x)
.ToArray();

2
Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj
View File

@ -8,7 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="YamlDotNet" Version="11.2.1" />
</ItemGroup>

2
Wabbajack.Paths/AbsolutePath.cs
View File

@ -66,6 +66,8 @@ namespace Wabbajack.Paths
}
}
public int Depth => Parts.Length;
public AbsolutePath ReplaceExtension(Extension newExtension)
{
var paths = new string[Parts.Length];

4
Wabbajack.Server/Wabbajack.Server.csproj
View File

@ -19,9 +19,9 @@
<PackageReference Include="Discord.Net.WebSocket" Version="2.4.0" />
<PackageReference Include="FluentFTP" Version="35.0.5" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.0-rc.1.21452.15" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.0-rc.2.21480.10" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Nettle" Version="1.3.0" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.3" />
</ItemGroup>

4
Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
View File

@ -3,10 +3,9 @@ using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Compiler;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.Logins;
using Wabbajack.Installer;
@ -59,6 +58,7 @@ namespace Wabbajack.Services.OSIntegrated
service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});
service.AddAllSingleton<IResource, IResource<DownloadDispatcher>>(s => new Resource<DownloadDispatcher>(12));
service.AddAllSingleton<IResource, IResource<HttpClient>>(s => new Resource<HttpClient>(12));
service.AddAllSingleton<IResource, IResource<Context>>(s => new Resource<Context>(12));
service.AddAllSingleton<IResource, IResource<FileExtractor.FileExtractor>>(s =>
new Resource<FileExtractor.FileExtractor>(12));

1
Wabbajack.Services.OSIntegrated/StubbedGameLocator.cs
View File

@ -1,3 +1,4 @@
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.Installer;
using Wabbajack.Paths;

2
Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj
View File

@ -13,7 +13,7 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.2.21480.5" />
</ItemGroup>
<ItemGroup>

6
Wabbajack.VFS.Test/Wabbajack.VFS.Test.csproj
View File

@ -7,10 +7,10 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="System.Data.SQLite.Core" Version="1.0.115" />
<PackageReference Include="Xunit.DependencyInjection" Version="7.6.0" />
<PackageReference Include="Xunit.DependencyInjection" Version="7.7.0" />
<PackageReference Include="Xunit.DependencyInjection.Logging" Version="7.5.1" />
</ItemGroup>

7
Wabbajack.VFS/Context.cs
View File

@ -10,6 +10,7 @@ using Wabbajack.FileExtractor.ExtractedFiles;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
namespace Wabbajack.VFS
{
@ -25,10 +26,12 @@ namespace Wabbajack.VFS
public readonly FileHashCache HashCache;
public readonly ILogger<Context> Logger;
public readonly VFSCache VfsCache;
public readonly IResource<Context> Limiter;
public Context(ILogger<Context> logger, ParallelOptions parallelOptions, TemporaryFileManager manager, VFSCache vfsCache,
FileHashCache hashCache, FileExtractor.FileExtractor extractor)
FileHashCache hashCache, IResource<Context> limiter, FileExtractor.FileExtractor extractor)
{
Limiter = limiter;
Logger = logger;
_manager = manager;
Extractor = extractor;
@ -49,7 +52,7 @@ namespace Wabbajack.VFS
var filesToIndex = root.EnumerateFiles().Distinct().ToList();
var allFiles = await filesToIndex
.PMap(_parallelOptions, async f =>
.PMapAll(async f =>
{
if (byPath.TryGetValue(f, out var found))
if (found.LastModified == f.LastModifiedUtc().AsUnixTime() && found.Size == f.Size())

5
Wabbajack.VFS/FileHashCache.cs
View File

@ -5,6 +5,7 @@ using System.Threading.Tasks;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
namespace Wabbajack.VFS
{
@ -113,13 +114,13 @@ namespace Wabbajack.VFS
WriteHashCache(file, hash);
}
public async Task<Hash> FileHashCachedAsync(AbsolutePath file, CancellationToken token)
public async Task<Hash> FileHashCachedAsync(AbsolutePath file, CancellationToken token, IJob? job = null)
{
if (TryGetHashCache(file, out var foundHash)) return foundHash;
await using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
var hash = await fs.HashingCopy(Stream.Null, token);
var hash = await fs.HashingCopy(Stream.Null, token, job);
if (hash != default)
WriteHashCache(file, hash);
return hash;

22
Wabbajack.VFS/VirtualFile.cs
View File

@ -162,14 +162,20 @@ namespace Wabbajack.VFS
IPath relPath, CancellationToken token, int depth = 0)
{
Hash hash;
if (extractedFile is NativeFileStreamFactory)
using (var job = await context.Limiter.Begin("Hash file", 0, token))
{
hash = await context.HashCache.FileHashCachedAsync((AbsolutePath)extractedFile.Name, token);
}
else
{
await using var hstream = await extractedFile.GetStream();
hash = await hstream.HashingCopy(Stream.Null, token);
if (extractedFile is NativeFileStreamFactory)
{
var absPath = (AbsolutePath)extractedFile.Name;
job.Size = absPath.Size();
hash = await context.HashCache.FileHashCachedAsync(absPath, token, job);
}
else
{
await using var hstream = await extractedFile.GetStream();
job.Size = hstream.Length;
hash = await hstream.HashingCopy(Stream.Null, token, job);
}
}
if (context.VfsCache.TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
@ -195,7 +201,9 @@ namespace Wabbajack.VFS
if (TextureExtensions.Contains(relPath.FileName.Extension) && await DDSSig.MatchesAsync(stream) != null)
try
{
using var job = await context.Limiter.Begin("Perceptual hash", self.Size, token);
self.ImageState = await ImageLoader.Load(stream);
await job.Report((int)self.Size, token);
stream.Position = 0;
}
catch (Exception)

2
Wabbajack.VFS/Wabbajack.VFS.csproj
View File

@ -8,7 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.1.21451.13" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0-rc.2.21480.5" />
<PackageReference Include="System.Data.SQLite.Core" Version="1.0.115" />
</ItemGroup>

7
Wabbajack.sln
View File

@ -112,6 +112,8 @@ ProjectSection(SolutionItems) = preProject
nuget.config = nuget.config
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloaders.GameFile", "Wabbajack.Downloaders.GameFile\Wabbajack.Downloaders.GameFile.csproj", "{4F252332-CA77-41DE-95A8-9DF38A81D675}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Launcher", "Wabbajack.Launcher\Wabbajack.Launcher.csproj", "{23D49FCC-A6CB-4873-879B-F90DA1871AA3}"
EndProject
Global
@ -312,6 +314,10 @@ Global
{DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A}.Release|Any CPU.Build.0 = Release|Any CPU
{4F252332-CA77-41DE-95A8-9DF38A81D675}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4F252332-CA77-41DE-95A8-9DF38A81D675}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4F252332-CA77-41DE-95A8-9DF38A81D675}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4F252332-CA77-41DE-95A8-9DF38A81D675}.Release|Any CPU.Build.0 = Release|Any CPU
{23D49FCC-A6CB-4873-879B-F90DA1871AA3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{23D49FCC-A6CB-4873-879B-F90DA1871AA3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{23D49FCC-A6CB-4873-879B-F90DA1871AA3}.Release|Any CPU.ActiveCfg = Release|Any CPU
@ -351,5 +357,6 @@ Global
{5D792FA8-8120-4A5C-A969-2258D3D05F1A} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C}
{29AC8A68-D5EC-43F5-B2CC-72A75545E418} = {98B731EE-4FC0-4482-A069-BCBA25497871}
{DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A} = {F01F8595-5FD7-4506-8469-F4A5522DACC1}
{4F252332-CA77-41DE-95A8-9DF38A81D675} = {98B731EE-4FC0-4482-A069-BCBA25497871}
EndGlobalSection
EndGlobal