Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1286 from wabbajack-tools/dont-write-game-file-metas

Browse Source 
Dont write game file metas
This commit is contained in:
Timothy Baldridge 2021-01-29 07:43:34 -07:00 committed by GitHub
commit 48eac013d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 108 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@ -1,5 +1,12 @@
### Changelog
#### Version - 2.4.1.2 - 1/29/2020
* Don't install .meta files for files sourced from the game folder
* Fix bug MO2 archive name detection in .meta files (rare bug with FO4VR and other like games)
* Catch exceptions when ECS downloads manifest data
* Don't double-index game files in some situations (duplicate game names in config files)
* Update all deps
#### Version - 2.4.1.1 - 1/13/2020
* HOTFIX: Fix game file sources that don't have MO2 specific names

2
Compression.BSA.Test/BSATests.cs
View File

@ -70,6 +70,7 @@ namespace Compression.BSA.Test
}
[Theory]
//[InlineData(Game.SkyrimSpecialEdition, 29194)] // 3D NPCS This fails not sure why
[InlineData(Game.SkyrimSpecialEdition, 12604)] // SkyUI
[InlineData(Game.Skyrim, 3863)] // SkyUI
[InlineData(Game.Skyrim, 51473)] // INeed
@ -150,6 +151,7 @@ namespace Compression.BSA.Test
Assert.Equal(pair.ai.Path, pair.bi.Path);
//Equal(pair.ai.Compressed, pair.bi.Compressed);
Assert.Equal(pair.ai.Size, pair.bi.Size);
Utils.Log($"Comparing {pair.ai.Path} to {pair.bi.Path}");
Assert.Equal(await GetData(pair.ai), await GetData(pair.bi));
});
}

20
Compression.BSA/Utils.cs
View File

@ -6,6 +6,9 @@ using System.Threading.Tasks;
using Wabbajack.Common;
using Path = Alphaleonis.Win32.Filesystem.Path;
// Yeah, we know, but BSAs use UTF7, that's how old they are
#pragma warning disable 618
namespace Compression.BSA
{
public static class BSAUtils
@ -20,11 +23,12 @@ namespace Compression.BSA
private static Encoding GetEncoding(VersionType version)
{
if (version == VersionType.TES3)
return Encoding.ASCII;
if (version == VersionType.SSE)
return Windows1252;
return Encoding.UTF7;
return version switch
{
VersionType.TES3 => Encoding.ASCII,
VersionType.SSE => Windows1252,
_ => Encoding.UTF7
};
}
public static string ReadStringLen(this BinaryReader rdr, VersionType version)
@ -72,9 +76,10 @@ namespace Compression.BSA
}
/// <summary>
/// Returns bytes for a \0 terminated string
/// Returns \0 terminated bytes for a string encoded with a given BSA version's encoding format
/// </summary>
/// <param name="val"></param>
/// <param name="version"></param>
/// <returns></returns>
public static byte[] ToBZString(this RelativePath val, VersionType version)
{
@ -101,9 +106,10 @@ namespace Compression.BSA
}
/// <summary>
/// Returns bytes for a \0 terminated string prefixed by a length
/// Returns bytes for a string with a length prefix, version is the BSA version
/// </summary>
/// <param name="val"></param>
/// <param name="version"></param>
/// <returns></returns>
public static byte[] ToTermString(this string val, VersionType version)
{

2
Wabbajack.CLI/Wabbajack.CLI.csproj
View File

@ -19,7 +19,7 @@
<ItemGroup>
<PackageReference Include="CommandLineParser" Version="2.9.0-preview1" />
<PackageReference Include="F23.StringSimilarity" Version="3.1.0" />
<PackageReference Include="Markdig" Version="0.22.1" />
<PackageReference Include="Markdig" Version="0.23.0" />
<PackageReference Include="System.Reactive" Version="5.0.0" />
<PackageReference Include="System.Reactive.Linq" Version="5.0.0" />
</ItemGroup>

4
Wabbajack.Common/Paths/AbsolutePath.cs
View File

@ -89,13 +89,13 @@ namespace Wabbajack.Common
public ValueTask<FileStream> Create()
{
var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => File.Open(path, FileMode.Create, FileAccess.ReadWrite));
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => File.Open(path, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 1024 * 32));
}
public ValueTask<FileStream> OpenWrite()
{
var path = _path;
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite));
return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read, 1024 * 32));
}
public async Task WriteAllTextAsync(string text)

4
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -49,7 +49,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.29" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.30" />
<PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
<PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
@ -61,7 +61,7 @@
<PackageReference Include="System.Reactive" Version="5.0.0" />
<PackageReference Include="System.Security.Cryptography.ProtectedData" Version="5.0.0" />
<PackageReference Include="System.Security.Principal.Windows" Version="5.0.0" />
<PackageReference Include="YamlDotNet" Version="8.1.2" />
<PackageReference Include="YamlDotNet" Version="9.1.4" />
</ItemGroup>
<ItemGroup>
<Compile Update="Serialization\PrimitiveHandlers.cs">

2
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -151,7 +151,7 @@ TOP:
{
read = await webs.ReadAsync(buffer, 0, bufferSize);
}
catch (Exception ex)
catch (Exception)
{
if (readThisCycle == 0)
throw;

2
Wabbajack.Lib/MO2Installer.cs
View File

@ -262,7 +262,7 @@ namespace Wabbajack.Lib
if (HashedArchives.TryGetValue(archive.Hash, out var paths))
{
var metaPath = paths.WithExtension(Consts.MetaFileExtension);
if (!metaPath.Exists)
if (!metaPath.Exists && !(archive.State is GameFileSourceDownloader.State))
{
Status($"Writing {metaPath.FileName}");
var meta = AddInstalled(archive.State.GetMetaIni()).ToArray();

24
Wabbajack.Lib/NexusApi/HtmlInterface.cs
View File

@ -1,8 +1,12 @@
using System.Linq;
using System;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using HtmlAgilityPack;
using Wabbajack.Common;
using Wabbajack.Lib.LibCefHelpers;
using Wabbajack.Lib.WebAutomation;
namespace Wabbajack.Lib.NexusApi
{
@ -10,15 +14,21 @@ namespace Wabbajack.Lib.NexusApi
{
public static async Task<PermissionValue> GetUploadPermissions(Game game, long modId)
{
var client = new Lib.Http.Client();
if (Utils.HaveEncryptedJson("nexus-cookies"))
HtmlDocument response;
using (var driver = await Driver.Create())
{
var cookies = await Utils.FromEncryptedJson<Helpers.Cookie[]>("nexus-cookies");
client.AddCookies(cookies);
await driver.NavigateTo(new Uri($"https://nexusmods.com/{game.MetaData().NexusName}/mods/{modId}"));
TOP:
response = await driver.GetHtmlAsync();
if (response!.Text!.Contains("This process is automatic. Your browser will redirect to your requested content shortly."))
{
await Task.Delay(5000);
goto TOP;
}
}
var response = await client.GetHtmlAsync($"https://nexusmods.com/{game.MetaData().NexusName}/mods/{modId}");
var hidden = response.DocumentNode
.Descendants()
.Any(n => n.Id == $"{modId}-title" && n.InnerText == "Hidden mod");

8
Wabbajack.Lib/Wabbajack.Lib.csproj
View File

@ -25,7 +25,7 @@
<Version>2.2.2.1</Version>
</PackageReference>
<PackageReference Include="HtmlAgilityPack">
<Version>1.11.29</Version>
<Version>1.11.30</Version>
</PackageReference>
<PackageReference Include="MegaApiClient">
<Version>1.8.2</Version>
@ -37,13 +37,13 @@
<Version>2.1.1</Version>
</PackageReference>
<PackageReference Include="ReactiveUI">
<Version>13.0.27</Version>
<Version>13.1.1</Version>
</PackageReference>
<PackageReference Include="ReactiveUI.Fody">
<Version>13.0.27</Version>
<Version>13.1.1</Version>
</PackageReference>
<PackageReference Include="SharpCompress">
<Version>0.26.0</Version>
<Version>0.27.1</Version>
</PackageReference>
<PackageReference Include="System.Collections.Immutable">
<Version>5.0.0</Version>

9
Wabbajack.Lib/WebAutomation/WebAutomation.cs
View File

@ -5,6 +5,7 @@ using System.Threading.Tasks;
using System.Windows;
using CefSharp;
using CefSharp.OffScreen;
using HtmlAgilityPack;
using Wabbajack.Common;
using Wabbajack.Lib.LibCefHelpers;
@ -83,6 +84,14 @@ namespace Wabbajack.Lib.WebAutomation
return await _browser.GetSourceAsync();
}
public async ValueTask<HtmlDocument> GetHtmlAsync()
{
var body = await GetSourceAsync();
var doc = new HtmlDocument();
doc.LoadHtml(body);
return doc;
}
public Action<Uri?> DownloadHandler {
set => _driver.DownloadHandler = value;
}

5
Wabbajack.Server.Test/Wabbajack.Server.Test.csproj
View File

@ -10,7 +10,10 @@
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.3" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
<PackageReference Include="coverlet.collector" Version="1.3.0" />
<PackageReference Include="coverlet.collector" Version="3.0.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="XunitContext" Version="3.0.0" />
</ItemGroup>

2
Wabbajack.Server/Wabbajack.Server.csproj
View File

@ -18,7 +18,7 @@
<PackageReference Include="Discord.Net.WebSocket" Version="2.2.0" />
<PackageReference Include="FluentFTP" Version="33.0.3" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="5.0.1" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="5.0.2" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Nettle" Version="1.3.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.6.3" />

21
Wabbajack.Test/ContentRightsManagementTests.cs
View File

@ -9,10 +9,11 @@ using Wabbajack.Common;
using System.Threading.Tasks;
using Wabbajack.Lib.NexusApi;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Test
{
public class ContentRightsManagementTests : IDisposable
public class ContentRightsManagementTests : ATestBase
{
private ValidateModlist validate;
private WorkQueue queue;
@ -27,17 +28,12 @@ namespace Wabbajack.Test
";
public ContentRightsManagementTests()
{
queue = new WorkQueue();
validate = new ValidateModlist();
validate.LoadServerWhitelist(server_whitelist);
}
public void Dispose()
public override void Dispose()
{
queue?.Dispose();
base.Dispose();
}
@ -127,5 +123,12 @@ namespace Wabbajack.Test
Assert.Equal(HTMLInterface.PermissionValue.NotFound, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 24287));
}
public ContentRightsManagementTests(ITestOutputHelper output) : base(output)
{
queue = new WorkQueue();
validate = new ValidateModlist();
validate.LoadServerWhitelist(server_whitelist);
}
}
}

5
Wabbajack.Test/Wabbajack.Test.csproj
View File

@ -33,7 +33,10 @@
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.3" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
<PackageReference Include="coverlet.collector" Version="1.3.0" />
<PackageReference Include="coverlet.collector" Version="3.0.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="XunitContext" Version="3.0.0" />
</ItemGroup>

13
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -201,14 +201,17 @@ namespace Wabbajack.VirtualFileSystem
}
}
/// <summary>
/// Extract the set of files and call the callback for each, handing it a stream factory and the virtual file,
/// top level archives (native archives) will be processed in parallel. Duplicate files will not be
/// Extracts a file
/// </summary>
/// <<param name="queue"></param>
/// <param name="files"></param>
/// <param name="callback"></param>
/// <param name="queue">Work queue to use when required by some formats</param>
/// <param name="files">Predefined list of files to extract, all others will be skipped</param>
/// <param name="callback">Func called for each file extracted</param>
/// <param name="tempFolder">Optional: folder to use for temporary storage</param>
/// <param name="updateTracker">Optional: Status update tracker</param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
public async Task Extract(WorkQueue queue, HashSet<VirtualFile> files, Func<VirtualFile, IExtractedFile, ValueTask> callback, AbsolutePath? tempFolder = null, StatusUpdateTracker updateTracker = null)
{
var top = new VirtualFile();

22
Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs
View File

@ -17,7 +17,7 @@ namespace Wabbajack.VirtualFileSystem
{
public static class FileExtractor2
{
public static readonly SignatureChecker ArchiveSigs = new SignatureChecker(Definitions.FileType.TES3,
public static readonly SignatureChecker ArchiveSigs = new(Definitions.FileType.TES3,
Definitions.FileType.BSA,
Definitions.FileType.BA2,
Definitions.FileType.ZIP,
@ -26,19 +26,19 @@ namespace Wabbajack.VirtualFileSystem
Definitions.FileType.RAR_NEW,
Definitions.FileType._7Z);
private static Extension OMODExtension = new Extension(".omod");
private static Extension FOMODExtension = new Extension(".fomod");
private static Extension OMODExtension = new(".omod");
private static Extension FOMODExtension = new(".fomod");
private static Extension BSAExtension = new Extension(".bsa");
private static Extension BSAExtension = new(".bsa");
public static readonly HashSet<Extension> ExtractableExtensions = new HashSet<Extension>
{
new Extension(".bsa"),
new Extension(".ba2"),
new Extension(".7z"),
new Extension(".7zip"),
new Extension(".rar"),
new Extension(".zip"),
new(".bsa"),
new(".ba2"),
new(".7z"),
new(".7zip"),
new(".rar"),
new(".zip"),
OMODExtension,
FOMODExtension
};
@ -196,7 +196,7 @@ namespace Wabbajack.VirtualFileSystem
else
{
spoolFile = new TempFile(tempPath.Combine(Guid.NewGuid().ToString())
.WithExtension(source.Extension));
.WithExtension(sf.Name.FileName.Extension));
await using var s = await sf.GetStream();
await spoolFile.Path.WriteAllAsync(s);
source = spoolFile.Path;

2
Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj
View File

@ -17,7 +17,7 @@
<PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
<PackageReference Include="K4os.Hash.Crc" Version="1.1.4" />
<PackageReference Include="OMODFramework" Version="2.1.0.2" />
<PackageReference Include="SharpCompress" Version="0.26.0" />
<PackageReference Include="SharpCompress" Version="0.27.1" />
<PackageReference Include="System.Collections.Immutable" Version="5.0.0" />
</ItemGroup>
<ItemGroup>

2
Wabbajack/View Models/BackNavigatingVM.cs
View File

@ -29,7 +29,7 @@ namespace Wabbajack
public ViewModel NavigateBackTarget { get; set; }
public ReactiveCommand<Unit, Unit> BackCommand { get; protected set; }
private readonly ObservableAsPropertyHelper<bool> _IsActive;
protected ObservableAsPropertyHelper<bool> _IsActive;
public bool IsActive => _IsActive.Value;
public Subject<bool> IsBackEnabledSubject { get; } = new Subject<bool>();

1
Wabbajack/View Models/Compilers/CompilerVM.cs
View File

@ -45,7 +45,6 @@ namespace Wabbajack
public ObservableCollectionExtended<IStatusMessage> Log => MWVM.Log;
public ReactiveCommand<Unit, Unit> BackCommand { get; }
public ReactiveCommand<Unit, Unit> GoToCommand { get; }
public ReactiveCommand<Unit, Unit> CloseWhenCompleteCommand { get; }
public ReactiveCommand<Unit, Unit> BeginCommand { get; }

5
Wabbajack/View Models/Installers/InstallerVM.cs
View File

@ -85,15 +85,10 @@ namespace Wabbajack
private readonly ObservableAsPropertyHelper<bool> _LoadingModlist;
public bool LoadingModlist => _LoadingModlist.Value;
private readonly ObservableAsPropertyHelper<bool> _IsActive;
public bool IsActive => _IsActive.Value;
// Command properties
public ReactiveCommand<Unit, Unit> ShowManifestCommand { get; }
public ReactiveCommand<Unit, Unit> OpenReadmeCommand { get; }
public ReactiveCommand<Unit, Unit> VisitModListWebsiteCommand { get; }
public ReactiveCommand<Unit, Unit> BackCommand { get; }
public ReactiveCommand<Unit, Unit> CloseWhenCompleteCommand { get; }
public ReactiveCommand<Unit, Unit> GoToInstallCommand { get; }

2
Wabbajack/View Models/Installers/MO2InstallerVM.cs
View File

@ -90,7 +90,7 @@ namespace Wabbajack
// Load settings
_CurrentSettings = installerVM.WhenAny(x => x.ModListLocation.TargetPath)
.Select(path => path == null ? null : installerVM.MWVM.Settings.Installer.Mo2ModlistSettings.TryCreate(path))
.Select(path => path == default ? null : installerVM.MWVM.Settings.Installer.Mo2ModlistSettings.TryCreate(path))
.ToGuiProperty(this, nameof(CurrentSettings));
this.WhenAny(x => x.CurrentSettings)
.Pairwise()

5
Wabbajack/View Models/Settings/AuthorFilesVM.cs
View File

@ -16,9 +16,6 @@ namespace Wabbajack
private readonly ObservableAsPropertyHelper<Visibility> _isVisible;
public Visibility IsVisible => _isVisible.Value;
private readonly ObservableAsPropertyHelper<string> _selectedFile;
public ICommand SelectFile { get; }
public ICommand HyperlinkCommand { get; }
public IReactiveCommand Upload { get; }
@ -69,7 +66,7 @@ namespace Wabbajack
_isUploading.OnNext(false);
}
}, IsUploading.StartWith(false).Select(u => !u)
.CombineLatest(Picker.WhenAnyValue(t => t.TargetPath).Select(f => f != null),
.CombineLatest(Picker.WhenAnyValue(t => t.TargetPath).Select(f => f != default),
(a, b) => a && b));
}
}

3
Wabbajack/View Models/WebBrowserVM.cs
View File

@ -45,9 +45,10 @@ namespace Wabbajack
return new WebBrowserVM(url);
}
public void Dispose()
public override void Dispose()
{
Browser.Dispose();
base.Dispose();
}
}
}

6
Wabbajack/Wabbajack.csproj
View File

@ -71,9 +71,9 @@
<PackageReference Include="MahApps.Metro.IconPacks" Version="4.8.0" />
<PackageReference Include="PInvoke.Gdi32" Version="0.7.78" />
<PackageReference Include="PInvoke.User32" Version="0.7.78" />
<PackageReference Include="ReactiveUI" Version="13.0.27" />
<PackageReference Include="ReactiveUI.Fody" Version="13.0.27" />
<PackageReference Include="ReactiveUI.WPF" Version="13.0.27" />
<PackageReference Include="ReactiveUI" Version="13.1.1" />
<PackageReference Include="ReactiveUI.Fody" Version="13.1.1" />
<PackageReference Include="ReactiveUI.WPF" Version="13.1.1" />
<PackageReference Include="SharpDX.DXGI" Version="4.2.0" />
<PackageReference Include="WindowsAPICodePack-Shell" Version="1.1.1" />
<PackageReference Include="WPFThemes.DarkBlend" Version="1.0.8" />