Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #198 from erri120/nitpicking-3

Browse Source 
Cleanup
This commit is contained in:
Timothy Baldridge 2019-11-21 09:53:11 -07:00 committed by GitHub
commit a2f0deea33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
147 changed files with 643 additions and 2250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
.gitmodules vendored
View File

54
Compression.BSA.Test/BSATests.cs
View File

@ -18,27 +18,27 @@ namespace Compression.BSA.Test
[TestClass]
public class BSATests
{
private static string StagingFolder = "NexusDownloads";
private static string BSAFolder = "BSAs";
private static string TestDir = "BSA Test Dir";
private static string TempDir = "BSA Temp Dir";
private static string _stagingFolder = "NexusDownloads";
private static string _bsaFolder = "BSAs";
private static string _testDir = "BSA Test Dir";
private static string _tempDir = "BSA Temp Dir";
public TestContext TestContext { get; set; }
private static WorkQueue Queue { get; set; }
[ClassInitialize]
public static void Setup(TestContext TestContext)
public static void Setup(TestContext testContext)
{
Queue = new WorkQueue();
Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f));
if (!Directory.Exists(StagingFolder))
Directory.CreateDirectory(StagingFolder);
Utils.LogMessages.Subscribe(f => testContext.WriteLine(f));
if (!Directory.Exists(_stagingFolder))
Directory.CreateDirectory(_stagingFolder);
if (!Directory.Exists(BSAFolder))
Directory.CreateDirectory(BSAFolder);
if (!Directory.Exists(_bsaFolder))
Directory.CreateDirectory(_bsaFolder);
var mod_ids = new[]
var modIDs = new[]
{
(Game.SkyrimSpecialEdition, 12604), // SkyUI
(Game.Skyrim, 3863), // SkyUI
@ -46,10 +46,10 @@ namespace Compression.BSA.Test
(Game.Fallout4, 22223) // 10mm SMG
};
foreach (var info in mod_ids)
foreach (var info in modIDs)
{
var filename = DownloadMod(info);
var folder = Path.Combine(BSAFolder, info.Item1.ToString(), info.Item2.ToString());
var folder = Path.Combine(_bsaFolder, info.Item1.ToString(), info.Item2.ToString());
if (!Directory.Exists(folder))
Directory.CreateDirectory(folder);
FileExtractor.ExtractAll(Queue, filename, folder);
@ -66,7 +66,7 @@ namespace Compression.BSA.Test
var results = client.GetModFiles(info.Item1, info.Item2);
var file = results.FirstOrDefault(f => f.is_primary) ??
results.OrderByDescending(f => f.uploaded_timestamp).First();
var src = Path.Combine(StagingFolder, file.file_name);
var src = Path.Combine(_stagingFolder, file.file_name);
if (File.Exists(src)) return src;
@ -83,7 +83,7 @@ namespace Compression.BSA.Test
public static IEnumerable<object[]> BSAs()
{
return Directory.EnumerateFiles(BSAFolder, "*", DirectoryEnumerationOptions.Recursive)
return Directory.EnumerateFiles(_bsaFolder, "*", DirectoryEnumerationOptions.Recursive)
.Where(f => Consts.SupportedBSAs.Contains(Path.GetExtension(f)))
.Select(nm => new object[] {nm});
}
@ -95,29 +95,29 @@ namespace Compression.BSA.Test
{
TestContext.WriteLine($"From {bsa}");
TestContext.WriteLine("Cleaning Output Dir");
if (Directory.Exists(TempDir)) Directory.Delete(TempDir, true);
if (Directory.Exists(_tempDir)) Directory.Delete(_tempDir, true);
//if (Directory.Exists(ArchiveTempDir)) Directory.Delete(ArchiveTempDir, true);
Directory.CreateDirectory(TempDir);
Directory.CreateDirectory(_tempDir);
TestContext.WriteLine($"Reading {bsa}");
string TempFile = Path.Combine("tmp.bsa");
string tempFile = Path.Combine("tmp.bsa");
using (var a = BSADispatch.OpenRead(bsa))
{
a.Files.PMap(Queue, file =>
{
var abs_name = Path.Combine(TempDir, file.Path);
var absName = Path.Combine(_tempDir, file.Path);
ViaJson(file.State);
if (!Directory.Exists(Path.GetDirectoryName(abs_name)))
Directory.CreateDirectory(Path.GetDirectoryName(abs_name));
if (!Directory.Exists(Path.GetDirectoryName(absName)))
Directory.CreateDirectory(Path.GetDirectoryName(absName));
using (var fs = File.OpenWrite(abs_name))
using (var fs = File.OpenWrite(absName))
{
file.CopyDataTo(fs);
}
Assert.AreEqual(file.Size, new FileInfo(abs_name).Length);
Assert.AreEqual(file.Size, new FileInfo(absName).Length);
});
Console.WriteLine($"Building {bsa}");
@ -126,17 +126,17 @@ namespace Compression.BSA.Test
{
a.Files.PMap(Queue, file =>
{
var abs_path = Path.Combine(TempDir, file.Path);
using (var str = File.OpenRead(abs_path))
var absPath = Path.Combine(_tempDir, file.Path);
using (var str = File.OpenRead(absPath))
{
w.AddFile(ViaJson(file.State), str);
}
});
w.Build(TempFile);
w.Build(tempFile);
}
Console.WriteLine($"Verifying {bsa}");
using (var b = BSADispatch.OpenRead(TempFile))
using (var b = BSADispatch.OpenRead(tempFile))
{
Console.WriteLine($"Performing A/B tests on {bsa}");

1
Compression.BSA.Test/Properties/AssemblyInfo.cs
View File

@ -1,5 +1,4 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
[assembly: AssemblyTitle("Compression.BSA.Test")]

2
Compression.BSA/BA2Reader.cs
View File

@ -294,7 +294,7 @@ namespace Compression.BSA
ddsHeader.dwPitchOrLinearSize = (uint)(_width * _height); // 8bpp
break;
default:
throw new Exception("Unsupported DDS header format. File: " + this.FullPath);
throw new Exception("Unsupported DDS header format. File: " + FullPath);
}
bw.Write((uint)DDS.DDS_MAGIC);

3
Compression.BSA/Readme.md Normal file
View File

@ -0,0 +1,3 @@
# Compression.BSA
BSA Compression gets it's own project to remove cluttering.

16
VirtualFileSystem.Test/App.config
View File

@ -1,16 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
</startup>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="System.Buffers" publicKeyToken="cc7b13ffcd2ddd51" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.2.0" newVersion="4.0.2.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

19
VirtualFileSystem.Test/Program.cs
View File

@ -1,19 +0,0 @@
using System;
using Wabbajack.Common;
using Microsoft.Win32;
using System.Reactive;
namespace VirtualFileSystem.Test
{
internal class Program
{
private static void Main(string[] args)
{
var result = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\7-zip\");
Utils.LogMessages.Subscribe(s => Console.WriteLine(s));
Utils.StatusUpdates.Subscribe((i) => Console.Write(i.Message + "\r"));
VFS.VirtualFileSystem.VFS.AddRoot(@"D:\tmp\archivetests");
}
}
}

35
VirtualFileSystem.Test/Properties/AssemblyInfo.cs
View File

@ -1,35 +0,0 @@
using System.Reflection;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("VirtualFileSystem.Test")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("VirtualFileSystem.Test")]
[assembly: AssemblyCopyright("Copyright © 2019")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("a2913dfe-18ff-468b-a6c1-55f7c0cc0ce8")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

106
VirtualFileSystem.Test/VirtualFileSystem.Test.csproj
View File

@ -1,106 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{A2913DFE-18FF-468B-A6C1-55F7C0CC0CE8}</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>VirtualFileSystem.Test</RootNamespace>
<AssemblyName>VirtualFileSystem.Test</AssemblyName>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<Deterministic>true</Deterministic>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
<OutputPath>bin\x64\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x86\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x86</PlatformTarget>
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
<OutputPath>bin\x86\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x86</PlatformTarget>
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
<Project>{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}</Project>
<Name>Wabbajack.Common</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="System.Reactive">
<Version>4.2.0</Version>
</PackageReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>

35
VirtualFileSystem/Properties/AssemblyInfo.cs
View File

@ -1,35 +0,0 @@
using System.Reflection;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("VirtualFileSystem")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("VirtualFileSystem")]
[assembly: AssemblyCopyright("Copyright © 2019")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("5128b489-bc28-4f66-9f0b-b4565af36cbc")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

752
VirtualFileSystem/VirtualFileSystem.cs
View File

@ -1,752 +0,0 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using Ceras;
using Compression.BSA;
using ICSharpCode.SharpZipLib.Zip;
using Newtonsoft.Json;
using Wabbajack.Common;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace VFS
{
public class VirtualFileSystem
{
public const ulong FileVersion = 0x01;
public const string Magic = "WABBAJACK VFS FILE";
internal static string _stagedRoot;
public static VirtualFileSystem VFS;
private bool _disableDiskCache;
private Dictionary<string, VirtualFile> _files = new Dictionary<string, VirtualFile>();
private volatile bool _isSyncing;
static VirtualFileSystem()
{
VFS = new VirtualFileSystem();
var entry = Assembly.GetEntryAssembly();
if (entry != null && !string.IsNullOrEmpty(entry.Location))
{
RootFolder = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
_stagedRoot = Path.Combine(RootFolder, "vfs_staged_files");
}
}
public static void Reconfigure(string root)
{
RootFolder = root;
if (RootFolder != null)
_stagedRoot = Path.Combine(RootFolder, "vfs_staged_files");
else
_stagedRoot = "vfs_staged_files";
}
public static void Clean()
{
if (Directory.Exists(_stagedRoot))
{
Directory.EnumerateDirectories(_stagedRoot)
.PMap(f => DeleteDirectory(f));
DeleteDirectory(_stagedRoot);
}
Directory.CreateDirectory(_stagedRoot);
}
public VirtualFileSystem()
{
LoadFromDisk();
}
public static string RootFolder { get; private set; }
public Dictionary<string, IEnumerable<VirtualFile>> HashIndex { get; private set; }
public VirtualFile this[string path] => Lookup(path);
public static void DeleteDirectory(string path)
{
var info = new ProcessStartInfo
{
FileName = "cmd.exe",
Arguments = $"/c del /f /q /s \"{path}\" && rmdir /q /s \"{path}\" ",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
};
var p = new Process
{
StartInfo = info
};
p.Start();
ChildProcessTracker.AddProcess(p);
try
{
p.PriorityClass = ProcessPriorityClass.BelowNormal;
}
catch (Exception)
{
}
while (!p.HasExited)
{
var line = p.StandardOutput.ReadLine();
if (line == null) break;
Utils.Status(line);
}
p.WaitForExit();
}
public void Reset()
{
LoadFromDisk();
}
private void LoadFromDisk()
{
try
{
HashIndex = new Dictionary<string, IEnumerable<VirtualFile>>();
Utils.Log("Loading VFS Cache");
if (!File.Exists("vfs_cache.bin")) return;
_files = new Dictionary<string, VirtualFile>();
try
{
using (var fs = File.OpenRead("vfs_cache.bin"))
using (var br = new BinaryReader(fs))
{
var magic = Encoding.ASCII.GetString(br.ReadBytes(Magic.Length));
if (magic != Magic || br.ReadUInt64() != FileVersion)
{
fs.Close();
File.Delete("vfs_cache.bin");
return;
}
while (true)
{
var fr = VirtualFile.Read(br);
_files.Add(fr.FullPath, fr);
}
}
}
catch (EndOfStreamException)
{
}
CleanDB();
}
catch (Exception ex)
{
Utils.Log($"Purging cache due to {ex}");
File.Delete("vfs_cache.bson");
_files.Clear();
}
}
public void SyncToDisk()
{
if (!_disableDiskCache)
{
Utils.Status("Syncing VFS Cache");
lock (this)
{
try
{
_isSyncing = true;
if (File.Exists("vfs_cache.bin_new"))
File.Delete("vfs_cache.bin_new");
using (var fs = File.OpenWrite("vfs_cache.bin_new"))
using (var bw = new BinaryWriter(fs))
{
bw.Write(Encoding.ASCII.GetBytes(Magic));
bw.Write(FileVersion);
Utils.Log($"Syncing VFS to Disk: {_files.Count} entries");
foreach (var f in _files.Values) f.Write(bw);
}
if (File.Exists("vfs_cache.bin"))
File.Delete("vfs_cache.bin");
File.Move("vfs_cache.bin_new", "vfs_cache.bin");
}
finally
{
_isSyncing = false;
}
}
}
}
public IList<VirtualFile> FilesInArchive(VirtualFile f)
{
var path = f.FullPath + "|";
return _files.Values
.Where(v => v.FullPath.StartsWith(path))
.ToList();
}
public void Purge(VirtualFile f)
{
var path = f.FullPath + "|";
lock (this)
{
_files.Values
.Where(v => v.FullPath.StartsWith(path) || v.FullPath == f.FullPath)
.ToList()
.Do(r =>
{
_files.Remove(r.FullPath);
});
}
}
public void Add(VirtualFile f)
{
lock (this)
{
if (_files.ContainsKey(f.FullPath))
Purge(f);
_files.Add(f.FullPath, f);
}
}
public VirtualFile Lookup(string f)
{
lock (this)
{
if (_files.TryGetValue(f, out var found))
return found;
return null;
}
}
/// <summary>
/// Remove any orphaned files in the DB.
/// </summary>
private void CleanDB()
{
Utils.Log("Cleaning VFS cache");
lock (this)
{
_files.Values
.Where(f =>
{
if (f.IsConcrete)
return !File.Exists(f.StagedPath);
if (f.Hash == null)
return true;
while (f.ParentPath != null)
{
if (Lookup(f.ParentPath) == null)
return true;
if (f.Hash == null)
return true;
f = Lookup(f.ParentPath);
}
return false;
})
.ToList()
.Do(f =>
{
_files.Remove(f.FullPath);
});
SyncToDisk();
}
}
public void BackfillMissing()
{
lock (this)
{
_files.Values
.Select(f => f.ParentPath)
.Where(s => s != null)
.Where(s => !_files.ContainsKey(s))
.ToHashSet()
.Do(s => { AddKnown(new VirtualFile {Paths = s.Split('|')}); });
}
}
/// <summary>
/// Add a known file to the index, bit of a hack as we won't assume that all the fields for the archive are filled in.
/// you will need to manually update the SHA hash when you are done adding files, by calling `RefreshIndexes`
/// </summary>
/// <param name="virtualFile"></param>
public void AddKnown(VirtualFile virtualFile)
{
lock (this)
{
// We don't know enough about these files to be able to store them in the disk cache
_disableDiskCache = true;
_files[virtualFile.FullPath] = virtualFile;
}
}
/// <summary>
/// Adds the root path to the filesystem. This may take quite some time as every file in the folder will be hashed,
/// and every archive examined.
/// </summary>
/// <param name="path"></param>
public void AddRoot(string path)
{
if (!Directory.Exists(path)) return;
IndexPath(path);
RefreshIndexes();
}
public void RefreshIndexes()
{
Utils.Log("Building Hash Index");
lock (this)
{
HashIndex = _files.Values
.GroupBy(f => f.Hash)
.Where(f => f.Key != null)
.ToDictionary(f => f.Key, f => (IEnumerable<VirtualFile>) f);
}
}
private void IndexPath(string path)
{
var file_list = Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories).ToList();
Utils.Log($"Updating the cache for {file_list.Count} files");
file_list.PMap(f => UpdateFile(f));
SyncToDisk();
}
private void UpdateFile(string f)
{
TOP:
var lv = Lookup(f);
if (lv == null)
{
Utils.Status($"Analyzing {f}");
lv = new VirtualFile
{
Paths = new[] {f}
};
lv.Analyze();
Add(lv);
if (lv.IsArchive)
{
UpdateArchive(lv);
// Upsert after extraction incase extraction fails
lv.FinishedIndexing = true;
}
}
if (lv.IsOutdated)
{
Purge(lv);
goto TOP;
}
}
private void UpdateArchive(VirtualFile f)
{
if (!f.IsStaged)
throw new InvalidDataException("Can't analyze an unstaged file");
var tmp_dir = Path.Combine(_stagedRoot, Guid.NewGuid().ToString());
Utils.Status($"Extracting Archive {Path.GetFileName(f.StagedPath)}");
FileExtractor.ExtractAll(f.StagedPath, tmp_dir).Wait();
Utils.Status($"Updating Archive {Path.GetFileName(f.StagedPath)}");
var entries = Directory.EnumerateFiles(tmp_dir, "*", SearchOption.AllDirectories)
.Select(path => path.RelativeTo(tmp_dir));
var new_files = entries.Select(e =>
{
var new_path = new string[f.Paths.Length + 1];
f.Paths.CopyTo(new_path, 0);
new_path[f.Paths.Length] = e;
var nf = new VirtualFile
{
Paths = new_path
};
nf._stagedPath = Path.Combine(tmp_dir, e);
Add(nf);
return nf;
}).ToList();
// Analyze them
new_files.PMap(file =>
{
Utils.Status($"Analyzing {Path.GetFileName(file.StagedPath)}");
file.Analyze();
});
// Recurse into any archives in this archive
new_files.Where(file => file.IsArchive).Do(file => UpdateArchive(file));
f.FinishedIndexing = true;
if (!_isSyncing)
SyncToDisk();
Utils.Status("Cleaning Directory");
DeleteDirectory(tmp_dir);
}
public Action Stage(IEnumerable<VirtualFile> files)
{
var grouped = files.SelectMany(f => f.FilesInPath)
.Distinct()
.Where(f => f.ParentArchive != null)
.GroupBy(f => f.ParentArchive)
.OrderBy(f => f.Key == null ? 0 : f.Key.Paths.Length)
.ToList();
var Paths = new List<string>();
foreach (var group in grouped)
{
var tmp_path = Path.Combine(_stagedRoot, Guid.NewGuid().ToString());
FileExtractor.ExtractAll(group.Key.StagedPath, tmp_path).Wait();
Paths.Add(tmp_path);
foreach (var file in group)
file._stagedPath = Path.Combine(tmp_path, file.Paths[group.Key.Paths.Length]);
}
return () =>
{
Paths.Do(p =>
{
if (Directory.Exists(p)) DeleteDirectory(p);
});
};
}
public StagingGroup StageWith(IEnumerable<VirtualFile> files)
{
var grp = new StagingGroup(files);
grp.Stage();
return grp;
}
internal List<string> GetArchiveEntryNames(VirtualFile file)
{
if (!file.IsStaged)
throw new InvalidDataException("File is not staged");
if (file.Extension == ".bsa")
using (var ar = new BSAReader(file.StagedPath))
{
return ar.Files.Select(f => f.Path).ToList();
}
if (file.Extension == ".zip")
using (var s = new ZipFile(File.OpenRead(file.StagedPath)))
{
s.IsStreamOwner = true;
s.UseZip64 = UseZip64.On;
if (s.OfType<ZipEntry>().FirstOrDefault(e => !e.CanDecompress) == null)
return s.OfType<ZipEntry>()
.Where(f => f.IsFile)
.Select(f => f.Name.Replace('/', '\\'))
.ToList();
}
/*
using (var e = new ArchiveFile(file.StagedPath))
{
return e.Entries
.Where(f => !f.IsFolder)
.Select(f => f.FileName).ToList();
}*/
return null;
}
/// <summary>
/// Given a path that starts with a HASH, return the Virtual file referenced
/// </summary>
/// <param name="archiveHashPath"></param>
/// <returns></returns>
public VirtualFile FileForArchiveHashPath(string[] archiveHashPath)
{
if (archiveHashPath.Length == 1)
return HashIndex[archiveHashPath[0]].First();
var archive = HashIndex[archiveHashPath[0]].Where(a => a.IsArchive).OrderByDescending(a => a.LastModified)
.First();
var fullPath = archive.FullPath + "|" + string.Join("|", archiveHashPath.Skip(1));
return Lookup(fullPath);
}
public IDictionary<VirtualFile, IEnumerable<VirtualFile>> GroupedByArchive()
{
return _files.Values
.Where(f => f.TopLevelArchive != null)
.GroupBy(f => f.TopLevelArchive)
.ToDictionary(f => f.Key, f => (IEnumerable<VirtualFile>) f);
}
}
public class StagingGroup : List<VirtualFile>, IDisposable
{
public StagingGroup(IEnumerable<VirtualFile> files) : base(files)
{
}
public void Dispose()
{
this.Do(f => f.Unstage());
}
internal void Stage()
{
VirtualFileSystem.VFS.Stage(this);
}
}
[MemberConfig(TargetMember.None)]
public class VirtualFile
{
private string _fullPath;
private bool? _isArchive;
private string _parentPath;
public string[] _paths;
internal string _stagedPath;
[Include]
public string[] Paths
{
get => _paths;
set
{
for (var idx = 0; idx < value.Length; idx += 1)
value[idx] = string.Intern(value[idx]);
_paths = value;
}
}
[Include] public string Hash { get; set; }
[Include] public long Size { get; set; }
[Include] public ulong LastModified { get; set; }
[Include]
public bool? FinishedIndexing { get; set; }
public string FullPath
{
get
{
if (_fullPath != null) return _fullPath;
_fullPath = string.Join("|", Paths);
return _fullPath;
}
}
public string Extension => Path.GetExtension(Paths.Last());
/// <summary>
/// If this file is in an archive, return the Archive File, otherwise return null.
/// </summary>
public VirtualFile TopLevelArchive
{
get
{
if (Paths.Length == 0) return null;
return VirtualFileSystem.VFS[Paths[0]];
}
}
public VirtualFile ParentArchive
{
get
{
if (ParentPath == null) return null;
return VirtualFileSystem.VFS.Lookup(ParentPath);
}
}
public bool IsArchive
{
get
{
if (_isArchive == null)
_isArchive = FileExtractor.CanExtract(Extension);
return (bool) _isArchive;
}
}
public bool IsStaged
{
get
{
if (IsConcrete) return true;
return _stagedPath != null;
}
}
public string StagedPath
{
get
{
if (!IsStaged)
throw new InvalidDataException("File is not staged");
if (IsConcrete) return Paths[0];
return _stagedPath;
}
set
{
if (IsStaged && value != null)
throw new InvalidDataException("Can't change the path of a already staged file");
_stagedPath = value;
}
}
/// <summary>
/// Returns true if this file always exists on-disk, and doesn't need to be staged.
/// </summary>
public bool IsConcrete => Paths.Length == 1;
public bool IsOutdated
{
get
{
if (IsStaged)
{
var fi = new FileInfo(StagedPath);
if (fi.LastWriteTime.ToMilliseconds() != LastModified || fi.Length != Size)
return true;
if (IsArchive)
if (!FinishedIndexing ?? true)
return true;
}
return false;
}
}
public string ParentPath
{
get
{
if (_parentPath == null && !IsConcrete)
_parentPath = string.Join("|", Paths.Take(Paths.Length - 1));
return _parentPath;
}
}
public IEnumerable<VirtualFile> FileInArchive => VirtualFileSystem.VFS.FilesInArchive(this);
public IEnumerable<VirtualFile> FilesInPath
{
get
{
return Enumerable.Range(1, Paths.Length)
.Select(i => Paths.Take(i))
.Select(path => VirtualFileSystem.VFS.Lookup(string.Join("|", path)));
}
}
public FileStream OpenRead()
{
if (!IsStaged)
throw new InvalidDataException("File is not staged, cannot open");
return File.OpenRead(_stagedPath);
}
/// <summary>
/// Calculate the file's SHA, size and last modified
/// </summary>
internal void Analyze()
{
if (!IsStaged)
throw new InvalidDataException("Cannot analyze an unstaged file");
var fio = new FileInfo(StagedPath);
Size = fio.Length;
Hash = StagedPath.FileHash();
LastModified = fio.LastWriteTime.ToMilliseconds();
}
/// <summary>
/// Delete the temoporary file associated with this file
/// </summary>
internal void Unstage()
{
if (IsStaged && !IsConcrete)
{
File.Delete(_stagedPath);
_stagedPath = null;
}
}
internal string GenerateStagedName()
{
if (_stagedPath != null) return _stagedPath;
_stagedPath = Path.Combine(VirtualFileSystem._stagedRoot, Guid.NewGuid() + Path.GetExtension(Paths.Last()));
return _stagedPath;
}
public string[] MakeRelativePaths()
{
var path_copy = (string[]) Paths.Clone();
path_copy[0] = VirtualFileSystem.VFS.Lookup(Paths[0]).Hash;
return path_copy;
}
public void Write(BinaryWriter bw)
{
bw.Write(FullPath);
bw.Write(Hash ?? "");
bw.Write(Size);
bw.Write(LastModified);
bw.Write(FinishedIndexing ?? false);
}
public static VirtualFile Read(BinaryReader rdr)
{
var vf = new VirtualFile();
var full_path = rdr.ReadString();
vf.Paths = full_path.Split('|');
for (var x = 0; x < vf.Paths.Length; x++)
vf.Paths[x] = string.Intern(vf.Paths[x]);
vf._fullPath = full_path;
vf.Hash = rdr.ReadString();
if (vf.Hash == "") vf.Hash = null;
vf.Size = rdr.ReadInt64();
vf.LastModified = rdr.ReadUInt64();
vf.FinishedIndexing = rdr.ReadBoolean();
if (vf.FinishedIndexing == false) vf.FinishedIndexing = null;
return vf;
}
}
}

119
VirtualFileSystem/VirtualFileSystem.csproj
View File

@ -1,119 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{5128B489-BC28-4F66-9F0B-B4565AF36CBC}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>VirtualFileSystem</RootNamespace>
<AssemblyName>VirtualFileSystem</AssemblyName>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<Deterministic>true</Deterministic>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
<OutputPath>bin\x64\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x86\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x86</PlatformTarget>
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
<OutputPath>bin\x86\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x86</PlatformTarget>
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.IO.Compression.FileSystem" />
<Reference Include="System.Transactions" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="VirtualFileSystem.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj">
<Project>{ff5d892f-8ff4-44fc-8f7f-cd58f307ad1b}</Project>
<Name>Compression.BSA</Name>
</ProjectReference>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
<Project>{b3f3fb6e-b9eb-4f49-9875-d78578bc7ae5}</Project>
<Name>Wabbajack.Common</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="app.config" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AlphaFS">
<Version>2.2.6</Version>
</PackageReference>
<PackageReference Include="Ceras">
<Version>4.1.7</Version>
</PackageReference>
<PackageReference Include="Newtonsoft.Json">
<Version>12.0.2</Version>
</PackageReference>
<PackageReference Include="SharpZipLib">
<Version>1.2.0</Version>
</PackageReference>
<PackageReference Include="System.Collections.Immutable">
<Version>1.6.0</Version>
</PackageReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>

11
VirtualFileSystem/app.config
View File

@ -1,11 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="System.Buffers" publicKeyToken="cc7b13ffcd2ddd51" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.2.0" newVersion="4.0.2.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

12
Wabbajack.CacheServer.Test/CacheServerTests.cs
View File

@ -1,14 +1,6 @@
using System;
using System.Net.Configuration;
using System.Net.Http;
using System.Windows;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.CacheServer.Test
namespace Wabbajack.CacheServer.Test
{
[TestClass]
//[TestClass]
public class CacheServerTests
{
// The server works, we just need to figure out proper testing.

1
Wabbajack.CacheServer.Test/Properties/AssemblyInfo.cs
View File

@ -1,5 +1,4 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
[assembly: AssemblyTitle("Wabbajack.CacheServer.Test")]

3
Wabbajack.CacheServer/Readme.md Normal file
View File

@ -0,0 +1,3 @@
# Wabbajack.CacheServer
CacheServer for caching mod information to reduce the amount of API calls a user has to account for when using Wabbajack to compiler/install a ModList.

3
Wabbajack.Common.CSP/AChannel.cs
View File

@ -1,8 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

8
Wabbajack.Common.CSP/AsyncResult.cs
View File

@ -1,10 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common.CSP
namespace Wabbajack.Common.CSP
{
public enum AsyncResult : int
{

42
Wabbajack.Common.CSP/CSP Readme.md
View File

@ -1,42 +0,0 @@
### Overview of CSP (Communicating Sequential Processes) for the C# programmer
#### What is CSP?
Communicating Sequential processes is a programming model invented in 1978 by Tony Hoare, who described a process
of computation where hundreds or thousands of small processes communicate via channels. Think of this process like
a assembly line. Each worker in the factory is a process, and the conveyor belts are the channels. The workers don't need
to know where a part came from, or where it's going, they simply take one item off the belt, perform an operation and pass
the item on to another belt. This analogy works quite well, and the following observations about a factory also apply to
CSP:
* Multiple workers can pull from the same belt (channel/queue)
* Multiple workers can put work onto the belt
* Belts can buffer items, for slow consumers, but at some point they backup and block the writer
* A worker can pull/push to multiple belts.
#### What does this look like in C#?
The basic unit of CSP in this library is the channel:
```
var chan = Channel.Create()
```
Without any other parameters this creates a channel with a size of 0, so every pending put must be matched
1:1 with a take. This creates a syncronization point. Channels are fully async and thread-safe:
```
public async Task TestTakePutBlocking()
{
var channel = Channel.Create<int>();
// Channel size is 0, so we can't await, because we'd never complete
var ptask = channel.Put(1);
// Now the put is dispatched to the scheduler because we've taken the value
var (open, val) = await channel.Take();
Assert.AreEqual(1, val);
Assert.IsTrue(open);
Assert.IsTrue(await ptask);
}
```

4
Wabbajack.Common.CSP/Channel.cs
View File

@ -3,10 +3,6 @@ using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common.CSP
{

6
Wabbajack.Common.CSP/EnumeratorBuffer.cs
View File

@ -1,9 +1,5 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common.CSP
{

2
Wabbajack.Common.CSP/Extensions.cs
View File

@ -1,7 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

8
Wabbajack.Common.CSP/FixedSizeBuffer.cs
View File

@ -1,10 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common.CSP
namespace Wabbajack.Common.CSP
{
public class FixedSizeBuffer<T> : IBuffer<T>
{

9
Wabbajack.Common.CSP/Handler.cs
View File

@ -1,11 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Threading.Tasks.Sources;
namespace Wabbajack.Common.CSP
namespace Wabbajack.Common.CSP
{
public interface Handler<T>
{

8
Wabbajack.Common.CSP/IChannel.cs
View File

@ -1,10 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common.CSP
namespace Wabbajack.Common.CSP
{
public interface IChannel<TIn, TOut> : IReadPort<TOut>, IWritePort<TIn>
{

8
Wabbajack.Common.CSP/ICloseable.cs
View File

@ -1,10 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common.CSP
namespace Wabbajack.Common.CSP
{
public interface ICloseable
{

3
Wabbajack.Common.CSP/IReadPort.cs
View File

@ -1,7 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common.CSP

3
Wabbajack.Common.CSP/IWritePort.cs
View File

@ -1,7 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common.CSP

1
Wabbajack.Common.CSP/PIpelines.cs
View File

@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Subjects;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

1
Wabbajack.Common.CSP/Properties/AssemblyInfo.cs
View File

@ -1,5 +1,4 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following

3
Wabbajack.Common.CSP/PutTaskHandler.cs
View File

@ -1,7 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common.CSP

38
Wabbajack.Common.CSP/Readme.md Normal file
View File

@ -0,0 +1,38 @@
# Wabbajack.Common.CSP
## Overview of CSP (Communicating Sequential Processes) for the C# programmer
### What is CSP?
Communicating Sequential processes is a programming model invented in 1978 by Tony Hoare, who described a process of computation where hundreds or thousands of small processes communicate via channels. Think of this process like a assembly line. Each worker in the factory is a process, and the conveyor belts are the channels. The workers don't need to know where a part came from, or where it's going, they simply take one item off the belt, perform an operation and pass the item on to another belt. This analogy works quite well, and the following observations about a factory also apply to CSP:
* Multiple workers can pull from the same belt (channel/queue)
* Multiple workers can put work onto the belt
* Belts can buffer items, for slow consumers, but at some point they backup and block the writer
* A worker can pull/push to multiple belts.
#### What does this look like in C#?
The basic unit of CSP in this library is the channel:
```cSharp
var chan = Channel.Create()
```
Without any other parameters this creates a channel with a size of 0, so every pending put must be matched 1:1 with a take. This creates a syncronization point. Channels are fully async and thread-safe:
```cSharp
public async Task TestTakePutBlocking()
{
var channel = Channel.Create<int>();
// Channel size is 0, so we can't await, because we'd never complete
var ptask = channel.Put(1);
// Now the put is dispatched to the scheduler because we've taken the value
var (open, val) = await channel.Take();
Assert.AreEqual(1, val);
Assert.IsTrue(open);
Assert.IsTrue(await ptask);
}
```

1
Wabbajack.Common.CSP/RingBuffer.cs
View File

@ -1,7 +1,6 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
namespace Wabbajack.Common.CSP

3
Wabbajack.Common.CSP/RxBuffer.cs
View File

@ -1,10 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common.CSP
{

4
Wabbajack.Common.CSP/TakeTaskHandler.cs
View File

@ -1,8 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

6
Wabbajack.Common/DynamicIniData.cs
View File

@ -7,11 +7,11 @@ namespace Wabbajack.Common
{
public class DynamicIniData : DynamicObject
{
private readonly IniData value;
private readonly IniData _value;
public DynamicIniData(IniData value) //
{
this.value = value;
this._value = value;
}
public static dynamic FromIni(IniData data)
@ -27,7 +27,7 @@ namespace Wabbajack.Common
public override bool TryGetMember(GetMemberBinder binder, out object result)
{
result = new SectionData(value[binder.Name]);
result = new SectionData(_value[binder.Name]);
return true;
}
}

8
Wabbajack.Common/Enums/ModManager.cs
View File

@ -1,10 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common
namespace Wabbajack.Common
{
public enum ModManager
{

8
Wabbajack.Common/Enums/RunMode.cs
View File

@ -1,10 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack
namespace Wabbajack
{
public enum RunMode
{

20
Wabbajack.Common/Error States/ErrorResponse.cs
View File

@ -1,8 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack
{
@ -20,25 +16,25 @@ namespace Wabbajack
{
get
{
if (this.Exception != null)
if (Exception != null)
{
return this.Exception.ToString();
return Exception.ToString();
}
return _reason;
}
}
bool IErrorResponse.Succeeded => this.Succeeded;
Exception IErrorResponse.Exception => this.Exception;
bool IErrorResponse.Succeeded => Succeeded;
Exception IErrorResponse.Exception => Exception;
private ErrorResponse(
bool succeeded,
string reason = null,
Exception ex = null)
{
this.Succeeded = succeeded;
this._reason = reason;
this.Exception = ex;
Succeeded = succeeded;
_reason = reason;
Exception = ex;
}
public override string ToString()
@ -80,7 +76,7 @@ namespace Wabbajack
public static ErrorResponse Convert(IErrorResponse err, bool nullIsSuccess = true)
{
if (err == null) return ErrorResponse.Create(nullIsSuccess);
if (err == null) return Create(nullIsSuccess);
return new ErrorResponse(err.Succeeded, err.Reason, err.Exception);
}
}

42
Wabbajack.Common/Error States/GetResponse.cs
View File

@ -1,8 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack
{
@ -20,16 +16,16 @@ namespace Wabbajack
{
get
{
if (this.Exception != null)
if (Exception != null)
{
return this.Exception.ToString();
return Exception.ToString();
}
return _reason;
}
}
bool IErrorResponse.Succeeded => this.Succeeded;
Exception IErrorResponse.Exception => this.Exception;
bool IErrorResponse.Succeeded => Succeeded;
Exception IErrorResponse.Exception => Exception;
private GetResponse(
bool succeeded,
@ -37,16 +33,16 @@ namespace Wabbajack
string reason = null,
Exception ex = null)
{
this.Value = val;
this.Succeeded = succeeded;
this._reason = reason;
this.Exception = ex;
Value = val;
Succeeded = succeeded;
_reason = reason;
Exception = ex;
}
public bool Equals(GetResponse<T> other)
{
return this.Succeeded == other.Succeeded
&& object.Equals(this.Value, other.Value);
return Succeeded == other.Succeeded
&& Equals(Value, other.Value);
}
public override bool Equals(object obj)
@ -70,26 +66,26 @@ namespace Wabbajack
{
return new GetResponse<R>(
succeeded: false,
reason: this._reason,
ex: this.Exception);
reason: _reason,
ex: Exception);
}
public GetResponse<R> Bubble<R>(Func<T, R> conv)
{
return new GetResponse<R>(
succeeded: this.Succeeded,
val: conv(this.Value),
reason: this._reason,
ex: this.Exception);
succeeded: Succeeded,
val: conv(Value),
reason: _reason,
ex: Exception);
}
public T EvaluateOrThrow()
{
if (this.Succeeded)
if (Succeeded)
{
return this.Value;
return Value;
}
throw new ArgumentException(this.Reason);
throw new ArgumentException(Reason);
}
#region Factories

3
Wabbajack.Common/Extensions/DictionaryExt.cs
View File

@ -1,8 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack
{

2
Wabbajack.Common/Extensions/EnumExt.cs
View File

@ -2,8 +2,6 @@
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Common
{

2
Wabbajack.Common/Extensions/RxExt.cs
View File

@ -1,11 +1,9 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reactive;
using System.Reactive.Concurrency;
using System.Reactive.Disposables;
using System.Reactive.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack

3
Wabbajack.Common/Extensions/TaskExt.cs
View File

@ -1,7 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack

6
Wabbajack.Common/FileExtractor.cs
View File

@ -73,13 +73,13 @@ namespace Wabbajack.Common
if (f.Path.StartsWith("\\"))
path = f.Path.Substring(1);
Utils.Status($"Extracting {path}");
var out_path = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(out_path);
var outPath = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(outPath);
if (!Directory.Exists(parent))
Directory.CreateDirectory(parent);
using (var fs = File.OpenWrite(out_path))
using (var fs = File.OpenWrite(outPath))
{
f.CopyDataTo(fs);
}

3
Wabbajack.Common/Readme.md Normal file
View File

@ -0,0 +1,3 @@
# Wabbajack.Common
The `Common` part of the project name tells it all: Here you will find our Utility functions, the Constants, File Extractors and Game Handlers.

14
Wabbajack.Common/SplittingStream.cs
View File

@ -7,15 +7,15 @@ namespace Wabbajack.Common
{
private readonly Stream _a;
private readonly Stream _b;
private readonly bool _leave_a_open;
private readonly bool _leave_b_open;
private readonly bool _leaveAOpen;
private readonly bool _leaveBOpen;
public SplittingStream(Stream a, bool leave_a_open, Stream b, bool leave_b_open)
public SplittingStream(Stream a, bool leaveAOpen, Stream b, bool leaveBOpen)
{
_a = a;
_b = b;
_leave_a_open = leave_a_open;
_leave_b_open = leave_b_open;
_leaveAOpen = leaveAOpen;
_leaveBOpen = leaveBOpen;
}
public override bool CanRead => false;
@ -63,8 +63,8 @@ namespace Wabbajack.Common
{
if (disposing)
{
if (!_leave_a_open) _a.Dispose();
if (!_leave_b_open) _b.Dispose();
if (!_leaveAOpen) _a.Dispose();
if (!_leaveBOpen) _b.Dispose();
}
}
}

8
Wabbajack.Common/StatusFileStream.cs
View File

@ -1,10 +1,4 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using System.IO;
namespace Wabbajack.Common
{

5
Wabbajack.Common/StatusUpdate.cs
View File

@ -1,10 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Subjects;
using System.Text;
using System.Threading.Tasks;
using ReactiveUI;
namespace Wabbajack.Common
{

66
Wabbajack.Common/Utils.cs
View File

@ -4,7 +4,6 @@ using System.Data.HashFunction.xxHash;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Configuration;
using System.Net.Http;
using System.Reactive.Subjects;
using System.Reflection;
@ -17,7 +16,6 @@ using Ceras;
using ICSharpCode.SharpZipLib.BZip2;
using IniParser;
using Newtonsoft.Json;
using Newtonsoft.Json.Bson;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
using Directory = System.IO.Directory;
@ -38,18 +36,18 @@ namespace Wabbajack.Common
public static string LogFile { get; private set; }
static Utils()
{
var program_name = Assembly.GetEntryAssembly()?.Location ?? "Wabbajack";
LogFile = program_name + ".log";
var programName = Assembly.GetEntryAssembly()?.Location ?? "Wabbajack";
LogFile = programName + ".log";
_startTime = DateTime.Now;
if (LogFile.FileExists())
File.Delete(LogFile);
}
private static readonly Subject<string> _loggerSubj = new Subject<string>();
public static IObservable<string> LogMessages => _loggerSubj;
private static readonly Subject<(string Message, int Progress)> _statusSubj = new Subject<(string Message, int Progress)>();
public static IObservable<(string Message, int Progress)> StatusUpdates => _statusSubj;
private static readonly Subject<string> LoggerSubj = new Subject<string>();
public static IObservable<string> LogMessages => LoggerSubj;
private static readonly Subject<(string Message, int Progress)> StatusSubj = new Subject<(string Message, int Progress)>();
public static IObservable<(string Message, int Progress)> StatusUpdates => StatusSubj;
private static readonly string[] Suffix = {"B", "KB", "MB", "GB", "TB", "PB", "EB"}; // Longs run out around EB
@ -65,7 +63,7 @@ namespace Wabbajack.Common
File.AppendAllText(LogFile, msg + "\r\n");
}
_loggerSubj.OnNext(msg);
LoggerSubj.OnNext(msg);
}
public static void LogToFile(string msg)
@ -83,7 +81,7 @@ namespace Wabbajack.Common
if (WorkQueue.CurrentQueue != null)
WorkQueue.CurrentQueue.Report(msg, progress);
else
_statusSubj.OnNext((msg, progress));
StatusSubj.OnNext((msg, progress));
}
/// <summary>
@ -169,14 +167,14 @@ namespace Wabbajack.Common
{
var buffer = new byte[1024 * 64];
if (maxSize == 0) maxSize = 1;
long total_read = 0;
long totalRead = 0;
while (true)
{
var read = istream.Read(buffer, 0, buffer.Length);
if (read == 0) break;
total_read += read;
totalRead += read;
ostream.Write(buffer, 0, read);
Status(status, (int) (total_read * 100 / maxSize));
Status(status, (int) (totalRead * 100 / maxSize));
}
}
@ -212,7 +210,7 @@ namespace Wabbajack.Common
public static DateTime AsUnixTime(this long timestamp)
{
System.DateTime dtDateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, System.DateTimeKind.Utc);
DateTime dtDateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
dtDateTime = dtDateTime.AddSeconds(timestamp).ToLocalTime();
return dtDateTime;
}
@ -448,7 +446,7 @@ namespace Wabbajack.Common
{
var colllst = coll.ToList();
var remaining_tasks = colllst.Count;
var remainingTasks = colllst.Count;
var tasks = coll.Select(i =>
{
@ -463,14 +461,14 @@ namespace Wabbajack.Common
{
tc.SetException(ex);
}
Interlocked.Decrement(ref remaining_tasks);
Interlocked.Decrement(ref remainingTasks);
});
return tc.Task;
}).ToList();
// To avoid thread starvation, we'll start to help out in the work queue
if (WorkQueue.WorkerThread)
while (remaining_tasks > 0)
while (remainingTasks > 0)
if (queue.Queue.TryTake(out var a, 500))
a();
@ -583,31 +581,31 @@ namespace Wabbajack.Common
public static void CreatePatch(byte[] a, byte[] b, Stream output)
{
var data_a = a.SHA256().FromBase64().ToHex();
var data_b = b.SHA256().FromBase64().ToHex();
var cache_file = Path.Combine("patch_cache", $"{data_a}_{data_b}.patch");
var dataA = a.SHA256().FromBase64().ToHex();
var dataB = b.SHA256().FromBase64().ToHex();
var cacheFile = Path.Combine("patch_cache", $"{dataA}_{dataB}.patch");
if (!Directory.Exists("patch_cache"))
Directory.CreateDirectory("patch_cache");
while (true)
{
if (File.Exists(cache_file))
if (File.Exists(cacheFile))
{
using (var f = File.OpenRead(cache_file))
using (var f = File.OpenRead(cacheFile))
{
f.CopyTo(output);
}
}
else
{
var tmp_name = Path.Combine("patch_cache", Guid.NewGuid() + ".tmp");
var tmpName = Path.Combine("patch_cache", Guid.NewGuid() + ".tmp");
using (var f = File.OpenWrite(tmp_name))
using (var f = File.OpenWrite(tmpName))
{
BSDiff.Create(a, b, f);
}
File.Move(tmp_name, cache_file);
File.Move(tmpName, cacheFile);
continue;
}
@ -617,9 +615,9 @@ namespace Wabbajack.Common
public static void TryGetPatch(string foundHash, string fileHash, out byte[] ePatch)
{
var patch_name = Path.Combine("patch_cache",
var patchName = Path.Combine("patch_cache",
$"{foundHash.FromBase64().ToHex()}_{fileHash.FromBase64().ToHex()}.patch");
ePatch = File.Exists(patch_name) ? File.ReadAllBytes(patch_name) : null;
ePatch = File.Exists(patchName) ? File.ReadAllBytes(patchName) : null;
}
public static void Warning(string s)
@ -701,7 +699,7 @@ namespace Wabbajack.Common
private static long TestDiskSpeedInner(WorkQueue queue, string path)
{
var start_time = DateTime.Now;
var startTime = DateTime.Now;
var seconds = 2;
return Enumerable.Range(0, queue.ThreadCount)
.PMap(queue, idx =>
@ -714,7 +712,7 @@ namespace Wabbajack.Common
random.NextBytes(buffer);
using (var fs = File.OpenWrite(file))
{
while (DateTime.Now < start_time + new TimeSpan(0, 0, seconds))
while (DateTime.Now < startTime + new TimeSpan(0, 0, seconds))
{
fs.Write(buffer, 0, buffer.Length);
// Flush to make sure large buffers don't cause the rate to be higher than it should
@ -730,11 +728,11 @@ namespace Wabbajack.Common
private static Dictionary<string, long> _cachedDiskSpeeds = new Dictionary<string, long>();
public static long TestDiskSpeed(WorkQueue queue, string path)
{
var drive_name = Volume.GetUniqueVolumeNameForPath(path);
if (_cachedDiskSpeeds.TryGetValue(drive_name, out long speed))
var driveName = Volume.GetUniqueVolumeNameForPath(path);
if (_cachedDiskSpeeds.TryGetValue(driveName, out long speed))
return speed;
speed = TestDiskSpeedInner(queue, path);
_cachedDiskSpeeds[drive_name] = speed;
_cachedDiskSpeeds[driveName] = speed;
return speed;
}
@ -753,7 +751,7 @@ namespace Wabbajack.Common
{
return ErrorResponse.Fail(ex.Message);
}
catch (System.IO.PathTooLongException ex)
catch (PathTooLongException ex)
{
return ErrorResponse.Fail(ex.Message);
}
@ -778,7 +776,7 @@ namespace Wabbajack.Common
{
return ErrorResponse.Fail(ex.Message);
}
catch (System.IO.PathTooLongException ex)
catch (PathTooLongException ex)
{
return ErrorResponse.Fail(ex.Message);
}

2
Wabbajack.Common/WorkQueue.cs
View File

@ -2,8 +2,6 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Reactive;
using System.Reactive.Concurrency;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Threading;

4
Wabbajack.Lib/ABatchProcessor.cs
View File

@ -1,12 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reactive.Subjects;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using ReactiveUI;
using Wabbajack.Common;
using Wabbajack.VirtualFileSystem;

4
Wabbajack.Lib/AInstaller.cs
View File

@ -1,17 +1,13 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Eventing.Reader;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Windows.Navigation;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
using Context = Wabbajack.VirtualFileSystem.Context;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using DriveInfo = Alphaleonis.Win32.Filesystem.DriveInfo;
using File = System.IO.File;
using FileInfo = System.IO.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;

10
Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs
View File

@ -10,7 +10,7 @@ namespace Wabbajack.Lib.CompilationSteps
{
public class DeconstructBSAs : ACompilationStep
{
private readonly IEnumerable<string> _include_directly;
private readonly IEnumerable<string> _includeDirectly;
private readonly List<ICompilationStep> _microstack;
private readonly List<ICompilationStep> _microstackWithInclude;
private readonly MO2Compiler _mo2Compiler;
@ -18,7 +18,7 @@ namespace Wabbajack.Lib.CompilationSteps
public DeconstructBSAs(ACompiler compiler) : base(compiler)
{
_mo2Compiler = (MO2Compiler) compiler;
_include_directly = _mo2Compiler.ModInis.Where(kv =>
_includeDirectly = _mo2Compiler.ModInis.Where(kv =>
{
var general = kv.Value.General;
if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE)) return true;
@ -54,16 +54,16 @@ namespace Wabbajack.Lib.CompilationSteps
var defaultInclude = false;
if (source.Path.StartsWith("mods"))
if (_include_directly.Any(path => source.Path.StartsWith(path)))
if (_includeDirectly.Any(path => source.Path.StartsWith(path)))
defaultInclude = true;
var source_files = source.File.Children;
var sourceFiles = source.File.Children;
var stack = defaultInclude ? _microstackWithInclude : _microstack;
var id = Guid.NewGuid().ToString();
var matches = source_files.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e)
var matches = sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e)
{
Path = Path.Combine(Consts.BSACreationDir, id, e.Name)
}));

3
Wabbajack.Lib/CompilationSteps/IgnoreDisabledVortexMods.cs
View File

@ -1,5 +1,4 @@
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
using Wabbajack.Common;
namespace Wabbajack.Lib.CompilationSteps
{

4
Wabbajack.Lib/CompilationSteps/IgnoreVortex.cs
View File

@ -1,7 +1,5 @@
using System;
using System.IO;
using System.IO;
using Newtonsoft.Json;
using Wabbajack.Common;
namespace Wabbajack.Lib.CompilationSteps
{

1
Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs
View File

@ -2,7 +2,6 @@
using System.Linq;
using Alphaleonis.Win32.Filesystem;
using Newtonsoft.Json;
using Wabbajack.Common;
namespace Wabbajack.Lib.CompilationSteps
{

8
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -1,10 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms.VisualStyles;
using Alphaleonis.Win32.Filesystem;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Lib.Validation;
namespace Wabbajack.Lib.Downloaders

3
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -1,9 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Policy;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
namespace Wabbajack.Lib.Downloaders

4
Wabbajack.Lib/Downloaders/DownloaderUtils.cs
View File

@ -1,8 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Lib.Downloaders
{

8
Wabbajack.Lib/Downloaders/DropboxDownloader.cs
View File

@ -1,17 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Web;
namespace Wabbajack.Lib.Downloaders
{
public class DropboxDownloader : IDownloader, IUrlDownloader
{
public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
{
var urlstring = archive_ini?.General?.directURL;
var urlstring = archiveINI?.General?.directURL;
return GetDownloaderState(urlstring);
}

20
Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs
View File

@ -1,11 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
@ -13,9 +7,9 @@ namespace Wabbajack.Lib.Downloaders
{
public class GoogleDriveDownloader : IDownloader, IUrlDownloader
{
public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
{
var url = archive_ini?.General?.directURL;
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url);
}
@ -53,14 +47,14 @@ namespace Wabbajack.Lib.Downloaders
private HTTPDownloader.State ToHttpState()
{
var initial_url = $"https://drive.google.com/uc?id={Id}&export=download";
var initialURL = $"https://drive.google.com/uc?id={Id}&export=download";
var client = new HttpClient();
var result = client.GetStringSync(initial_url);
var result = client.GetStringSync(initialURL);
var regex = new Regex("(?<=/uc\\?export=download&amp;confirm=).*(?=;id=)");
var confirm = regex.Match(result);
var url = $"https://drive.google.com/uc?export=download&confirm={confirm}&id={Id}";
var http_state = new HTTPDownloader.State {Url = url, Client = client};
return http_state;
var httpState = new HTTPDownloader.State {Url = url, Client = client};
return httpState;
}
public override bool Verify()

30
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -13,10 +13,10 @@ namespace Wabbajack.Lib.Downloaders
public class HTTPDownloader : IDownloader, IUrlDownloader
{
public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
{
var url = archive_ini?.General?.directURL;
return GetDownloaderState(url, archive_ini);
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url, archiveINI);
}
@ -25,7 +25,7 @@ namespace Wabbajack.Lib.Downloaders
return GetDownloaderState(uri, null);
}
public AbstractDownloadState GetDownloaderState(string url, dynamic archive_ini)
public AbstractDownloadState GetDownloaderState(string url, dynamic archiveINI)
{
if (url != null)
{
@ -33,10 +33,10 @@ namespace Wabbajack.Lib.Downloaders
{
Url = url
};
if (archive_ini?.General?.directURLHeaders != null)
if (archiveINI?.General?.directURLHeaders != null)
{
tmp.Headers = new List<string>();
tmp.Headers.AddRange(archive_ini?.General.directURLHeaders.Split('|'));
tmp.Headers.AddRange(archiveINI?.General.directURLHeaders.Split('|'));
}
return tmp;
}
@ -82,8 +82,8 @@ namespace Wabbajack.Lib.Downloaders
client.DefaultRequestHeaders.Add(k, v);
}
long total_read = 0;
var buffer_size = 1024 * 32;
long totalRead = 0;
var bufferSize = 1024 * 32;
var response = client.GetSync(Url);
var stream = response.Content.ReadAsStreamAsync();
@ -105,25 +105,25 @@ namespace Wabbajack.Lib.Downloaders
if (!download)
return true;
var header_var = a.Size == 0 ? "1" : a.Size.ToString();
var headerVar = a.Size == 0 ? "1" : a.Size.ToString();
if (response.Content.Headers.Contains("Content-Length"))
header_var = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
var content_size = header_var != null ? long.Parse(header_var) : 1;
var contentSize = headerVar != null ? long.Parse(headerVar) : 1;
using (var webs = stream.Result)
using (var fs = File.OpenWrite(destination))
{
var buffer = new byte[buffer_size];
var buffer = new byte[bufferSize];
while (true)
{
var read = webs.Read(buffer, 0, buffer_size);
var read = webs.Read(buffer, 0, bufferSize);
if (read == 0) break;
Utils.Status($"Downloading {a.Name}", (int)(total_read * 100 / content_size));
Utils.Status($"Downloading {a.Name}", (int)(totalRead * 100 / contentSize));
fs.Write(buffer, 0, read);
total_read += read;
totalRead += read;
}
}

11
Wabbajack.Lib/Downloaders/IDownloader.cs
View File

@ -1,15 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Lib.Validation;
namespace Wabbajack.Lib.Downloaders
namespace Wabbajack.Lib.Downloaders
{
public interface IDownloader
{
AbstractDownloadState GetDownloaderState(dynamic archive_ini);
AbstractDownloadState GetDownloaderState(dynamic archiveINI);
/// <summary>
/// Called before any downloads are inacted by the installer;

8
Wabbajack.Lib/Downloaders/IUrlDownloader.cs
View File

@ -1,10 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Lib.Downloaders
namespace Wabbajack.Lib.Downloaders
{
public interface IUrlDownloader : IDownloader
{

23
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -1,24 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Security.Policy;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using CG.Web.MegaApiClient;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
namespace Wabbajack.Lib.Downloaders
{
public class MegaDownloader : IDownloader, IUrlDownloader
{
public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
{
var url = archive_ini?.General?.directURL;
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url);
}
@ -40,10 +31,10 @@ namespace Wabbajack.Lib.Downloaders
var client = new MegaApiClient();
Utils.Status("Logging into MEGA (as anonymous)");
client.LoginAnonymous();
var file_link = new Uri(Url);
var node = client.GetNodeFromLink(file_link);
var fileLink = new Uri(Url);
var node = client.GetNodeFromLink(fileLink);
Utils.Status($"Downloading MEGA file: {a.Name}");
client.DownloadFile(file_link, destination);
client.DownloadFile(fileLink, destination);
}
public override bool Verify()
@ -51,10 +42,10 @@ namespace Wabbajack.Lib.Downloaders
var client = new MegaApiClient();
Utils.Status("Logging into MEGA (as anonymous)");
client.LoginAnonymous();
var file_link = new Uri(Url);
var fileLink = new Uri(Url);
try
{
var node = client.GetNodeFromLink(file_link);
var node = client.GetNodeFromLink(fileLink);
return true;
}
catch (Exception)

16
Wabbajack.Lib/Downloaders/ManualDownloader.cs
View File

@ -1,13 +1,9 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Reactive.Threading.Tasks;
using System.Text;
using System.Threading.Tasks;
using Syroot.Windows.IO;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
@ -62,9 +58,9 @@ namespace Wabbajack.Lib.Downloaders
}
}
public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
{
var url = archive_ini?.General?.manualURL;
var url = archiveINI?.General?.manualURL;
return url != null ? new State { Url = url} : null;
}
@ -83,7 +79,7 @@ namespace Wabbajack.Lib.Downloaders
public override void Download(Archive a, string destination)
{
var downloader = (ManualDownloader)GetDownloader();
var abs_path = Path.Combine(downloader._downloadfolder.Path, a.Name);
var absPath = Path.Combine(downloader._downloadfolder.Path, a.Name);
lock (downloader)
{
try
@ -99,10 +95,10 @@ namespace Wabbajack.Lib.Downloaders
.FirstOrDefaultAsync();
Process.Start(Url);
abs_path = watcher.Wait()?.FullPath;
if (!File.Exists(abs_path))
absPath = watcher.Wait()?.FullPath;
if (!File.Exists(absPath))
throw new InvalidDataException($"File not found after manual download operation");
File.Move(abs_path, destination);
File.Move(absPath, destination);
}
finally
{

14
Wabbajack.Lib/Downloaders/MediaFireDownloader.cs
View File

@ -1,10 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Lib.Validation;
using Wabbajack.Lib.WebAutomation;
@ -13,9 +9,9 @@ namespace Wabbajack.Lib.Downloaders
{
public class MediaFireDownloader : IUrlDownloader
{
public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
{
Uri url = DownloaderUtils.GetDirectURL(archive_ini);
Uri url = DownloaderUtils.GetDirectURL(archiveINI);
if (url == null || url.Host != "www.mediafire.com") return null;
return new State
@ -50,12 +46,12 @@ namespace Wabbajack.Lib.Downloaders
await d.NavigateTo(new Uri("http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.tx"));
// MediaFire creates the link after all the JS loads
await Task.Delay(1000);
var new_url = await d.GetAttr("a.input", "href");
if (new_url == null || !new_url.StartsWith("http")) return null;
var newURL = await d.GetAttr("a.input", "href");
if (newURL == null || !newURL.StartsWith("http")) return null;
return new HTTPDownloader.State()
{
Client = new HttpClient(),
Url = new_url
Url = newURL
};
}
}

23
Wabbajack.Lib/Downloaders/ModDBDownloader.cs
View File

@ -1,10 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
@ -12,9 +7,9 @@ namespace Wabbajack.Lib.Downloaders
{
public class ModDBDownloader : IDownloader, IUrlDownloader
{
public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
{
var url = archive_ini?.General?.directURL;
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url);
}
@ -46,8 +41,8 @@ namespace Wabbajack.Lib.Downloaders
public override void Download(Archive a, string destination)
{
var new_url = GetDownloadUrl();
new HTTPDownloader.State {Url = new_url}.Download(a, destination);
var newURL = GetDownloadUrl();
new HTTPDownloader.State {Url = newURL}.Download(a, destination);
}
private string GetDownloadUrl()
@ -56,14 +51,14 @@ namespace Wabbajack.Lib.Downloaders
var result = client.GetStringSync(Url);
var regex = new Regex("https:\\/\\/www\\.moddb\\.com\\/downloads\\/mirror\\/.*(?=\\\")");
var match = regex.Match(result);
var new_url = match.Value;
return new_url;
var newURL = match.Value;
return newURL;
}
public override bool Verify()
{
var new_url = GetDownloadUrl();
return new HTTPDownloader.State { Url = new_url }.Verify();
var newURL = GetDownloadUrl();
return new HTTPDownloader.State { Url = newURL }.Verify();
}
public override IDownloader GetDownloader()

8
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -1,20 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Game = Wabbajack.Common.Game;
namespace Wabbajack.Lib.Downloaders
{
public class NexusDownloader : IDownloader
{
public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
{
var general = archive_ini?.General;
var general = archiveINI?.General;
if (general.modID != null && general.fileID != null && general.gameName != null)
{

3
Wabbajack.Lib/IBatchProcessor.cs
View File

@ -1,7 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;

71
Wabbajack.Lib/MO2Compiler.cs
View File

@ -9,7 +9,6 @@ using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Wabbajack.VirtualFileSystem;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
@ -25,11 +24,11 @@ namespace Wabbajack.Lib
public string MO2Profile;
public MO2Compiler(string mo2_folder)
public MO2Compiler(string mo2Folder)
{
ModManager = ModManager.MO2;
MO2Folder = mo2_folder;
MO2Folder = mo2Folder;
MO2Ini = Path.Combine(MO2Folder, "ModOrganizer.ini").LoadIniFile();
GamePath = ((string)MO2Ini.General.gamePath).Replace("\\\\", "\\");
@ -69,9 +68,9 @@ namespace Wabbajack.Lib
UpdateTracker.Reset();
UpdateTracker.NextStep("Gathering information");
Info("Looking for other profiles");
var other_profiles_path = Path.Combine(MO2ProfileDir, "otherprofiles.txt");
var otherProfilesPath = Path.Combine(MO2ProfileDir, "otherprofiles.txt");
SelectedProfiles = new HashSet<string>();
if (File.Exists(other_profiles_path)) SelectedProfiles = File.ReadAllLines(other_profiles_path).ToHashSet();
if (File.Exists(otherProfilesPath)) SelectedProfiles = File.ReadAllLines(otherProfilesPath).ToHashSet();
SelectedProfiles.Add(MO2Profile);
Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));
@ -105,31 +104,31 @@ namespace Wabbajack.Lib
UpdateTracker.NextStep("Finding Install Files");
Directory.CreateDirectory(ModListOutputFolder);
var mo2_files = Directory.EnumerateFiles(MO2Folder, "*", SearchOption.AllDirectories)
var mo2Files = Directory.EnumerateFiles(MO2Folder, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) { Path = p.RelativeTo(MO2Folder) });
var game_files = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
var gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{ Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) });
var loot_path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
var lootPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"LOOT");
// TODO: make this generic so we can add more paths
IEnumerable<RawSourceFile> loot_files = new List<RawSourceFile>();
if (Directory.Exists(loot_path))
IEnumerable<RawSourceFile> lootFiles = new List<RawSourceFile>();
if (Directory.Exists(lootPath))
{
Info($"Indexing {loot_path}");
VFS.AddRoot(loot_path);
Info($"Indexing {lootPath}");
VFS.AddRoot(lootPath);
VFS.WriteToFile(_vfsCacheName);
loot_files = Directory.EnumerateFiles(loot_path, "userlist.yaml", SearchOption.AllDirectories)
lootFiles = Directory.EnumerateFiles(lootPath, "userlist.yaml", SearchOption.AllDirectories)
.Where(p => p.FileExists())
.Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
{ Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(loot_path)) });
{ Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)) });
}
IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder)
@ -148,8 +147,8 @@ namespace Wabbajack.Lib
.GroupBy(f => f.Hash)
.ToDictionary(f => f.Key, f => f.AsEnumerable());
AllFiles = mo2_files.Concat(game_files)
.Concat(loot_files)
AllFiles = mo2Files.Concat(gameFiles)
.Concat(lootFiles)
.DistinctBy(f => f.Path)
.ToList();
@ -174,10 +173,10 @@ namespace Wabbajack.Lib
ModInis = Directory.EnumerateDirectories(Path.Combine(MO2Folder, "mods"))
.Select(f =>
{
var mod_name = Path.GetFileName(f);
var meta_path = Path.Combine(f, "meta.ini");
if (File.Exists(meta_path))
return (mod_name, meta_path.LoadIniFile());
var modName = Path.GetFileName(f);
var metaPath = Path.Combine(f, "meta.ini");
if (File.Exists(metaPath))
return (mod_name: modName, metaPath.LoadIniFile());
return (null, null);
})
.Where(f => f.Item2 != null)
@ -295,48 +294,48 @@ namespace Wabbajack.Lib
.ToList();
Info($"Patching building patches from {groups.Count} archives");
var absolute_paths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath);
groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolute_paths));
var absolutePaths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath);
groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths));
if (InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == null) != null)
Error("Missing patches after generation, this should not happen");
}
private void BuildArchivePatches(string archive_sha, IEnumerable<PatchedFromArchive> group,
Dictionary<string, string> absolute_paths)
private void BuildArchivePatches(string archiveSha, IEnumerable<PatchedFromArchive> group,
Dictionary<string, string> absolutePaths)
{
using (var files = VFS.StageWith(group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath))))
{
var by_path = files.GroupBy(f => string.Join("|", f.FilesInFullPath.Skip(1).Select(i => i.Name)))
var byPath = files.GroupBy(f => string.Join("|", f.FilesInFullPath.Skip(1).Select(i => i.Name)))
.ToDictionary(f => f.Key, f => f.First());
// Now Create the patches
group.PMap(Queue, entry =>
{
Info($"Patching {entry.To}");
Status($"Patching {entry.To}");
using (var origin = by_path[string.Join("|", entry.ArchiveHashPath.Skip(1))].OpenRead())
using (var origin = byPath[string.Join("|", entry.ArchiveHashPath.Skip(1))].OpenRead())
using (var output = new MemoryStream())
{
var a = origin.ReadAll();
var b = LoadDataForTo(entry.To, absolute_paths).Result;
var b = LoadDataForTo(entry.To, absolutePaths).Result;
Utils.CreatePatch(a, b, output);
entry.PatchID = IncludeFile(output.ToArray());
var file_size = File.GetSize(Path.Combine(ModListOutputFolder, entry.PatchID));
Info($"Patch size {file_size} for {entry.To}");
var fileSize = File.GetSize(Path.Combine(ModListOutputFolder, entry.PatchID));
Info($"Patch size {fileSize} for {entry.To}");
}
});
}
}
private async Task<byte[]> LoadDataForTo(string to, Dictionary<string, string> absolute_paths)
private async Task<byte[]> LoadDataForTo(string to, Dictionary<string, string> absolutePaths)
{
if (absolute_paths.TryGetValue(to, out var absolute))
if (absolutePaths.TryGetValue(to, out var absolute))
return File.ReadAllBytes(absolute);
if (to.StartsWith(Consts.BSACreationDir))
{
var bsa_id = to.Split('\\')[1];
var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsa_id);
var bsaID = to.Split('\\')[1];
var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaID);
using (var a = BSADispatch.OpenRead(Path.Combine(MO2Folder, bsa.To)))
{
@ -356,9 +355,9 @@ namespace Wabbajack.Lib
public override IEnumerable<ICompilationStep> GetStack()
{
var user_config = Path.Combine(MO2ProfileDir, "compilation_stack.yml");
if (File.Exists(user_config))
return Serialization.Deserialize(File.ReadAllText(user_config), this);
var userConfig = Path.Combine(MO2ProfileDir, "compilation_stack.yml");
if (File.Exists(userConfig))
return Serialization.Deserialize(File.ReadAllText(userConfig), this);
var stack = MakeStack();

51
Wabbajack.Lib/MO2Installer.cs
View File

@ -3,15 +3,12 @@ using System.IO;
using System.Linq;
using System.Text;
using System.Windows;
using System.Windows.Forms.VisualStyles;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Wabbajack.VirtualFileSystem;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Lib
@ -20,13 +17,13 @@ namespace Wabbajack.Lib
{
public bool WarnOnOverwrite { get; set; } = true;
public MO2Installer(string archive, ModList mod_list, string output_folder)
public MO2Installer(string archive, ModList modList, string outputFolder)
{
ModManager = ModManager.MO2;
ModListArchive = archive;
OutputFolder = output_folder;
OutputFolder = outputFolder;
DownloadFolder = Path.Combine(OutputFolder, "downloads");
ModList = mod_list;
ModList = modList;
}
public string GameFolder { get; set; }
@ -112,9 +109,9 @@ namespace Wabbajack.Lib
.PMap(Queue, directive =>
{
Status($"Writing included .meta file {directive.To}");
var out_path = Path.Combine(DownloadFolder, directive.To);
if (File.Exists(out_path)) File.Delete(out_path);
File.WriteAllBytes(out_path, LoadBytesFromPath(directive.SourceDataID));
var outPath = Path.Combine(DownloadFolder, directive.To);
if (File.Exists(outPath)) File.Delete(outPath);
File.WriteAllBytes(outPath, LoadBytesFromPath(directive.SourceDataID));
});
}
@ -123,9 +120,9 @@ namespace Wabbajack.Lib
foreach (var esm in ModList.Directives.OfType<CleanedESM>().ToList())
{
var filename = Path.GetFileName(esm.To);
var game_file = Path.Combine(GameFolder, "Data", filename);
var gameFile = Path.Combine(GameFolder, "Data", filename);
Utils.Log($"Validating {filename}");
var hash = game_file.FileHash();
var hash = gameFile.FileHash();
if (hash != esm.SourceESMHash)
{
Utils.Error("Game ESM hash doesn't match, is the ESM already cleaned? Please verify your local game files.");
@ -176,14 +173,14 @@ namespace Wabbajack.Lib
bsas.Do(bsa =>
{
Status($"Building {bsa.To}");
var source_dir = Path.Combine(OutputFolder, Consts.BSACreationDir, bsa.TempID);
var sourceDir = Path.Combine(OutputFolder, Consts.BSACreationDir, bsa.TempID);
using (var a = bsa.State.MakeBuilder())
{
bsa.FileStates.PMap(Queue, state =>
{
Status($"Adding {state.Path} to BSA");
using (var fs = File.OpenRead(Path.Combine(source_dir, state.Path)))
using (var fs = File.OpenRead(Path.Combine(sourceDir, state.Path)))
{
a.AddFile(state, fs);
}
@ -195,11 +192,11 @@ namespace Wabbajack.Lib
});
var bsa_dir = Path.Combine(OutputFolder, Consts.BSACreationDir);
if (Directory.Exists(bsa_dir))
var bsaDir = Path.Combine(OutputFolder, Consts.BSACreationDir);
if (Directory.Exists(bsaDir))
{
Info($"Removing temp folder {Consts.BSACreationDir}");
Directory.Delete(bsa_dir, true, true);
Directory.Delete(bsaDir, true, true);
}
}
@ -211,36 +208,36 @@ namespace Wabbajack.Lib
.PMap(Queue, directive =>
{
Status($"Writing included file {directive.To}");
var out_path = Path.Combine(OutputFolder, directive.To);
if (File.Exists(out_path)) File.Delete(out_path);
var outPath = Path.Combine(OutputFolder, directive.To);
if (File.Exists(outPath)) File.Delete(outPath);
if (directive is RemappedInlineFile)
WriteRemappedFile((RemappedInlineFile)directive);
else if (directive is CleanedESM)
GenerateCleanedESM((CleanedESM)directive);
else
File.WriteAllBytes(out_path, LoadBytesFromPath(directive.SourceDataID));
File.WriteAllBytes(outPath, LoadBytesFromPath(directive.SourceDataID));
});
}
private void GenerateCleanedESM(CleanedESM directive)
{
var filename = Path.GetFileName(directive.To);
var game_file = Path.Combine(GameFolder, "Data", filename);
var gameFile = Path.Combine(GameFolder, "Data", filename);
Info($"Generating cleaned ESM for {filename}");
if (!File.Exists(game_file)) throw new InvalidDataException($"Missing {filename} at {game_file}");
if (!File.Exists(gameFile)) throw new InvalidDataException($"Missing {filename} at {gameFile}");
Status($"Hashing game version of {filename}");
var sha = game_file.FileHash();
var sha = gameFile.FileHash();
if (sha != directive.SourceESMHash)
throw new InvalidDataException(
$"Cannot patch {filename} from the game folder hashes don't match have you already cleaned the file?");
var patch_data = LoadBytesFromPath(directive.SourceDataID);
var to_file = Path.Combine(OutputFolder, directive.To);
var patchData = LoadBytesFromPath(directive.SourceDataID);
var toFile = Path.Combine(OutputFolder, directive.To);
Status($"Patching {filename}");
using (var output = File.OpenWrite(to_file))
using (var input = File.OpenRead(game_file))
using (var output = File.OpenWrite(toFile))
using (var input = File.OpenRead(gameFile))
{
BSDiff.Apply(input, () => new MemoryStream(patch_data), output);
BSDiff.Apply(input, () => new MemoryStream(patchData), output);
}
}

14
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -1,18 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Collections.Generic;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Media.Imaging;
using CommonMark.Syntax;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.Validation;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
using File = System.IO.File;
using Game = Wabbajack.Common.Game;
@ -32,7 +22,7 @@ namespace Wabbajack.Lib.ModListRegistry
[JsonProperty("game")]
public Game Game { get; set; }
[JsonIgnore] public string GameName => this.Game.ToDescriptionString();
[JsonIgnore] public string GameName => Game.ToDescriptionString();
[JsonProperty("official")]
public bool Official { get; set; }

3
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -1,7 +1,6 @@
using ReactiveUI;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
@ -9,12 +8,10 @@ using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Reflection;
using System.Runtime.Serialization.Json;
using System.Security.Authentication;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using Microsoft.WindowsAPICodePack.Shell;
using Syroot.Windows.IO;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;

1
Wabbajack.Lib/Properties/AssemblyInfo.cs
View File

@ -1,5 +1,4 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following

3
Wabbajack.Lib/Readme.md Normal file
View File

@ -0,0 +1,3 @@
# Wabbajack.Lib
While `Wabbajack` is the front end, `Wabbajack.Lib` is the back end and contains all functionality from the Compilers, the Installers to our NexusAPI and Downloaders.

29
Wabbajack.Lib/ReportBuilder.cs
View File

@ -3,7 +3,6 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
@ -12,19 +11,19 @@ namespace Wabbajack.Lib
public class ReportBuilder : IDisposable
{
private const int WRAP_SIZE = 80;
private readonly StreamWriter wtr;
private string _output_folder;
private readonly StreamWriter _wtr;
private string _outputFolder;
public ReportBuilder(Stream str, string output_folder)
public ReportBuilder(Stream str, string outputFolder)
{
_output_folder = output_folder;
wtr = new StreamWriter(str);
_outputFolder = outputFolder;
_wtr = new StreamWriter(str);
}
public void Dispose()
{
wtr.Flush();
wtr?.Dispose();
_wtr.Flush();
_wtr?.Dispose();
}
public void Text(string txt)
@ -32,16 +31,16 @@ namespace Wabbajack.Lib
var offset = 0;
while (offset + WRAP_SIZE < txt.Length)
{
wtr.WriteLine(txt.Substring(offset, WRAP_SIZE));
_wtr.WriteLine(txt.Substring(offset, WRAP_SIZE));
offset += WRAP_SIZE;
}
if (offset < txt.Length) wtr.WriteLine(txt.Substring(offset, txt.Length - offset));
if (offset < txt.Length) _wtr.WriteLine(txt.Substring(offset, txt.Length - offset));
}
public void NoWrapText(string txt)
{
wtr.WriteLine(txt);
_wtr.WriteLine(txt);
}
public void Build(ACompiler c, ModList lst)
@ -59,9 +58,9 @@ namespace Wabbajack.Lib
if (lst.ModManager == ModManager.MO2)
{
var readme_file = Path.Combine(compiler?.MO2ProfileDir, "readme.md");
if (File.Exists(readme_file))
File.ReadAllLines(readme_file)
var readmeFile = Path.Combine(compiler?.MO2ProfileDir, "readme.md");
if (File.Exists(readmeFile))
File.ReadAllLines(readmeFile)
.Do(NoWrapText);
}
@ -127,7 +126,7 @@ namespace Wabbajack.Lib
private long SizeForID(string id)
{
return File.GetSize(Path.Combine(_output_folder, id));
return File.GetSize(Path.Combine(_outputFolder, id));
}
private IEnumerable<Archive> SortArchives(List<Archive> lstArchives)

2
Wabbajack.Lib/UI/UIUtils.cs
View File

@ -1,13 +1,11 @@
using Microsoft.WindowsAPICodePack.Dialogs;
using System;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Windows.Media.Imaging;
using System.Windows.Threading;
using Wabbajack.Common;
namespace Wabbajack.Lib

12
Wabbajack.Lib/ViewModel.cs
View File

@ -1,25 +1,21 @@
using ReactiveUI;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Reactive.Disposables;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace Wabbajack.Lib
{
public class ViewModel : ReactiveObject, IDisposable
{
private readonly Lazy<CompositeDisposable> _CompositeDisposable = new Lazy<CompositeDisposable>();
public CompositeDisposable CompositeDisposable => _CompositeDisposable.Value;
private readonly Lazy<CompositeDisposable> _compositeDisposable = new Lazy<CompositeDisposable>();
public CompositeDisposable CompositeDisposable => _compositeDisposable.Value;
public virtual void Dispose()
{
if (_CompositeDisposable.IsValueCreated)
if (_compositeDisposable.IsValueCreated)
{
_CompositeDisposable.Value.Dispose();
_compositeDisposable.Value.Dispose();
}
}

1
Wabbajack.Lib/VortexCompiler.cs
View File

@ -9,7 +9,6 @@ using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.NexusApi;
using Wabbajack.VirtualFileSystem;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.Lib

1
Wabbajack.Lib/VortexInstaller.cs
View File

@ -1,6 +1,5 @@
using System.Linq;
using Wabbajack.Common;
using Wabbajack.VirtualFileSystem;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;

3
Wabbajack.Lib/WebAutomation/WebAutomation.cs
View File

@ -1,7 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows;

14
Wabbajack.Lib/WebAutomation/WebAutomationWindow.xaml.cs
View File

@ -1,16 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Shapes;
using System.Windows;
using Wabbajack.Lib.WebAutomation;
namespace Wabbajack

3
Wabbajack.Lib/WebAutomation/WebAutomationWindowViewModel.cs
View File

@ -1,7 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Toolkit.Win32.UI.Controls.Interop.WinRT;
using Microsoft.Toolkit.Wpf.UI.Controls;

3
Wabbajack.Test.ListValidation/ListValidation.cs
View File

@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Windows.Documents;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
using Wabbajack.Lib;
@ -64,7 +63,7 @@ namespace Wabbajack.Test.ListValidation
Log($"Loading {modlist_path}");
var installer = MO2Installer.LoadFromFile(modlist_path);
var installer = AInstaller.LoadFromFile(modlist_path);
Log($"{installer.Archives.Count} archives to validate");

1
Wabbajack.Test.ListValidation/Properties/AssemblyInfo.cs
View File

@ -1,5 +1,4 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
[assembly: AssemblyTitle("Wabbajack.Test.ListValidation")]

4
Wabbajack.Test.ListValidation/Readme.md Normal file
View File

@ -0,0 +1,4 @@
# Wabbajack.Test.ListValidation
This project is not part of `Wabbajack.Test` as the `ListValidation` test validates every ModList from [this](https://github.com/wabbajack-tools/mod-lists) repository and checks if all ModLists are valid.
The gets called when you push to master or to the repo linked above.

6
Wabbajack.Test/ACompilerTest.cs
View File

@ -1,8 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
using Wabbajack.Lib;
@ -55,7 +51,7 @@ namespace Wabbajack.Test
protected void Install(MO2Compiler compiler)
{
var modlist = MO2Installer.LoadFromFile(compiler.ModListOutputFile);
var modlist = AInstaller.LoadFromFile(compiler.ModListOutputFile);
var installer = new MO2Installer(compiler.ModListOutputFile, modlist, utils.InstallFolder);
installer.WarnOnOverwrite = false;
installer.DownloadFolder = utils.DownloadsFolder;

2
Wabbajack.Test/AVortexCompilerTest.cs
View File

@ -60,7 +60,7 @@ namespace Wabbajack.Test
protected void Install(VortexCompiler vortexCompiler)
{
var modList = MO2Installer.LoadFromFile(vortexCompiler.ModListOutputFile);
var modList = AInstaller.LoadFromFile(vortexCompiler.ModListOutputFile);
var installer = new MO2Installer(vortexCompiler.ModListOutputFile, modList, utils.InstallFolder)
{
DownloadFolder = utils.DownloadsFolder,

4
Wabbajack.Test/CSP/CSPTests.cs
View File

@ -1,9 +1,7 @@
using System;
using System.Linq;
using System.Linq;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common.CSP;

5
Wabbajack.Test/CompilationStackTests.cs
View File

@ -1,7 +1,4 @@
using System;
using System.Windows.Forms;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Lib;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Lib.CompilationSteps;
namespace Wabbajack.Test

1
Wabbajack.Test/ContentRightsManagementTests.cs
View File

@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.Linq;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib;
using Wabbajack.Lib.Validation;

3
Wabbajack.Test/DownloaderTests.cs
View File

@ -1,8 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
using Wabbajack.Lib;

4
Wabbajack.Test/EndToEndTests.cs
View File

@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
@ -146,7 +144,7 @@ namespace Wabbajack.Test
private void Install(MO2Compiler compiler)
{
var modlist = MO2Installer.LoadFromFile(compiler.ModListOutputFile);
var modlist = AInstaller.LoadFromFile(compiler.ModListOutputFile);
var installer = new MO2Installer(compiler.ModListOutputFile, modlist, utils.InstallFolder);
installer.DownloadFolder = utils.DownloadsFolder;
installer.GameFolder = utils.GameFolder;

7
Wabbajack.Test/Extensions.cs
View File

@ -1,9 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Microsoft.VisualStudio.TestTools.UnitTesting;
namespace Wabbajack.Test
{

8
Wabbajack.Test/MiscTests.cs
View File

@ -1,10 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using MahApps.Metro.Controls;
using Wabbajack.Common;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Utils = Wabbajack.Common.Utils;

6
Wabbajack.Test/ModlistMetadataTests.cs
View File

@ -1,8 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Linq;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;

1
Wabbajack.Test/Properties/AssemblyInfo.cs
View File

@ -1,5 +1,4 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
[assembly: AssemblyTitle("Wabbajack.Test")]

3
Wabbajack.Test/Readme.md Normal file
View File

@ -0,0 +1,3 @@
# Wabbajack.Test
This project contains almost all tests for Wabbajack. If you create a new PR, push to master or update your PR than the tests in this project will be started at Azure DevOps. It is recommended that you run the test(s) for the part of the application you have changed. You can also create new tests if you have added something that other tests do not cover.

Some files were not shown because too many files have changed in this diff Show More