Re-add OMOD support

Compression.BSA/BA2Reader.cs

@@ -350,7 +350,7 @@ namespace Compression.BSA
             var ms = new MemoryStream();
             await CopyDataTo(ms);
             ms.Position = 0;
-            return new MemoryStreamFactory(ms);
+            return new MemoryStreamFactory(ms, Path);
         }
     }
 
@@ -517,7 +517,7 @@ namespace Compression.BSA
             var ms = new MemoryStream();
             await CopyDataTo(ms);
             ms.Position = 0;
-            return new MemoryStreamFactory(ms);
+            return new MemoryStreamFactory(ms, Path);
         }
     }
 

Compression.BSA/BSA/Reader/FileRecord.cs

@@ -171,7 +171,7 @@ namespace Compression.BSA
             var ms = new MemoryStream();
             await CopyDataTo(ms);
             ms.Position = 0;
-            return new MemoryStreamFactory(ms);
+            return new MemoryStreamFactory(ms, Path);
         }
     }
 }

Compression.BSA/MemoryStreamFactory.cs

@@ -9,9 +9,10 @@ namespace Compression.BSA
     {
         private readonly MemoryStream _data;
 
-        public MemoryStreamFactory(MemoryStream data)
+        public MemoryStreamFactory(MemoryStream data, IPath path)
         {
             _data = data;
+            Name = path;
         }
         public async ValueTask<Stream> GetStream()
         {
@@ -19,7 +20,7 @@ namespace Compression.BSA
         }
 
         public DateTime LastModifiedUtc => DateTime.UtcNow;
-        public IPath Name => (RelativePath)"BSA Memory Stream";
+        public IPath Name { get; }
     }
     
     public class MemoryBufferFactory : IStreamFactory
@@ -27,10 +28,11 @@ namespace Compression.BSA
         private readonly byte[] _data;
         private int _size;
 
-        public MemoryBufferFactory(byte[] data, int size)
+        public MemoryBufferFactory(byte[] data, int size, IPath path)
         {
             _data = data;
             _size = size;
+            Name = path;
         }
         public async ValueTask<Stream> GetStream()
         {
@@ -38,6 +40,6 @@ namespace Compression.BSA
         }
 
         public DateTime LastModifiedUtc => DateTime.UtcNow;
-        public IPath Name => (RelativePath)"BSA Memory Stream";
+        public IPath Name { get; }
     }
 }

Compression.BSA/TES3Reader.cs

@@ -135,7 +135,7 @@ namespace Compression.BSA
             var ms = new MemoryStream();
             await CopyDataTo(ms);
             ms.Position = 0;
-            return new MemoryStreamFactory(ms);
+            return new MemoryStreamFactory(ms, Path);
         }
 
 

Wabbajack.Common/IStreamFactory.cs

@@ -18,9 +18,16 @@ namespace Wabbajack.Common
     {
         private AbsolutePath _file;
 
+        public NativeFileStreamFactory(AbsolutePath file, IPath path)
+        {
+            _file = file;
+            Name = path;
+        }
+        
         public NativeFileStreamFactory(AbsolutePath file)
         {
             _file = file;
+            Name = file;
         }
         public async ValueTask<Stream> GetStream()
         {
@@ -28,7 +35,7 @@ namespace Wabbajack.Common
         }
 
         public DateTime LastModifiedUtc => _file.LastModifiedUtc;
-        public IPath Name => _file;
+        public IPath Name { get; }
     }
     
 }

Wabbajack.Common/Paths/FileCompaction.cs

@@ -57,5 +57,15 @@ namespace Wabbajack.Common
                 return await proc.Start() == 0;
             }
         }
+
+        public static async Task CompactFolder(this AbsolutePath folder, WorkQueue queue, Algorithm algorithm)
+        {
+            await folder.EnumerateFiles(true)
+                .PMap(queue, async path =>
+                {
+                    Utils.Status($"Compacting {path.FileName}");
+                    await path.Compact(algorithm);
+                });
+        }
     }
 }

Wabbajack.Lib/MO2Installer.cs

@@ -40,7 +40,7 @@ namespace Wabbajack.Lib
                   outputFolder: outputFolder, 
                   downloadFolder: downloadFolder,
                   parameters: parameters,
-                  steps: 21,
+                  steps: 22,
                   game: modList.GameType)
         {
             var gameExe = Consts.GameFolderFilesDir.Combine(modList.GameType.MetaData().MainExecutable!);
@@ -178,6 +178,9 @@ namespace Wabbajack.Lib
 
             UpdateTracker.NextStep("Updating System-specific ini settings");
             SetScreenSizeInPrefs();
+            
+            UpdateTracker.NextStep("Compacting files");
+            await CompactFiles();
 
             UpdateTracker.NextStep("Installation complete! You may exit the program.");
             await ExtractedModlistFolder!.DisposeAsync();
@@ -186,6 +189,14 @@ namespace Wabbajack.Lib
             return true;
         }
 
+        private async Task CompactFiles()
+        {
+            if (this.UseCompression)
+            {
+                await OutputFolder.CompactFolder(Queue, FileCompaction.Algorithm.XPRESS16K);
+            }
+        }
+
         private void CreateOutputMods()
         {
             OutputFolder.Combine("profiles")

Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs

@@ -1,7 +1,10 @@
 using System;
 using System.IO.Compression;
+using System.Linq;
 using System.Threading.Tasks;
 using Wabbajack.Common;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.NexusApi;
 using Xunit;
 
 namespace Wabbajack.VirtualFileSystem.Test
@@ -33,8 +36,26 @@ namespace Wabbajack.VirtualFileSystem.Test
             {
                 Assert.Equal(await temp.Dir.Combine(path).FileHashAsync(), hash);
             }
+        }
+        
+        private static Extension OMODExtension = new Extension(".omod");
+        private static Extension CRCExtension = new Extension(".crc");
 
+        [Fact]
+        public async Task CanGatherDataFromOMODFiles()
+        {
+            var src = await DownloadMod(Game.Oblivion, 18498);
 
+            await FileExtractor2.GatheringExtract(new NativeFileStreamFactory(src),
+                p => p.Extension == OMODExtension, async (path, sfn) =>
+                {
+                    await FileExtractor2.GatheringExtract(sfn, _ => true, async (ipath, isfn) => {
+                    // We shouldn't have any .crc files because this file should be recognized as a OMOD and extracted correctly
+                    Assert.NotEqual(CRCExtension, ipath.Extension);
+                    return 0;
+                    });
+                    return 0;
+                });
         }
 
 
@@ -59,5 +80,27 @@ namespace Wabbajack.VirtualFileSystem.Test
                 await folder.DeleteDirectory();
         }
         
+        
+        private static AbsolutePath _stagingFolder = ((RelativePath)"NexusDownloads").RelativeToEntryPoint();
+        private static async Task<AbsolutePath> DownloadMod(Game game, int mod)
+        {
+            using var client = await NexusApiClient.Get();
+            var results = await client.GetModFiles(game, mod);
+            var file = results.files.FirstOrDefault(f => f.is_primary) ??
+                       results.files.OrderByDescending(f => f.uploaded_timestamp).First();
+            var src = _stagingFolder.Combine(file.file_name);
+
+            if (src.Exists) return src;
+
+            var state = new NexusDownloader.State
+            {
+                ModID = mod,
+                Game = game,
+                FileID = file.file_id
+            };
+            await state.Download(src);
+            return src;
+        }
+        
     }
 }

Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj

@@ -15,6 +15,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\Wabbajack.Lib\Wabbajack.Lib.csproj" />
     <ProjectReference Include="..\Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj" />
   </ItemGroup>
 

Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs

@@ -4,11 +4,13 @@ using System.IO;
 using System.Threading.Tasks;
 using Compression.BSA;
 using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
+using OMODFramework;
 using SharpCompress.Archives.SevenZip;
 using SharpCompress.Readers;
 using Wabbajack.Common;
 using Wabbajack.Common.FileSignatures;
 using Wabbajack.VirtualFileSystem.SevenZipExtractor;
+using Utils = Wabbajack.Common.Utils;
 
 namespace Wabbajack.VirtualFileSystem
 {
@@ -22,6 +24,8 @@ namespace Wabbajack.VirtualFileSystem
             Definitions.FileType.RAR_OLD,
             Definitions.FileType.RAR_NEW,
             Definitions.FileType._7Z);
+        
+        private static Extension OMODExtension = new Extension(".omod");
 
 
         public static async Task<Dictionary<RelativePath, T>> GatheringExtract<T>(IStreamFactory sFn,
@@ -39,10 +43,21 @@ namespace Wabbajack.VirtualFileSystem
             {
                 case Definitions.FileType.RAR_OLD:
                 case Definitions.FileType.RAR_NEW:
-                case Definitions.FileType._7Z:    
+                case Definitions.FileType._7Z:
                 case Definitions.FileType.ZIP:
-                    return await GatheringExtractWith7Zip<T>(archive, (Definitions.FileType)sig, shouldExtract, mapfn);
+                {
-                
+                    if (sFn.Name.FileName.Extension == OMODExtension)
+                    {
+                        return await GatheringExtractWithOMOD(archive, shouldExtract, mapfn);
+
+                    }
+                    else
+                    {
+                        return await GatheringExtractWith7Zip<T>(archive, (Definitions.FileType)sig, shouldExtract,
+                            mapfn);
+                    }
+                }
+
                 case Definitions.FileType.TES3:
                 case Definitions.FileType.BSA:
                 case Definitions.FileType.BA2:
@@ -54,6 +69,54 @@ namespace Wabbajack.VirtualFileSystem
             }
         }
 
+        private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithOMOD<T>(Stream archive, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
+        {
+            var tmpFile = new TempFile();
+            await tmpFile.Path.WriteAllAsync(archive);
+            var dest = await TempFolder.Create();
+            Utils.Log($"Extracting {(string)tmpFile.Path}");
+
+            Framework.Settings.TempPath = (string)dest.Dir;
+            Framework.Settings.CodeProgress = new OMODProgress();
+
+            var omod = new OMOD((string)tmpFile.Path);
+            omod.GetDataFiles();
+            omod.GetPlugins();
+            
+            var results = new Dictionary<RelativePath, T>();
+            foreach (var file in dest.Dir.EnumerateFiles())
+            {
+                var path = file.RelativeTo(dest.Dir);
+                if (!shouldExtract(path)) continue;
+
+                var result = await mapfn(path, new NativeFileStreamFactory(file, path));
+                results.Add(path, result);
+            }
+
+            return results;
+        }
+        
+        private class OMODProgress : ICodeProgress
+        {
+            private long _total;
+
+            public void SetProgress(long inSize, long outSize)
+            {
+                Utils.Status("Extracting OMOD", Percent.FactoryPutInRange(inSize, _total));
+            }
+
+            public void Init(long totalSize, bool compressing)
+            {
+                _total = totalSize;
+            }
+
+            public void Dispose()
+            {
+                //
+            }
+        }
+
+
         private static async Task<Dictionary<RelativePath,T>> GatheringExtractWithBSA<T>(IStreamFactory sFn, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
         {
             var archive = await BSADispatch.OpenRead(sFn, sig);
@@ -73,31 +136,6 @@ namespace Wabbajack.VirtualFileSystem
         private static async Task<Dictionary<RelativePath,T>> GatheringExtractWith7Zip<T>(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory,ValueTask<T>> mapfn)
         {
             return await new GatheringExtractor<T>(stream, sig, shouldExtract, mapfn).Extract();
-            /*
-            IReader reader;
-            if (sig == Definitions.FileType._7Z)
-                reader = SevenZipArchive.Open(stream).ExtractAllEntries();
-            else
-            {
-                reader = ReaderFactory.Open(stream);
-            }
-
-            var results = new Dictionary<RelativePath, T>();
-            while (reader.MoveToNextEntry())
-            {
-                var path = (RelativePath)reader.Entry.Key;
-                if (!reader.Entry.IsDirectory && shouldExtract(path))
-                {
-                    var ms = new MemoryStream();
-                    reader.WriteEntryTo(ms);
-                    ms.Position = 0;
-                    var result = await mapfn(path, new MemoryStreamFactory(ms));
-                    results.Add(path, result);
-                }
-            }
-
-            return results;
-            */
         }
 
         public static async Task ExtractAll(AbsolutePath src, AbsolutePath dest)

Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs

@@ -126,6 +126,11 @@ namespace Wabbajack.VirtualFileSystem
                 _diskCached = _totalSize >= 500_000_000;
             }
 
+            private IPath GetPath()
+            {
+                return _extractor._indexes[_index].Item1;
+            }
+
             public int Write(byte[] data, uint size, IntPtr processedSize)
             {
                 try
@@ -155,7 +160,7 @@ namespace Wabbajack.VirtualFileSystem
 
             private void WriteSingleCall(byte[] data, in uint size)
             {
-                var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryBufferFactory(data, (int)size)).Result;
+                var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryBufferFactory(data, (int)size, GetPath())).Result;
                 AddResult(result);
                 Cleanup();
             }
@@ -181,7 +186,7 @@ namespace Wabbajack.VirtualFileSystem
 
                 _tmpStream.Flush();
                 _tmpStream.Position = 0;
-                var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory((MemoryStream)_tmpStream)).Result;
+                var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory((MemoryStream)_tmpStream, GetPath())).Result;
                 AddResult(result);
                 Cleanup();
             }
@@ -201,7 +206,7 @@ namespace Wabbajack.VirtualFileSystem
                 _tmpStream.Flush();
                 _tmpStream.Close();
                 
-                var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new NativeFileStreamFactory(_tmpFile.Path)).Result;
+                var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new NativeFileStreamFactory(_tmpFile.Path, GetPath())).Result;
                 AddResult(result);
                 Cleanup();
             }

Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj

@@ -31,4 +31,7 @@
         <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       </None>
     </ItemGroup>
+    <ItemGroup>
+      <Folder Include="Extractors\OMOD" />
+    </ItemGroup>
 </Project>