Merge pull request #786 from wabbajack-tools/ba2-duplicate-file-fixes

Ba2 duplicate file fixes

Compression.BSA.Test/BSATests.cs

@@ -75,6 +75,7 @@ namespace Compression.BSA.Test
         [InlineData(Game.Fallout4, 22223)] // 10mm SMG
         [InlineData(Game.Fallout4, 4472)] // True Storms
         [InlineData(Game.Morrowind, 44537)]
+        [InlineData(Game.Fallout4, 43474)] // EM 2 Rifle
         public async Task BSACompressionRecompression(Game game, int modid)
         {
             var filename = await DownloadMod(game, modid);
@@ -109,6 +110,11 @@ namespace Compression.BSA.Test
 
                     Assert.Equal(file.Size, absName.Size);
                 });
+                
+                
+                // Check Files should be case insensitive
+                Assert.Equal(a.Files.Count(), a.Files.Select(f => f.Path).ToHashSet().Count);
+                Assert.Equal(a.Files.Count(), a.Files.Select(f => f.Path.ToString().ToLowerInvariant()).ToHashSet().Count);
 
                 TestContext.WriteLine($"Building {bsa}");
 
@@ -132,6 +138,7 @@ namespace Compression.BSA.Test
 
                 // Check same number of files
                 Assert.Equal(a.Files.Count(), b.Files.Count());
+                
 
                 await a.Files.Zip(b.Files, (ai, bi) => (ai, bi))
                     .PMap(Queue, pair =>

Compression.BSA/BA2Reader.cs

@@ -108,6 +108,18 @@ namespace Compression.BSA
 
         public IEnumerable<IFile> Files { get; private set; }
         public ArchiveStateObject State => new BA2StateObject(this);
+        public void Dump(Action<string> print)
+        {
+            print($"HeaderMagic: {_headerMagic}");
+            print($"Number of Files: {_numFiles}");
+            print($"----------------------------");
+            
+            foreach (var file in Files)
+            {
+                print("\n");
+                file.Dump(print);
+            }
+        }
     }
 
     [JsonName("BA2State")]
@@ -218,6 +230,12 @@ namespace Compression.BSA
             }
 
         }
+
+        public void Dump(Action<string> print)
+        {
+            throw new NotImplementedException();
+        }
+
         private void WriteHeader(BinaryWriter bw)
         {
             var ddsHeader = new DDS_HEADER();
@@ -426,6 +444,17 @@ namespace Compression.BSA
         internal int _index;
 
         public bool Compressed => _size != 0;
+        
+        public void Dump(Action<string> print)
+        {
+            print($"FullPath: {FullPath}");
+            print($"Name Hash: {_nameHash}");
+            print($"Offset: {_offset}");
+            print($"Flags: {_flags:x}");
+            print($"Real Size: {_realSize}");
+            print($"Index: {_index}");
+        }
+
 
         public BA2FileEntry(BA2Reader ba2Reader, int index)
         {
@@ -442,6 +471,8 @@ namespace Compression.BSA
             _realSize = _rdr.ReadUInt32();
             _align = _rdr.ReadUInt32();
         }
+        
+        
 
         public string FullPath { get; set; }
 
@@ -473,6 +504,7 @@ namespace Compression.BSA
                 }
             }
         }
+
     }
 
     [JsonName("BA2FileEntryState")]

Compression.BSA/BSAReader.cs

@@ -65,6 +65,20 @@ namespace Compression.BSA
         internal uint _totalFileNameLength;
         internal uint _totalFolderNameLength;
         internal uint _version;
+        
+        public void Dump(Action<string> print)
+        {
+            print($"File Name: {_fileName}");
+            print($"File Count: {_fileCount}");
+            print($"Magic: {_magic}");
+
+            foreach (var file in Files)
+            {
+                print("\n");
+                file.Dump(print);
+            }
+        }
+
 
         public BSAReader(AbsolutePath filename)
         {
@@ -225,6 +239,18 @@ namespace Compression.BSA
         private readonly uint _originalSize;
         private readonly uint _size;
         internal readonly int _index;
+        
+        
+        public void Dump(Action<string> print)
+        {
+            print($"Name: {_name}");
+            print($"Offset: {_offset}");
+            print($"On Disk Size: {_onDiskSize}");
+            print($"Original Size: {_originalSize}");
+            print($"Size: {_size}");
+            print($"Index: {_index}");
+        }
+
 
         public FileRecord(BSAReader bsa, FolderRecord folderRecord, BinaryReader src, int index)
         {

Compression.BSA/IBSAReader.cs

@@ -14,6 +14,8 @@ namespace Compression.BSA
         IEnumerable<IFile> Files { get; }
 
         ArchiveStateObject State { get; }
+
+        void Dump(Action<string> print);
     }
 
     public interface IBSABuilder : IAsyncDisposable
@@ -59,5 +61,7 @@ namespace Compression.BSA
         /// </summary>
         /// <param name="output"></param>
         void CopyDataTo(Stream output);
+
+        void Dump(Action<string> print);
     }
 }

Compression.BSA/TES3Reader.cs

@@ -81,6 +81,11 @@ namespace Compression.BSA
                 };
             }
         }
+
+        public void Dump(Action<string> print)
+        {
+            throw new NotImplementedException();
+        }
     }
 
     [JsonName("TES3Archive")]
@@ -120,6 +125,11 @@ namespace Compression.BSA
             fs.CopyToLimit(output, (int)Size);
         }
 
+        public void Dump(Action<string> print)
+        {
+            throw new NotImplementedException();
+        }
+
         public uint Offset { get; set; }
         public uint NameOffset { get; set; }
         public uint Hash1 { get; set; }

Wabbajack.CLI/OptionsDefinition.cs

@@ -18,7 +18,8 @@ namespace Wabbajack.CLI
             typeof(MyFiles),
             typeof(DeleteFile),
             typeof(Changelog),
-            typeof(FindSimilar)
+            typeof(FindSimilar),
+            typeof(BSADump)
         };
     }
 }

Wabbajack.CLI/Program.cs

@@ -21,6 +21,7 @@ namespace Wabbajack.CLI
                     (DeleteFile opts) => opts.Execute(),
                     (Changelog opts) => opts.Execute(),
                     (FindSimilar opts) => opts.Execute(),
+                    (BSADump opts) => opts.Execute(),
                     errs => 1);
         }
     }

Wabbajack.CLI/Verbs/BSADump.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Threading.Tasks;
+using CommandLine;
+using Compression.BSA;
+using Wabbajack.Common;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("bsa-dump", HelpText = "Print detailed info about the contents of a BSA")]
+    public class BSADump : AVerb
+    {
+        [Option('i', "input", Required = true, HelpText = "Input BSA to dump")]
+        public string Input { get; set; } = "";
+        
+        protected override async Task<ExitCode> Run()
+        {
+            await using var bsa = BSADispatch.OpenRead(Input.RelativeTo(AbsolutePath.GetCurrentDirectory()));
+            bsa.Dump(line => Console.WriteLine(line));
+            return ExitCode.Ok;
+        }
+    }
+}

Wabbajack.Test/EndToEndTests.cs

@@ -61,6 +61,7 @@ namespace Wabbajack.Test
                 DownloadAndInstall(Game.SkyrimSpecialEdition, 12604, "SkyUI"), 
                 DownloadAndInstall(Game.Fallout4, 11925, "Anti-Tank Rifle"), 
                 DownloadAndInstall(Game.SkyrimSpecialEdition, 4783, "Frost Armor UNP"), 
+                DownloadAndInstall(Game.Fallout4, 43474, "EM 2 Rifle No.9 Mk1"),
                 DownloadAndInstall(Game.SkyrimSpecialEdition, 32359, "Frost Armor HDT"));
             
             // We're going to fully patch this mod from another source.

Wabbajack.VirtualFileSystem/Context.cs

@@ -359,7 +359,7 @@ namespace Wabbajack.VirtualFileSystem
         public async Task<IndexRoot> Integrate(ICollection<VirtualFile> files)
         {
             Utils.Log($"Integrating {files.Count} files");
-            var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList();
+            var allFiles = AllFiles.Concat(files).GroupBy(f => f.FullPath).Select(g => g.Last()).ToImmutableList();
 
             var byFullPath = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
                                      .ToDictionary(f => f.FullPath));