From d28719bd748887df9dfda1707fbb1357f97524a5 Mon Sep 17 00:00:00 2001 From: Unnoen Date: Sat, 9 Jan 2021 13:47:00 +1100 Subject: [PATCH 1/2] Filter for filesize before hashing archives --- Wabbajack.Lib/AInstaller.cs | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 0c1c5e1d..228135e0 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -279,12 +279,41 @@ namespace Wabbajack.Lib public async Task HashArchives() { Utils.Log("Looking for files to hash"); - var toHash = DownloadFolder.EnumerateFiles() + // Enumerate all files. + var allFiles = DownloadFolder.EnumerateFiles() .Concat(Game.GameLocation().EnumerateFiles()) .ToList(); + + // Create a Dictionary with filesize as the key. + var hashDict = new Dictionary>(); + + // Populate the Dictionary with all files we found, using a list for the paths to allow for duplicated filesizes. + allFiles.ForEach(o => + { + if (hashDict.ContainsKey(o.Size)) + { + hashDict[o.Size].Add(o); + } + else + { + hashDict.Add(o.Size, new List { o }); + } + }); + + // Create a list for the files we filter. + var toHash = new List(); + + // Iterate through required archives, adding any files we have that match filesize. + ModList.Archives.ForEach(o => { + if (hashDict.ContainsKey(o.Size)) + { + hashDict[o.Size].ForEach(o => toHash.Add(o)); + } + }); - Utils.Log($"Found {toHash.Count} files to hash"); + Utils.Log($"Found {allFiles.Count} total files, {toHash.Count} matching filesize"); + // Hash all the files we added. var hashResults = await toHash .PMap(Queue, UpdateTracker,async e => (await e.FileHashCachedAsync(), e)); From 5d857386f2158f1ed7534885ad0fb2f8cf5a6cfd Mon Sep 17 00:00:00 2001 From: Unnoen Date: Sat, 9 Jan 2021 14:57:11 +1100 Subject: [PATCH 2/2] Simplify filtering and remove comments Thanks Linq --- Wabbajack.Lib/AInstaller.cs | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 228135e0..6f1c5c13 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; @@ -279,41 +279,17 @@ namespace Wabbajack.Lib public async Task HashArchives() { Utils.Log("Looking for files to hash"); - // Enumerate all files. + var allFiles = DownloadFolder.EnumerateFiles() .Concat(Game.GameLocation().EnumerateFiles()) .ToList(); - // Create a Dictionary with filesize as the key. - var hashDict = new Dictionary>(); + var hashDict = allFiles.GroupBy(f => f.Size).ToDictionary(g => g.Key); - // Populate the Dictionary with all files we found, using a list for the paths to allow for duplicated filesizes. - allFiles.ForEach(o => - { - if (hashDict.ContainsKey(o.Size)) - { - hashDict[o.Size].Add(o); - } - else - { - hashDict.Add(o.Size, new List { o }); - } - }); - - // Create a list for the files we filter. - var toHash = new List(); - - // Iterate through required archives, adding any files we have that match filesize. - ModList.Archives.ForEach(o => { - if (hashDict.ContainsKey(o.Size)) - { - hashDict[o.Size].ForEach(o => toHash.Add(o)); - } - }); + var toHash = ModList.Archives.Where(a => hashDict.ContainsKey(a.Size)).SelectMany(a => hashDict[a.Size]).ToList(); Utils.Log($"Found {allFiles.Count} total files, {toHash.Count} matching filesize"); - // Hash all the files we added. var hashResults = await toHash .PMap(Queue, UpdateTracker,async e => (await e.FileHashCachedAsync(), e));