Merge pull request #227 from erri120/steam-workshop-support

Steam workshop support

Wabbajack.Common/GameMetaData.cs

@@ -96,6 +96,12 @@ namespace Wabbajack.Common
             return Games.Values.FirstOrDefault(g => g.NexusName == gameName.ToLower());
         }
 
+        public static GameMetaData GetBySteamID(int id)
+        {
+            return Games.Values
+                .FirstOrDefault(g => g.SteamIDs != null && g.SteamIDs.Count > 0 && g.SteamIDs.Any(i => i == id));
+        }
+
         public static IReadOnlyDictionary<Game, GameMetaData> Games = new Dictionary<Game, GameMetaData>
         {
             {

Wabbajack.Lib/CerasConfig.cs

@@ -23,7 +23,7 @@ namespace Wabbajack.Lib
                 typeof(MegaDownloader.State), typeof(ModDBDownloader.State), typeof(NexusDownloader.State),
                 typeof(BSAStateObject), typeof(BSAFileStateObject), typeof(BA2StateObject), typeof(BA2DX10EntryState),
                 typeof(BA2FileEntryState), typeof(MediaFireDownloader.State), typeof(ArchiveMeta),
-                typeof(PropertyFile)
+                typeof(PropertyFile), typeof(SteamMeta), typeof(SteamWorkshopDownloader), typeof(SteamWorkshopDownloader.State)
 
             }
         };

Wabbajack.Lib/CompilationSteps/IncludeSteamWorkshopItems.cs

@@ -0,0 +1,70 @@
+using System;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Alphaleonis.Win32.Filesystem;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.CompilationSteps
+{
+    public class IncludeSteamWorkshopItems : ACompilationStep
+    {
+        private readonly SteamGame _game;
+        private readonly Regex _regex = new Regex("steamWorkshopItem_\\d*\\.meta$");
+
+        public IncludeSteamWorkshopItems(ACompiler compiler, SteamGame steamGame) : base(compiler)
+        {
+            _game = steamGame;
+        }
+
+        public override Directive Run(RawSourceFile source)
+        {
+            if (!_regex.IsMatch(source.Path)) 
+                return null;
+
+            try
+            {
+                var lines = File.ReadAllLines(source.AbsolutePath);
+                var id = 0;
+                lines.Where(l => l.StartsWith("itemID=")).Do(l => int.TryParse(l.Replace("itemID=", ""), out id));
+                if (id == 0)
+                    return null;
+
+                SteamWorkshopItem item = null;
+                _game.WorkshopItems.Where(i => i.ItemID == id).Do(i => item = i);
+                if (item == null)
+                    return null;
+
+                var fromSteam = source.EvolveTo<SteamMeta>();
+                fromSteam.SourceDataID = _compiler.IncludeFile(source.AbsolutePath);
+                fromSteam.ItemID = item.ItemID;
+                fromSteam.Size = item.Size;
+                return fromSteam;
+            }
+            catch (Exception e)
+            {
+                Utils.LogToFile($"Exception while trying to evolve source to FromSteam\n{e}");
+                return null;
+            }
+        }
+
+        public override IState GetState()
+        {
+            return new State(_game);
+        }
+
+        public class State : IState
+        {
+            private readonly SteamGame _game;
+
+            public State(SteamGame game)
+            {
+                _game = game;
+            }
+
+            public ICompilationStep CreateStep(ACompiler compiler)
+            {
+                return new IncludeSteamWorkshopItems(compiler, _game);
+            }
+        }
+    }
+}

Wabbajack.Lib/Data.cs

@@ -169,6 +169,15 @@ namespace Wabbajack.Lib
     {
     }
 
+    public class SteamMeta : ArchiveMeta
+    {
+        public int ItemID;
+        /// <summary>
+        /// Size is in bytes
+        /// </summary>
+        public int Size;
+    }
+
     [MemberConfig(TargetMember.All)]
     public class FromArchive : Directive
     {

Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs

@@ -0,0 +1,97 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Lib.Validation;
+
+namespace Wabbajack.Lib.Downloaders
+{
+    public class SteamWorkshopDownloader : IUrlDownloader
+    {
+        private SteamWorkshopItem _item;
+
+        public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
+        {
+            var id = archiveINI?.General?.itemID;
+            var steamID = archiveINI?.General?.steamID;
+            var size = archiveINI?.General?.itemSize;
+            _item = new SteamWorkshopItem
+            {
+                ItemID = id != null ? int.Parse(id) : 0,
+                Size = size != null ? int.Parse(size) : 0,
+                Game = steamID != null ? GameRegistry.GetBySteamID(int.Parse(steamID)) : null
+            };
+            return new State {Item = _item};
+        }
+
+        public void Prepare()
+        {
+        }
+
+        public AbstractDownloadState GetDownloaderState(string url)
+        {
+            throw new NotImplementedException();
+        }
+
+        public class State : AbstractDownloadState
+        {
+            public SteamWorkshopItem Item { get; set; }
+            public override bool IsWhitelisted(ServerWhitelist whitelist)
+            {
+                return true;
+            }
+
+            public override void Download(Archive a, string destination)
+            {
+                var currentLib = "";
+                SteamHandler.Instance.InstallFolders.Where(f => f.Contains(Item.Game.InstallDir)).Do(s => currentLib = s);
+
+                var downloadFolder = Path.Combine(currentLib, "workshop", "downloads", Item.Game.AppId.ToString());
+                var contentFolder = Path.Combine(currentLib, "workshop", "content", Item.Game.AppId.ToString());
+                var p = new Process
+                {
+                    StartInfo = new ProcessStartInfo
+                    {
+                        FileName = Path.Combine(SteamHandler.Instance.SteamPath, "steam.exe"),
+                        CreateNoWindow = true,
+                        Arguments = $"console +workshop_download_item {Item.Game.AppId} {Item.ItemID}"
+                    }
+                };
+
+                p.Start();
+
+                //TODO: async
+                var finished = false;
+                while (!finished)
+                {
+                    if(!Directory.Exists(Path.Combine(downloadFolder, Item.ItemID.ToString())))
+                        if (Directory.Exists(Path.Combine(contentFolder, Item.ItemID.ToString())))
+                            finished = true;
+
+                    Thread.Sleep(1000);
+                }
+            }
+
+            public override bool Verify()
+            {
+                //TODO: find a way to verify steam workshop items
+                throw new NotImplementedException();
+            }
+
+            public override IDownloader GetDownloader()
+            {
+                return DownloadDispatcher.GetInstance<SteamWorkshopDownloader>();
+            }
+
+            public override string GetReportEntry(Archive a)
+            {
+                return $"* Steam - [{Item.ItemID}]";
+            }
+        }
+    }
+}

Wabbajack.Lib/ReportBuilder.cs

@@ -73,6 +73,14 @@ namespace Wabbajack.Lib
                 NoWrapText($"    * Size : {archive.Size.ToFileSizeString()}");
                 NoWrapText($"    * SHA256 : [{hash}](https://www.virustotal.com/gui/file/{hash})");
             }
+            lst.Directives.Where(d => d is SteamMeta).Do(f =>
+            {
+                if (f is SteamMeta s)
+                {
+                    var link = $"https://steamcommunity.com/sharedfiles/filedetails/?id={s.ItemID}";
+                    NoWrapText($"* Steam Workshop Item: [{s.ItemID}]({link}) | Size: {s.Size}");
+                }
+            });
 
             Text("\n\n");
             var patched = lst.Directives.OfType<PatchedFromArchive>().OrderBy(p => p.To).ToList();
@@ -134,4 +142,4 @@ namespace Wabbajack.Lib
             return lstArchives.OrderByDescending(a => a.Size);
         }
     }
-}
+}

Wabbajack.Lib/VortexCompiler.cs

@@ -40,6 +40,10 @@ namespace Wabbajack.Lib
         public const string StagingMarkerName = "__vortex_staging_folder";
         public const string DownloadMarkerName = "__vortex_downloads_folder";
 
+        private bool _isSteamGame;
+        private SteamGame _steamGame;
+        private bool _hasSteamWorkshopItems;
+
         public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder, string stagingFolder, string outputFile)
         {
             Game = game;
@@ -52,6 +56,15 @@ namespace Wabbajack.Lib
             ModListOutputFolder = "output_folder";
             ModListOutputFile = outputFile;
 
+            // there can be max one game after filtering
+            SteamHandler.Instance.Games.Where(g => g.Game != null && g.Game == game).Do(g =>
+            {
+                _isSteamGame = true;
+                _steamGame = g;
+                SteamHandler.Instance.LoadWorkshopItems(_steamGame);
+                _hasSteamWorkshopItems = _steamGame.WorkshopItems.Count > 0;
+            });
+
             ActiveArchives = new List<string>();
         }
 
@@ -310,6 +323,38 @@ namespace Wabbajack.Lib
                         ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
                     }
                 });
+
+            Utils.Log($"Checking for Steam Workshop Items...");
+            if (!_isSteamGame || _steamGame == null || _steamGame.WorkshopItems.Count <= 0)
+                return;
+
+            _steamGame.WorkshopItems.Do(item =>
+            {
+                Utils.Log($"Creating meta file for {item.ItemID}");
+                var metaString = "[General]\n" +
+                                 "repository=Steam\n" +
+                                 "installed=true\n" +
+                                 $"gameName={GameName}\n" +
+                                 $"steamID={_steamGame.AppId}\n" +
+                                 $"itemID={item.ItemID}\n" +
+                                 $"itemSize={item.Size}\n";
+
+                var filePath = Path.Combine(DownloadsFolder, $"steamWorkshopItem_{item.ItemID}.meta");
+                if (File.Exists(filePath))
+                {
+                    Utils.Log($"File {filePath} already exists, skipping...");
+                    return;
+                }
+
+                try
+                {
+                    File.WriteAllText(filePath, metaString);
+                }
+                catch (Exception e)
+                {
+                    Utils.LogToFile($"Exception while writing to disk at {filePath}\n{e}");
+                }
+            });
         }
 
         public override IEnumerable<ICompilationStep> GetStack()
@@ -333,6 +378,8 @@ namespace Wabbajack.Lib
             return new List<ICompilationStep>
             {
                 new IncludePropertyFiles(this),
+                new IncludeSteamWorkshopItems(this, _steamGame),
+                _hasSteamWorkshopItems ? new IncludeRegex(this, "^steamWorkshopItem_\\d*\\.meta$") : null,
                 new IgnoreDisabledVortexMods(this),
                 new IncludeVortexDeployment(this),
                 new IgnoreVortex(this),

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -103,6 +103,7 @@
     <Compile Include="CompilationSteps\IncludePatches.cs" />
     <Compile Include="CompilationSteps\IncludePropertyFiles.cs" />
     <Compile Include="CompilationSteps\IncludeRegex.cs" />
+    <Compile Include="CompilationSteps\IncludeSteamWorkshopItems.cs" />
     <Compile Include="CompilationSteps\IncludeStubbedConfigFiles.cs" />
     <Compile Include="CompilationSteps\IncludeTaggedMods.cs" />
     <Compile Include="CompilationSteps\IncludeThisProfile.cs" />
@@ -110,6 +111,7 @@
     <Compile Include="CompilationSteps\IStackStep.cs" />
     <Compile Include="CompilationSteps\PatchStockESMs.cs" />
     <Compile Include="CompilationSteps\Serialization.cs" />
+    <Compile Include="Downloaders\SteamWorkshopDownloader.cs" />
     <Compile Include="MO2Compiler.cs" />
     <Compile Include="Data.cs" />
     <Compile Include="Downloaders\AbstractDownloadState.cs" />