Merge pull request #1905 from wabbajack-tools/add-repositories

add repositories

Wabbajack.App.Wpf/View Models/Gallery/ModListGalleryVM.cs

@@ -26,7 +26,7 @@ namespace Wabbajack
     {
         public MainWindowVM MWVM { get; }
         
-        private readonly SourceCache<ModListMetadataVM, string> _modLists = new(x => x.Metadata.Links.MachineURL);
+        private readonly SourceCache<ModListMetadataVM, string> _modLists = new(x => x.Metadata.NamespacedName);
         public ReadOnlyObservableCollection<ModListMetadataVM> _filteredModLists;
         public ReadOnlyObservableCollection<ModListMetadataVM> ModLists => _filteredModLists;
 

Wabbajack.App.Wpf/View Models/Gallery/ModListMetadataVM.cs

@@ -100,7 +100,7 @@ namespace Wabbajack
             _maintainer = maintainer;
             Metadata = metadata;
             _wjClient = wjClient;
-            Location = LauncherUpdater.CommonFolder.Value.Combine("downloaded_mod_lists", Metadata.Links.MachineURL).WithExtension(Ext.Wabbajack);
+            Location = LauncherUpdater.CommonFolder.Value.Combine("downloaded_mod_lists", Metadata.NamespacedName).WithExtension(Ext.Wabbajack);
             ModListTagList = new List<ModListTag>();
             
             UpdateStatus().FireAndForget();
@@ -116,7 +116,7 @@ namespace Wabbajack
             VersionText = "Modlist version : " + Metadata.Version;
             IsBroken = metadata.ValidationSummary.HasFailures || metadata.ForceDown;
             //https://www.wabbajack.org/#/modlists/info?machineURL=eldersouls
-            OpenWebsiteCommand = ReactiveCommand.Create(() => UIUtils.OpenWebsite(new Uri($"https://www.wabbajack.org/#/modlists/info?machineURL={Metadata.Links.MachineURL}")));
+            OpenWebsiteCommand = ReactiveCommand.Create(() => UIUtils.OpenWebsite(new Uri($"https://www.wabbajack.org/#/modlists/info?machineURL={Metadata.NamespacedName}")));
 
             IsLoadingIdle = new Subject<bool>();
             
@@ -126,7 +126,7 @@ namespace Wabbajack
                 IsLoadingIdle.OnNext(false);
                 try
                 {
-                    var status = await wjClient.GetDetailedStatus(metadata.Links.MachineURL);
+                    var status = await wjClient.GetDetailedStatus(metadata.NamespacedName);
                     var coll = _parent.MWVM.ModListContentsVM.Value.Status;
                     coll.Clear();
                     coll.AddRange(status.Archives.Select(a => new DetailedStatusItem

Wabbajack.CLI/Program.cs

@@ -69,6 +69,7 @@ internal class Program
                 services.AddSingleton<IVerb, SteamDownloadFile>();
                 services.AddSingleton<IVerb, UploadToNexus>();
                 services.AddSingleton<IVerb, ListCreationClubContent>();
+                services.AddSingleton<IVerb, ListModlists>();
                 services.AddSingleton<IVerb, Extract>();
 
                 services.AddSingleton<IUserInterventionHandler, UserInterventionHandler>();

Wabbajack.CLI/Verbs/ListModlists.cs

@@ -0,0 +1,47 @@
+using System;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using FluentFTP.Helpers;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Downloaders.Bethesda;
+using Wabbajack.DTOs;
+using Wabbajack.Networking.WabbajackClientApi;
+
+namespace Wabbajack.CLI.Verbs;
+
+public class ListModlists : IVerb
+{
+    private readonly ILogger<ListCreationClubContent> _logger;
+    private readonly Client _client;
+    private readonly BethesdaDownloader _downloader;
+
+    public ListModlists(ILogger<ListCreationClubContent> logger, Client wjClient)
+    {
+        _logger = logger;
+        _client = wjClient;
+    }
+    public Command MakeCommand()
+    {
+        var command = new Command("list-modlists");
+        command.Description = "Lists all known modlists";
+        command.Handler = CommandHandler.Create(Run);
+        return command;
+    }
+
+    public async Task<int> Run(CancellationToken token)
+    {
+        _logger.LogInformation("Loading all modlist definitions");
+        var modlists = await _client.LoadLists();
+        _logger.LogInformation("Loaded {Count} lists", modlists.Length);
+
+        foreach (var modlist in modlists.OrderBy(l => l.NamespacedName))
+        {
+            _logger.LogInformation("{Url} {Game} {Size}", modlist.NamespacedName, modlist.Game.MetaData().HumanFriendlyGameName, modlist.DownloadMetadata!.Size.FileSizeToString());
+        }
+        
+        return 0;
+    }
+}

Wabbajack.CLI/Verbs/ValidateLists.cs

@@ -112,7 +112,7 @@ public class ValidateLists : IVerb
             {
                 Name = modList.Title,
                 ModListHash = modList.DownloadMetadata?.Hash ?? default,
-                MachineURL = modList.Links.MachineURL,
+                MachineURL = modList.NamespacedName,
                 Version = modList.Version
             };
 
@@ -123,8 +123,8 @@ public class ValidateLists : IVerb
                 return validatedList;
             }
 
-            using var scope = _logger.BeginScope("MachineURL: {MachineUrl}", modList.Links.MachineURL);
+            using var scope = _logger.BeginScope("MachineURL: {MachineUrl}", modList.NamespacedName);
-            _logger.LogInformation("Verifying {MachineUrl} - {Title}", modList.Links.MachineURL, modList.Title);
+            _logger.LogInformation("Verifying {MachineUrl} - {Title}", modList.NamespacedName, modList.Title);
             //await DownloadModList(modList, archiveManager, CancellationToken.None);
 
             ModList modListData;
@@ -426,14 +426,14 @@ public class ValidateLists : IVerb
     {
         if (archiveManager.HaveArchive(modList.DownloadMetadata!.Hash))
         {
-            _logger.LogInformation("Previously downloaded {hash} not re-downloading", modList.Links.MachineURL);
+            _logger.LogInformation("Previously downloaded {hash} not re-downloading", modList.NamespacedName);
             return modList.DownloadMetadata!.Hash;
         }
         else
         {
-            _logger.LogInformation("Downloading {hash}", modList.Links.MachineURL);
+            _logger.LogInformation("Downloading {hash}", modList.NamespacedName);
             await _discord.SendAsync(Channel.Ham,
-                $"Downloading and ingesting {modList.Title} ({modList.Links.MachineURL}) v{modList.Version}", token);
+                $"Downloading and ingesting {modList.Title} ({modList.NamespacedName}) v{modList.Version}", token);
             return await DownloadWabbajackFile(modList, archiveManager, token);
         }
     }

Wabbajack.DTOs/ModList/Links.cs

@@ -12,7 +12,7 @@ public class LinksObject
 
     [JsonPropertyName("download")] public string Download { get; set; } = string.Empty;
 
-    [JsonPropertyName("machineURL")] public string MachineURL { get; set; } = string.Empty;
+    [JsonPropertyName("machineURL")] internal string MachineURL { get; set; } = string.Empty;
     
     [JsonPropertyName("discordURL")] public string DiscordURL { get; set; } = string.Empty;
 }

Wabbajack.DTOs/ModList/ModListMetadata.cs

@@ -44,4 +44,7 @@ public class ModlistMetadata
     [JsonPropertyName("dateCreated")] public DateTime DateCreated { get; set; } = DateTime.UnixEpoch;
 
     [JsonPropertyName("dateUpdated")] public DateTime DateUpdated { get; set; }
+
+    [JsonPropertyName("repositoryName")] public string RepositoryName { get; set; } = string.Empty;
+    [JsonIgnore] public string NamespacedName => $"{RepositoryName}/{Links.MachineURL}";
 }

Wabbajack.Networking.GitHub/Client.cs

@@ -56,7 +56,7 @@ public class Client
         {
             var data = await GetData(file);
             var found = data.Lists.FirstOrDefault(l =>
-                l.Links.MachineURL == newData.MachineUrl && l.Maintainers.Contains(maintainer));
+                l.NamespacedName == newData.MachineUrl && l.Maintainers.Contains(maintainer));
             if (found == null) continue;
 
             found.DownloadMetadata = newData.DownloadMetadata;

Wabbajack.Networking.WabbajackClientApi/Client.cs

@@ -187,23 +187,29 @@ public class Client
         return definition;
     }
 
-    public async Task<ModlistMetadata[]> LoadLists(bool includeUnlisted = false)
+    public async Task<ModlistMetadata[]> LoadLists()
     {
-        var lists = new[]
+        var repos = await LoadRepositories();
-            {
-                "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/modlists.json",
-                "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/utility_modlists.json",
-                "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/unlisted_modlists.json"
-            }
-            .Take(includeUnlisted ? 3 : 2);
 
-        return await lists.PMapAll(async url =>
+        return await repos.PMapAll(async url =>
-                await _client.GetFromJsonAsync<ModlistMetadata[]>(_limiter, new HttpRequestMessage(HttpMethod.Get, url),
+                 (await _client.GetFromJsonAsync<ModlistMetadata[]>(_limiter, new HttpRequestMessage(HttpMethod.Get, url.Value),
-                    _dtos.Options)!)
+                    _dtos.Options))!.Select(meta =>
+                 {
+                     meta.RepositoryName = url.Key;
+                     return meta;
+                 }))
             .SelectMany(x => x)
             .ToArray();
     }
 
+    public async Task<Dictionary<string, Uri>> LoadRepositories()
+    {
+        var repositories = await _client.GetFromJsonAsync<Dictionary<string, Uri>>(_limiter,
+            new HttpRequestMessage(HttpMethod.Get,
+                "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/repositories.json"), _dtos.Options);
+        return repositories!;
+    }
+
     public Uri GetPatchUrl(Hash upgradeHash, Hash archiveHash)
     {
         return new Uri($"{_configuration.PatchBaseAddress}{upgradeHash.ToHex()}_{archiveHash.ToHex()}");

Wabbajack.Server/Controllers/AuthorControls.cs

@@ -62,7 +62,7 @@ public class AuthorControls : ControllerBase
         List<string> lists = new();
         foreach (var file in Enum.GetValues<List>())
             lists.AddRange((await _gitHubClient.GetData(file)).Lists.Where(l => l.Maintainers.Contains(user))
-                .Select(lst => lst.Links.MachineURL));
+                .Select(lst => lst.NamespacedName));
 
         return Ok(lists);
     }

Wabbajack.Services.OSIntegrated/Services/ModListDownloadMaintainer.cs

@@ -35,7 +35,7 @@ public class ModListDownloadMaintainer
 
     public AbsolutePath ModListPath(ModlistMetadata metadata)
     {
-        return _configuration.ModListsDownloadLocation.Combine(metadata.Links.MachineURL).WithExtension(Ext.Wabbajack);
+        return _configuration.ModListsDownloadLocation.Combine(metadata.NamespacedName.Replace("/", "_@@_")).WithExtension(Ext.Wabbajack);
     }
 
     public async Task<bool> HaveModList(ModlistMetadata metadata, CancellationToken? token = null)