Merge #3661 GitLab kref

CHANGELOG.md

@@ -37,6 +37,7 @@ All notable changes to this project will be documented in this file.
 - [Tooling] Self review option for merge script (#3650 by: HebaruSan; reviewed: DasSkelett)
 - [Infra] Improve translation process (#3648 by: Olympic1; reviewed: HebaruSan)
 - [Netkan] Inflation error for `version_min` and `version_max` (#3657 by: HebaruSan; reviewed: DasSkelett)
+- [Netkan] GitLab kref (#3661 by: HebaruSan; reviewed: techman83)
 
 ## v1.31.0 (IKAROS)
 

Netkan/CKAN-netkan.csproj

@@ -89,6 +89,12 @@
     <Compile Include="Sources\Github\GithubReleaseAsset.cs" />
     <Compile Include="Sources\Github\GithubRepo.cs" />
     <Compile Include="Sources\Github\IGithubApi.cs" />
+    <Compile Include="Sources\Gitlab\GitlabApi.cs" />
+    <Compile Include="Sources\Gitlab\GitlabOptions.cs" />
+    <Compile Include="Sources\Gitlab\GitlabProject.cs" />
+    <Compile Include="Sources\Gitlab\GitlabRef.cs" />
+    <Compile Include="Sources\Gitlab\GitlabRelease.cs" />
+    <Compile Include="Sources\Gitlab\IGitlabApi.cs" />
     <Compile Include="Sources\Jenkins\IJenkinsApi.cs" />
     <Compile Include="Sources\Jenkins\JenkinsApi.cs" />
     <Compile Include="Sources\Jenkins\JenkinsArtifact.cs" />
@@ -110,6 +116,7 @@
     <Compile Include="Transformers\ForcedVTransformer.cs" />
     <Compile Include="Transformers\GeneratedByTransformer.cs" />
     <Compile Include="Transformers\GithubTransformer.cs" />
+    <Compile Include="Transformers\GitlabTransformer.cs" />
     <Compile Include="Transformers\HttpTransformer.cs" />
     <Compile Include="Transformers\InternalCkanTransformer.cs" />
     <Compile Include="Transformers\ITransformer.cs" />

Netkan/CmdLineOptions.cs

@@ -25,6 +25,9 @@ internal class CmdLineOptions
         [Option("github-token", HelpText = "GitHub OAuth token for API access")]
         public string GitHubToken { get; set; }
 
+        [Option("gitlab-token", HelpText = "GitLab OAuth token for API access")]
+        public string GitLabToken { get; set; }
+
         [Option("net-useragent", DefaultValue = null, HelpText = "Set the default User-Agent string for HTTP requests")]
         public string NetUserAgent { get; set; }
 

Netkan/Processors/Inflator.cs

@@ -15,7 +15,7 @@ namespace CKAN.NetKAN.Processors
 {
     public class Inflator
     {
-        public Inflator(string cacheDir, bool overwriteCache, string githubToken, bool prerelease)
+        public Inflator(string cacheDir, bool overwriteCache, string githubToken, string gitlabToken, bool prerelease)
         {
             log.Debug("Initializing inflator");
             cache = FindCache(
@@ -28,7 +28,7 @@ public Inflator(string cacheDir, bool overwriteCache, string githubToken, bool p
             IFileService   fileService   = new FileService(cache);
             http          = new CachingHttpService(cache, overwriteCache);
             ckanValidator = new CkanValidator(http, moduleService);
-            transformer   = new NetkanTransformer(http, fileService, moduleService, githubToken, prerelease, netkanValidator);
+            transformer   = new NetkanTransformer(http, fileService, moduleService, githubToken, gitlabToken, prerelease, netkanValidator);
         }
 
         internal IEnumerable<Metadata> Inflate(string filename, Metadata netkan, TransformOptions opts)
@@ -56,8 +56,15 @@ internal IEnumerable<Metadata> Inflate(string filename, Metadata netkan, Transfo
             }
             catch (Exception)
             {
-                // Purge anything we download for a failed indexing attempt from the cache to allow re-downloads
-                PurgeDownloads(http, cache);
+                try
+                {
+                    // Purge anything we download for a failed indexing attempt from the cache to allow re-downloads
+                    PurgeDownloads(http, cache);
+                }
+                catch
+                {
+                    // Don't freak out if we can't delete
+                }
                 throw;
             }
         }

Netkan/Processors/QueueHandler.cs

@@ -22,13 +22,13 @@ namespace CKAN.NetKAN.Processors
 {
     public class QueueHandler
     {
-        public QueueHandler(string inputQueueName, string outputQueueName, string cacheDir, bool overwriteCache, string githubToken, bool prerelease)
+        public QueueHandler(string inputQueueName, string outputQueueName, string cacheDir, bool overwriteCache, string githubToken, string gitlabToken, bool prerelease)
         {
             warningAppender = GetQueueLogAppender();
             (LogManager.GetRepository() as Hierarchy)?.Root.AddAppender(warningAppender);
 
             log.Debug("Initializing SQS queue handler");
-            inflator = new Inflator(cacheDir, overwriteCache, githubToken, prerelease);
+            inflator = new Inflator(cacheDir, overwriteCache, githubToken, gitlabToken, prerelease);
 
             inputQueueURL  = getQueueUrl(inputQueueName);
             outputQueueURL = getQueueUrl(outputQueueName);

Netkan/Program.cs

@@ -54,6 +54,7 @@ public static int Main(string[] args)
                         Options.CacheDir,
                         Options.OverwriteCache,
                         Options.GitHubToken,
+                        Options.GitLabToken,
                         Options.PreRelease
                     );
                     inf.ValidateCkan(ckan);
@@ -72,6 +73,7 @@ public static int Main(string[] args)
                         Options.CacheDir,
                         Options.OverwriteCache,
                         Options.GitHubToken,
+                        Options.GitLabToken,
                         Options.PreRelease
                     );
                     qh.Process();
@@ -89,6 +91,7 @@ public static int Main(string[] args)
                         Options.CacheDir,
                         Options.OverwriteCache,
                         Options.GitHubToken,
+                        Options.GitLabToken,
                         Options.PreRelease
                     );
                     var ckans = inf.Inflate(

Netkan/Sources/Gitlab/GitlabApi.cs

@@ -0,0 +1,63 @@
+using System;
+using System.Linq;
+using System.Collections.Generic;
+
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+using CKAN.NetKAN.Services;
+
+namespace CKAN.NetKAN.Sources.Gitlab
+{
+    /// <summary>
+    /// Provides convenient access to the GitLab API
+    /// https://docs.gitlab.com/ee/api/
+    /// </summary>
+    internal sealed class GitlabApi : IGitlabApi
+    {
+        /// <summary>
+        /// Initialize the API object
+        /// </summary>
+        /// <param name="http">HTTP service for getting URLs</param>
+        /// <param name="token">GitLab API token</param>
+        public GitlabApi(IHttpService http, string token = null)
+        {
+            this.http  = http;
+            this.token = token;
+        }
+
+        /// <summary>
+        /// Retrieve info about a GitLab project from the API
+        /// </summary>
+        /// <param name="reference">Specification of which project to retrieve</param>
+        /// <returns>A project object</returns>
+        public GitlabProject GetProject(GitlabRef reference)
+        {
+            // https://docs.gitlab.com/ee/api/projects
+            return JsonConvert.DeserializeObject<GitlabProject>(
+                http.DownloadText(
+                    new Uri(apiBase, $"{reference.Account}%2F{reference.Project}"),
+                    token, null));
+        }
+
+        /// <summary>
+        /// Retrieve info about a GitLab project's releases from the API
+        /// </summary>
+        /// <param name="reference">Specification of which project's releases to retrieve</param>
+        /// <returns>Sequence of release objects from the API</returns>
+        public IEnumerable<GitlabRelease> GetAllReleases(GitlabRef reference)
+        {
+            // https://docs.gitlab.com/ee/api/releases/
+            return JArray.Parse(
+                http.DownloadText(
+                    new Uri(apiBase, $"{reference.Account}%2F{reference.Project}/releases"),
+                        token, null))
+                .Select(releaseJson => releaseJson.ToObject<GitlabRelease>());
+        }
+
+        private IHttpService http;
+        private string       token;
+
+        private static readonly Uri apiBase = new Uri("https://gitlab.com/api/v4/projects/");
+    }
+}

Netkan/Sources/Gitlab/GitlabOptions.cs

@@ -0,0 +1,18 @@
+using Newtonsoft.Json;
+
+namespace CKAN.NetKAN.Sources.Gitlab
+{
+    /// <summary>
+    /// Represents the x_netkan_gitlab object from a netkan
+    /// </summary>
+    internal sealed class GitlabOptions
+    {
+        /// <summary>
+        /// True to use source ZIP for a release.
+        /// Note that this MUST be true because GitLab only provides source ZIPs!
+        /// If they add other assets in the future, this requirement can be relaxed.
+        /// </summary>
+        [JsonProperty("use_source_archive")]
+        public readonly bool UseSourceArchive = false;
+    }
+}

Netkan/Sources/Gitlab/GitlabProject.cs

@@ -0,0 +1,25 @@
+using Newtonsoft.Json;
+
+namespace CKAN.NetKAN.Sources.Gitlab
+{
+    /// <summary>
+    /// Represents a project from the GitLab API
+    /// </summary>
+    public sealed class GitlabProject
+    {
+        [JsonProperty("name")]
+        public readonly string Name;
+
+        [JsonProperty("description")]
+        public readonly string Description;
+
+        [JsonProperty("web_url")]
+        public readonly string WebURL;
+
+        [JsonProperty("issues_enabled")]
+        public readonly bool IssuesEnabled;
+
+        [JsonProperty("readme_url")]
+        public readonly string ReadMeURL;
+    }
+}

Netkan/Sources/Gitlab/GitlabRef.cs

@@ -0,0 +1,44 @@
+using System.Text.RegularExpressions;
+
+using CKAN.NetKAN.Model;
+
+namespace CKAN.NetKAN.Sources.Gitlab
+{
+    /// <summary>
+    /// Represents a GitLab $kref
+    /// </summary>
+    internal sealed class GitlabRef : RemoteRef
+    {
+        /// <summary>
+        /// Initialize the GitLab reference
+        /// </summary>
+        /// <param name="reference">The base $kref object from a netkan</param>
+        public GitlabRef(RemoteRef reference)
+            : base(reference)
+        {
+            var match = Pattern.Match(reference.Id);
+            if (match.Success)
+            {
+                Account = match.Groups["account"].Value;
+                Project = match.Groups["project"].Value;
+            }
+            else
+            {
+                throw new Kraken(string.Format(@"Could not parse reference: ""{0}""", reference));
+            }
+        }
+
+        /// <summary>
+        /// The first part of the "account/project" path from GitLab
+        /// </summary>
+        public readonly string Account;
+        /// <summary>
+        /// The second part of the "account/project" path from GitLab
+        /// </summary>
+        public readonly string Project;
+
+        private static readonly Regex Pattern = new Regex(
+            @"^(?<account>[^/]+)/(?<project>[^/]+)$",
+            RegexOptions.Compiled);
+    }
+}

Netkan/Sources/Gitlab/GitlabRelease.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Collections.Generic;
+
+using Newtonsoft.Json;
+
+namespace CKAN.NetKAN.Sources.Gitlab
+{
+    /// <summary>
+    /// Represents a release from the GitLab API
+    /// </summary>
+    internal sealed class GitlabRelease
+    {
+        [JsonProperty("tag_name")]
+        public readonly string TagName;
+
+        [JsonProperty("author")]
+        public readonly GitlabReleaseAuthor Author = new GitlabReleaseAuthor();
+
+        [JsonProperty("released_at")]
+        public readonly DateTime ReleasedAt;
+
+        [JsonProperty("assets")]
+        public readonly GitlabReleaseAssets Assets = new GitlabReleaseAssets();
+    }
+
+    /// <summary>
+    /// Represents an author from the GitLab API
+    /// </summary>
+    internal sealed class GitlabReleaseAuthor
+    {
+        [JsonProperty("name")]
+        public readonly string Name;
+    }
+
+    /// <summary>
+    /// Represents an assets object from the GitLab API
+    /// </summary>
+    internal sealed class GitlabReleaseAssets
+    {
+        [JsonProperty("sources")]
+        public readonly List<GitlabReleaseAssetSource> Sources = new List<GitlabReleaseAssetSource>();
+    }
+
+    /// <summary>
+    /// Represents an assets source object from the GitLab API
+    /// </summary>
+    internal sealed class GitlabReleaseAssetSource
+    {
+        [JsonProperty("format")]
+        public readonly string Format;
+
+        [JsonProperty("url")]
+        public readonly string URL;
+    }
+}

Netkan/Sources/Gitlab/IGitlabApi.cs

@@ -0,0 +1,25 @@
+using System.Collections.Generic;
+
+namespace CKAN.NetKAN.Sources.Gitlab
+{
+    /// <summary>
+    /// Interface for classes providing access to the GitLab API
+    /// Allows mocking up in tests
+    /// </summary>
+    internal interface IGitlabApi
+    {
+        /// <summary>
+        /// Retrieve info about a GitLab project from the API
+        /// </summary>
+        /// <param name="reference">Specification of which project to retrieve</param>
+        /// <returns>A project object</returns>
+        GitlabProject GetProject(GitlabRef reference);
+
+        /// <summary>
+        /// Retrieve info about a GitLab project's releases from the API
+        /// </summary>
+        /// <param name="reference">Specification of which project's releases to retrieve</param>
+        /// <returns>Sequence of release objects from the API</returns>
+        IEnumerable<GitlabRelease> GetAllReleases(GitlabRef reference);
+    }
+}