Artifactsv4

.github/workflows/nuget.yml

@@ -156,6 +156,7 @@ jobs:
         gharun -C testworkflows/workflow_ref_and_job_workflow_ref
         gharun -C testworkflows/reusable-workflows-secrets-inherit-with-required-secrets -s TEST=topsecret -s OPT=testsec
         gharun -C testworkflows/inherit_vars --var ACTIONS_STEP_DEBUG=true
+        gharun -C testworkflows/actions_artifacts_v4 -s ACTIONS_STEP_DEBUG=true -s ACTIONS_RUNNER_DEBUG=true --runner-version v2.311.0
         gharun --event azpipelines -C testworkflows/azpipelines/cross-repo-checkout -W testworkflows/azpipelines/cross-repo-checkout/pipeline.yml --local-repository az/containermatrix@main=testworkflows/azpipelines/containermatrix
         gharun --event azpipelines -C testworkflows/azpipelines/typedtemplates -W testworkflows/azpipelines/typedtemplates/pipeline.yml
         gharun --event azpipelines -C testworkflows/azpipelines/untypedtemplates -W testworkflows/azpipelines/untypedtemplates/pipeline.yml

src/Runner.Sdk/GharunUtil.cs

@@ -4,14 +4,27 @@
 
 namespace GitHub.Runner.Sdk {
     public class GharunUtil {
+        private static bool IsUsableLocalStorage(string localStorage) {
+            return localStorage != "" && !localStorage.Contains(' ') && !localStorage.Contains('"') && !localStorage.Contains('\'');
+        }
+
         public static string GetLocalStorage() {
             var localStorage = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
-            if(localStorage == "") {
+            if(!IsUsableLocalStorage(localStorage)) {
                 localStorage = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData);
             }
-            if(localStorage == "") {
+            if(!IsUsableLocalStorage(localStorage)) {
+                localStorage = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
+                if(IsUsableLocalStorage(localStorage)) {
+                    localStorage = Path.Join(localStorage, ".local", "share");
+                }
+            }
+            if(!IsUsableLocalStorage(localStorage)) {
                 localStorage = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
             }
+            if(!IsUsableLocalStorage(localStorage)) {
+                localStorage = Path.GetTempPath();
+            }
             return Path.GetFullPath(Path.Join(localStorage, "gharun"));
         }
 

src/Runner.Server/Controllers/ArtifactControllerV2.cs

@@ -0,0 +1,183 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.Extensions.Configuration;
+using Google.Protobuf;
+using System;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc.ModelBinding;
+using Microsoft.AspNetCore.Http;
+using System.IO;
+using System.Reflection;
+using Google.Protobuf.Reflection;
+using Microsoft.AspNetCore.Http.Extensions;
+using Runner.Server.Models;
+using System.Linq;
+using Microsoft.EntityFrameworkCore;
+using GitHub.Actions.Pipelines.WebApi;
+using System.Security.Cryptography;
+using Microsoft.IdentityModel.Tokens;
+using System.Text;
+
+namespace Runner.Server.Controllers
+{
+
+    [ApiController]
+    [Route("twirp/github.actions.results.api.v1.ArtifactService")]
+    [Authorize(AuthenticationSchemes = "Bearer", Policy = "AgentJob")]
+    public class ArtifactControllerV2 : VssControllerBase{
+        private readonly SqLiteDb _context;
+        private readonly JsonFormatter formatter;
+
+        public ArtifactControllerV2(SqLiteDb _context, IConfiguration configuration) : base(configuration)
+        {
+            this._context = _context;
+            formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithIndentation().WithPreserveProtoFieldNames(true).WithFormatDefaultValues(false));
+        }
+
+        private string CreateSignature(int id) {
+            using(var rsa = RSA.Create(Startup.AccessTokenParameter))
+            return Base64UrlEncoder.Encode(rsa.SignData(Encoding.UTF8.GetBytes(id.ToString()), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1));
+        }
+
+        private bool VerifySignature(int id, string sig) {
+            using(var rsa = RSA.Create(Startup.AccessTokenParameter))
+            return rsa.VerifyData(Encoding.UTF8.GetBytes(id.ToString()), Base64UrlEncoder.DecodeBytes(sig), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
+        }
+
+        [HttpPost("CreateArtifact")]
+        public async Task<string> CreateArtifact([FromBody, Protobuf] Github.Actions.Results.Api.V1.CreateArtifactRequest body) {
+            var guid = Guid.Parse(body.WorkflowJobRunBackendId);
+            var jobInfo = (from j in _context.Jobs where j.JobId == guid select new { j.runid, j.WorkflowRunAttempt.Attempt }).FirstOrDefault();
+            var artifacts = new ArtifactController(_context, Configuration);
+            var fname = $"{body.Name}.zip";
+            var container = await artifacts.CreateContainer(jobInfo.runid, jobInfo.Attempt, new CreateActionsStorageArtifactParameters() { Name = body.Name }, jobInfo.Attempt);
+            if(_context.Entry(container).Collection(c => c.Files).Query().Any()) {
+                //var files = _context.Entry(container).Collection(c => c.Files).Query().ToList();
+                // Duplicated Artifact of the same name in the same Attempt => fail
+                return formatter.Format(new Github.Actions.Results.Api.V1.CreateArtifactResponse() {
+                    Ok = false
+                });
+            }
+            var record = new ArtifactRecord() {FileName = fname, StoreName = Path.GetRandomFileName(), GZip = false, FileContainer = container} ;
+            _context.ArtifactRecords.Add(record);
+            await _context.SaveChangesAsync();
+
+            var resp = new Github.Actions.Results.Api.V1.CreateArtifactResponse
+            {
+                Ok = true,
+                SignedUploadUrl = new Uri(new Uri(ServerUrl), $"twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact?id={record.Id}&sig={CreateSignature(record.Id)}").ToString()
+            };
+            return formatter.Format(resp);
+        }
+
+        [HttpPut("UploadArtifact")]
+        [AllowAnonymous]
+        public async Task<IActionResult> UploadArtifact(int id, string sig, string comp = null, bool seal = false) {
+            if(string.IsNullOrEmpty(sig) || !VerifySignature(id, sig)) {
+                return NotFound();
+            }
+            if(comp == "block" || comp == "appendBlock") {
+                var record = await _context.ArtifactRecords.FindAsync(id);
+                var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");
+                using(var targetStream = new FileStream(Path.Combine(_targetFilePath, record.StoreName), FileMode.OpenOrCreate | FileMode.Append, FileAccess.Write, FileShare.Write)) {
+                    await Request.Body.CopyToAsync(targetStream);
+                }
+                return Created(HttpContext.Request.GetEncodedUrl(), null);
+            }
+            if(comp == "blocklist") {
+                return Created(HttpContext.Request.GetEncodedUrl(), null);
+            }
+            return Ok();
+        }
+
+        [HttpPost("FinalizeArtifact")]
+        public string FinalizeArtifact([FromBody, Protobuf] Github.Actions.Results.Api.V1.FinalizeArtifactRequest body) {
+            var attempt = long.Parse(User.FindFirst("attempt")?.Value ?? "-1");
+            var artifactsMinAttempt = long.Parse(User.FindFirst("artifactsMinAttempt")?.Value ?? "-1");
+            var runid = long.Parse(body.WorkflowRunBackendId);
+
+            var container = (from fileContainer in _context.ArtifactFileContainer where (fileContainer.Container.Attempt.Attempt >= artifactsMinAttempt || artifactsMinAttempt == -1) && (fileContainer.Container.Attempt.Attempt <= attempt || attempt == -1) && fileContainer.Container.Attempt.WorkflowRun.Id == runid && fileContainer.Files.Count == 1 && !fileContainer.Files.FirstOrDefault().FileName.Contains('/') && fileContainer.Files.FirstOrDefault().FileName.EndsWith(".zip") && body.Name.ToLower() == fileContainer.Name.ToLower() orderby fileContainer.Container.Attempt.Attempt descending select fileContainer).First();
+            container.Size = body.Size;
+            var resp = new Github.Actions.Results.Api.V1.FinalizeArtifactResponse
+            {
+                Ok = true,
+                ArtifactId = container.Id
+            };
+            return formatter.Format(resp);
+        }
+
+        [HttpPost("ListArtifacts")]
+        public string ListArtifacts([FromBody, Protobuf] Github.Actions.Results.Api.V1.ListArtifactsRequest body) {
+            var resp = new Github.Actions.Results.Api.V1.ListArtifactsResponse();
+
+            var attempt = long.Parse(User.FindFirst("attempt")?.Value ?? "-1");
+            var artifactsMinAttempt = long.Parse(User.FindFirst("artifactsMinAttempt")?.Value ?? "-1");
+            var runid = long.Parse(body.WorkflowRunBackendId);
+            resp.Artifacts.AddRange(from fileContainer in _context.ArtifactFileContainer where (fileContainer.Container.Attempt.Attempt >= artifactsMinAttempt || artifactsMinAttempt == -1) && (fileContainer.Container.Attempt.Attempt <= attempt || attempt == -1) && fileContainer.Container.Attempt.WorkflowRun.Id == runid && fileContainer.Files.Count == 1 && !fileContainer.Files.FirstOrDefault().FileName.Contains('/') && fileContainer.Files.FirstOrDefault().FileName.EndsWith(".zip") && (body.IdFilter == null || body.IdFilter == fileContainer.Id)  && (body.NameFilter == null || body.NameFilter.ToLower() == fileContainer.Name.ToLower()) orderby fileContainer.Container.Attempt.Attempt descending select new Github.Actions.Results.Api.V1.ListArtifactsResponse_MonolithArtifact
+            {
+                CreatedAt = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(System.DateTimeOffset.UtcNow),
+                DatabaseId = fileContainer.Id,
+                Name = fileContainer.Name,
+                Size = fileContainer.Size ?? 0,
+                WorkflowRunBackendId = body.WorkflowRunBackendId,
+                WorkflowJobRunBackendId = body.WorkflowJobRunBackendId
+            });
+            return formatter.Format(resp);
+        }
+
+        [HttpPost("GetSignedArtifactURL")]
+        public async Task<string> GetSignedArtifactURL([FromBody, Protobuf] Github.Actions.Results.Api.V1.GetSignedArtifactURLRequest body) {
+            var attempt = long.Parse(User.FindFirst("attempt")?.Value ?? "-1");
+            var artifactsMinAttempt = long.Parse(User.FindFirst("artifactsMinAttempt")?.Value ?? "-1");
+            var runid = long.Parse(body.WorkflowRunBackendId);
+            var file = await (from fileContainer in _context.ArtifactFileContainer where (fileContainer.Container.Attempt.Attempt >= artifactsMinAttempt || artifactsMinAttempt == -1) && (fileContainer.Container.Attempt.Attempt <= attempt || attempt == -1) && fileContainer.Container.Attempt.WorkflowRun.Id == runid && fileContainer.Files.Count == 1 && fileContainer.Files.FirstOrDefault().FileName.ToLower() == $"{body.Name}.zip".ToLower() orderby fileContainer.Container.Attempt.Attempt descending select fileContainer.Files.FirstOrDefault()).FirstAsync();
+            var resp = new Github.Actions.Results.Api.V1.GetSignedArtifactURLResponse
+            {
+                SignedUrl = new Uri(new Uri(ServerUrl), $"twirp/github.actions.results.api.v1.ArtifactService/DownloadArtifact?id={file.Id}&sig={CreateSignature(file.Id)}").ToString()
+            };
+            return formatter.Format(resp);
+        }
+
+        [AllowAnonymous]
+        [HttpGet("DownloadArtifact")]
+        public IActionResult DownloadArtifact(int id, string sig) {
+            if(string.IsNullOrEmpty(sig) || !VerifySignature(id, sig)) {
+                return NotFound();
+            }
+            var container = _context.ArtifactRecords.Find(id);
+            var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");
+            return new FileStreamResult(System.IO.File.OpenRead(Path.Combine(_targetFilePath, container.StoreName)), "application/octet-stream") { EnableRangeProcessing = true };
+        }
+
+
+        public class ProtobufBinder : IModelBinder
+        {
+            public async Task BindModelAsync(ModelBindingContext bindingContext)
+            {
+                if (!bindingContext.HttpContext.Request.HasJsonContentType())
+                {
+                    throw new BadHttpRequestException(
+                        "Request content type was not a recognized JSON content type.",
+                        StatusCodes.Status415UnsupportedMediaType);
+                }
+
+                using var sr = new StreamReader(bindingContext.HttpContext.Request.Body);
+                var str = await sr.ReadToEndAsync();
+
+                var valueType = bindingContext.ModelType;
+                var parser = new JsonParser(JsonParser.Settings.Default.WithIgnoreUnknownFields(true));
+
+                var descriptor = (MessageDescriptor)bindingContext.ModelType.GetProperty("Descriptor", BindingFlags.Public | BindingFlags.Static).GetValue(null, null);
+                var obj = parser.Parse(str, descriptor);
+
+                bindingContext.Result = ModelBindingResult.Success(obj);
+            }
+        }
+
+        public class ProtobufAttribute : ModelBinderAttribute {
+            public ProtobufAttribute() : base(typeof(ProtobufBinder)) {
+
+            }
+        }
+    }
+}

src/Runner.Server/Controllers/MessageController.cs

@@ -5581,6 +5581,7 @@ private Func<bool, Job> queueJob(GitHub.DistributedTask.ObjectTemplating.ITraceW
                         var feedStreamUrl = new UriBuilder(new Uri(new Uri(apiUrl), $"_apis/v1/TimeLineWebConsoleLog/feedstream/{Uri.EscapeDataString(timelineId.ToString())}/ws"));
                         feedStreamUrl.Scheme = feedStreamUrl.Scheme == "http" ? "ws" : "wss";
                         systemVssConnection.Data["FeedStreamUrl"] = feedStreamUrl.ToString();
+                        systemVssConnection.Data["ResultsServiceUrl"] = apiUrl;
                         if(calculatedPermissions.TryGetValue("id_token", out var p_id_token) && p_id_token == "write") {
                             var environment = deploymentEnvironmentValue?.Name ?? ("");
                             var claims = new Dictionary<string, string>();
@@ -5711,6 +5712,7 @@ private Func<bool, Job> queueJob(GitHub.DistributedTask.ObjectTemplating.ITraceW
                                 new Claim("localcheckout", localcheckout ? "actions/checkout" : ""),
                                 new Claim("runid", runid.ToString()),
                                 new Claim("github_token", variables.TryGetValue("github_token", out var ghtoken) ? ghtoken.Value : ""),
+                                new Claim("scp", $"Actions.Results:{runid}:{job.JobId}")
                             }),
                             Expires = DateTime.UtcNow.AddMinutes(timeoutMinutes + 10),
                             Issuer = myIssuer,

src/Runner.Server/Controllers/TimelineController.cs

@@ -89,9 +89,10 @@ public async Task<IActionResult> GetTimeline(Guid timelineId) {
         public async Task<IActionResult> PutAttachment(Guid timelineId, Guid recordId, string type, string name) {
             var jobInfo = (from j in _context.Jobs where j.TimeLineId == timelineId select new { j.runid, j.Attempt }).FirstOrDefault();
             var artifacts = new ArtifactController(_context, Configuration);
-            var fname = $"Attachment_{timelineId}_{recordId}_{type}";
-            var container = await artifacts.CreateContainer(jobInfo.runid, jobInfo.Attempt, new CreateActionsStorageArtifactParameters() { Name = fname });
-            var record = new ArtifactRecord() {FileName = Path.Join(fname, name), StoreName = Path.GetRandomFileName(), GZip = false, FileContainer = container} ;
+            var prefix = $"Attachment_{timelineId}_{recordId}";
+            var fname = $"{prefix}_{type}_{name}";
+            var container = await artifacts.CreateContainer(jobInfo.runid, jobInfo.Attempt, new CreateActionsStorageArtifactParameters() { Name = prefix });
+            var record = new ArtifactRecord() {FileName = fname, StoreName = Path.GetRandomFileName(), GZip = false, FileContainer = container} ;
             _context.ArtifactRecords.Add(record);
             await _context.SaveChangesAsync();
             var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");