Skip to content

Commit

Permalink
Expermental Artifacts v4
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChristopherHX
ChristopherHX committed Dec 20, 2023
1 parent 57cdb1b commit fa53b2a
Show file tree
Hide file tree
Showing 6 changed files with 2,809 additions and 2 deletions.
183 changes: 183 additions & 0 deletions src/Runner.Server/Controllers/ArtifactControllerV2.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Authorization;
using Microsoft.Extensions.Configuration;
using Google.Protobuf;
using System;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc.ModelBinding;
using Microsoft.AspNetCore.Http;
using System.IO;
using System.Reflection;
using Google.Protobuf.Reflection;
using Microsoft.AspNetCore.Http.Extensions;
using Runner.Server.Models;
using System.Linq;
using Microsoft.EntityFrameworkCore;
using GitHub.Actions.Pipelines.WebApi;
using System.Security.Cryptography;
using Microsoft.IdentityModel.Tokens;
using System.Text;

namespace Runner.Server.Controllers
{

[ApiController]
[Route("twirp/github.actions.results.api.v1.ArtifactService")]
[Authorize(AuthenticationSchemes = "Bearer", Policy = "AgentJob")]
public class ArtifactControllerV2 : VssControllerBase{
private readonly SqLiteDb _context;
private readonly JsonFormatter formatter;

public ArtifactControllerV2(SqLiteDb _context, IConfiguration configuration) : base(configuration)
{
this._context = _context;
formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithIndentation().WithPreserveProtoFieldNames(true).WithFormatDefaultValues(false));
}

private string CreateSignature(int id) {
using(var rsa = RSA.Create(Startup.AccessTokenParameter))
return Base64UrlEncoder.Encode(rsa.SignData(Encoding.UTF8.GetBytes(id.ToString()), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1));
}

private bool VerifySignature(int id, string sig) {
using(var rsa = RSA.Create(Startup.AccessTokenParameter))
return rsa.VerifyData(Encoding.UTF8.GetBytes(id.ToString()), Base64UrlEncoder.DecodeBytes(sig), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
}

[HttpPost("CreateArtifact")]
public async Task<string> CreateArtifact([FromBody, Protobuf] Github.Actions.Results.Api.V1.CreateArtifactRequest body) {
var guid = Guid.Parse(body.WorkflowJobRunBackendId);
var jobInfo = (from j in _context.Jobs where j.JobId == guid select new { j.runid, Attempt = j.WorkflowRunAttempt.Id }).FirstOrDefault();
var artifacts = new ArtifactController(_context, Configuration);
var fname = $"{body.Name}.zip";
var container = await artifacts.CreateContainer(jobInfo.runid, jobInfo.Attempt, new CreateActionsStorageArtifactParameters() { Name = body.Name }, jobInfo.Attempt);
if(_context.Entry(container).Collection(c => c.Files).Query().Any()) {
//var files = _context.Entry(container).Collection(c => c.Files).Query().ToList();
// Duplicated Artifact of the same name in the same Attempt => fail
return formatter.Format(new Github.Actions.Results.Api.V1.CreateArtifactResponse() {
Ok = false
});
}
var record = new ArtifactRecord() {FileName = fname, StoreName = Path.GetRandomFileName(), GZip = false, FileContainer = container} ;
_context.ArtifactRecords.Add(record);
await _context.SaveChangesAsync();

var resp = new Github.Actions.Results.Api.V1.CreateArtifactResponse
{
Ok = true,
SignedUploadUrl = new Uri(new Uri(ServerUrl), $"twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact?id={record.Id}&sig={CreateSignature(record.Id)}").ToString()
};
return formatter.Format(resp);
}

[HttpPut("UploadArtifact")]
[AllowAnonymous]
public async Task<IActionResult> UploadArtifact(int id, string sig, string comp = null, bool seal = false) {
if(string.IsNullOrEmpty(sig) || !VerifySignature(id, sig)) {
return NotFound();
}
if(comp == "block" || comp == "appendBlock") {
var record = await _context.ArtifactRecords.FindAsync(id);
var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");
using(var targetStream = new FileStream(Path.Combine(_targetFilePath, record.StoreName), FileMode.OpenOrCreate | FileMode.Append, FileAccess.Write, FileShare.Write)) {
await Request.Body.CopyToAsync(targetStream);
}
return Created(HttpContext.Request.GetEncodedUrl(), null);
}
if(comp == "blocklist") {
return Created(HttpContext.Request.GetEncodedUrl(), null);
}
return Ok();
}

[HttpPost("FinalizeArtifact")]
public string FinalizeArtifact([FromBody, Protobuf] Github.Actions.Results.Api.V1.FinalizeArtifactRequest body) {
var attempt = long.Parse(User.FindFirst("attempt")?.Value ?? "-1");
var artifactsMinAttempt = long.Parse(User.FindFirst("artifactsMinAttempt")?.Value ?? "-1");
var runid = long.Parse(body.WorkflowRunBackendId);

var container = (from fileContainer in _context.ArtifactFileContainer where (fileContainer.Container.Attempt.Attempt >= artifactsMinAttempt || artifactsMinAttempt == -1) && (fileContainer.Container.Attempt.Attempt <= attempt || attempt == -1) && fileContainer.Container.Attempt.WorkflowRun.Id == runid && fileContainer.Files.Count == 1 && !fileContainer.Files.FirstOrDefault().FileName.Contains('/') && fileContainer.Files.FirstOrDefault().FileName.EndsWith(".zip") && body.Name.ToLower() == fileContainer.Name.ToLower() orderby fileContainer.Container.Attempt.Attempt descending select fileContainer).First();
container.Size = body.Size;
var resp = new Github.Actions.Results.Api.V1.FinalizeArtifactResponse
{
Ok = true,
ArtifactId = container.Id
};
return formatter.Format(resp);
}

[HttpPost("ListArtifacts")]
public string ListArtifacts([FromBody, Protobuf] Github.Actions.Results.Api.V1.ListArtifactsRequest body) {
var resp = new Github.Actions.Results.Api.V1.ListArtifactsResponse();

var attempt = long.Parse(User.FindFirst("attempt")?.Value ?? "-1");
var artifactsMinAttempt = long.Parse(User.FindFirst("artifactsMinAttempt")?.Value ?? "-1");
var runid = long.Parse(body.WorkflowRunBackendId);
resp.Artifacts.AddRange(from fileContainer in _context.ArtifactFileContainer where (fileContainer.Container.Attempt.Attempt >= artifactsMinAttempt || artifactsMinAttempt == -1) && (fileContainer.Container.Attempt.Attempt <= attempt || attempt == -1) && fileContainer.Container.Attempt.WorkflowRun.Id == runid && fileContainer.Files.Count == 1 && !fileContainer.Files.FirstOrDefault().FileName.Contains('/') && fileContainer.Files.FirstOrDefault().FileName.EndsWith(".zip") && (body.IdFilter == null || body.IdFilter == fileContainer.Id) && (body.NameFilter == null || body.NameFilter.ToLower() == fileContainer.Name.ToLower()) orderby fileContainer.Container.Attempt.Attempt descending select new Github.Actions.Results.Api.V1.ListArtifactsResponse_MonolithArtifact
{
CreatedAt = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(System.DateTimeOffset.UtcNow),
DatabaseId = fileContainer.Id,
Name = fileContainer.Name,
Size = fileContainer.Size ?? 0,
WorkflowRunBackendId = body.WorkflowRunBackendId,
WorkflowJobRunBackendId = body.WorkflowJobRunBackendId
});
return formatter.Format(resp);
}

[HttpPost("GetSignedArtifactURL")]
public async Task<string> GetSignedArtifactURL([FromBody, Protobuf] Github.Actions.Results.Api.V1.GetSignedArtifactURLRequest body) {
var attempt = long.Parse(User.FindFirst("attempt")?.Value ?? "-1");
var artifactsMinAttempt = long.Parse(User.FindFirst("artifactsMinAttempt")?.Value ?? "-1");
var runid = long.Parse(body.WorkflowRunBackendId);
var file = await (from fileContainer in _context.ArtifactFileContainer where (fileContainer.Container.Attempt.Attempt >= artifactsMinAttempt || artifactsMinAttempt == -1) && (fileContainer.Container.Attempt.Attempt <= attempt || attempt == -1) && fileContainer.Container.Attempt.WorkflowRun.Id == runid && fileContainer.Files.Count == 1 && fileContainer.Files.FirstOrDefault().FileName.ToLower() == $"{body.Name}.zip".ToLower() orderby fileContainer.Container.Attempt.Attempt descending select fileContainer.Files.FirstOrDefault()).FirstAsync();
var resp = new Github.Actions.Results.Api.V1.GetSignedArtifactURLResponse
{
SignedUrl = new Uri(new Uri(ServerUrl), $"twirp/github.actions.results.api.v1.ArtifactService/DownloadArtifact?id={file.Id}&sig={CreateSignature(file.Id)}").ToString()
};
return formatter.Format(resp);
}

[AllowAnonymous]
[HttpGet("DownloadArtifact")]
public IActionResult DownloadArtifact(int id, string sig) {
if(string.IsNullOrEmpty(sig) || !VerifySignature(id, sig)) {
return NotFound();
}
var container = _context.ArtifactRecords.Find(id);
var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");
return new FileStreamResult(System.IO.File.OpenRead(Path.Combine(_targetFilePath, container.StoreName)), "application/octet-stream") { EnableRangeProcessing = true };
}


public class ProtobufBinder : IModelBinder
{
public async Task BindModelAsync(ModelBindingContext bindingContext)
{
if (!bindingContext.HttpContext.Request.HasJsonContentType())
{
throw new BadHttpRequestException(
"Request content type was not a recognized JSON content type.",
StatusCodes.Status415UnsupportedMediaType);
}

using var sr = new StreamReader(bindingContext.HttpContext.Request.Body);
var str = await sr.ReadToEndAsync();

var valueType = bindingContext.ModelType;
var parser = new JsonParser(JsonParser.Settings.Default.WithIgnoreUnknownFields(true));

var descriptor = (MessageDescriptor)bindingContext.ModelType.GetProperty("Descriptor", BindingFlags.Public | BindingFlags.Static).GetValue(null, null);
var obj = parser.Parse(str, descriptor);

bindingContext.Result = ModelBindingResult.Success(obj);
}
}

public class ProtobufAttribute : ModelBinderAttribute {
public ProtobufAttribute() : base(typeof(ProtobufBinder)) {

}
}
}
}
2 changes: 2 additions & 0 deletions src/Runner.Server/Controllers/MessageController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5581,6 +5581,7 @@ private Func<bool, Job> queueJob(GitHub.DistributedTask.ObjectTemplating.ITraceW
var feedStreamUrl = new UriBuilder(new Uri(new Uri(apiUrl), $"_apis/v1/TimeLineWebConsoleLog/feedstream/{Uri.EscapeDataString(timelineId.ToString())}/ws"));
feedStreamUrl.Scheme = feedStreamUrl.Scheme == "http" ? "ws" : "wss";
systemVssConnection.Data["FeedStreamUrl"] = feedStreamUrl.ToString();
systemVssConnection.Data["ResultsServiceUrl"] = apiUrl;
if(calculatedPermissions.TryGetValue("id_token", out var p_id_token) && p_id_token == "write") {
var environment = deploymentEnvironmentValue?.Name ?? ("");
var claims = new Dictionary<string, string>();
Expand Down Expand Up @@ -5711,6 +5712,7 @@ private Func<bool, Job> queueJob(GitHub.DistributedTask.ObjectTemplating.ITraceW
new Claim("localcheckout", localcheckout ? "actions/checkout" : ""),
new Claim("runid", runid.ToString()),
new Claim("github_token", variables.TryGetValue("github_token", out var ghtoken) ? ghtoken.Value : ""),
new Claim("scp", $"Actions.Results:{runid}:{job.JobId}")
}),
Expires = DateTime.UtcNow.AddMinutes(timeoutMinutes + 10),
Issuer = myIssuer,
Expand Down
4 changes: 2 additions & 2 deletions src/Runner.Server/Controllers/TimelineController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,9 @@ public async Task<IActionResult> GetTimeline(Guid timelineId) {
public async Task<IActionResult> PutAttachment(Guid timelineId, Guid recordId, string type, string name) {
var jobInfo = (from j in _context.Jobs where j.TimeLineId == timelineId select new { j.runid, j.Attempt }).FirstOrDefault();
var artifacts = new ArtifactController(_context, Configuration);
var fname = $"Attachment_{timelineId}_{recordId}_{type}";
var fname = $"Attachment_{timelineId}_{recordId}_{type}_{name}";
var container = await artifacts.CreateContainer(jobInfo.runid, jobInfo.Attempt, new CreateActionsStorageArtifactParameters() { Name = fname });
var record = new ArtifactRecord() {FileName = Path.Join(fname, name), StoreName = Path.GetRandomFileName(), GZip = false, FileContainer = container} ;
var record = new ArtifactRecord() {FileName = fname, StoreName = Path.GetRandomFileName(), GZip = false, FileContainer = container} ;
_context.ArtifactRecords.Add(record);
await _context.SaveChangesAsync();
var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");
Expand Down
Loading

0 comments on commit fa53b2a

Please sign in to comment.