RSTUF initial implementation (#4167)

CONTRIBUTING.md

@@ -205,6 +205,14 @@ and prefixing your commands with `script/dev`.
 For example, running `script/dev bin/rails s` will launch the development server with development secrets set in
 the environment.
 
+#### Running with local RSTUF
+
+There is experimental [RSTUF](https://repository-service-tuf.readthedocs.io/en/stable/) support in RubyGems.org. When `RSTUF_API_URL` environment variable is set, RSTUF functionality is enabled. Easiest way to setup RSTUF locally is to follow [official docker guide](https://repository-service-tuf.readthedocs.io/en/latest/guide/deployment/guide/docker.html). It starts RSTUF API available at `http://localhost:80` by default and app can be locally started using following command.
+
+```bash
+RSTUF_API_URL="http://localhost:80" bin/rails s
+```
+
 ---
 
 When everything is set up, start the web server with `rails server` and browse to

app/jobs/rstuf/add_job.rb

@@ -0,0 +1,18 @@
+class Rstuf::AddJob < Rstuf::ApplicationJob
+  queue_with_priority PRIORITIES.fetch(:push)
+
+  def perform(version:)
+    target = {
+      info: {
+        length: version.size,
+        hashes: {
+          sha256: version.sha256_hex
+        }
+      },
+      path: version.gem_file_name
+    }
+
+    task_id = Rstuf::Client.post_artifacts([target])
+    Rstuf::CheckJob.set(wait: Rstuf.wait_for).perform_later(task_id)
+  end
+end

app/jobs/rstuf/application_job.rb

@@ -0,0 +1,5 @@
+class Rstuf::ApplicationJob < ApplicationJob
+  before_enqueue do
+    throw :abort unless Rstuf.enabled?
+  end
+end

app/jobs/rstuf/check_job.rb

@@ -0,0 +1,24 @@
+class Rstuf::CheckJob < Rstuf::ApplicationJob
+  RetryException = Class.new(StandardError)
+  FailureException = Class.new(StandardError)
+  ErrorException = Class.new(StandardError)
+  retry_on RetryException, wait: :polynomially_longer, attempts: 10
+
+  queue_with_priority PRIORITIES.fetch(:push)
+
+  def perform(task_id)
+    case status = Rstuf::Client.task_state(task_id)
+    when "SUCCESS"
+      # no-op, all good
+    when "FAILURE"
+      raise FailureException, "RSTUF job failed, please check payload and retry"
+    when "ERRORED", "REVOKED", "REJECTED"
+      raise ErrorException, "RSTUF internal problem, please check RSTUF health"
+    when "PENDING", "RUNNING", "RECEIVED", "STARTED"
+      raise RetryException
+    else
+      Rails.logger.info "RSTUF job returned unexpected state #{status}"
+      raise RetryException
+    end
+  end
+end

app/jobs/rstuf/remove_job.rb

@@ -0,0 +1,8 @@
+class Rstuf::RemoveJob < Rstuf::ApplicationJob
+  queue_with_priority PRIORITIES.fetch(:push)
+
+  def perform(version:)
+    task_id = Rstuf::Client.delete_artifacts([version.gem_file_name])
+    Rstuf::CheckJob.set(wait: Rstuf.wait_for).perform_later(task_id)
+  end
+end

app/models/deletion.rb

@@ -57,11 +57,13 @@ def expire_cache
   def remove_from_index
     version.update!(indexed: false, yanked_at: Time.now.utc)
     reindex
+    Rstuf::RemoveJob.perform_later(version:)
   end
 
   def restore_to_index
     version.update!(indexed: true, yanked_at: nil, yanked_info_checksum: nil)
     reindex
+    Rstuf::AddJob.perform_later(version:)
   end
 
   def reindex

app/models/pusher.rb

@@ -149,6 +149,7 @@ def after_write
     RackAttackReset.gem_push_backoff(@request.remote_ip, owner.to_gid) if @request&.remote_ip.present?
     AfterVersionWriteJob.new(version:).perform(version:)
     StatsD.increment "push.success"
+    Rstuf::AddJob.perform_later(version:)
   end
 
   def notify(message, code)

config/initializers/rstuf.rb

@@ -0,0 +1,7 @@
+require 'rstuf'
+
+if ENV['RSTUF_API_URL'].presence
+  Rstuf.base_url = ENV['RSTUF_API_URL']
+  Rstuf.enabled = true
+  Rstuf.wait_for = 10.seconds
+end

lib/rstuf.rb

@@ -0,0 +1,9 @@
+module Rstuf
+  mattr_accessor :base_url
+  mattr_accessor :enabled, default: false
+  mattr_accessor :wait_for, default: 1
+
+  def self.enabled?
+    enabled
+  end
+end

lib/rstuf/client.rb

@@ -0,0 +1,39 @@
+class Rstuf::Client
+  include SemanticLogger::Loggable
+
+  Error = Class.new(StandardError)
+
+  def self.post_artifacts(targets)
+    response = connection.post("/api/v1/artifacts/", { targets: targets })
+
+    return response.body.dig("data", "task_id") if response.success?
+    raise Error, "Error posting artifacts: #{response.body}"
+  end
+
+  def self.delete_artifacts(targets)
+    response = connection.post("/api/v1/artifacts/delete", { targets: targets }, {})
+
+    return response.body.dig("data", "task_id") if response.success?
+    raise Error, "Error deleting artifacts: #{response.body}"
+  end
+
+  def self.task_state(task_id)
+    result = get_task(task_id)
+    result.dig("data", "state")
+  end
+
+  def self.connection
+    Faraday.new(url: Rstuf.base_url) do |f|
+      f.request :json
+      f.response :json
+      f.response :logger, logger
+    end
+  end
+
+  def self.get_task(task_id)
+    response = connection.get("/api/v1/task/", task_id: task_id)
+
+    return response.body if response.success?
+    raise Error, "Error fetching task: #{response.body}"
+  end
+end

test/jobs/rstuf/add_job_test.rb

@@ -0,0 +1,27 @@
+require "test_helper"
+
+class Rstuf::AddJobTest < ActiveJob::TestCase
+  setup do
+    setup_rstuf
+
+    @version = create(:version)
+    @task_id = "12345"
+
+    stub_request(:post, "#{Rstuf.base_url}/api/v1/artifacts/")
+      .to_return(
+        status: 200,
+        body: { data: { task_id: @task_id } }.to_json,
+        headers: { "Content-Type" => "application/json" }
+      )
+  end
+
+  test "perform_later enqueues CheckJob with task_id" do
+    assert_enqueued_with(at: Time.zone.now + Rstuf.wait_for, job: Rstuf::CheckJob, args: [@task_id]) do
+      Rstuf::AddJob.perform_now(version: @version)
+    end
+  end
+
+  teardown do
+    teardown_rstuf
+  end
+end

test/jobs/rstuf/application_job_test.rb

@@ -0,0 +1,31 @@
+require "test_helper"
+
+class Rstuf::ApplicationJobTest < ActiveJob::TestCase
+  class MockJob < Rstuf::ApplicationJob
+    def perform
+      # no-op
+    end
+  end
+
+  setup do
+    setup_rstuf
+  end
+
+  test "job is not performed if Rstuf is disabled" do
+    Rstuf.enabled = false
+    assert_no_enqueued_jobs only: MockJob do
+      MockJob.perform_later
+    end
+  end
+
+  test "job is performed if Rstuf is enabled" do
+    Rstuf.enabled = true
+    assert_enqueued_jobs 1, only: MockJob do
+      MockJob.perform_later
+    end
+  end
+
+  teardown do
+    teardown_rstuf
+  end
+end

test/jobs/rstuf/check_job_test.rb

@@ -0,0 +1,63 @@
+require "test_helper"
+
+class Rstuf::CheckJobTest < ActiveJob::TestCase
+  setup do
+    setup_rstuf
+
+    @task_id = "task123"
+  end
+
+  test "perform does not raise on success" do
+    success_response = { "data" => { "state" => "SUCCESS" } }
+    stub_request(:get, "#{Rstuf.base_url}/api/v1/task/?task_id=#{@task_id}")
+      .to_return(status: 200, body: success_response.to_json, headers: { "Content-Type" => "application/json" })
+
+    assert_nothing_raised do
+      Rstuf::CheckJob.perform_now(@task_id)
+    end
+  end
+
+  test "perform raises an error on error" do
+    failure_response = { "data" => { "state" => "ERRORED" } }
+    stub_request(:get, "#{Rstuf.base_url}/api/v1/task/?task_id=#{@task_id}")
+      .to_return(status: 200, body: failure_response.to_json, headers: { "Content-Type" => "application/json" })
+
+    assert_raises(Rstuf::CheckJob::ErrorException) do
+      Rstuf::CheckJob.new.perform(@task_id)
+    end
+  end
+
+  test "perform raises an error on failure" do
+    failure_response = { "data" => { "state" => "FAILURE" } }
+    stub_request(:get, "#{Rstuf.base_url}/api/v1/task/?task_id=#{@task_id}")
+      .to_return(status: 200, body: failure_response.to_json, headers: { "Content-Type" => "application/json" })
+
+    assert_raises(Rstuf::CheckJob::FailureException) do
+      Rstuf::CheckJob.new.perform(@task_id)
+    end
+  end
+
+  test "perform raises a retry exception on pending state and retries" do
+    retry_response = { "data" => { "state" => "PENDING" } }
+    stub_request(:get, "#{Rstuf.base_url}/api/v1/task/?task_id=#{@task_id}")
+      .to_return(status: 200, body: retry_response.to_json, headers: { "Content-Type" => "application/json" })
+
+    assert_enqueued_with(job: Rstuf::CheckJob, args: [@task_id]) do
+      Rstuf::CheckJob.perform_now(@task_id)
+    end
+  end
+
+  test "perform raises a retry exception on retry state and retries" do
+    retry_response = { "data" => { "state" => "UNKNOWN" } }
+    stub_request(:get, "#{Rstuf.base_url}/api/v1/task/?task_id=#{@task_id}")
+      .to_return(status: 200, body: retry_response.to_json, headers: { "Content-Type" => "application/json" })
+
+    assert_enqueued_with(job: Rstuf::CheckJob, args: [@task_id]) do
+      Rstuf::CheckJob.perform_now(@task_id)
+    end
+  end
+
+  teardown do
+    teardown_rstuf
+  end
+end

test/jobs/rstuf/remove_job_test.rb

@@ -0,0 +1,27 @@
+require "test_helper"
+
+class Rstuf::RemoveJobTest < ActiveJob::TestCase
+  setup do
+    setup_rstuf
+
+    @version = create(:version)
+    @task_id = "67890"
+
+    stub_request(:post, "#{Rstuf.base_url}/api/v1/artifacts/delete")
+      .to_return(
+        status: 200,
+        body: { data: { task_id: @task_id } }.to_json,
+        headers: { "Content-Type" => "application/json" }
+      )
+  end
+
+  test "perform_later enqueues CheckJob with task_id" do
+    assert_enqueued_with(at: Time.zone.now + Rstuf.wait_for, job: Rstuf::CheckJob, args: [@task_id]) do
+      Rstuf::RemoveJob.perform_now(version: @version)
+    end
+  end
+
+  teardown do
+    teardown_rstuf
+  end
+end

test/models/deletion_test.rb

@@ -95,6 +95,22 @@ class DeletionTest < ActiveSupport::TestCase
     end
   end
 
+  context "when rstuf is enabled" do
+    setup do
+      setup_rstuf
+    end
+
+    should "enqueue rstuf removal" do
+      assert_enqueued_with(job: Rstuf::RemoveJob, args: [{ version: @version }]) do
+        delete_gem
+      end
+    end
+
+    teardown do
+      teardown_rstuf
+    end
+  end
+
   should "enque job for updating ES index, spec index and purging cdn" do
     assert_enqueued_jobs 1, only: ActionMailer::MailDeliveryJob do
       assert_enqueued_jobs 8, only: FastlyPurgeJob do
@@ -189,6 +205,23 @@ class DeletionTest < ActiveSupport::TestCase
       end
     end
 
+    context "with rstuf enabled" do
+      setup do
+        setup_rstuf
+      end
+
+      should "enqueue rstuf addition" do
+        @deletion = delete_gem
+        assert_enqueued_jobs 1, only: Rstuf::AddJob do
+          @deletion.restore!
+        end
+      end
+
+      teardown do
+        teardown_rstuf
+      end
+    end
+
     should "enqueue indexing jobs" do
       @deletion = delete_gem
       assert_enqueued_jobs 1, only: Indexer do

test/models/pusher_test.rb

@@ -677,6 +677,22 @@ def two_cert_chain(signing_key:, root_not_before: Time.current, cert_not_before:
       @cutter.save
     end
 
+    context "with rstuf enabled" do
+      setup do
+        setup_rstuf
+      end
+
+      should "enqueue rstuf addition" do
+        assert_enqueued_with(job: Rstuf::AddJob, args: [{ version: @cutter.version }]) do
+          @cutter.save
+        end
+      end
+
+      teardown do
+        teardown_rstuf
+      end
+    end
+
     should "update rubygem attributes when saved" do
       @rubygem.expects(:update_attributes_from_gem_specification!).with(@cutter.version, @spec)
       @cutter.save

test/test_helper.rb

@@ -173,6 +173,18 @@ def create_webauthn_credential
     @user.reload
     @authenticator
   end
+
+  def setup_rstuf
+    @original_rstuf_enabled = Rstuf.enabled
+    @original_base_url = Rstuf.base_url
+    Rstuf.base_url = "https://rstuf.example.com"
+    Rstuf.enabled = true
+  end
+
+  def teardown_rstuf
+    Rstuf.enabled = @original_rstuf_enabled
+    Rstuf.base_url = @original_base_url
+  end
 end
 
 class ActionDispatch::IntegrationTest