diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1279c746b66..6e5581d1339 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -205,6 +205,14 @@ and prefixing your commands with `script/dev`. For example, running `script/dev bin/rails s` will launch the development server with development secrets set in the environment. +#### Running with local RSTUF + +There is experimental [RSTUF](https://repository-service-tuf.readthedocs.io/en/stable/) support in RubyGems.org. When `RSTUF_API_URL` environment variable is set, RSTUF functionality is enabled. Easiest way to setup RSTUF locally is to follow [official docker guide](https://repository-service-tuf.readthedocs.io/en/latest/guide/deployment/guide/docker.html). It starts RSTUF API available at `http://localhost:80` by default and app can be locally started using following command. + +```bash +RSTUF_API_URL="http://localhost:80" bin/rails s +``` + --- When everything is set up, start the web server with `rails server` and browse to diff --git a/app/jobs/rstuf/add_job.rb b/app/jobs/rstuf/add_job.rb new file mode 100644 index 00000000000..fc76f65e4a5 --- /dev/null +++ b/app/jobs/rstuf/add_job.rb @@ -0,0 +1,18 @@ +class Rstuf::AddJob < Rstuf::ApplicationJob + queue_with_priority PRIORITIES.fetch(:push) + + def perform(version:) + target = { + info: { + length: version.size, + hashes: { + sha256: version.sha256_hex + } + }, + path: version.gem_file_name + } + + task_id = Rstuf::Client.post_artifacts([target]) + Rstuf::CheckJob.set(wait: Rstuf.wait_for).perform_later(task_id) + end +end diff --git a/app/jobs/rstuf/application_job.rb b/app/jobs/rstuf/application_job.rb new file mode 100644 index 00000000000..21b6f01d992 --- /dev/null +++ b/app/jobs/rstuf/application_job.rb @@ -0,0 +1,5 @@ +class Rstuf::ApplicationJob < ApplicationJob + before_enqueue do + throw :abort unless Rstuf.enabled? + end +end diff --git a/app/jobs/rstuf/check_job.rb b/app/jobs/rstuf/check_job.rb new file mode 100644 index 00000000000..c66da399fe2 --- /dev/null +++ b/app/jobs/rstuf/check_job.rb @@ -0,0 +1,24 @@ +class Rstuf::CheckJob < Rstuf::ApplicationJob + RetryException = Class.new(StandardError) + FailureException = Class.new(StandardError) + ErrorException = Class.new(StandardError) + retry_on RetryException, wait: :polynomially_longer, attempts: 10 + + queue_with_priority PRIORITIES.fetch(:push) + + def perform(task_id) + case status = Rstuf::Client.task_state(task_id) + when "SUCCESS" + # no-op, all good + when "FAILURE" + raise FailureException, "RSTUF job failed, please check payload and retry" + when "ERRORED", "REVOKED", "REJECTED" + raise ErrorException, "RSTUF internal problem, please check RSTUF health" + when "PENDING", "RUNNING", "RECEIVED", "STARTED" + raise RetryException + else + Rails.logger.info "RSTUF job returned unexpected state #{status}" + raise RetryException + end + end +end diff --git a/app/jobs/rstuf/remove_job.rb b/app/jobs/rstuf/remove_job.rb new file mode 100644 index 00000000000..14a0b600cae --- /dev/null +++ b/app/jobs/rstuf/remove_job.rb @@ -0,0 +1,8 @@ +class Rstuf::RemoveJob < Rstuf::ApplicationJob + queue_with_priority PRIORITIES.fetch(:push) + + def perform(version:) + task_id = Rstuf::Client.delete_artifacts([version.gem_file_name]) + Rstuf::CheckJob.set(wait: Rstuf.wait_for).perform_later(task_id) + end +end diff --git a/app/models/deletion.rb b/app/models/deletion.rb index 0441164b1a1..056cd20ae68 100644 --- a/app/models/deletion.rb +++ b/app/models/deletion.rb @@ -57,11 +57,13 @@ def expire_cache def remove_from_index version.update!(indexed: false, yanked_at: Time.now.utc) reindex + Rstuf::RemoveJob.perform_later(version:) end def restore_to_index version.update!(indexed: true, yanked_at: nil, yanked_info_checksum: nil) reindex + Rstuf::AddJob.perform_later(version:) end def reindex diff --git a/app/models/pusher.rb b/app/models/pusher.rb index 02d023c6dd2..be573e6b995 100644 --- a/app/models/pusher.rb +++ b/app/models/pusher.rb @@ -149,6 +149,7 @@ def after_write RackAttackReset.gem_push_backoff(@request.remote_ip, owner.to_gid) if @request&.remote_ip.present? AfterVersionWriteJob.new(version:).perform(version:) StatsD.increment "push.success" + Rstuf::AddJob.perform_later(version:) end def notify(message, code) diff --git a/config/initializers/rstuf.rb b/config/initializers/rstuf.rb new file mode 100644 index 00000000000..768d2ccee9d --- /dev/null +++ b/config/initializers/rstuf.rb @@ -0,0 +1,7 @@ +require 'rstuf' + +if ENV['RSTUF_API_URL'].presence + Rstuf.base_url = ENV['RSTUF_API_URL'] + Rstuf.enabled = true + Rstuf.wait_for = 10.seconds +end diff --git a/lib/rstuf.rb b/lib/rstuf.rb new file mode 100644 index 00000000000..bb09c89a47b --- /dev/null +++ b/lib/rstuf.rb @@ -0,0 +1,9 @@ +module Rstuf + mattr_accessor :base_url + mattr_accessor :enabled, default: false + mattr_accessor :wait_for, default: 1 + + def self.enabled? + enabled + end +end diff --git a/lib/rstuf/client.rb b/lib/rstuf/client.rb new file mode 100644 index 00000000000..20e6ea098bf --- /dev/null +++ b/lib/rstuf/client.rb @@ -0,0 +1,39 @@ +class Rstuf::Client + include SemanticLogger::Loggable + + Error = Class.new(StandardError) + + def self.post_artifacts(targets) + response = connection.post("/api/v1/artifacts/", { targets: targets }) + + return response.body.dig("data", "task_id") if response.success? + raise Error, "Error posting artifacts: #{response.body}" + end + + def self.delete_artifacts(targets) + response = connection.post("/api/v1/artifacts/delete", { targets: targets }, {}) + + return response.body.dig("data", "task_id") if response.success? + raise Error, "Error deleting artifacts: #{response.body}" + end + + def self.task_state(task_id) + result = get_task(task_id) + result.dig("data", "state") + end + + def self.connection + Faraday.new(url: Rstuf.base_url) do |f| + f.request :json + f.response :json + f.response :logger, logger + end + end + + def self.get_task(task_id) + response = connection.get("/api/v1/task/", task_id: task_id) + + return response.body if response.success? + raise Error, "Error fetching task: #{response.body}" + end +end diff --git a/test/jobs/rstuf/add_job_test.rb b/test/jobs/rstuf/add_job_test.rb new file mode 100644 index 00000000000..56acaa31660 --- /dev/null +++ b/test/jobs/rstuf/add_job_test.rb @@ -0,0 +1,27 @@ +require "test_helper" + +class Rstuf::AddJobTest < ActiveJob::TestCase + setup do + setup_rstuf + + @version = create(:version) + @task_id = "12345" + + stub_request(:post, "#{Rstuf.base_url}/api/v1/artifacts/") + .to_return( + status: 200, + body: { data: { task_id: @task_id } }.to_json, + headers: { "Content-Type" => "application/json" } + ) + end + + test "perform_later enqueues CheckJob with task_id" do + assert_enqueued_with(at: Time.zone.now + Rstuf.wait_for, job: Rstuf::CheckJob, args: [@task_id]) do + Rstuf::AddJob.perform_now(version: @version) + end + end + + teardown do + teardown_rstuf + end +end diff --git a/test/jobs/rstuf/application_job_test.rb b/test/jobs/rstuf/application_job_test.rb new file mode 100644 index 00000000000..4bcf8be4be8 --- /dev/null +++ b/test/jobs/rstuf/application_job_test.rb @@ -0,0 +1,31 @@ +require "test_helper" + +class Rstuf::ApplicationJobTest < ActiveJob::TestCase + class MockJob < Rstuf::ApplicationJob + def perform + # no-op + end + end + + setup do + setup_rstuf + end + + test "job is not performed if Rstuf is disabled" do + Rstuf.enabled = false + assert_no_enqueued_jobs only: MockJob do + MockJob.perform_later + end + end + + test "job is performed if Rstuf is enabled" do + Rstuf.enabled = true + assert_enqueued_jobs 1, only: MockJob do + MockJob.perform_later + end + end + + teardown do + teardown_rstuf + end +end diff --git a/test/jobs/rstuf/check_job_test.rb b/test/jobs/rstuf/check_job_test.rb new file mode 100644 index 00000000000..d45a535de96 --- /dev/null +++ b/test/jobs/rstuf/check_job_test.rb @@ -0,0 +1,63 @@ +require "test_helper" + +class Rstuf::CheckJobTest < ActiveJob::TestCase + setup do + setup_rstuf + + @task_id = "task123" + end + + test "perform does not raise on success" do + success_response = { "data" => { "state" => "SUCCESS" } } + stub_request(:get, "#{Rstuf.base_url}/api/v1/task/?task_id=#{@task_id}") + .to_return(status: 200, body: success_response.to_json, headers: { "Content-Type" => "application/json" }) + + assert_nothing_raised do + Rstuf::CheckJob.perform_now(@task_id) + end + end + + test "perform raises an error on error" do + failure_response = { "data" => { "state" => "ERRORED" } } + stub_request(:get, "#{Rstuf.base_url}/api/v1/task/?task_id=#{@task_id}") + .to_return(status: 200, body: failure_response.to_json, headers: { "Content-Type" => "application/json" }) + + assert_raises(Rstuf::CheckJob::ErrorException) do + Rstuf::CheckJob.new.perform(@task_id) + end + end + + test "perform raises an error on failure" do + failure_response = { "data" => { "state" => "FAILURE" } } + stub_request(:get, "#{Rstuf.base_url}/api/v1/task/?task_id=#{@task_id}") + .to_return(status: 200, body: failure_response.to_json, headers: { "Content-Type" => "application/json" }) + + assert_raises(Rstuf::CheckJob::FailureException) do + Rstuf::CheckJob.new.perform(@task_id) + end + end + + test "perform raises a retry exception on pending state and retries" do + retry_response = { "data" => { "state" => "PENDING" } } + stub_request(:get, "#{Rstuf.base_url}/api/v1/task/?task_id=#{@task_id}") + .to_return(status: 200, body: retry_response.to_json, headers: { "Content-Type" => "application/json" }) + + assert_enqueued_with(job: Rstuf::CheckJob, args: [@task_id]) do + Rstuf::CheckJob.perform_now(@task_id) + end + end + + test "perform raises a retry exception on retry state and retries" do + retry_response = { "data" => { "state" => "UNKNOWN" } } + stub_request(:get, "#{Rstuf.base_url}/api/v1/task/?task_id=#{@task_id}") + .to_return(status: 200, body: retry_response.to_json, headers: { "Content-Type" => "application/json" }) + + assert_enqueued_with(job: Rstuf::CheckJob, args: [@task_id]) do + Rstuf::CheckJob.perform_now(@task_id) + end + end + + teardown do + teardown_rstuf + end +end diff --git a/test/jobs/rstuf/remove_job_test.rb b/test/jobs/rstuf/remove_job_test.rb new file mode 100644 index 00000000000..5bfa65bee6d --- /dev/null +++ b/test/jobs/rstuf/remove_job_test.rb @@ -0,0 +1,27 @@ +require "test_helper" + +class Rstuf::RemoveJobTest < ActiveJob::TestCase + setup do + setup_rstuf + + @version = create(:version) + @task_id = "67890" + + stub_request(:post, "#{Rstuf.base_url}/api/v1/artifacts/delete") + .to_return( + status: 200, + body: { data: { task_id: @task_id } }.to_json, + headers: { "Content-Type" => "application/json" } + ) + end + + test "perform_later enqueues CheckJob with task_id" do + assert_enqueued_with(at: Time.zone.now + Rstuf.wait_for, job: Rstuf::CheckJob, args: [@task_id]) do + Rstuf::RemoveJob.perform_now(version: @version) + end + end + + teardown do + teardown_rstuf + end +end diff --git a/test/models/deletion_test.rb b/test/models/deletion_test.rb index 2c0a4ef89b0..703a9e6e1a1 100644 --- a/test/models/deletion_test.rb +++ b/test/models/deletion_test.rb @@ -95,6 +95,22 @@ class DeletionTest < ActiveSupport::TestCase end end + context "when rstuf is enabled" do + setup do + setup_rstuf + end + + should "enqueue rstuf removal" do + assert_enqueued_with(job: Rstuf::RemoveJob, args: [{ version: @version }]) do + delete_gem + end + end + + teardown do + teardown_rstuf + end + end + should "enque job for updating ES index, spec index and purging cdn" do assert_enqueued_jobs 1, only: ActionMailer::MailDeliveryJob do assert_enqueued_jobs 8, only: FastlyPurgeJob do @@ -189,6 +205,23 @@ class DeletionTest < ActiveSupport::TestCase end end + context "with rstuf enabled" do + setup do + setup_rstuf + end + + should "enqueue rstuf addition" do + @deletion = delete_gem + assert_enqueued_jobs 1, only: Rstuf::AddJob do + @deletion.restore! + end + end + + teardown do + teardown_rstuf + end + end + should "enqueue indexing jobs" do @deletion = delete_gem assert_enqueued_jobs 1, only: Indexer do diff --git a/test/models/pusher_test.rb b/test/models/pusher_test.rb index 66d1adab13b..a73355cccc5 100644 --- a/test/models/pusher_test.rb +++ b/test/models/pusher_test.rb @@ -677,6 +677,22 @@ def two_cert_chain(signing_key:, root_not_before: Time.current, cert_not_before: @cutter.save end + context "with rstuf enabled" do + setup do + setup_rstuf + end + + should "enqueue rstuf addition" do + assert_enqueued_with(job: Rstuf::AddJob, args: [{ version: @cutter.version }]) do + @cutter.save + end + end + + teardown do + teardown_rstuf + end + end + should "update rubygem attributes when saved" do @rubygem.expects(:update_attributes_from_gem_specification!).with(@cutter.version, @spec) @cutter.save diff --git a/test/test_helper.rb b/test/test_helper.rb index 6c99b81b4a4..4dc511c8f03 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -173,6 +173,18 @@ def create_webauthn_credential @user.reload @authenticator end + + def setup_rstuf + @original_rstuf_enabled = Rstuf.enabled + @original_base_url = Rstuf.base_url + Rstuf.base_url = "https://rstuf.example.com" + Rstuf.enabled = true + end + + def teardown_rstuf + Rstuf.enabled = @original_rstuf_enabled + Rstuf.base_url = @original_base_url + end end class ActionDispatch::IntegrationTest diff --git a/test/unit/rstuf/client_test.rb b/test/unit/rstuf/client_test.rb new file mode 100644 index 00000000000..7776bc8dd7c --- /dev/null +++ b/test/unit/rstuf/client_test.rb @@ -0,0 +1,79 @@ +require "test_helper" + +class Rstuf::ClientTest < ActiveSupport::TestCase + setup do + setup_rstuf + end + + teardown do + teardown_rstuf + end + + test "post_artifacts should post targets and return task_id on success" do + task_id = "12345" + stub_request(:post, "#{Rstuf.base_url}/api/v1/artifacts/") + .with(body: { targets: %w[artifact1 artifact2] }) + .to_return(body: { data: { task_id: task_id } }.to_json, status: 200, headers: { "Content-Type" => "application/json" }) + + response_task_id = Rstuf::Client.post_artifacts(%w[artifact1 artifact2]) + + assert_equal task_id, response_task_id + end + + test "post_artifacts should raise Error on failure" do + error_message = "Invalid targets" + stub_request(:post, "#{Rstuf.base_url}/api/v1/artifacts/") + .with(body: { targets: %w[artifact1 artifact2] }) + .to_return(body: { error: error_message }.to_json, status: 400, headers: { "Content-Type" => "application/json" }) + + assert_raises(Rstuf::Client::Error) do + Rstuf::Client.post_artifacts(%w[artifact1 artifact2]) + end + end + + test "delete_artifacts should post targets for deletion and return task_id on success" do + task_id = "67890" + stub_request(:post, "#{Rstuf.base_url}/api/v1/artifacts/delete") + .with(body: { targets: %w[artifact1 artifact2] }) + .to_return(body: { data: { task_id: task_id } }.to_json, status: 200, headers: { "Content-Type" => "application/json" }) + + response_task_id = Rstuf::Client.delete_artifacts(%w[artifact1 artifact2]) + + assert_equal task_id, response_task_id + end + + test "delete_artifacts should raise Error on failure" do + error_message = "Could not delete" + stub_request(:post, "#{Rstuf.base_url}/api/v1/artifacts/delete") + .with(body: { targets: %w[artifact1 artifact2] }) + .to_return(body: { error: error_message }.to_json, status: 400, headers: { "Content-Type" => "application/json" }) + + assert_raises(Rstuf::Client::Error) do + Rstuf::Client.delete_artifacts(%w[artifact1 artifact2]) + end + end + + test "task_state should return the status of the task" do + task_id = "12345" + state = "processing" + stub_request(:get, "#{Rstuf.base_url}/api/v1/task/") + .with(query: { task_id: task_id }) + .to_return(body: { data: { state: state } }.to_json, status: 200, headers: { "Content-Type" => "application/json" }) + + status = Rstuf::Client.task_state(task_id) + + assert_equal state, status + end + + test "task_state should raise Error if task retrieval fails" do + task_id = "12345" + error_message = "Task not found" + stub_request(:get, "#{Rstuf.base_url}/api/v1/task/") + .with(query: { task_id: task_id }) + .to_return(body: { error: error_message }.to_json, status: 404, headers: { "Content-Type" => "application/json" }) + + assert_raises(Rstuf::Client::Error) do + Rstuf::Client.task_state(task_id) + end + end +end diff --git a/test/unit/rstuf_test.rb b/test/unit/rstuf_test.rb new file mode 100644 index 00000000000..5f9e78ffc7d --- /dev/null +++ b/test/unit/rstuf_test.rb @@ -0,0 +1,49 @@ +require "test_helper" + +class RstufTest < ActiveSupport::TestCase + def setup + @original_base_url = Rstuf.base_url + @original_enabled = Rstuf.enabled + @original_wait_for = Rstuf.wait_for + end + + def teardown + Rstuf.base_url = @original_base_url + Rstuf.enabled = @original_enabled + Rstuf.wait_for = @original_wait_for + end + + test "default values are set correctly" do + refute Rstuf.enabled + assert_equal 1, Rstuf.wait_for + end + + test "base_url can be set and retrieved" do + new_url = "http://example.com" + Rstuf.base_url = new_url + + assert_equal new_url, Rstuf.base_url + end + + test "enabled can be set and retrieved" do + Rstuf.enabled = true + + assert Rstuf.enabled + end + + test "enabled? returns the value of enabled" do + Rstuf.enabled = false + + refute_predicate Rstuf, :enabled? + Rstuf.enabled = true + + assert_predicate Rstuf, :enabled? + end + + test "wait_for can be set and retrieved" do + new_wait = 5 + Rstuf.wait_for = new_wait + + assert_equal new_wait, Rstuf.wait_for + end +end