From e6d40e5f23c031b06ef2bab338c82f586a9f59a8 Mon Sep 17 00:00:00 2001 From: 0xevolve Date: Fri, 15 Dec 2023 19:49:52 +0100 Subject: [PATCH] feat: add bucket as env var --- .env.example | 2 ++ .github/workflows/coverage.yml | 3 ++- src/constants/s3.rs | 1 - src/handlers/indexers/create_indexer.rs | 5 +++-- src/handlers/indexers/start_indexer.rs | 6 ++++-- src/tests/server/postgres.rs | 5 +++-- src/tests/server/webhook.rs | 5 +++-- 7 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 9b2a2e9..deb4400 100644 --- a/.env.example +++ b/.env.example @@ -8,3 +8,5 @@ PORT=8080 LOCALSTACK_ENDPOINT=http://localhost:4566 BINARY_BASE_PATH= APIBARA_POSTGRES_CONNECTION_STRING= +APIBARA_ETCD_URL=http://localhost:2379 +INDEXER_SERVICE_BUCKET= \ No newline at end of file diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 039c71e..928b26d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -21,6 +21,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} APIBARA_AUTH_TOKEN: ${{ secrets.APIBARA_AUTH_TOKEN }} APIBARA_ETCD_URL: http://localhost:2379 + INDEXER_SERVICE_BUCKET: indexer-service services: postgres: @@ -86,7 +87,7 @@ jobs: - name: Create S3 Bucket and SQS Queues run: | - awslocal s3api create-bucket --bucket indexer-service --region eu-west-3 --create-bucket-configuration LocationConstraint=eu-west-3 + awslocal s3api create-bu7ket --bucket indexer-service --region eu-west-3 --create-bucket-configuration LocationConstraint=eu-west-3 awslocal s3api put-object --bucket indexer-service --key apibara-scripts/ awslocal s3api list-buckets awslocal sqs create-queue --queue-name indexer-service-start-indexer diff --git a/src/constants/s3.rs b/src/constants/s3.rs index bcbe1aa..d012729 100644 --- a/src/constants/s3.rs +++ b/src/constants/s3.rs @@ -1,2 +1 @@ -pub const INDEXER_SERVICE_BUCKET: &str = "indexer-service"; pub const INDEXER_SERVICE_SCRIPTS_FOLDER: &str = "apibara-scripts"; diff --git a/src/handlers/indexers/create_indexer.rs b/src/handlers/indexers/create_indexer.rs index 0bd99c6..b166495 100644 --- a/src/handlers/indexers/create_indexer.rs +++ b/src/handlers/indexers/create_indexer.rs @@ -9,13 +9,13 @@ use diesel_async::{AsyncConnection, RunQueryDsl}; use uuid::Uuid; use crate::config::config; -use crate::constants::s3::INDEXER_SERVICE_BUCKET; use crate::domain::models::indexer::{IndexerError, IndexerModel, IndexerStatus, IndexerType}; use crate::handlers::indexers::utils::get_s3_script_key; use crate::infra::db::schema::indexers; use crate::infra::errors::InfraError; use crate::infra::repositories::indexer_repository::{self, IndexerDb}; use crate::publishers::indexers::publish_start_indexer; +use crate::utils::env::get_environment_variable; use crate::AppState; #[derive(Default)] @@ -93,6 +93,7 @@ pub async fn create_indexer( }; let config = config().await; + let bucket_name = get_environment_variable("INDEXER_SERVICE_BUCKET"); let connection = &mut state.pool.get().await.expect("Failed to get a connection from the pool"); let created_indexer = connection @@ -109,7 +110,7 @@ pub async fn create_indexer( config .s3_client() .put_object() - .bucket(INDEXER_SERVICE_BUCKET) + .bucket(bucket_name) .key(get_s3_script_key(id)) .body(create_indexer_request.data.into()) .send() diff --git a/src/handlers/indexers/start_indexer.rs b/src/handlers/indexers/start_indexer.rs index 02e3eae..4333293 100644 --- a/src/handlers/indexers/start_indexer.rs +++ b/src/handlers/indexers/start_indexer.rs @@ -5,7 +5,6 @@ use axum::extract::State; use uuid::Uuid; use crate::config::config; -use crate::constants::s3::INDEXER_SERVICE_BUCKET; use crate::domain::models::indexer::{IndexerError, IndexerStatus}; use crate::handlers::indexers::indexer_types::get_indexer_handler; use crate::handlers::indexers::utils::{get_s3_script_key, get_script_tmp_directory}; @@ -13,6 +12,7 @@ use crate::infra::repositories::indexer_repository::{ IndexerFilter, IndexerRepository, Repository, UpdateIndexerStatusAndProcessIdDb, }; use crate::publishers::indexers::publish_start_indexer; +use crate::utils::env::get_environment_variable; use crate::utils::PathExtractor; use crate::AppState; @@ -38,10 +38,12 @@ pub async fn start_indexer(id: Uuid, attempt: u32) -> Result<(), IndexerError> { _ => return Err(IndexerError::InvalidIndexerStatus(indexer_model.status)), } + let bucket_name = get_environment_variable("INDEXER_SERVICE_BUCKET"); + let data = config .s3_client() .get_object() - .bucket(INDEXER_SERVICE_BUCKET) + .bucket(bucket_name) .key(get_s3_script_key(id)) .send() .await diff --git a/src/tests/server/postgres.rs b/src/tests/server/postgres.rs index 9d576a8..6cdf621 100644 --- a/src/tests/server/postgres.rs +++ b/src/tests/server/postgres.rs @@ -5,7 +5,6 @@ use mpart_async::client::MultipartRequest; use rstest::rstest; use crate::config::config; -use crate::constants::s3::INDEXER_SERVICE_BUCKET; use crate::constants::sqs::START_INDEXER_QUEUE; use crate::domain::models::indexer::{IndexerModel, IndexerStatus, IndexerType}; use crate::domain::models::types::AxumErrorResponse; @@ -17,6 +16,7 @@ use crate::tests::common::utils::{ }; use crate::tests::server::common::setup_server; use crate::types::sqs::StartIndexerRequest; +use crate::utils::env::get_environment_variable; #[rstest] #[tokio::test] @@ -41,7 +41,8 @@ async fn create_postgres_indexer(#[future] setup_server: SocketAddr) { assert_eq!(body.indexer_type, IndexerType::Postgres); // check if the file exists on S3 - assert_s3_contains_key(INDEXER_SERVICE_BUCKET, get_s3_script_key(body.id).as_str()).await; + let bucket_name = get_environment_variable("INDEXER_SERVICE_BUCKET"); + assert_s3_contains_key(&bucket_name, get_s3_script_key(body.id).as_str()).await; // check if the message is present on the queue let request = StartIndexerRequest { id: body.id, attempt_no: 1 }; diff --git a/src/tests/server/webhook.rs b/src/tests/server/webhook.rs index d07fac8..63a8913 100644 --- a/src/tests/server/webhook.rs +++ b/src/tests/server/webhook.rs @@ -5,7 +5,6 @@ use mpart_async::client::MultipartRequest; use rstest::rstest; use crate::config::config; -use crate::constants::s3::INDEXER_SERVICE_BUCKET; use crate::constants::sqs::START_INDEXER_QUEUE; use crate::domain::models::indexer::{IndexerModel, IndexerStatus, IndexerType}; use crate::domain::models::types::AxumErrorResponse; @@ -17,6 +16,7 @@ use crate::tests::common::utils::{ }; use crate::tests::server::common::setup_server; use crate::types::sqs::StartIndexerRequest; +use crate::utils::env::get_environment_variable; #[rstest] #[tokio::test] @@ -41,7 +41,8 @@ async fn create_webhook_indexer(#[future] setup_server: SocketAddr) { assert_eq!(body.indexer_type, IndexerType::Webhook); // check if the file exists on S3 - assert_s3_contains_key(INDEXER_SERVICE_BUCKET, get_s3_script_key(body.id).as_str()).await; + let bucket_name = get_environment_variable("INDEXER_SERVICE_BUCKET"); + assert_s3_contains_key(&bucket_name, get_s3_script_key(body.id).as_str()).await; // check if the message is present on the queue let request = StartIndexerRequest { id: body.id, attempt_no: 1 };