added eth,s3,snowflake

.github/workflows/ci.yaml

@@ -45,5 +45,17 @@ jobs:
           sudo dpkg -i dozer-linux-amd64.deb
 
       - name: Run tests
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          BUCKET_NAME: ${{ secrets.BUCKET_NAME }}
+          SN_SERVER: ${{ secrets.SN_SERVER }}
+          SN_USER: ${{ secrets.SN_USER }}
+          SN_PASSWORD: ${{ secrets.SN_PASSWORD }}
+          SN_DATABASE: ${{ secrets.SN_DATABASE }}
+          SN_WAREHOUSE: ${{ secrets.SN_WAREHOUSE }}
+          SN_ROLE: ${{ secrets.SN_ROLE }}
+          ETH_WSS_URL: ${{ secrets.ETH_WSS_URL }}
         run: |
           pnpm test

connectors/aws-s3/dozer-config.yaml

@@ -1,17 +1,17 @@
 app_name: aws-s3-sample
 version: 1
 connections:
-  - config : !S3Storage
+  - config: !S3Storage
       details:
-        access_key_id: {{YOUR_ACCESS_KEY}}
-        secret_access_key: {{YOUR_SECRET_KEY}}
-        region: {{YOUR_REGION}}
-        bucket_name: {{YOUR_BUCKET_NAME}}
+        access_key_id: "{{AWS_ACCESS_KEY_ID}}"
+        secret_access_key: "{{AWS_SECRET_ACCESS_KEY}}"
+        region: "{{AWS_DEFAULT_REGION}}"
+        bucket_name: "{{BUCKET_NAME}}"
       tables:
         - !Table
           name: stocks
           config: !CSV
-            path: . # path to files or folder inside a bucket
+            path: stock_data_monthly_100_rows
             extension: .csv
     name: s3
 
@@ -62,6 +62,5 @@ endpoints:
   - name: lowest_daily_close
     path: /analysis/lowest_daily_close
     table_name: lowest_daily_close
-
 # telemetry:
 #   metrics: !Prometheus # You can check the metrics at http://localhost:9000

connectors/ethereum/dozer-config.yml

@@ -1,4 +1,5 @@
 app_name: eth-smartcontracts-sample
+version: 1
 connections:
   - config: !Ethereum
       provider: !Log

connectors/snowflake/dozer-config.yaml

@@ -57,7 +57,3 @@ endpoints:
       primary_key:
         - C_CUSTKEY
         - C_NAME
-
-api:
-  app_grpc:
-    host: "{{APP_HOST}}"

test/connectors/ethereum.ts

@@ -0,0 +1,27 @@
+import { execSync } from 'child_process';
+import path from 'path';
+
+import {
+  assertEndpointsWithRetry, initDozer,
+} from '../helper';
+
+const TEST_PATH = './connectors/ethereum';
+
+describe('Connectors: ethereum', async () => {
+  beforeEach(async () => {
+    const baseDir = path.join(__dirname, '../../');
+    const fullPath = path.join(baseDir, TEST_PATH);
+    process.chdir(fullPath);
+
+    console.log(`Current directory: ${process.cwd()}`);
+
+    execSync('rm -rf .dozer && rm -f dozer.lock', { stdio: 'inherit' });
+  });
+
+  it('should run and return API endpoints', async () => {
+    const dozer = await initDozer();
+    await assertEndpointsWithRetry();
+    dozer.kill(9);
+    console.log('Killed dozer ethereum');
+  });
+});

test/connectors/local-storage.ts

@@ -1,4 +1,5 @@
 import { execSync } from 'child_process';
+import path from 'path';
 import {
   assertEndpointsWithRetry, initDozer,
 } from '../helper';
@@ -7,18 +8,22 @@ const TEST_PATH = './connectors/local-storage';
 
 describe('Connectors: Local Storage', async () => {
   beforeEach(async () => {
-    console.log(`Starting directory: ${process.cwd()}`);
-
     // Copy test data to local-storage connector
     execSync(`${__dirname}/init.sh`, { stdio: 'inherit' });
 
-    process.chdir(TEST_PATH);
+    const baseDir = path.join(__dirname, '../../');
+    const fullPath = path.join(baseDir, TEST_PATH);
+    process.chdir(fullPath);
+
+    console.log(`Current directory: ${process.cwd()}`);
+
     execSync('rm -rf .dozer && rm -f dozer.lock', { stdio: 'inherit' });
   });
 
   it('should run and return API endpoints', async () => {
     const dozer = await initDozer();
     await assertEndpointsWithRetry();
     dozer.kill(9);
+    console.log('Killed dozer local-storage');
   });
 });

test/connectors/s3.ts

@@ -0,0 +1,27 @@
+import { execSync } from 'child_process';
+import path from 'path';
+
+import {
+  assertEndpointsWithRetry, initDozer,
+} from '../helper';
+
+const TEST_PATH = './connectors/aws-s3';
+
+describe('Connectors: aws-s3', async () => {
+  beforeEach(async () => {
+    const baseDir = path.join(__dirname, '../../');
+    const fullPath = path.join(baseDir, TEST_PATH);
+    process.chdir(fullPath);
+
+    console.log(`Current directory: ${process.cwd()}`);
+
+    execSync('rm -rf .dozer && rm -f dozer.lock', { stdio: 'inherit' });
+  });
+
+  it('should run and return API endpoints', async () => {
+    const dozer = await initDozer();
+    await assertEndpointsWithRetry();
+    dozer.kill(9);
+    console.log('Killed dozer aws-s3');
+  });
+});

test/connectors/snowflake.ts

@@ -0,0 +1,27 @@
+import { execSync } from 'child_process';
+import path from 'path';
+
+import {
+  assertEndpointsWithRetry, initDozer,
+} from '../helper';
+
+const TEST_PATH = './connectors/snowflake';
+
+describe('Connectors: Snowflake', async () => {
+  beforeEach(async () => {
+    const baseDir = path.join(__dirname, '../../');
+    const fullPath = path.join(baseDir, TEST_PATH);
+    process.chdir(fullPath);
+
+    console.log(`Current directory: ${process.cwd()}`);
+
+    execSync('rm -rf .dozer && rm -f dozer.lock', { stdio: 'inherit' });
+  });
+
+  it('should run and return API endpoints', async () => {
+    const dozer = await initDozer();
+    await assertEndpointsWithRetry();
+    dozer.kill(9);
+    console.log('Killed dozer snowflake');
+  });
+});

test/helper.ts

@@ -33,6 +33,10 @@ export class CommonGrpc {
   ): Promise<GetEndpointsResponse> {
     return promisify(this.client.getEndpoints.bind(this.client, params, metadata, {}))();
   }
+
+  public close() {
+    this.client.close();
+  }
 }
 
 export async function checkEndpoints() {
@@ -43,6 +47,7 @@ export async function checkEndpoints() {
     const { endpoints } = endpointsRes;
     console.log(endpoints);
     assert(endpoints.length > 0);
+    client.close();
     return true;
   } catch (err) {
     console.log('dozer is not up yet, retrying...');
@@ -51,8 +56,8 @@ export async function checkEndpoints() {
 }
 
 export async function assertEndpointsWithRetry(
-  maxRetries: number = 10,
-  retryInterval: number = 500,
+  maxRetries: number = 60,
+  retryInterval: number = 1000,
 ) {
   let i = 0;
   let endpointsReady = false;