init analytics sdk

.cargo/config.toml

@@ -1,3 +1,3 @@
 [alias]
 tests-relay = "test"
-tests-with-db = "test --features cloud_db_tests -- --test-threads=1"
+test-integration = "test --features cloud_integration_tests -- --test-threads=1"

.env

@@ -1,74 +1,16 @@
-ENV=DEV # PROD or DEV
-ONLY_RELAY_SERVICE=TRUE # TRUE - This will start only nightly relay service without cloud service
+ENV=PROD # PROD or DEV
+ONLY_RELAY_SERVICE=FALSE # TRUE - This will start only nightly relay service without cloud service
 NONCE=VERY_SECRET_NONCE
-# Generated so it can work with grafana 
-# ssh-keygen -t rsa -b 4096 -m PEM -f grafana.key -N ""
-# openssl rsa -in grafana.key -pubout -outform PEM -out grafana.key.pub
-# TEST KEY DO NO USE IN PRODUCTION
-JWT_SECRET="-----BEGIN RSA PRIVATE KEY-----
-MIIJKQIBAAKCAgEAzlUYIpqSUAyLJaf8ZUef06YBh5DcmaTrwGcVwC57VtywY7bH
-XQUtGooULQjiYgnyOPxHDt2W+gQW1axiMxOQew0MF0kXYQhg0+WA1dcWOsxCDpyL
-c4z+E7EmdV7MTuvsDR2TLhZ6DFmVbN4ca64Wvftz//3Ptc/I1/VpfGsHWb0Vpmph
-kbE9vWGHzzJVp/Lvtk7ybcpfxiiNWEi7unr7/TIEsqy93lCRfpLKIvO06ITH1GZz
-7+VPL/8q8lrH5kSwxqo5szq1mEa0pNN6hTpocz00x5oBfL9K/TNLQNgdb5uOOfjQ
-9pYtZ49UDyMLs5lCsZ6ue+SIbWrJXDEPMIEgLzx9nnvytL9wbiwo5GQdIaDAZXMw
-VVdKQGyqx+UhaHmrb1u0RVRcFzO/QPWzNCWvoxJGfg9dVTxfjaa1HzQ0irPs86zC
-QUnt0OyTuq0SmUgTCNXLSQaN8MigN++zwT4cIRDB0WMgmu8sxtnmA3Zq+2GYT97j
-mbdVMB34DVbXI4UigD3UieVe4rO95cEssoUTAFG0pthv64BRKIIRNch+eiBBjkGp
-nWKRybzsmqnEqpMf9v7KQU9hmpBJDbi6KA6w3fg3KSroJZwosbAwdBuylCfQgq1j
-Z6UfR/ss9kYGhaWbCB3ARmY9XHFc+OZuFW0pohgfBfs9RDMDVkTa4tzBAGsCAwEA
-AQKCAgB/cQHCRdS/SrAbZyG28MFgObXSJQCMidfvc/x0QoF+CjuMq1WCN/M+5DBI
-DKe/RENvPSsqnGVnMWVlPt88iwX/avSf4U5maYVc+/FSo1IgqQ7X9YsQqPI5+X42
-moH48TFQfIbuvYVO4XowzxOg0ttLJufkIsHLm3aDJRqtdXIJ1IdtzWFMWrw2n+oB
-aSaM4Ll80lBczdXQEDYVZH9HVDUWk3FhZt6zVPI4yrO9WOfRgknnAAz2DJ2XmLTP
-EpFLwfuqCXQwJsAJOQrUj6aqaYQgQZq6+n2WrpTDxY3ilFnCGMOFPRDcba/HujiC
-TPQqTyxphwhgj6xQYNfUcJzgKbFVPkcp2Y72feu4dPwRn9v/9zflsezPg33mjNLD
-ls2XrA1S+UKQpiLbF5Mv6U0MxuFRoZIVhSflA/FGHgIE+IGiMDWPLQVpTBJYDjWG
-9giRRBOm4qf9CPPJVnJVEkibDbsKnRTMOPSRBgNBYvpiMO06hWsNktntJ/iqvgf0
-2L//kQEPUTx5+RdSjarQNw0tU7nQid0mf1e3rEXfitr77E6dp+T+2POXQzr2lTFA
-QzaWkIRNNJqCBI/aUhEkAnx2kVlzgLBGYVtAwlCZVF9r0VphmAIQdSeaiI/4N3zh
-jPl0jVYQCKUPe7FslC6f/hks6Y+uMJIeph9XHdYBvcDWZcZCCQKCAQEA/vhsO+vR
-vFRu4q6cvruFoEqODjZnwYUdYKnSa7O2ZTn3VkUE197pL5tSc2At9txZ6AFZ28ye
-ZDUsx+IOvt2PUfJAo7cAL6Vx9TTEMlDcSs0LFRKIrE5lI9hMHOu1evJxN9s3jivZ
-rp0PLEGrllcKIOg5+Zh/yXQue0R2b3H3/f7BdvMS4gjZq8NF0SDLe/WL2PQ0baSL
-MLC8048CDM3BVzjxx2u6ghFambJ0fJIID6DAvkdrNZ/loie71sEVJdJiY3jra/Mq
-JCOZJlRBTevVSfI0uqtybv1QYaU+Mqth8fZxWvhC14Rf30VLEWp9GV0u7Q9p0baj
-nXay/kwXE7uvJQKCAQEAzypkQ6BodHU2FRPpU4wytsowTVbcQs5DNr9ayrTyuPGa
-ydGRzJ0/qHzAv/rWjImny7r8uuvDfNXAxRI5kbAJMImrbG6zjmLHwYKCrB1GFsx1
-QRRuDiMFbblVitA1XHs/sSM0VdO18nLYPY71uIEaTKZTxd1rYuxam3SwZ7prTUGH
-zfI8S4G5vnAn2nWUHmbSNKdtBPqMf65pPGPkKmzBUd++1mvmdsVroGxi37n5vL66
-cUKlvQ5u+l1cqV/EWC6wIoJhrsgkfAO7aFSZQIXIdUeh2C57oFoVdCWULhN09quB
-XIRularP7IC/0e7dMAf5tyQ7Z90/zyMqEpMyX6XkTwKCAQEA5JbMUpyFmRcJoulx
-Vf7BOogc+9kDBJjXUcZvBGhGalKh0RJn2THcVfYm4ZGlI+FCKaKscUbt6mFSGJIv
-LidtSsap9R0oaY8mr/PywjmEhvt1qBrJtdFDE/PjqToZpnSd1LoRAMGHk+jzpXXq
-5Ap85ivNblyulGo5EgK7PActkuZHhOFAWK4emMOlYGzKggQOsR+fsX0H3UtWv0VD
-TT9ay1weR+/pcpskYw9/J/+0gm5Y3z8gex8zvUFqQosw7ovD8fKC/nEvot7Xe2mm
-crmwq28enwz+t0scOa7wKHVGhquvzSMuqhHf8kgpmR+jsI2+eIKNGJtp7M5yg1Ks
-jeCCkQKCAQAP3/G69OnMMscoKlRw4IdqVmgJJSTPwbqI0XUFn4QSBAGWgYaopUwh
-fx3OGEykjE/dXsDLGhHq2P5im5jpvxGVNJd8Qadku9EO1Q9qXPvn91bs28HrN2fN
-FqylbHsKUS96RXZXNVf18jL71J6jutDnGr/Eo8j81ZvD2ddCu5hJXUIo1+0i5Bf1
-reZ/6Q6mnb5x5nqGLSTjC9xokkcDsT3HJlwbVj1c0JgEvQl+l2O5wOvMjgzhRd/f
-M5RMLlh/YWSB4HfXyuJw1mBgCEuOFDJeOlT+meFDUmPeeJq4RSlrVY0eJ8/JjENO
-njcUwTcV3SaXkCE1PlELcGhi8ACmL7IlAoIBAQDRR/cQe7/6jIfEXbCemkWQ5Drs
-CF6jk7gUREbNiTpfWUYSa4DcsyB5phvn2MTM/rsunWtKBrRl7FwDPZcIHrNnsp1U
-w/OPG9nl9ScULy34hwksYShrFZPGeWM8dg+zB1LpwgzsirQiXqmu5FDUrzmCqLM6
-8GYuZHYXowAU0LDUtn/z8Mqv05HR/NwTZxDlxyJMIdVdoUYyWtYUAHb5p839N1Qx
-tj4b9QGK2SBDcOQwq8eCnK/DUjIBUTL+4DNSDeQCr8gc82SwkoZGBCwIWQH+ZwDz
-noL/o/3a5OnNQI0eB7/0g6ElR8BmkeZy1sZXaxZ3caatctEUuUgVFYuVkf9m
------END RSA PRIVATE KEY-----
-"
-JWT_PUBLIC_KEY="-----BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAzlUYIpqSUAyLJaf8ZUef
-06YBh5DcmaTrwGcVwC57VtywY7bHXQUtGooULQjiYgnyOPxHDt2W+gQW1axiMxOQ
-ew0MF0kXYQhg0+WA1dcWOsxCDpyLc4z+E7EmdV7MTuvsDR2TLhZ6DFmVbN4ca64W
-vftz//3Ptc/I1/VpfGsHWb0VpmphkbE9vWGHzzJVp/Lvtk7ybcpfxiiNWEi7unr7
-/TIEsqy93lCRfpLKIvO06ITH1GZz7+VPL/8q8lrH5kSwxqo5szq1mEa0pNN6hTpo
-cz00x5oBfL9K/TNLQNgdb5uOOfjQ9pYtZ49UDyMLs5lCsZ6ue+SIbWrJXDEPMIEg
-Lzx9nnvytL9wbiwo5GQdIaDAZXMwVVdKQGyqx+UhaHmrb1u0RVRcFzO/QPWzNCWv
-oxJGfg9dVTxfjaa1HzQ0irPs86zCQUnt0OyTuq0SmUgTCNXLSQaN8MigN++zwT4c
-IRDB0WMgmu8sxtnmA3Zq+2GYT97jmbdVMB34DVbXI4UigD3UieVe4rO95cEssoUT
-AFG0pthv64BRKIIRNch+eiBBjkGpnWKRybzsmqnEqpMf9v7KQU9hmpBJDbi6KA6w
-3fg3KSroJZwosbAwdBuylCfQgq1jZ6UfR/ss9kYGhaWbCB3ARmY9XHFc+OZuFW0p
-ohgfBfs9RDMDVkTa4tzBAGsCAwEAAQ==
------END PUBLIC KEY-----
-"
+[email protected]
+# Different than db address specified in infra/.env due to docker shenanigans, used in setting up datasource in grafana
+# This is a address of the whole docker interface not just container with database
+# Can be found by looking for docker0 entry by sing command ifconfig/ip -a
+DATABASE_ADDRESS=172.17.0.1
+GRAFANA_BASE_PATH=http://localhost:3005/api
+# TEST LOGIN DO NO USE IN PRODUCTION
+GF_SECURITY_ADMIN_USER=admin
+# TEST PASSWORD DO NO USE IN PRODUCTION
+GF_SECURITY_ADMIN_PASSWORD=admin
+# TEST PASSWORD DO NO USE IN PRODUCTION
+MAILER_PASSWORD=ZtA5gFKMsXzHmEm
+MAILER_ACTIVE=TRUE

.github/workflows/connect-test-local.yml

@@ -13,6 +13,8 @@ jobs:
     timeout-minutes: 10
     runs-on: [testing]
     steps:
+      - name: Set CI Environment Variable
+        run: echo "CI_ENVIRONMENT=true" >> $GITHUB_ENV
       - name: Fix permissions
         continue-on-error: true
         run: |
@@ -31,9 +33,24 @@ jobs:
       - name: Setup Infra
         working-directory: ./infra
         run: |
-          docker-compose down
+          docker-compose --profile full down
           rm -rf .infra/target
+          rm -rf .infra/backups
+          rm -rf .infra/config
+          rm -rf .infra/logs
+          rm -rf .infra/ofelia_logs
+          echo ${{secrets.TESTING_LAPTOP_PASSWORD}} | sudo -S ./scripts/clean_start.sh
+      - name: Setup Grafana
+        working-directory: ./grafana
+        run: |
+          echo "GRAFANA_USER_ID=$(id -u):$(id -g)" >> $GITHUB_ENV
+          rm -rf ./grafana-data
+          mkdir -p ./grafana-data
+          echo ${{secrets.TESTING_LAPTOP_PASSWORD}} | sudo -S chmod 777 -R ${{ env.GRAFANA_USER_ID }} ./grafana-data
+          docker-compose down
           docker-compose up -d --no-deps --force-recreate --remove-orphans
+        env:
+          GRAFANA_USER_ID: ${{ env.GRAFANA_USER_ID }}
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
@@ -50,13 +67,25 @@ jobs:
         working-directory: ./infra
         run: |
           ( docker-compose logs --follow  &  ) | grep -q "database system is ready to accept connections"
+
+          if [[ "$FAILURE" -eq 1 ]]; then
+            echo "Database init failed. Printing last 20 log entries:"
+            docker-compose logs | tail -n 20
+          fi
+      - name: Wait for Grafana
+        continue-on-error: true
+        timeout-minutes: 2
+        working-directory: ./grafana
+        run: |
+          ( docker-compose logs --follow  &  ) | grep -q "Update check succeeded"
       - name: Prepare db tables
         run: |
           cargo run --bin tables_migration
       - name: run cargo test
         run: |
-          cargo tests-with-db
+          cargo test-integration
           cargo run --bin nightly-connect-server &
+          sleep 50
       - name: test base local
         run: |
           pnpm test:ci
@@ -69,8 +98,29 @@ jobs:
         run: |
           pnpm test:ci
         working-directory: ./sdk/packages/sui
+      - name: kill-nightly-connect
+        run: |
+          kill $(pgrep -f 'nightly-connect-server')
+      - name: run connect with db
+        run: |
+          cargo run --bin nightly-connect-server &
+          sleep 5
+        env:
+          ONLY_RELAY_SERVICE: FALSE
+      - name: test cloud local
+        run: |
+          pnpm test:ci
+        working-directory: ./sdk/packages/cloud
+      - name: test analytics local
+        run: |
+          pnpm test:ci
+        working-directory: ./sdk/packages/analytics
       - name: Clean target
         working-directory: ./infra
         run: |
-          docker-compose down
+          docker-compose --profile down
           rm -rf .infra/target
+          rm -rf .infra/backups
+          rm -rf .infra/config
+          rm -rf .infra/logs
+          rm -rf .infra/ofelia_logs

.gitignore

@@ -1,3 +1,9 @@
 /target
 /Cargo.lock
-/.vscode
+/.vscode
+/grafana-client-gen/build
+/infra/target
+/infra/config
+/infra/logs
+/infra/backups
+/infra/ofelia_logs

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = ["server", "database"]
+members = ["server", "database", "openapi"]
 
 [workspace.dependencies]
 ctrlc = "3.4.2"
@@ -27,9 +27,17 @@ tower = { version = "0.4.13", features = [
   "limit",
 ] }
 tower-http = { version = "0.5.2", features = ["cors", "trace"] }
-reqwest = {version = "0.11.24", features = ["json"]}
+reqwest = { version = "0.11.24", features = ["json"] }
 tokio = { version = "1.35.1", features = ["full"] }
 async-trait = "0.1.77"
+r-cache = "0.5.0"
+lettre = "0.11.6"
+addr = "0.15.6"
+hickory-resolver = "0.24.0"
+webauthn-rs = { version = "0.4.8", features = [
+  "danger-allow-state-serialisation",
+] }
+sha256 = "1.5.0"
 
 # If you're updating sqlx, make sure that chrono version below is the same as the one in sqlx
 sqlx = { version = "0.7.3", features = [
@@ -39,4 +47,9 @@ sqlx = { version = "0.7.3", features = [
   "postgres",
   "chrono",
 ] }
-chrono = { version = "0.4.22", features = ["serde"] }
+chrono = { version = "0.4.22", features = ["serde"] }
+configparser = "3.1.0"
+
+[profile.dev]
+debug = 0
+strip = "debuginfo"

database/Cargo.toml

@@ -16,6 +16,12 @@ log = { workspace = true }
 strum = { workspace = true }
 anyhow = { workspace = true }
 chrono = { workspace = true }
+webauthn-rs = { workspace = true }
+uuid7 = { workspace = true }
+
+[dev-dependencies]
+rand = { workspace = true }
+futures = { workspace = true }
 
 [features]
-cloud_db_tests = []
+cloud_integration_tests = []

server/bindings/DeviceMetadata.ts → database/bindings/DeviceMetadata.ts

@@ -2,4 +2,4 @@
 import type { MobileMetadata } from "./MobileMetadata";
 import type { WebMetadata } from "./WebMetadata";
 
-export type DeviceMetadata = { mobile: MobileMetadata } | { web: WebMetadata };
+export type DeviceMetadata = { mobile: MobileMetadata } | { web: WebMetadata } | "unknown";

server/bindings/TransactionToSign.ts → database/bindings/DomainVerificationStatus.ts

@@ -1,3 +1,3 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface TransactionToSign { transaction: string, metadata?: string, }
+export type DomainVerificationStatus = "Pending" | "Verified" | "Cancelled";

server/bindings/HttpAddUserToTeamResponse.ts → database/bindings/PaginationCursor.ts

@@ -1,3 +1,3 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type HttpAddUserToTeamResponse = null;
+export type PaginationCursor = string;

server/bindings/WebMetadata.ts → database/bindings/WebMetadata.ts

@@ -1,3 +1,3 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface WebMetadata { browser: string, version: string, os: string, osVersion: string, }
+export interface WebMetadata { browser: string, browserVersion: string, os: string, osVersion: string, }

database/bindings/WhitelistedDomain.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { DomainVerificationStatus } from "./DomainVerificationStatus";
+
+export interface WhitelistedDomain { domain: string, status: DomainVerificationStatus, }

database/migrations/0000_extensions.sql

@@ -0,0 +1 @@
+CREATE EXTENSION postgis;

database/migrations/0001_types.sql

@@ -35,12 +35,13 @@ CREATE TYPE geo_location AS (
 CREATE TYPE event_type_enum AS ENUM (
     'AppConnect',
     'AppDisconnect',
-    'ClientConnectInit',
-    'ClientConnectResolve'
+    'ClientConnect',
     'ClientDisconnect',
-    'SingMessage',
+    'SignMessage',
     'SignTransaction',
     'SignAndSendTransaction',
     'ChangeWallet',
     'ChangeNetwork'
-);
+);
+
+CREATE TYPE device_medium_type_enum AS ENUM ('Browser', 'Mobile', 'Unknown');

database/migrations/0002_team.sql

@@ -1,9 +1,14 @@
 CREATE TABLE team(
     team_id TEXT NOT NULL UNIQUE,
+    grafana_id TEXT UNIQUE,
     team_name TEXT NOT NULL,
     personal BOOLEAN NOT NULL,
     subscription subscription,
     team_admin_id TEXT NOT NULL,
     registration_timestamp TIMESTAMPTZ NOT NULL,
-    PRIMARY KEY (team_name, team_admin_id)
-);
+    deactivated_at TIMESTAMPTZ
+);
+
+CREATE UNIQUE INDEX unique_active_team_name 
+ON team (team_admin_id, team_name) 
+WHERE deactivated_at IS NULL;

database/migrations/0003_users.sql

@@ -0,0 +1,11 @@
+CREATE TABLE users(
+    user_id TEXT NOT NULL UNIQUE,
+    email TEXT NOT NULL UNIQUE,
+    password_hash TEXT,
+    passkeys TEXT,
+    creation_timestamp TIMESTAMPTZ NOT NULL,
+    deactivated_at TIMESTAMPTZ
+);
+
+CREATE INDEX users_name_idx ON users(user_id);
+CREATE INDEX users_email_idx ON users(email);

database/migrations/0004_registered_apps.sql

@@ -4,7 +4,10 @@ CREATE TABLE registered_apps(
     app_name TEXT NOT NULL,
     whitelisted_domains TEXT [] NOT NULL,
     ack_public_keys TEXT [] NOT NULL,
-    registration_timestamp TIMESTAMPTZ NOT NULL
+    registration_timestamp TIMESTAMPTZ NOT NULL,
+    deactivated_at TIMESTAMPTZ
 );
 
-CREATE UNIQUE INDEX app_id_idx ON registered_apps(app_id);
+CREATE UNIQUE INDEX unique_active_app_name 
+ON registered_apps (team_id, app_name) 
+WHERE deactivated_at IS NULL;

database/migrations/0005_user_app_privilages.sql

@@ -4,6 +4,6 @@ CREATE TABLE user_app_privileges (
     creation_timestamp TIMESTAMPTZ NOT NULL,
     privilege_level privilege_level_enum NOT NULL,
     PRIMARY KEY (user_id, app_id),
-    FOREIGN KEY (user_id) REFERENCES grafana_users (user_id) ON DELETE CASCADE,
+    FOREIGN KEY (user_id) REFERENCES users (user_id) ON DELETE CASCADE,
     FOREIGN KEY (app_id) REFERENCES registered_apps (app_id) ON DELETE CASCADE
 );

database/migrations/0007_sessions.sql

@@ -1,3 +1,7 @@
+CREATE TABLE networks (
+    network TEXT PRIMARY KEY
+);
+
 CREATE TABLE sessions (
     session_id TEXT NOT NULL,
     app_id TEXT NOT NULL,

database/migrations/0009_connection_events.sql

@@ -1,6 +1,7 @@
 CREATE TABLE connection_events (
     event_id BIGSERIAL NOT NULL,
     app_id TEXT NOT NULL, -- Always references the related app, whether an app or client connection
+    network TEXT NOT NULL,
     session_id TEXT NOT NULL,
     entity_id TEXT NOT NULL, -- The ID of the connecting entity (could be the same app_id or client_profile_id or client_id)
     entity_type entity_type_enum NOT NULL, -- Distinguishes between 'client' and 'app'