diff --git a/docker-compose.yml b/docker-compose.yml index 602641ca7..2e2c99837 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,9 @@ services: depends_on: - db volumes: - - ./src/demo_hic_et_nunc/dipdup-docker.yml:/home/dipdup/dipdup.yml + - ./src/demo_hic_et_nunc/dipdup.yml:/home/dipdup/dipdup.yml + - ./src/demo_hic_et_nunc/dipdup-docker.yml:/home/dipdup/dipdup-docker.yml + command: ["-c", "dipdup.yml", "-c", "dipdup-docker.yml", "run"] restart: always environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-changeme} @@ -17,7 +19,9 @@ services: depends_on: - db volumes: - - ./src/demo_quipuswap/dipdup-docker.yml:/home/dipdup/dipdup.yml + - ./src/demo_quipuswap/dipdup.yml:/home/dipdup/dipdup.yml + - ./src/demo_quipuswap/dipdup-docker.yml:/home/dipdup/dipdup-docker.yml + command: ["-c", "dipdup.yml", "-c", "dipdup-docker.yml", "run"] restart: always environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-changeme} @@ -28,18 +32,22 @@ services: depends_on: - db volumes: - - ./src/demo_tzcolors/dipdup-docker.yml:/home/dipdup/dipdup.yml + - ./src/demo_tzcolors/dipdup.yml:/home/dipdup/dipdup.yml + - ./src/demo_tzcolors/dipdup-docker.yml:/home/dipdup/dipdup-docker.yml + command: ["-c", "dipdup.yml", "-c", "dipdup-docker.yml", "run"] restart: always environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-changeme} - ADMIN_SECRET=${ADMIN_SECRET:-changeme} - domains: + tezos_domains: build: . depends_on: - db volumes: - - ./src/demo_tezos_domains/dipdup-docker.yml:/home/dipdup/dipdup.yml + - ./src/demo_tezos_domains/dipdup.yml:/home/dipdup/dipdup.yml + - ./src/demo_tezos_domains/dipdup-docker.yml:/home/dipdup/dipdup-docker.yml + command: ["-c", "dipdup.yml", "-c", "dipdup-docker.yml", "run"] restart: always environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-changeme} diff --git a/src/demo_hic_et_nunc/dipdup-docker.yml b/src/demo_hic_et_nunc/dipdup-docker.yml index c892147cc..41d373eea 100644 --- a/src/demo_hic_et_nunc/dipdup-docker.yml +++ b/src/demo_hic_et_nunc/dipdup-docker.yml @@ -1,6 +1,3 @@ -spec_version: 0.0.1 -package: demo_hic_et_nunc - database: kind: postgres host: db @@ -13,41 +10,3 @@ database: hasura: url: http://hasura:8080 admin_secret: ${ADMIN_SECRET:-changeme} - -contracts: - HEN_objkts: - address: ${HEN_OBJKTS:-KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton} - typename: hen_objkts - HEN_minter: - address: ${HEN_MINTER:-KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9} - typename: hen_minter - -datasources: - tzkt_mainnet: - kind: tzkt - url: ${TZKT_URL:-https://staging.api.tzkt.io} - -indexes: - hen_mainnet: - kind: operation - datasource: tzkt_mainnet - contract: HEN_minter - handlers: - - callback: on_mint - pattern: - - destination: HEN_minter - entrypoint: mint_OBJKT - - destination: HEN_objkts - entrypoint: mint - - callback: on_swap - pattern: - - destination: HEN_minter - entrypoint: swap - - callback: on_cancel_swap - pattern: - - destination: HEN_minter - entrypoint: cancel_swap - - callback: on_collect - pattern: - - destination: HEN_minter - entrypoint: collect diff --git a/src/demo_quipuswap/dipdup-docker.yml b/src/demo_quipuswap/dipdup-docker.yml index e8ef33a6b..3623a96f1 100644 --- a/src/demo_quipuswap/dipdup-docker.yml +++ b/src/demo_quipuswap/dipdup-docker.yml @@ -1,134 +1,12 @@ -spec_version: 0.0.1 -package: demo_quipuswap - database: kind: postgres host: db port: 5432 - user: dipdup - password: changeme - database: dipdup + user: ${POSTGRES_USER:-dipdup} + password: ${POSTGRES_PASSWORD:-changeme} + database: ${POSTGRES_DB:-dipdup} schema_name: quipuswap hasura: url: http://hasura:8080 - admin_secret: changeme - -contracts: - kusd_dex_mainnet: - address: KT1CiSKXR68qYSxnbzjwvfeMCRburaSDonT2 - typename: quipu_fa12 - tzbtc_dex_mainnet: - address: KT1N1wwNPqT5jGhM91GQ2ae5uY8UzFaXHMJS - typename: quipu_fa12 - kusd_token_mainnet: - address: KT1K9gCRgaLRFKTErYt1wVxA3Frb9FjasjTV - typename: fa12_token - tzbtc_token_mainnet: - address: KT1PWx2mnDueood7fEmfbBDKx1D9BAnnXitn - typename: fa12_token - hdao_dex_mainnet: - address: KT1V41fGzkdTJki4d11T1Rp9yPkCmDhB7jph - typename: quipu_fa2 - hdao_token_mainnet: - address: KT1AFA2mwNUMNd4SsujE1YYp29vd8BZejyKW - typename: fa2_token - -datasources: - tzkt_staging_mainnet: - kind: tzkt - url: https://staging.api.tzkt.io - -templates: - quipuswap_fa12: - kind: operation - datasource: tzkt_staging_mainnet - contract: - handlers: - - callback: on_fa12_token_to_tez - pattern: - - destination: - entrypoint: tokenToTezPayment - - destination: - entrypoint: transfer - - callback: on_fa12_tez_to_token - pattern: - - destination: - entrypoint: tezToTokenPayment - - destination: - entrypoint: transfer - - callback: on_fa12_invest_liquidity - pattern: - - destination: - entrypoint: investLiquidity - - destination: - entrypoint: transfer - - callback: on_fa12_divest_liquidity - pattern: - - destination: - entrypoint: divestLiquidity - - destination: - entrypoint: transfer - - callback: on_fa12_withdraw_profit - pattern: - - destination: - entrypoint: withdrawProfit - - quipuswap_fa2: - kind: operation - datasource: tzkt_staging_mainnet - contract: - handlers: - - callback: on_fa2_token_to_tez - pattern: - - destination: - entrypoint: tokenToTezPayment - - destination: - entrypoint: transfer - - callback: on_fa2_tez_to_token - pattern: - - destination: - entrypoint: tezToTokenPayment - - destination: - entrypoint: transfer - - callback: on_fa20_invest_liquidity - pattern: - - destination: - entrypoint: investLiquidity - - destination: - entrypoint: transfer - - callback: on_fa20_divest_liquidity - pattern: - - destination: - entrypoint: divestLiquidity - - destination: - entrypoint: transfer - - callback: on_fa20_withdraw_profit - pattern: - - destination: - entrypoint: withdrawProfit - -indexes: - kusd_mainnet: - template: quipuswap_fa12 - values: - dex_contract: kusd_dex_mainnet - token_contract: kusd_token_mainnet - symbol: kUSD - decimals: 18 - - # tzbtc_mainnet: - # template: quipuswap_fa12 - # values: - # dex_contract: tzbtc_dex_mainnet - # token_contract: tzbtc_token_mainnet - # symbol: tzBTC - # decimals: 8 - - hdao_mainnet: - template: quipuswap_fa2 - values: - dex_contract: hdao_dex_mainnet - token_contract: hdao_token_mainnet - symbol: hDAO - decimals: 6 + admin_secret: ${ADMIN_SECRET:-changeme} diff --git a/src/demo_registrydao/dipdup-docker.yml b/src/demo_registrydao/dipdup-docker.yml index d40394dd2..738808a41 100644 --- a/src/demo_registrydao/dipdup-docker.yml +++ b/src/demo_registrydao/dipdup-docker.yml @@ -1,43 +1,12 @@ -spec_version: 0.0.1 -package: demo_registrydao - database: kind: postgres host: db port: 5432 - user: dipdup - password: changeme - database: dipdup + user: ${POSTGRES_USER:-dipdup} + password: ${POSTGRES_PASSWORD:-changeme} + database: ${POSTGRES_DB:-dipdup} schema_name: registrydao hasura: url: http://hasura:8080 - admin_secret: changeme - -contracts: - registry: - address: KT1QMdCTqzmY4QKHntV1nZEinLPU1GbxUFQu - typename: registry - -datasources: - tzkt: - kind: tzkt - url: ${TZKT_URL:-https://staging.api.edo2net.tzkt.io} - -templates: - - registry_dao: - kind: operation - datasource: tzkt - contract: - handlers: - - callback: on_propose - pattern: - - destination: - entrypoint: propose - -indexes: - registry: - template: registry_dao - values: - contract: registry + admin_secret: ${ADMIN_SECRET:-changeme} diff --git a/src/demo_tezos_domains/dipdup-docker.yml b/src/demo_tezos_domains/dipdup-docker.yml index bea2a9a96..3da4a01e8 100644 --- a/src/demo_tezos_domains/dipdup-docker.yml +++ b/src/demo_tezos_domains/dipdup-docker.yml @@ -1,6 +1,3 @@ -spec_version: 0.1 -package: demo_tezos_domains - database: kind: postgres host: db @@ -8,36 +5,8 @@ database: user: ${POSTGRES_USER:-dipdup} password: ${POSTGRES_PASSWORD:-changeme} database: ${POSTGRES_DB:-dipdup} - schema_name: domains - -contracts: - edo_name_registry: - address: KT1JJbWfW8CHUY95hG9iq2CEMma1RiKhMHDR - typename: name_registry - -datasources: - tzkt_staging_edo: - kind: tzkt - url: https://staging.api.edo2net.tzkt.io - -templates: - tezos_domains: - kind: operation - datasource: - contract: - handlers: - - callback: on_admin_update - pattern: - - destination: - entrypoint: admin_update - - callback: on_execute - pattern: - - destination: - entrypoint: execute + schema_name: tezos_domains -indexes: - tezos_domains_edo: - template: tezos_domains - values: - datasource: tzkt_staging_edo - name_registry: edo_name_registry \ No newline at end of file +hasura: + url: http://hasura:8080 + admin_secret: ${ADMIN_SECRET:-changeme} diff --git a/src/demo_tzcolors/dipdup-docker.yml b/src/demo_tzcolors/dipdup-docker.yml index 623244d49..fcffd641f 100644 --- a/src/demo_tzcolors/dipdup-docker.yml +++ b/src/demo_tzcolors/dipdup-docker.yml @@ -1,57 +1,12 @@ -spec_version: 0.0.1 -package: demo_tzcolors - database: kind: postgres host: db port: 5432 - user: dipdup - password: changeme - database: dipdup + user: ${POSTGRES_USER:-dipdup} + password: ${POSTGRES_PASSWORD:-changeme} + database: ${POSTGRES_DB:-dipdup} schema_name: tzcolors hasura: url: http://hasura:8080 - admin_secret: changeme - -contracts: - tzcolors_minter: - address: KT1FyaDqiMQWg7Exo7VUiXAgZbd2kCzo3d4s - typename: tzcolors_minter - tzcolors_auction: - address: KT1CpeSQKdkhWi4pinYcseCFKmDhs5M74BkU - typename: tzcolors_auction - -datasources: - tzkt_staging: - kind: tzkt - url: ${TZKT_URL:-https://staging.api.tzkt.io} - -templates: - - tzcolors_auction: - kind: operation - datasource: - contract: - handlers: - - callback: on_create_auction - pattern: - - destination: - entrypoint: create_auction - - callback: on_bid - pattern: - - destination: - entrypoint: bid - - callback: on_withdraw - pattern: - - destination: - entrypoint: withdraw - -indexes: - - tzcolors_auction: - template: tzcolors_auction - values: - datasource: tzkt_staging - minter: tzcolors_minter - auction: tzcolors_auction \ No newline at end of file + admin_secret: ${ADMIN_SECRET:-changeme} diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index fe4ad1c9d..f351d32ed 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from functools import wraps from os.path import dirname, join +from typing import List import click @@ -30,11 +31,11 @@ class CLIContext: @click.group() @click.version_option(__version__) -@click.option('--config', '-c', type=str, help='Path to dipdup YAML config', default='dipdup.yml') +@click.option('--config', '-c', type=str, multiple=True, help='Path to dipdup YAML config', default='dipdup.yml') @click.option('--logging-config', '-l', type=str, help='Path to logging YAML config', default='logging.yml') @click.pass_context @click_async -async def cli(ctx, config: str, logging_config: str): +async def cli(ctx, config: List[str], logging_config: str): try: path = join(os.getcwd(), logging_config) _logging_config = LoggingConfig.load(path) diff --git a/src/dipdup/config.py b/src/dipdup/config.py index ee2621ebb..4542b5cb0 100644 --- a/src/dipdup/config.py +++ b/src/dipdup/config.py @@ -422,24 +422,31 @@ def package_path(self) -> str: @classmethod def load( cls, - filename: str, + filenames: List[str], ) -> 'DipDupConfig': current_workdir = os.path.join(os.getcwd()) - filename = os.path.join(current_workdir, filename) - _logger.info('Loading config from %s', filename) - with open(filename) as file: - raw_config = file.read() + json_config: Dict[str, Any] = {} + for filename in filenames: + filename = os.path.join(current_workdir, filename) + + _logger.info('Loading config from %s', filename) + with open(filename) as file: + raw_config = file.read() + + _logger.info('Substituting environment variables') + for match in re.finditer(ENV_VARIABLE_REGEX, raw_config): + variable, default_value = match.group(1), match.group(2) + value = env.get(variable) + placeholder = '${' + variable + ':-' + default_value + '}' + raw_config = raw_config.replace(placeholder, value or default_value) - _logger.info('Substituting environment variables') - for match in re.finditer(ENV_VARIABLE_REGEX, raw_config): - variable, default_value = match.group(1), match.group(2) - value = env.get(variable) - placeholder = '${' + variable + ':-' + default_value + '}' - raw_config = raw_config.replace(placeholder, value or default_value) + json_config = { + **json_config, + **YAML(typ='base').load(raw_config), + } - json_config = YAML(typ='base').load(raw_config) config = cls(**json_config) return config diff --git a/tests/integration_tests/test_codegen.py b/tests/integration_tests/test_codegen.py index 1c471901b..f43a0c41b 100644 --- a/tests/integration_tests/test_codegen.py +++ b/tests/integration_tests/test_codegen.py @@ -33,7 +33,7 @@ async def test_codegen(self): for name in ['hic_et_nunc.yml', 'quipuswap.yml', 'tzcolors.yml']: with self.subTest(name): config_path = join(dirname(__file__), name) - config = DipDupConfig.load(config_path) + config = DipDupConfig.load([config_path]) config.package = 'tmp_test_dipdup' if config.package in sys.modules: diff --git a/tests/test_dipdup/test_config.py b/tests/test_dipdup/test_config.py index 1fa979312..bca531752 100644 --- a/tests/test_dipdup/test_config.py +++ b/tests/test_dipdup/test_config.py @@ -14,7 +14,7 @@ async def asyncSetUp(self): self.path = join(dirname(__file__), 'dipdup.yml') async def test_load_initialize(self): - config = DipDupConfig.load(self.path) + config = DipDupConfig.load([self.path]) async with tortoise_wrapper('sqlite://:memory:'): await Tortoise.generate_schemas()