Merge branch 'master' into osparc-tooling-upgrade

packages/postgres-database/src/simcore_postgres_database/migration/versions/2a4b4167e088_product_name_column_in_api_keys_table.py

@@ -24,7 +24,7 @@ def _find_default_product_name_or_none(conn):
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column("api_keys", sa.Column("product_name", sa.String(), nullable=False))
+    op.add_column("api_keys", sa.Column("product_name", sa.String(), nullable=True))
     op.create_foreign_key(
         "fk_api_keys_product_name",
         "api_keys",
@@ -39,7 +39,12 @@ def upgrade():
     conn = op.get_bind()
     default_product = _find_default_product_name_or_none(conn)
     if default_product:
-        op.execute(f"UPDATE api_keys SET product_name = '{default_product}'")
+        op.execute(sa.DDL(f"UPDATE api_keys SET product_name = '{default_product}'"))
+
+    # make it non nullable now
+    op.alter_column(
+        "api_keys", "product_name", existing_type=sa.String(), nullable=False
+    )
 
 
 def downgrade():

services/autoscaling/src/simcore_service_autoscaling/core/settings.py

@@ -50,23 +50,9 @@ class EC2InstancesSettings(BaseCustomSettings):
         min_length=1,
         description="Defines the AMI (Amazon Machine Image) ID used to start a new EC2 instance",
     )
-    EC2_INSTANCES_MAX_INSTANCES: int = Field(
-        default=10,
-        description="Defines the maximum number of instances the autoscaling app may create",
-    )
-    EC2_INSTANCES_SECURITY_GROUP_IDS: list[str] = Field(
-        ...,
-        min_items=1,
-        description="A security group acts as a virtual firewall for your EC2 instances to control incoming and outgoing traffic"
-        " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-security-groups.html), "
-        " this is required to start a new EC2 instance",
-    )
-    EC2_INSTANCES_SUBNET_ID: str = Field(
-        ...,
-        min_length=1,
-        description="A subnet is a range of IP addresses in your VPC "
-        " (https://docs.aws.amazon.com/vpc/latest/userguide/configure-subnets.html), "
-        "this is required to start a new EC2 instance",
+    EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS: list[str] = Field(
+        default_factory=list,
+        description="script(s) to run on EC2 instance startup (be careful!), each entry is run one after the other using '&&' operator",
     )
     EC2_INSTANCES_KEY_NAME: str = Field(
         ...,
@@ -75,42 +61,52 @@ class EC2InstancesSettings(BaseCustomSettings):
         " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html),"
         "this is required to start a new EC2 instance",
     )
-
-    EC2_INSTANCES_TIME_BEFORE_TERMINATION: datetime.timedelta = Field(
-        default=datetime.timedelta(minutes=1),
-        description="Time after which an EC2 instance may be terminated (repeat every hour, min 0, max 59 minutes)",
-    )
-
     EC2_INSTANCES_MACHINES_BUFFER: NonNegativeInt = Field(
         default=0,
         description="Constant reserve of drained ready machines for fast(er) usage,"
         "disabled when set to 0. Uses 1st machine defined in EC2_INSTANCES_ALLOWED_TYPES",
     )
-
+    EC2_INSTANCES_MAX_INSTANCES: int = Field(
+        default=10,
+        description="Defines the maximum number of instances the autoscaling app may create",
+    )
     EC2_INSTANCES_MAX_START_TIME: datetime.timedelta = Field(
         default=datetime.timedelta(minutes=3),
-        description="Usual time taken an EC2 instance with the given AMI takes to be in 'running' mode",
+        description="Usual time taken an EC2 instance with the given AMI takes to be in 'running' mode "
+        "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
+    )
+    EC2_INSTANCES_NAME_PREFIX: str = Field(
+        default="autoscaling",
+        min_length=1,
+        description="prefix used to name the EC2 instances created by this instance of autoscaling",
     )
-
     EC2_INSTANCES_PRE_PULL_IMAGES: list[DockerGenericTag] = Field(
         default_factory=list,
         description="a list of docker image/tags to pull on instance cold start",
     )
-
     EC2_INSTANCES_PRE_PULL_IMAGES_CRON_INTERVAL: datetime.timedelta = Field(
         default=datetime.timedelta(minutes=30),
-        description="time interval between pulls of images (minimum is 1 minute)",
+        description="time interval between pulls of images (minimum is 1 minute) "
+        "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
     )
-
-    EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS: list[str] = Field(
-        default_factory=list,
-        description="script(s) to run on EC2 instance startup (be careful!), each entry is run one after the other using '&&' operator",
+    EC2_INSTANCES_SECURITY_GROUP_IDS: list[str] = Field(
+        ...,
+        min_items=1,
+        description="A security group acts as a virtual firewall for your EC2 instances to control incoming and outgoing traffic"
+        " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-security-groups.html), "
+        " this is required to start a new EC2 instance",
     )
-
-    EC2_INSTANCES_NAME_PREFIX: str = Field(
-        default="autoscaling",
+    EC2_INSTANCES_SUBNET_ID: str = Field(
+        ...,
         min_length=1,
-        description="prefix used to name the EC2 instances created by this instance of autoscaling",
+        description="A subnet is a range of IP addresses in your VPC "
+        " (https://docs.aws.amazon.com/vpc/latest/userguide/configure-subnets.html), "
+        "this is required to start a new EC2 instance",
+    )
+    EC2_INSTANCES_TIME_BEFORE_TERMINATION: datetime.timedelta = Field(
+        default=datetime.timedelta(minutes=1),
+        description="Time after which an EC2 instance may be terminated (repeat every hour, min 0, max 59 minutes)"
+        "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
     )
 
     @validator("EC2_INSTANCES_TIME_BEFORE_TERMINATION")
@@ -208,7 +204,8 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
 
     AUTOSCALING_POLL_INTERVAL: datetime.timedelta = Field(
         default=datetime.timedelta(seconds=10),
-        description="interval between each resource check (default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
+        description="interval between each resource check "
+        "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
     )
 
     AUTOSCALING_RABBITMQ: RabbitSettings | None = Field(auto_default_from_env=True)

services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py

@@ -67,9 +67,16 @@ class WorkersEC2InstancesSettings(BaseCustomSettings):
         "this is required to start a new EC2 instance",
     )
 
+    WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION: datetime.timedelta = Field(
+        default=datetime.timedelta(minutes=3),
+        description="Time after which an EC2 instance may be terminated (repeat every hour, min 0, max 59 minutes) "
+        "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
+    )
+
     WORKERS_EC2_INSTANCES_MAX_START_TIME: datetime.timedelta = Field(
         default=datetime.timedelta(minutes=3),
-        description="Usual time taken an EC2 instance with the given AMI takes to be in 'running' mode",
+        description="Usual time taken an EC2 instance with the given AMI takes to be in 'running' mode "
+        "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
     )
 
     WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS: list[str] = Field(
@@ -196,12 +203,14 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
 
     CLUSTERS_KEEPER_TASK_INTERVAL: datetime.timedelta = Field(
         default=datetime.timedelta(seconds=30),
-        description="interval between each clusters clean check (default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
+        description="interval between each clusters clean check "
+        "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
     )
 
     SERVICE_TRACKING_HEARTBEAT: datetime.timedelta = Field(
         default=datetime.timedelta(seconds=60),
-        description="Service heartbeat interval (everytime a heartbeat is sent into RabbitMQ)",
+        description="Service heartbeat interval (everytime a heartbeat is sent into RabbitMQ) "
+        "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
     )
 
     CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION: NonNegativeInt = Field(

services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml

@@ -53,11 +53,12 @@ services:
       EC2_INSTANCES_AMI_ID: ${WORKERS_EC2_INSTANCES_AMI_ID}
       EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS: ${WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS}
       EC2_INSTANCES_KEY_NAME: ${WORKERS_EC2_INSTANCES_KEY_NAME}
-      EC2_INSTANCES_MAX_START_TIME: ${WORKERS_EC2_INSTANCES_MAX_START_TIME}
       EC2_INSTANCES_MAX_INSTANCES: ${WORKERS_EC2_INSTANCES_MAX_INSTANCES}
+      EC2_INSTANCES_MAX_START_TIME: ${WORKERS_EC2_INSTANCES_MAX_START_TIME}
       EC2_INSTANCES_NAME_PREFIX: ${EC2_INSTANCES_NAME_PREFIX}
       EC2_INSTANCES_SECURITY_GROUP_IDS: ${WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS}
       EC2_INSTANCES_SUBNET_ID: ${WORKERS_EC2_INSTANCES_SUBNET_ID}
+      EC2_INSTANCES_TIME_BEFORE_TERMINATION: ${WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}
       EC2_REGION_NAME: ${EC2_CLUSTERS_KEEPER_REGION_NAME}
       EC2_SECRET_ACCESS_KEY: ${EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY}
       LOG_FORMAT_LOCAL_DEV_ENABLED: 1

services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py

@@ -51,6 +51,7 @@ def _convert_to_env_list(entries: list[Any]) -> str:
         f"EC2_INSTANCES_NAME_PREFIX={cluster_machines_name_prefix}",
         f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}",
         f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}",
+        f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}",
         f"EC2_CLUSTERS_KEEPER_REGION_NAME={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_CLUSTERS_KEEPER_REGION_NAME}",
         f"EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY}",
         f"LOG_LEVEL={app_settings.LOG_LEVEL}",

services/clusters-keeper/tests/unit/test_utils_clusters.py

@@ -69,7 +69,9 @@ def test_create_startup_script(
         if isinstance(v, str) and v.startswith("${")
     ] + ["DOCKER_IMAGE_TAG"]
     for env_key in expected_env_keys:
-        assert env_key in startup_script_env_keys_names
+        assert (
+            env_key in startup_script_env_keys_names
+        ), f"{env_key} is missing from startup script! please adjust"
 
     # check we do not define "too much"
     for env_key in startup_script_env_keys_names:

services/director-v2/src/simcore_service_director_v2/core/settings.py

@@ -545,7 +545,8 @@ class AppSettings(BaseCustomSettings, MixinLoggingSettings):
     SWARM_STACK_NAME: str = Field("undefined-please-check", env="SWARM_STACK_NAME")
     SERVICE_TRACKING_HEARTBEAT: datetime.timedelta = Field(
         default=DEFAULT_RESOURCE_USAGE_HEARTBEAT_INTERVAL,
-        description="Service scheduler heartbeat (everytime a heartbeat is sent into RabbitMQ)",
+        description="Service scheduler heartbeat (everytime a heartbeat is sent into RabbitMQ)"
+        " (default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
     )
 
     SIMCORE_SERVICES_NETWORK_NAME: str | None = Field(

services/docker-compose.yml

@@ -110,6 +110,7 @@ services:
       - WORKERS_EC2_INSTANCES_ALLOWED_TYPES=${WORKERS_EC2_INSTANCES_ALLOWED_TYPES}
       - WORKERS_EC2_INSTANCES_AMI_ID=${WORKERS_EC2_INSTANCES_AMI_ID}
       - WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS=${WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS}
+      - WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION=${WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}
       - WORKERS_EC2_INSTANCES_KEY_NAME=${WORKERS_EC2_INSTANCES_KEY_NAME}
       - WORKERS_EC2_INSTANCES_MAX_INSTANCES=${WORKERS_EC2_INSTANCES_MAX_INSTANCES}
       - WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS=${WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS}

services/static-webserver/client/source/class/osparc/desktop/credits/BillingCenter.js

@@ -91,7 +91,6 @@ qx.Class.define("osparc.desktop.credits.BillingCenter", {
     __activityPage: null,
     __transactionsPage: null,
     __usagePage: null,
-    __buyCredits: null,
     __transactionsTable: null,
 
     __getOverviewPage: function() {
@@ -103,17 +102,7 @@ qx.Class.define("osparc.desktop.credits.BillingCenter", {
       overview.set({
         margin: 10
       });
-      overview.addListener("buyCredits", e => {
-        this.__openBuyCredits();
-        const {
-          walletId
-        } = e.getData();
-        const store = osparc.store.Store.getInstance();
-        const found = store.getWallets().find(wallet => wallet.getWalletId() === parseInt(walletId));
-        if (found) {
-          this.__buyCredits.setWallet(found);
-        }
-      });
+      overview.addListener("buyCredits", () => this.__openBuyCredits());
       overview.addListener("toWallets", () => this.openWallets());
       overview.addListener("toActivity", () => this.__openActivity());
       overview.addListener("toTransactions", () => this.__openTransactions());
@@ -136,17 +125,7 @@ qx.Class.define("osparc.desktop.credits.BillingCenter", {
       walletsView.set({
         margin: 10
       });
-      walletsView.addListener("buyCredits", e => {
-        this.__openBuyCredits();
-        const {
-          walletId
-        } = e.getData();
-        const store = osparc.store.Store.getInstance();
-        const found = store.getWallets().find(wallet => wallet.getWalletId() === parseInt(walletId));
-        if (found) {
-          this.__buyCredits.setWallet(found);
-        }
-      });
+      walletsView.addListener("buyCredits", () => this.__openBuyCredits());
       page.add(walletsView);
       return page;
     },

services/static-webserver/client/source/class/osparc/desktop/credits/CurrentUsage.js

@@ -37,57 +37,47 @@ qx.Class.define("osparc.desktop.credits.CurrentUsage", {
     }
   },
 
-  statics: {
-    POLLING_INTERVAL: 10000
-  },
-
   members: {
-    __interval: null,
-
     __currentStudyChanged: function(currentStudy) {
       if (osparc.desktop.credits.Utils.areWalletsEnabled()) {
         if (currentStudy) {
-          this.__startRequesting();
+          const store = osparc.store.Store.getInstance();
+          const contextWallet = store.getContextWallet();
+          if (contextWallet) {
+            this.__fetchUsedCredits();
+            contextWallet.addListener("changeCreditsAvailable", () => this.__fetchUsedCredits());
+          }
         } else {
-          this.__stopRequesting();
+          this.setUsedCredits(null);
         }
       }
     },
 
-    __startRequesting: function() {
-      this.setUsedCredits(0);
-
-      this.__interval = setInterval(() => this.__fetchUsedCredits(), this.self().POLLING_INTERVAL);
-      this.__fetchUsedCredits();
-    },
-
-    __stopRequesting: function() {
-      this.setUsedCredits(null);
-
-      if (this.__interval) {
-        clearInterval(this.__interval);
-      }
-    },
-
     __fetchUsedCredits: function() {
-      const params = {
-        url: {
-          offset: 0,
-          limit: 10
-        }
-      };
-      osparc.data.Resources.fetch("resourceUsage", "getPage", params)
-        .then(data => {
-          const currentStudy = osparc.store.Store.getInstance().getCurrentStudy();
-          const currentTasks = data.filter(d => (d.project_id === currentStudy.getUuid()) && d.service_run_status === "RUNNING");
-          let cost = 0;
-          currentTasks.forEach(currentTask => {
-            if (currentTask["credit_cost"]) {
-              cost += currentTask["credit_cost"];
-            }
+      const store = osparc.store.Store.getInstance();
+      const currentStudy = store.getCurrentStudy();
+      const contextWallet = store.getContextWallet();
+      if (currentStudy && contextWallet) {
+        const walletId = contextWallet.getWalletId();
+        const params = {
+          url: {
+            walletId,
+            offset: 0,
+            limit: 10
+          }
+        };
+        osparc.data.Resources.fetch("resourceUsagePerWallet", "getPage", params)
+          .then(data => {
+            const currentTasks = data.filter(d => (d.project_id === currentStudy.getUuid()) && d.service_run_status === "RUNNING");
+            let cost = 0;
+            currentTasks.forEach(currentTask => {
+              if (currentTask["credit_cost"]) {
+                cost += currentTask["credit_cost"];
+              }
+            });
+            this.setUsedCredits(cost);
           });
-          this.setUsedCredits(cost);
-        });
+      }
     }
   }
 });