From 6fa2327d112a2fe8029e5f28405c893b8716f3e4 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 17 Jul 2024 16:42:53 -0400 Subject: [PATCH 1/5] Remove timeout on MessageOutboxDeliveryService --- .../Services/MessageOutboxDeliveryService.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs b/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs index 09f49fb6..91776006 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs @@ -8,8 +8,6 @@ public class MessageOutboxDeliveryService( ILogger logger ) : BackgroundService { - private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10); - private readonly IServiceProvider _services = services; private readonly Dictionary _outboxMessageHandlers = outboxMessageHandlers.ToDictionary(o => o.OutboxId); @@ -25,7 +23,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) using ISubscription subscription = await messages.SubscribeAsync(e => true, stoppingToken); while (true) { - await subscription.WaitForChangeAsync(timeout: Timeout, cancellationToken: stoppingToken); + await subscription.WaitForChangeAsync(cancellationToken: stoppingToken); if (stoppingToken.IsCancellationRequested) break; await ProcessMessagesAsync(messages, stoppingToken); From 32a20c83df7f28ddd144dc5522ad0e8ffd69da08 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 18 Jul 2024 09:12:30 -0400 Subject: [PATCH 2/5] I believe this is how multi-indexes should be specified - https://www.mongodb.com/docs/drivers/csharp/current/fundamentals/indexes/#compound-indexes --- .../IMachineBuilderExtensions.cs | 6 +---- .../IMongoDataAccessConfiguratorExtensions.cs | 26 ++++++------------- .../IMongoDataAccessConfiguratorExtensions.cs | 7 +++-- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs index 567a073e..f51d83c6 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs @@ -261,11 +261,7 @@ await c.Indexes.CreateOrUpdateAsync( Builders .IndexKeys.Ascending(e => e.EngineId) .Ascending("currentBuild._id") - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner) + .Ascending(e => e.CurrentBuild!.BuildJobRunner) ) ); } diff --git a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs index 175bceef..5fdf84f3 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -13,10 +13,9 @@ this IMongoDataAccessConfigurator configurator init: async c => { await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.Owner)) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("corpora._id")) + new CreateIndexModel( + Builders.IndexKeys.Ascending(e => e.Owner).Ascending("corpora._id") + ) ); } ); @@ -33,22 +32,13 @@ await c.Indexes.CreateOrUpdateAsync( { await c.Indexes.CreateOrUpdateAsync( new CreateIndexModel( - Builders.IndexKeys.Ascending(pt => pt.EngineRef) - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(pt => pt.ModelRevision) + Builders + .IndexKeys.Ascending(pt => pt.EngineRef) + .Ascending(pt => pt.ModelRevision) + .Ascending(pt => pt.CorpusRef) + .Ascending(pt => pt.TextId) ) ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(pt => pt.CorpusRef) - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(pt => pt.TextId)) - ); } ); return configurator; diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs index 2551de84..715ee94e 100644 --- a/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -11,10 +11,9 @@ public static IMongoDataAccessConfigurator AddWebhooksRepositories(this IMongoDa init: async c => { await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(h => h.Owner)) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending(h => h.Events)) + new CreateIndexModel( + Builders.IndexKeys.Ascending(h => h.Owner).Ascending(h => h.Events) + ) ); } ); From d8ac34eeafbd1886162f36bd6977e3ad4d0e0a8d Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Thu, 18 Jul 2024 10:14:10 -0400 Subject: [PATCH 3/5] Revert compounding of all indices. Remove unused indices. Specify useful compound indices on pretranslations (and remove redundant index on engineRef). --- .../IMachineBuilderExtensions.cs | 10 ++++++---- .../IMongoDataAccessConfiguratorExtensions.cs | 19 +++++++++++++++---- .../IMongoDataAccessConfiguratorExtensions.cs | 4 +--- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs index f51d83c6..2495da6e 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs @@ -258,10 +258,12 @@ public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, s { await c.Indexes.CreateOrUpdateAsync( new CreateIndexModel( - Builders - .IndexKeys.Ascending(e => e.EngineId) - .Ascending("currentBuild._id") - .Ascending(e => e.CurrentBuild!.BuildJobRunner) + Builders.IndexKeys.Ascending(e => e.EngineId) + ) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner) ) ); } diff --git a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs index 5fdf84f3..563a4142 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -13,9 +13,7 @@ this IMongoDataAccessConfigurator configurator init: async c => { await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => e.Owner).Ascending("corpora._id") - ) + new CreateIndexModel(Builders.IndexKeys.Ascending(e => e.Owner)) ); } ); @@ -30,12 +28,25 @@ await c.Indexes.CreateOrUpdateAsync( "translation.pretranslations", init: async c => { + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(pt => pt.ModelRevision) + ) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(pt => pt.CorpusRef) + ) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending(pt => pt.TextId)) + ); await c.Indexes.CreateOrUpdateAsync( new CreateIndexModel( Builders .IndexKeys.Ascending(pt => pt.EngineRef) - .Ascending(pt => pt.ModelRevision) .Ascending(pt => pt.CorpusRef) + .Ascending(pt => pt.ModelRevision) .Ascending(pt => pt.TextId) ) ); diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs index 715ee94e..8548cf5b 100644 --- a/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -11,9 +11,7 @@ public static IMongoDataAccessConfigurator AddWebhooksRepositories(this IMongoDa init: async c => { await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(h => h.Owner).Ascending(h => h.Events) - ) + new CreateIndexModel(Builders.IndexKeys.Ascending(h => h.Events)) ); } ); From 817c8a844eb31e907cc62a38134b5addb376650b Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Thu, 18 Jul 2024 10:22:18 -0400 Subject: [PATCH 4/5] Add other recommended index on pretranslations (should revisit this later) --- .../IMongoDataAccessConfiguratorExtensions.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs index 563a4142..ea016c0e 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs @@ -41,6 +41,13 @@ await c.Indexes.CreateOrUpdateAsync( await c.Indexes.CreateOrUpdateAsync( new CreateIndexModel(Builders.IndexKeys.Ascending(pt => pt.TextId)) ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders + .IndexKeys.Ascending(pt => pt.EngineRef) + .Ascending(pt => pt.ModelRevision) + ) + ); await c.Indexes.CreateOrUpdateAsync( new CreateIndexModel( Builders From 9bf9b091e5774422021fa44ca6ed728c967e727c Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 18 Jul 2024 17:02:15 -0400 Subject: [PATCH 5/5] Update docker-compose with Atlas. --- docker-compose.withatlas.yml | 40 ++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/docker-compose.withatlas.yml b/docker-compose.withatlas.yml index 0038da2c..c1f8382b 100644 --- a/docker-compose.withatlas.yml +++ b/docker-compose.withatlas.yml @@ -14,8 +14,8 @@ services: - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80 - ASPNETCORE_Kestrel__Endpoints__Http2__Url=http://*:81 - ASPNETCORE_Kestrel__Endpoints__Http2__Protocols=Http2 - - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING:?connection string needed}/${MONGO_CONNECTION_PREFIX:?mongo prefix needed}serval_jobs - - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}serval + - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING:?connection string needed}/${MONGO_CONNECTION_PREFIX:?mongo prefix needed}serval_jobs${MONGO_EXTENSIONS} + - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}serval${MONGO_EXTENSIONS} - ASPNETCORE_Translation__Engines__0__Type=Echo - ASPNETCORE_Translation__Engines__0__Address=http://echo - ASPNETCORE_Translation__Engines__1__Type=SmtTransfer @@ -74,22 +74,24 @@ services: hostname: machine-engine container_name: machine-engine-cntr build: - context: ${MACHINE_TESTING_DIR:-../machine} - dockerfile: ../machine/dockerfile.development + context: . + dockerfile: dockerfile.development environment: - ASPNETCORE_ENVIRONMENT=Staging - ASPNETCORE_Kestrel__Endpoints__Https__Url=http://*:80 - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2 - - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine_jobs - - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine + - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine_jobs${MONGO_EXTENSIONS} + - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine${MONGO_EXTENSIONS} - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81 - ClearML__ApiServer=https://api.sil.hosted.allegro.ai - - ClearML__Queue=lambert_24gb - - ClearML__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest} - ClearML__Project=docker-compose - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}" - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}" + - BuildJob__ClearML__0__Queue=lambert_24gb + - BuildJob__ClearML__0__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest} + - BuildJob__ClearML__1__Queue=lambert_24gb.cpu_only + - BuildJob__ClearML__1__DockerImage=${MACHINE_PY_CPU_IMAGE:-ghcr.io/sillsdev/machine.py:latest.cpu_only} - SharedFile__Uri=s3://aqua-ml-data/docker-compose/ - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}" - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}" @@ -100,11 +102,11 @@ services: depends_on: - serval-api volumes: - - ${MACHINE_TESTING_DIR:-../machine}:/app:ro + - .:/app:ro - ~/.nuget/packages:/root/.nuget/packages:ro - /var/lib/machine:/var/lib/machine - /var/lib/serval:/var/lib/serval - working_dir: '/app/src/SIL.Machine.Serval.EngineServer' + working_dir: '/app/src/Machine/src/Serval.Machine.EngineServer' entrypoint: - dotnet - run @@ -117,21 +119,23 @@ services: hostname: machine-job-server container_name: machine-job-cntr build: - context: ${MACHINE_TESTING_DIR:-../machine} - dockerfile: ../machine/dockerfile.development + context: . + dockerfile: dockerfile.development environment: - ASPNETCORE_ENVIRONMENT=Staging - - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine_jobs - - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine + - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine_jobs${MONGO_EXTENSIONS} + - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine${MONGO_EXTENSIONS} - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81 - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80 - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2 - ClearML__ApiServer=https://api.sil.hosted.allegro.ai - - ClearML__Queue=lambert_24gb - - ClearML__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest} - ClearML__Project=docker-compose - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}" - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}" + - BuildJob__ClearML__0__Queue=lambert_24gb + - BuildJob__ClearML__0__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest} + - BuildJob__ClearML__1__Queue=lambert_24gb.cpu_only + - BuildJob__ClearML__1__DockerImage=${MACHINE_PY_CPU_IMAGE:-ghcr.io/sillsdev/machine.py:latest.cpu_only} - SharedFile__Uri=s3://aqua-ml-data/docker-compose/ - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}" - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}" @@ -143,11 +147,11 @@ services: - machine-engine - serval-api volumes: - - ${MACHINE_TESTING_DIR:-../machine}:/app:ro + - .:/app:ro - ~/.nuget/packages:/root/.nuget/packages:ro - /var/lib/machine:/var/lib/machine - /var/lib/serval:/var/lib/serval - working_dir: '/app/src/SIL.Machine.Serval.JobServer' + working_dir: '/app/src/Machine/src/Serval.Machine.JobServer' entrypoint: - dotnet - run