From 3f2246eccd70264eca02ab53e5ee0f1a8d2f32e9 Mon Sep 17 00:00:00 2001 From: erikzaadi Date: Sun, 8 Dec 2024 15:19:23 +0200 Subject: [PATCH 1/2] [Core] Add optional mock port api for perf tests #1214 --- .github/workflows/core-test.yml | 4 +- .github/workflows/perf-test.yml | 23 ++- Makefile | 13 +- .../jira/.port/resources/port-app-config.yaml | 1 - port_ocean/tests/helpers/fake_port_api.py | 191 ++++++++++++++++++ scripts/run-local-perf-test.sh | 13 ++ 6 files changed, 238 insertions(+), 7 deletions(-) create mode 100644 port_ocean/tests/helpers/fake_port_api.py diff --git a/.github/workflows/core-test.yml b/.github/workflows/core-test.yml index afc67499ee..ef523187c2 100644 --- a/.github/workflows/core-test.yml +++ b/.github/workflows/core-test.yml @@ -56,7 +56,7 @@ jobs: PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }} SMOKE_TEST_SUFFIX: ${{ github.run_id }} run: | - make test/smoke + make smoke/test - name: Cleanup Smoke Test if: always() @@ -67,7 +67,7 @@ jobs: PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }} SMOKE_TEST_SUFFIX: ${{ github.run_id }} run: | - make clean/smoke + make smoke/clean - name: Install current core for all integrations run: | diff --git a/.github/workflows/perf-test.yml b/.github/workflows/perf-test.yml index 3f80c8710f..dbc92b685d 100644 --- a/.github/workflows/perf-test.yml +++ b/.github/workflows/perf-test.yml @@ -51,6 +51,18 @@ on: - "20000" - "25000" - "35000" + ocean_log_level: + type: choice + default: 'INFO' + options: + - 'DEBUG' + - 'INFO' + description: Log level to use (defaults to INFO) + mock_port_api: + type: boolean + default: false + description: Mock the Port API instead of using the real one + jobs: test: name: 🌊 Ocean Performance Tests @@ -82,6 +94,8 @@ jobs: THIRD_PARTY_LATENCY_MS: ${{ inputs.third_party_latency_ms }} ENTITY_AMOUNT: ${{ inputs.entities_amount }} ENTITY_KB_SIZE: ${{ inputs.entity_kb_size }} + OCEAN_LOG_LEVEL: ${{ inputs.ocean_log_level }} + MOCK_PORT_API: ${{ inputs.mock_port_api && '1' || '0' }} run: | ./scripts/run-local-perf-test.sh @@ -92,8 +106,15 @@ jobs: PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }} PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }} SMOKE_TEST_SUFFIX: ${{ github.run_id }} + MOCK_PORT_API: ${{ inputs.mock_port_api && '1' || '0' }} run: | - make clean/smoke + if [[ "${MOCK_PORT_API}" = "1" ]]; then + make smoke/start-mock-api + make smoke/clean + make smoke/stop-mock-api + else + make smoke/clean + fi - name: Publish Performance Test Summary run: | diff --git a/Makefile b/Makefile index 9b9d01d705..7e89beca8b 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ define deactivate_virtualenv fi endef -.SILENT: install install/all test/all test/smoke clean/smoke lint lint/fix build run new test test/watch clean bump/integrations bump/single-integration execute/all +.SILENT: install install/all test/all smoke/test smoke/clean lint lint/fix build run new test test/watch clean bump/integrations bump/single-integration execute/all smoke/start-mock-api smoke/stop-mock-api # Install dependencies @@ -122,10 +122,10 @@ new: test: $(ACTIVATE) && pytest -m 'not smoke' -test/smoke: +smoke/test: $(ACTIVATE) && SMOKE_TEST_SUFFIX=$${SMOKE_TEST_SUFFIX:-default_value} pytest -m smoke -clean/smoke: +smoke/clean: $(ACTIVATE) && SMOKE_TEST_SUFFIX=$${SMOKE_TEST_SUFFIX:-default_value} python ./scripts/clean-smoke-test.py test/watch: @@ -156,3 +156,10 @@ bump/integrations: # make bump/single-integration INTEGRATION=aws bump/single-integration: ./scripts/bump-single-integration.sh -i $(INTEGRATION) + +# run a mock port api server for perf / smoke tests +smoke/start-mock-api: + $(ACTIVATE) && SMOKE_TEST_SUFFIX=$${SMOKE_TEST_SUFFIX:-default_value} python ./port_ocean/tests/helpers/fake_port_api.py & + +smoke/stop-mock-api: + ps aux | grep fake_port_api | egrep -v grep | awk '{print $$2};' | xargs kill -9 diff --git a/integrations/jira/.port/resources/port-app-config.yaml b/integrations/jira/.port/resources/port-app-config.yaml index f83db206be..97139b55b9 100644 --- a/integrations/jira/.port/resources/port-app-config.yaml +++ b/integrations/jira/.port/resources/port-app-config.yaml @@ -58,4 +58,3 @@ resources: subtasks: .fields.subtasks | map(.key) assignee: .fields.assignee.accountId reporter: .fields.reporter.accountId - diff --git a/port_ocean/tests/helpers/fake_port_api.py b/port_ocean/tests/helpers/fake_port_api.py new file mode 100644 index 0000000000..d3112914d7 --- /dev/null +++ b/port_ocean/tests/helpers/fake_port_api.py @@ -0,0 +1,191 @@ +import uvicorn +import os +from typing import Dict, Any +from fastapi import FastAPI, Request + +SMOKE_TEST_SUFFIX = os.environ.get("SMOKE_TEST_SUFFIX", "smoke") + +app = FastAPI() + +FAKE_DEPARTMENT_BLUEPRINT = { + "identifier": f"fake-department-{SMOKE_TEST_SUFFIX}", + "title": "Fake Department", + "icon": "Blueprint", + "schema": {"properties": {"name": {"type": "string"}, "id": {"type": "string"}}}, + "relations": {}, +} +FAKE_PERSON_BLUEPRINT = { + "identifier": f"fake-person-{SMOKE_TEST_SUFFIX}", + "title": "Fake Person", + "icon": "Blueprint", + "schema": { + "properties": { + "status": { + "type": "string", + "enum": ["WORKING", "NOPE"], + "enumColors": {"WORKING": "green", "NOPE": "red"}, + "title": "Status", + }, + "email": {"type": "string", "format": "email", "title": "Email"}, + "age": {"type": "number", "title": "Age"}, + "bio": {"type": "string", "title": "Bio"}, + } + }, + "relations": { + "department": { + "title": "Department", + "description": "Fake Department", + "target": f"fake-department-{SMOKE_TEST_SUFFIX}", + "required": False, + "many": False, + } + }, +} + + +@app.router.get("/v1/blueprints/{blueprint_id}") +@app.router.patch("/v1/blueprints/{blueprint_id}") +async def get_blueprint(blueprint_id: str) -> Dict[str, Any]: + return { + "blueprint": ( + FAKE_PERSON_BLUEPRINT + if blueprint_id.startswith("fake-person") + else FAKE_DEPARTMENT_BLUEPRINT + ) + } + + +@app.router.post("/v1/entities/search") +async def search_entities() -> Dict[str, Any]: + return {"ok": True, "entities": []} + + +@app.router.get("/v1/integration/{integration_id}") +@app.router.patch("/v1/integration/{integration_id}") +@app.router.patch("/v1/integration/{integration_id}/resync-state") +async def get_integration(integration_id: str) -> Dict[str, Any]: + return { + "integration": { + "identifer": integration_id, + "resyncState": { + "status": "completed", + "lastResyncEnd": "2024-11-20T12:01:54.225362+00:00", + "lastResyncStart": "2024-11-20T12:01:45.483844+00:00", + "nextResync": None, + "intervalInMinuets": None, + "updatedAt": "2024-11-20T12:01:54.355Z", + }, + "config": { + "deleteDependentEntities": True, + "createMissingRelatedEntities": True, + "enableMergeEntity": True, + "resources": [ + { + "kind": "fake-department", + "selector": {"query": "true"}, + "port": { + "entity": { + "mappings": { + "identifier": ".id", + "title": ".name", + "blueprint": f'"fake-department-{SMOKE_TEST_SUFFIX}"', + "properties": {"name": ".name", "id": ".id"}, + } + } + }, + }, + { + "kind": "fake-person", + "selector": {"query": "true"}, + "port": { + "entity": { + "mappings": { + "identifier": ".id", + "title": ".name", + "blueprint": f'"fake-person-{SMOKE_TEST_SUFFIX}"', + "properties": { + "name": ".name", + "email": ".email", + "status": ".status", + "age": ".age", + "department": ".department.name", + }, + "relations": {"department": ".department.id"}, + } + } + }, + }, + ], + }, + "installationType": "OnPrem", + "_orgId": "org_ZOMGMYUNIQUEID", + "_id": "integration_0dOOhnlJQDjMPnfe", + "identifier": f"smoke-test-integration-{SMOKE_TEST_SUFFIX}", + "integrationType": "smoke-test", + "createdBy": "APSQAYsYoIwPXqjn6XpwCAgnPakkNO67", + "updatedBy": "APSQAYsYoIwPXqjn6XpwCAgnPakkNO67", + "createdAt": "2024-11-20T12:01:42.651Z", + "updatedAt": "2024-11-20T12:01:54.355Z", + "clientId": "", + "logAttributes": { + "ingestId": "DOHSAIDHOMER", + "ingestUrl": "http://localhost:5555/logs/integration/DOHSAIDHOMER", + }, + }, + } + + +@app.router.post("/v1/blueprints/{blueprint_id}/entities") +async def upsert_entities(blueprint_id: str, request: Request) -> Dict[str, Any]: + json = await request.json() + + return { + "ok": True, + "entity": json, + } + + +@app.router.post("/v1/auth/access_token") +async def auth_token() -> Dict[str, Any]: + return { + "accessToken": "ZOMG", + "expiresIn": 1232131231, + "tokenType": "adadad", + } + + +@app.router.delete("/v1/blueprints/{blueprint_id}/all-entities") +async def delete_blueprint(blueprint_id: str, request: Request) -> Dict[str, Any]: + return {"migrationId": "ZOMG"} + + +@app.router.get("/v1/migrations/{migration_id}") +async def migration(migration_id: str, request: Request) -> Dict[str, Any]: + return { + "migration": { + "id": migration_id, + "status": "COMPLETE", + "actor": "Dwayne Scissors Johnson", + "sourceBlueprint": "leBlue", + "mapping": {}, + } + } + + +CATCH_ALL = "/{full_path:path}" + + +@app.router.get(CATCH_ALL) +@app.router.post(CATCH_ALL) +@app.router.patch(CATCH_ALL) +@app.router.delete(CATCH_ALL) +async def catch_all(full_path: str, request: Request) -> str: + return f"Hello there from fake Port API - {full_path}, thanks for accessing me with {request.method}" + + +def start() -> None: + uvicorn.run(app, host="0.0.0.0", port=5555) + + +if __name__ == "__main__": + start() diff --git a/scripts/run-local-perf-test.sh b/scripts/run-local-perf-test.sh index d2b0e08fc7..e5a0436f3d 100755 --- a/scripts/run-local-perf-test.sh +++ b/scripts/run-local-perf-test.sh @@ -23,6 +23,12 @@ export OCEAN__INTEGRATION__CONFIG__ENTITY_KB_SIZE=${ENTITY_KB_SIZE:--1} export OCEAN__INTEGRATION__CONFIG__THIRD_PARTY_BATCH_SIZE=${THIRD_PARTY_BATCH_SIZE:--1} export OCEAN__INTEGRATION__CONFIG__THIRD_PARTY_LATENCY_MS=${THIRD_PARTY_LATENCY_MS:--1} export OCEAN__INTEGRATION__CONFIG__SINGLE_DEPARTMENT_RUN=1 +export APPLICATION__LOG_LEVEL=${OCEAN_LOG_LEVEL:-'INFO'} + +if [[ "${MOCK_PORT_API:-0}" = "1" ]]; then + export PORT_BASE_URL=http://localhost:5555 + make smoke/start-mock-api +fi LOG_FILE_MD="${SCRIPT_BASE}/../perf-test-results-${SMOKE_TEST_SUFFIX}.log.md" @@ -58,6 +64,8 @@ RUN_LOG_FILE="./perf-sync.log" END_NS=$(date +%s%N) ELAPSED_MS=$(((END_NS - START_NS) / 1000000)) _log "Duration $((ELAPSED_MS / 1000)) seconds" + + UPSERTED=$(ruby -ne 'puts "#{$1}" if /Upserting (\d*) entities/' <"${RUN_LOG_FILE}" | xargs) if [[ -n "${UPSERTED}" ]]; then TOTAL_UPSERTED=0 @@ -75,4 +83,9 @@ if [[ -n "${DELETED}" ]]; then _log "Deleted: ${TOTAL_DELETED} entities" fi + +if [[ "${MOCK_PORT_API:-0}" = "1" ]]; then + make smoke/stop-mock-api +fi + _log "Perf test complete" From bb37e139f21b7a2dadb05f78517d8b39feb3fafc Mon Sep 17 00:00:00 2001 From: Ivan <62664893+ivankalinovski@users.noreply.github.com> Date: Mon, 9 Dec 2024 22:49:14 +0200 Subject: [PATCH 2/2] [Core] Change token refresh condition and retry upserts (#1215) # Description What - 1. Update the condition upon which the JWT token is refreshed so it will refresh on expiration instead of only after. 2. Set upsert entenies as retryable. 3. Remove timeout in favor of global timeout. 4. Update `handle_request` to use method for identifying retryable requests. Why - User receives 401 errors because JWT is expired so entities are not inserted. ## Type of change Please leave one option from the following and delete the rest: - [x] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] New Integration (non-breaking change which adds a new integration) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Non-breaking change (fix of existing functionality that will not change current behavior) - [ ] Documentation (added/updated documentation)

All tests should be run against the port production environment(using a testing org).

### Core testing checklist - [x] Integration able to create all default resources from scratch - [x] Resync finishes successfully - [x] Resync able to create entities - [x] Resync able to update entities - [ ] Resync able to detect and delete entities - [ ] Scheduled resync able to abort existing resync and start a new one - [ ] Tested with at least 2 integrations from scratch - [ ] Tested with Kafka and Polling event listeners - [ ] Tested deletion of entities that don't pass the selector ### Integration testing checklist - [ ] Integration able to create all default resources from scratch - [ ] Resync able to create entities - [ ] Resync able to update entities - [ ] Resync able to detect and delete entities - [ ] Resync finishes successfully - [ ] If new resource kind is added or updated in the integration, add example raw data, mapping and expected result to the `examples` folder in the integration directory. - [ ] If resource kind is updated, run the integration with the example data and check if the expected result is achieved - [ ] If new resource kind is added or updated, validate that live-events for that resource are working as expected - [ ] Docs PR link [here](#) ### Preflight checklist - [ ] Handled rate limiting - [ ] Handled pagination - [ ] Implemented the code in async - [ ] Support Multi account ## Screenshots Include screenshots from your environment showing how the resources of the integration will look. ## API Documentation Provide links to the API documentation used for this integration. --------- Co-authored-by: Ivan Kalinovski --- CHANGELOG.md | 11 +++++++++++ port_ocean/clients/port/authentication.py | 2 +- port_ocean/clients/port/mixins/entities.py | 2 +- port_ocean/helpers/retry.py | 2 +- pyproject.toml | 2 +- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d1630d8b3..71ec5e9d00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +## 0.14.7 (2024-12-09) + + +### Bug Fixes + +- Remove specific timeout for search request in favor of global timeout. +- Update `handle_request` to use method for indentifying retryable requests. +- Set upsert entenies as retryable. +- Update the condition upon which the JWT token is refreshed so it will refresh on expiration instead of only after. + + ## 0.14.6 (2024-12-04) diff --git a/port_ocean/clients/port/authentication.py b/port_ocean/clients/port/authentication.py index b05fbdd59c..4855651816 100644 --- a/port_ocean/clients/port/authentication.py +++ b/port_ocean/clients/port/authentication.py @@ -18,7 +18,7 @@ class TokenResponse(BaseModel): @property def expired(self) -> bool: - return self._retrieved_time + self.expires_in < get_time() + return self._retrieved_time + self.expires_in <= get_time() @property def full_token(self) -> str: diff --git a/port_ocean/clients/port/mixins/entities.py b/port_ocean/clients/port/mixins/entities.py index f8d5f63fa3..80f8178849 100644 --- a/port_ocean/clients/port/mixins/entities.py +++ b/port_ocean/clients/port/mixins/entities.py @@ -48,6 +48,7 @@ async def upsert_entity( ).lower(), "validation_only": str(validation_only).lower(), }, + extensions={"retryable": True}, ) if response.is_error: @@ -205,7 +206,6 @@ async def search_entities( "include": ["blueprint", "identifier"], }, extensions={"retryable": True}, - timeout=30, ) handle_status_code(response) return [Entity.parse_obj(result) for result in response.json()["entities"]] diff --git a/port_ocean/helpers/retry.py b/port_ocean/helpers/retry.py index 0893235c4a..5b2058c86c 100644 --- a/port_ocean/helpers/retry.py +++ b/port_ocean/helpers/retry.py @@ -134,7 +134,7 @@ def handle_request(self, request: httpx.Request) -> httpx.Response: """ try: transport: httpx.BaseTransport = self._wrapped_transport # type: ignore - if request.method in self._retryable_methods: + if self._is_retryable_method(request): send_method = partial(transport.handle_request) response = self._retry_operation(request, send_method) else: diff --git a/pyproject.toml b/pyproject.toml index 2762c2a607..bb799d82da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "port-ocean" -version = "0.14.6" +version = "0.14.7" description = "Port Ocean is a CLI tool for managing your Port projects." readme = "README.md" homepage = "https://app.getport.io"