From a209095f99bb817549667fea46ed6f429795881a Mon Sep 17 00:00:00 2001 From: Subsidiebot Amsterdam Date: Wed, 18 Sep 2024 15:00:32 +0200 Subject: [PATCH 1/3] Add rudimentary monitoring tools. Fixes #321 --- docker-compose.yml | 2 +- manage.py | 34 ++++++++++++++++++++++++++- ocd_backend/sources/ori.go.yaml | 2 +- ocd_backend/utils/monitor.py | 41 +++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 ocd_backend/utils/monitor.py diff --git a/docker-compose.yml b/docker-compose.yml index a593b6b5..a45129b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: - ori volumes: - oridata:/opt/ori/data - #- .:/opt/ori + - .:/opt/ori - ./ocd_backend:/opt/ori/ocd_backend depends_on: elastic: diff --git a/manage.py b/manage.py index a69cf243..e3f07659 100755 --- a/manage.py +++ b/manage.py @@ -23,7 +23,7 @@ from ocd_backend.settings import SOURCES_CONFIG_FILE, \ DEFAULT_INDEX_PREFIX, DUMPS_DIR, REDIS_HOST, REDIS_PORT from ocd_backend.utils.misc import load_sources_config - +from ocd_backend.utils.monitor import get_recent_counts # NOTE: don't forget to change this value if you forked this repo and # renamed '/opt/oci' @@ -521,6 +521,37 @@ def extract_process(modus, source_path, sources_config): click.echo('Source %s in redis does not exist in available sources' % source) +@command('monthly_check') +@click.option('--token') +def es_monthly_check(token): + result = get_recent_counts() + #pprint(result) + lines = [] + for idx, total in result.items(): + if total < 20: + muni = ' '.join(idx.split('_')[1:-1]) + lines.append( + "%s heeft %d documenten erbij in de afgelopen maand" % (muni, total,)) + if len(lines) > 0: + # curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer 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" + # -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/openstate/open-raadsinformatie/issues + # -d '{"title":"Found a bug","body":"I'\''m having a problem with this.","assignees":["breyten"],"milestone":1,"labels":["bug"]}' + payload = { + "title":"Found a bug", + "body":"\n".join(lines), + "assignees":[ + "breyten"], + "labels":["bug"]} + requests.post( + 'https://api.github.com/repos/openstate/open-raadsinformatie/issues', + headers={ + 'X-GitHub-Api-Version': '2022-11-28', + 'Accept': 'application/vnd.github+json', + 'Authorization': 'Bearer ' + token + }, + data=json.dumps(payload) + ) + # Register commands explicitly with groups, so we can easily use the docstring # wrapper elasticsearch.add_command(es_put_template) @@ -529,6 +560,7 @@ def extract_process(modus, source_path, sources_config): elasticsearch.add_command(delete_indexes) elasticsearch.add_command(available_indices) elasticsearch.add_command(es_copy_data) +elasticsearch.add_command(es_monthly_check) extract.add_command(extract_list_sources) extract.add_command(extract_start) diff --git a/ocd_backend/sources/ori.go.yaml b/ocd_backend/sources/ori.go.yaml index d8688dee..c655de56 100644 --- a/ocd_backend/sources/ori.go.yaml +++ b/ocd_backend/sources/ori.go.yaml @@ -313,7 +313,7 @@ ori.go: woudenberg: key: woudenberg base_url: "https://gemeentebestuur.woudenberg.nl/api/" - api_version: "v2" + api_version: "v1" allmanak_id: 22403 cbs_id: GM0351 <<: *go diff --git a/ocd_backend/utils/monitor.py b/ocd_backend/utils/monitor.py new file mode 100644 index 00000000..873b5666 --- /dev/null +++ b/ocd_backend/utils/monitor.py @@ -0,0 +1,41 @@ +#!usr/bin/env python +from pprint import pprint + +import requests +import json + +from ocd_backend.es import elasticsearch as es +from ocd_backend.es import setup_elasticsearch + +def get_indices(): + return [l.strip().split(" ")[2] for l in es.cat.indices().split("\n") if l != ''] + +def get_recent_counts(): + payload = { + 'query': { + 'bool': { + 'filter': [ + {'term': {"@type": "MediaObject"}}, + {'range': { + 'last_discussed_at': { + 'gte': 'now-3000d' + } + }} + ] + } + }, + "aggs": { + "index": { + "terms": { + "field": "_index" + } + } + }, + "size": 0 + } + resp = es.search(body=payload) + indices = get_indices() + result = {i: 0 for i in indices} + for b in resp.get('aggregations', {}).get('index', {}).get('buckets', []): + result[b['key']] = b['doc_count'] + return result From 04850d4fd0e18c4f117dfab223e17a48b86d700b Mon Sep 17 00:00:00 2001 From: Subsidiebot Amsterdam Date: Wed, 18 Sep 2024 15:10:17 +0200 Subject: [PATCH 2/3] Uhm --- manage.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/manage.py b/manage.py index e3f07659..9991385f 100755 --- a/manage.py +++ b/manage.py @@ -533,9 +533,6 @@ def es_monthly_check(token): lines.append( "%s heeft %d documenten erbij in de afgelopen maand" % (muni, total,)) if len(lines) > 0: - # curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer 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" - # -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/openstate/open-raadsinformatie/issues - # -d '{"title":"Found a bug","body":"I'\''m having a problem with this.","assignees":["breyten"],"milestone":1,"labels":["bug"]}' payload = { "title":"Found a bug", "body":"\n".join(lines), From 249f355044cdd16a19413cb913d1d71507e5b89f Mon Sep 17 00:00:00 2001 From: Subsidiebot Amsterdam Date: Wed, 18 Sep 2024 15:03:24 +0200 Subject: [PATCH 3/3] Fix? --- docker-compose.yml | 2 +- ocd_backend/sources/ori.ibabs.yaml | 23 ++++++++---- ocd_backend/sources/ori.notubiz.yaml | 52 ++++++++++++++++++++-------- ocd_backend/sources/owi.ibabs.yaml | 9 +++++ 4 files changed, 63 insertions(+), 23 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a45129b5..916b2b00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,7 +77,7 @@ services: retries: 5 restart: always environment: - - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + - "ES_JAVA_OPTS=-Xms16g -Xmx16g" # Discovery - network.host=127.0.0.1 - http.host=0.0.0.0 diff --git a/ocd_backend/sources/ori.ibabs.yaml b/ocd_backend/sources/ori.ibabs.yaml index f0fc338d..b2463ff4 100644 --- a/ocd_backend/sources/ori.ibabs.yaml +++ b/ocd_backend/sources/ori.ibabs.yaml @@ -942,7 +942,7 @@ ori.ibabs: key: weststellingwerf ibabs_sitename: Weststellingwerf allmanak_id: 29037 - cbs_id: GM0085 + cbs_id: GM0098 <<: *ibabs westerkwartier: @@ -1046,12 +1046,13 @@ ori.ibabs: cbs_id: GM0888 <<: *ibabs - venlo: - key: venlo - ibabs_sitename: venlo - allmanak_id: 25285 - cbs_id: GM0983 - <<: *ibabs + # Swithced to GO + # venlo: + # key: venlo + # ibabs_sitename: venlo + # allmanak_id: 25285 + # cbs_id: GM0983 + # <<: *ibabs oostzaan: key: oostzaan @@ -1209,3 +1210,11 @@ ori.ibabs: allmanak_id: 36916 cbs_id: GM1876 <<: *ibabs + + # Added 2024-09-16 + borger-odoorn: + key: borger-odoorn + ibabs_sitename: borgerodoorn + allmanak_id: 21806 + cbs_id: gm1681 + <<: *ibabs diff --git a/ocd_backend/sources/ori.notubiz.yaml b/ocd_backend/sources/ori.notubiz.yaml index f5e3d365..b1680871 100644 --- a/ocd_backend/sources/ori.notubiz.yaml +++ b/ocd_backend/sources/ori.notubiz.yaml @@ -216,12 +216,13 @@ ori.notubiz: # notubiz_organization_id: 1674 # <<: *notubiz - noordwijk: - key: noordwijk - allmanak_id: 27985 - cbs_id: GM0575 - notubiz_organization_id: 574 - <<: *notubiz + # switched to ibabs + # noordwijk: + # key: noordwijk + # allmanak_id: 27985 + # cbs_id: GM0575 + # notubiz_organization_id: 574 + # <<: *notubiz amersfoort: key: amersfoort @@ -340,7 +341,7 @@ ori.notubiz: almere: key: almere allmanak_id: 28393 - cbs_id: GM0441 + cbs_id: GM0034 notubiz_organization_id: 952 <<: *notubiz @@ -365,13 +366,14 @@ ori.notubiz: notubiz_organization_id: 1781 <<: *notubiz - stichtse_vecht: - key: stichtse_vecht - source_name: "Stichtse Vecht" - allmanak_id: 34861 - cbs_id: GM1904 - notubiz_organization_id: 1770 - <<: *notubiz + # Switched to GO + # stichtse_vecht: + # key: stichtse_vecht + # source_name: "Stichtse Vecht" + # allmanak_id: 34861 + # cbs_id: GM1904 + # notubiz_organization_id: 1770 + # <<: *notubiz deventer: key: deventer @@ -827,7 +829,7 @@ ori.notubiz: bunschoten: key: bunschoten allmanak_id: 28326 - cbs_id: GM0047 + cbs_id: GM0313 notubiz_organization_id: 505 <<: *notubiz @@ -936,3 +938,23 @@ ori.notubiz: cbs_id: GM1992 notubiz_organization_id: 4236 <<: *notubiz + +# Added on 16-09-2024 + haarlem: + key: haarlem + allmanak_id: 38688 + cbs_id: GM0392 + notubiz_organization_id: 544 + <<: *notubiz + haaksbergen: + key: haaksbergen + allmanak_id: 32233 + cbs_id: GM0158 + notubiz_organization_id: 275 + <<: *notubiz + tilburg: + key: tilburg + allmanak_id: 38194 + cbs_id: GM0855 + notubiz_organization_id: 1736 + <<: *notubiz diff --git a/ocd_backend/sources/owi.ibabs.yaml b/ocd_backend/sources/owi.ibabs.yaml index 3295eab7..b541688f 100644 --- a/ocd_backend/sources/owi.ibabs.yaml +++ b/ocd_backend/sources/owi.ibabs.yaml @@ -75,3 +75,12 @@ owi.ibabs: key: aa-en-maas allmanak_id: 21099 <<: *ibabs + # Added 2024-09-16 + de_dommel: + key: de_dommel + allmanak_id: 21124 + <<: *ibabs + limburg: + key: limburg + allmanak_id: 21194 + <<: *ibabs