From 5fd6442b4a5aac211e3c6d17c7f1b62673ed236b Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 12:45:35 +0000 Subject: [PATCH 1/7] build: add TM_DEV_PORT variable for traefik port change --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 459512fdd6..2978165f32 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ services: image: traefik:v2.10 restart: always ports: - - "80:80" + - "${TM_DEV_PORT:-80}:80" volumes: - /var/run/docker.sock:/var/run/docker.sock:ro command: From aa25b3ad732c528d1f4d301d41b5cc8b53f4ba5d Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 12:45:53 +0000 Subject: [PATCH 2/7] build: add container_name for all compose services consistency --- docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2978165f32..62ddffca73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: # Main application backend: <<: *backend - container_name: backend + container_name: tm-backend restart: always labels: - traefik.http.routers.backend.rule=(Host(`127.0.0.1`) || Host(`localhost`)) && PathPrefix(`/api/`) @@ -23,12 +23,13 @@ services: migration: <<: *backend - container_name: migration + container_name: tm-migration restart: on-failure command: flask db upgrade frontend: image: hotosm-tasking-manager:frontend + container_name: tm-frontend restart: always networks: - tm-web @@ -38,7 +39,7 @@ services: postgresql: image: postgis/postgis:14-3.3 - container_name: postgresql + container_name: tm-postgresql restart: always env_file: ${ENV_FILE:-tasking-manager.env} networks: @@ -46,6 +47,7 @@ services: traefik: image: traefik:v2.10 + container_name: tm-traefik restart: always ports: - "${TM_DEV_PORT:-80}:80" From ec4b363844370faeb6d9a48fb5c655d137012c42 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 12:46:11 +0000 Subject: [PATCH 3/7] docs: add TM_DEV_PORT to example.env --- example.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/example.env b/example.env index 7425d3523a..dd5a77dfa0 100644 --- a/example.env +++ b/example.env @@ -19,6 +19,9 @@ TM_APP_API_URL=http://127.0.0.1:5000 # Defines the version of the API and will be used after /api/ on the url TM_APP_API_VERSION=v2 +# Override default port 80 during development +# TM_DEV_PORT=880 + # Information about the hosting organization TM_ORG_NAME="Humanitarian OpenStreetMap Team" TM_ORG_CODE=HOT From 28eb6db484c9d967a2b0849ea4081078d0943679 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 12:46:29 +0000 Subject: [PATCH 4/7] docs: update development setup info --- docs/developers/development-setup.md | 128 ++++++++++++++++++++++----- 1 file changed, 107 insertions(+), 21 deletions(-) diff --git a/docs/developers/development-setup.md b/docs/developers/development-setup.md index b941656d76..817053a168 100644 --- a/docs/developers/development-setup.md +++ b/docs/developers/development-setup.md @@ -1,7 +1,7 @@ -## Development setup +# Development setup -### Architecture +## Architecture The Tasking Manager is composed of two parts: @@ -10,13 +10,104 @@ The Tasking Manager is composed of two parts: The two parts can be developed independently of each other. +## OSM Auth + +The Tasking Manager uses OAuth2 with OSM to authenticate users. + +In order to use the frontend, you may need to create keys for OSM: + +1. [Login to OSM][1] + (_If you do not have an account yet, click the signup + button at the top navigation bar to create one_). + + Click the drop down arrow on the top right of the navigation bar + and select My Settings. + +2. Register your Tasking Manager instance to OAuth 2 applications. + + Put your login redirect url as `http://127.0.0.1:880/authorized/` + + > Note: `127.0.0.1` is required for debugging instead of `localhost` + > due to OSM restrictions. + +3. Permissions required: + - Read user preferences (read_prefs). + - Modify the map (write_api). + +4. Now save your Client ID and Client Secret for the next step. + +## Configure The Dot Env File + +1. Copy the `example.env` to `tasking-manager.env`. + + ```bash + cp example.env tasking-manager.env + ``` + +2. Uncomment or update the following variables + + ```dotenv + TM_DEV_PORT=880 + TM_APP_BASE_URL=http://127.0.0.1:880 + TM_APP_API_URL=http://127.0.0.1:880/api + # 'postgresql' if using docker, else 'localhost' or comment out + POSTGRES_ENDPOINT=postgresql + TM_REDIRECT_URI=http://127.0.0.1:880/authorized + TM_CLIENT_ID=from-previous-step + TM_CLIENT_SECRET=from-previous-step + ``` + + - Note that the port 880 can be swapped to any available port on + your system. + - If you change this, don't forget to update the OAuth login redirect + URL from the step above. + +> If you are a frontend developer and do not wish to configure the +> backend, you can use our staging server API. +> +> Update the variable: +> +>    `TM_APP_API_URL='https://tasking-manager-staging-api.hotosm.org'` +> +> before running the `yarn start` command. +> +> Be aware that the staging API can be offline while we are deploying +> newer versions to the staging server and that you'll not have access +> to some management views due to permissions. Check the +> [configuration](#configuration) section to learn more about how +> to configure Tasking Manager. + +For more details see the [configuration section](#configuration). + +## Docker + +The easiest option to get started with all components may be using Docker. + +### Requirements + +[Docker Engine](https://docs.docker.com/engine/install/) must be +available locally. + +### Running Tasking Manager + +Once the steps above have been complete, simply run: + +```bash +docker compose pull +docker compose build +docker compose --env-file tasking-manager.env up --detach +``` + +Tasking Manager should be available from: +[http://127.0.0.1:880](http://127.0.0.1:880) + +## Running Components Standalone + ### Frontend The client is the front-end user interface of the Tasking Manager. It is based on the React framework and you can find all files in the `frontend` directory. -If you don't want to setup a backend server, you can work on frontend development using our staging server API. Execute `export TM_APP_API_URL='https://tasking-manager-staging-api.hotosm.org'` before running the `yarn start` command. Be aware that the staging API can be offline while we are deploying newer versions to the staging server and that you'll not have access to some management views due to permissions. Check the [configuration](#configuration) section to learn more about how to configure Tasking Manager. - -**Dependencies** +#### Dependencies The following dependencies must be available _globally_ on your system: * Download and install [NodeJS LTS v12+](https://nodejs.org/en/) and [yarn](https://classic.yarnpkg.com/en/docs/install) @@ -149,7 +240,7 @@ We use create the database from the migrations directory. Check the instructions on how to setup a PostGIS database with [docker](#creating-a-local-postgis-database-with-docker) or on your -[local system](#non-docker). Then you can execute the following +[local system](#creating-a-local-postgis-database-without-docker). Then you can execute the following command to apply the migrations: ``` @@ -169,7 +260,7 @@ on your terminal (with the OS user that is the owner of the database): `psql -d -c "UPDATE users set role = 1 where username = ''"` -### API +## API If you plan to only work on the API you only have to build the backend architecture. Install the backend dependencies, and run the server: @@ -183,7 +274,7 @@ docs is also available on our [production](https://tasks.hotosm.org/api-docs) and [staging](https://tasks-stage.hotosm.org/api-docs/) instances. -#### API Authentication +### API Authentication In order to authenticate on the API, you need to have an Authorization Token. @@ -217,7 +308,7 @@ edit history of your user, selecting a changeset from the list, and then at the bottom link `Changeset XML` and it will be in the `uid` field of the XML returned. -#### API Authentication on remote instance +### API Authentication on remote instance To get your token on the production or staging Tasking Manager instances, sign in in the browser and then either: @@ -227,20 +318,14 @@ instances, sign in in the browser and then either: - inspect a network request and search for the `Authorization` field in the request headers section. -# Docker +## Additional Info -## Creating a local PostGIS database with Docker +### Creating a local PostGIS database with Docker If you're not able to connect to an existing tasking-manager DB, we have a [Dockerfile]() that will allow you to run PostGIS locally as follows. -### Dependencies - -Following must be available locally: - -* [Docker CE](https://www.docker.com/community-edition#/download) - ### Build & Run the PostGIS dockerfile 1. From the root of the project: @@ -279,21 +364,21 @@ export TM_DB=postgresql://hottm:hottm@localhost/tasking-manager 6. Refer to the rest of the instructions in the README to setup the DB and run the app. -# Non-Docker +## Creating a local PostGIS database without Docker -## Creating the PostGIS database +### Creating the PostGIS database It may be the case you would like to set up the database without using Docker for one reason or another. This provides you with a set of commands to create the database and export the database address to allow you to dive into backend development. -### Dependencies +#### Dependencies First, ensure that Postgresql and PostGIS are installed and running on your computer. -### Create the database user and database +#### Create the database user and database Assuming you have sudo access and the unix Postgresql owner is `postgres`: @@ -325,3 +410,4 @@ get a working version of the API running. * [Managing CI/CD with CircleCI](../sysadmins/ci-cd.md) * [Deployment Guide](../sysadmins/deployment.md) +[1]: https://www.openstreetmap.org/login "Login to OSM" From 576d007517d7c8fb79cc97f5f7d05ba8d8d12264 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 15:33:50 +0000 Subject: [PATCH 5/7] refactor: remove support for pep582 (PEP blocked) --- scripts/install/install_on_debian_10_buster.sh | 2 -- scripts/install/install_on_ubuntu_18_04.sh | 2 -- 2 files changed, 4 deletions(-) diff --git a/scripts/install/install_on_debian_10_buster.sh b/scripts/install/install_on_debian_10_buster.sh index 6590350837..a447c2bf42 100755 --- a/scripts/install/install_on_debian_10_buster.sh +++ b/scripts/install/install_on_debian_10_buster.sh @@ -32,8 +32,6 @@ git clone https://github.com/hotosm/tasking-manager.git && cd tasking-manager/ && pip install --upgrade pip && pip install --upgrade pdm && -pdm config --global python.use_venv False && -eval "$(pdm --pep582)" && pdm install && # Set up configuration diff --git a/scripts/install/install_on_ubuntu_18_04.sh b/scripts/install/install_on_ubuntu_18_04.sh index 1f4ce43b41..c7fa3a0540 100755 --- a/scripts/install/install_on_ubuntu_18_04.sh +++ b/scripts/install/install_on_ubuntu_18_04.sh @@ -32,8 +32,6 @@ git clone https://github.com/hotosm/tasking-manager.git && cd tasking-manager/ && pip install --upgrade pip && pip install --upgrade pdm && -pdm config --global python.use_venv False && -eval "$(pdm --pep582)" && pdm install && echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p && From a1a43856419212ae1e8ea87bec8eeb0018654f33 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 15:34:24 +0000 Subject: [PATCH 6/7] ci: pin pdm version to ==2.7.4 during functional tests --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3f4e3d146c..a67393bb63 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,8 +81,7 @@ jobs: - run: sudo apt-get update - run: sudo apt-get -y install libgeos-dev # Required for shapely - run: sudo apt-get -y install proj-bin libproj-dev - - run: pip install --upgrade pip pdm - - run: pdm config --global python.use_venv False + - run: pip install --upgrade pip pdm==2.7.4 - run: pdm export --dev --without-hashes > requirements.txt - run: pip install -r requirements.txt - run: mkdir --mode 766 -p /tmp/logs From 9e96f8f401a57e24687199c50fc5b88ffe9caa42 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 15:34:39 +0000 Subject: [PATCH 7/7] docs: clarify info about running backend api only --- docs/developers/development-setup.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/developers/development-setup.md b/docs/developers/development-setup.md index 817053a168..bbc95807fd 100644 --- a/docs/developers/development-setup.md +++ b/docs/developers/development-setup.md @@ -187,6 +187,7 @@ directory. To use that last option, follow the below instructions: - `TM_CLIENT_SECRET`=oauth-client-secret-key-from-openstreetmap - `TM_REDIRECT_URI`=oauth-client-redirect_uri - `TM_SCOPE`=oauth-client-scopes + - `TM_LOG_DIR=logs` In order to send email correctly, set these variables as well: - `TM_SMTP_HOST` @@ -196,20 +197,12 @@ In order to send email correctly, set these variables as well: - `TM_SMTP_USE_TLS=0` - `TM_SMTP_USE_SSL=1` (Either TLS or SSL can be set to 1 but not both) -#### Build +#### Install Dependencies * Install project dependencies: - * Linux/Mac (Option 1: pep582): - * First ensure the Python version in `pyproject.toml:requires-python` is installed on your system. - * ```pip install --upgrade pdm``` - * ```pdm config --global python.use_venv False``` - * ```pdm --pep582 >> ~/.bash_profile``` - * ```source ~/.bash_profile``` - * ```pdm install``` - * Linux/Mac (Option 2: pip (system/venv)): - * ```pip install --upgrade pdm``` - * ```pdm export --without-hashes > requirements.txt``` - * ```pip install requirements.txt``` + * First ensure the Python version in `pyproject.toml:requires-python` is installed on your system. + * ```pip install --upgrade pdm``` + * ```pdm install``` #### Tests @@ -265,7 +258,16 @@ on your terminal (with the OS user that is the owner of the database): If you plan to only work on the API you only have to build the backend architecture. Install the backend dependencies, and run the server: -`flask run --debug --reload` or `pdm run start` +```bash +# Install dependencies +pdm install + +# Run (Option 1) +pdm run start + +# Run (Option 2) +pdm run flask run --debug --reload +``` You can access the API documentation on [http://localhost:5000/api-docs](http://localhost:5000/api-docs), it