diff --git a/.env.example b/.env.example index 7eb794c..ba91fa3 100644 --- a/.env.example +++ b/.env.example @@ -1,18 +1,18 @@ +SECRET_KEY=[secret key string] +ACTIVE_PERIOD=[YYYY-T, for example: 2020-2] +SSO_UI_FORCE_HTTPS=True + # rabbit mq UPDATE_COURSE_LIST_EXCHANGE_NAME=update_course -RABBIT_HOST=rmq +RABBIT_HOST=rmq || localhost RABBIT_USERNAME=guest -RABBIT_PASSWORD=[Rabbit MQ Password] - -SECRET_KEY=[secret_key_string] -ACTIVE_PERIOD=[for example: 2020-2] -SSO_UI_FORCE_HTTPS=True +RABBIT_PASSWORD=[RabbitMQ password] # mongo MONGODB_DB=backend -MONGODB_HOST=mongo +MONGODB_HOST=mongo || localhost MONGODB_PORT=27017 -MONGODB_USERNAME=user +MONGODB_USERNAME=[MongoDB username] MONGODB_PASSWORD=[MongoDB password] # sentry diff --git a/.github/workflows/deploy-staging.yaml b/.github/workflows/deploy-staging.yaml index 0190a2e..81f26ed 100644 --- a/.github/workflows/deploy-staging.yaml +++ b/.github/workflows/deploy-staging.yaml @@ -2,6 +2,9 @@ name: Deploy (Staging) on: workflow_dispatch: + push: + branches: + - master jobs: deploy_service: diff --git a/Dockerfile b/Dockerfile index afb0e6b..5488822 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,14 +6,14 @@ ENV APP_ENV="container" COPY . . -COPY launch.sh /opt/app/launch.sh +COPY scripts/launch.sh /opt/app/launch.sh COPY sso/additional-info.json /opt/app/sso/additional-info.json COPY sso/faculty-base-additional-info.json /opt/app/sso/faculty-base-additional-info.json COPY sso/faculty_exchange_route.json /opt/app/sso/faculty_exchange_route.json -RUN apk add -u --no-cache tzdata gcc musl-dev libxml2 libxslt-dev && \ - pip install wheel && \ - pip install -r requirements.txt +RUN apk add -u --no-cache tzdata gcc musl-dev libxml2 libxslt-dev +RUN pip install wheel +RUN pip install -r requirements.txt ENV PORT=8006 diff --git a/MongoDB-Dockerfile b/MongoDB-Dockerfile deleted file mode 100644 index e6cbfde..0000000 --- a/MongoDB-Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM mongo:latest - -ADD init-mongo.sh /docker-entrypoint-initdb.d/ \ No newline at end of file diff --git a/README.md b/README.md index 275742c..acc0220 100644 --- a/README.md +++ b/README.md @@ -1,127 +1,113 @@ # Susun Jadwal -Susun Jadwal is an open source tool to plan class schedules for university students. +Susun Jadwal is an open source tool to plan class schedules for university students, +developed by Ristek Fasilkom UI. https://susunjadwal.cs.ui.ac.id/ -Susun Jadwal by Ristek Fasilkom UI. https://susunjadwal.cs.ui.ac.id/ +In the spirit of Open Source Software, *everyone* is welcome to contribute to Susun Jadwal! +See the Contributing Guide below for more. -Monorepo setup with React frontend and Flask backend. - -## Structure explained +## Structure ``` -app/ // general views -models/ // mongoDB models -scraper/ // courses (academic.ui.ac.id) scraper -sso/ // SSO UI authentication logic -README.md // important info -requirements.txt // dependency list -start.sh // script to start server -... -README.md // workspace-wide information shown in github +app/ // general views +models/ // mongoDB models +scraper/ // courses (academic.ui.ac.id) scraper +sso/ // SSO UI authentication logic +requirements.txt // python dependency list +scripts/ // (utility scripts) +├── init-mongo.sh // script to create non-root mongoDB user +├── launch.sh // main script to start flask +├── mongo_dump.sh // script to dump mongoDB data to .dump file +└── start.sh // alternative script to start flask +.env.example // template for .env file +dev.docker-compose.yml // docker-compose for mongo and rmq +docker-compose.yml // docker-compose for mongo, rmq, & server ``` ## Contributing Guide -Feel free to contribute by submitting a pull request. +*Everyone* is welcome to contribute to Susun Jadwal! +Feel free to make a contribution by submitting a pull request. +You can also report bugs and request features / changes by creating a new +[Issue](https://github.com/ristekoss/susunjadwal-backend/issues/new). + +For in-depth discussion, please join RistekOSS's Discord. -# Susun Jadwal Backend +## Development -## Requirements +### Requirements -1. `python 3.6` and `pip using` +1. `python` (tested on 3.6 and 3.9.18), and `pip` 2. `docker` -## Configuration +### Installing -### Development +The following steps will assume you have already set up a python virtual environment, +and will use `dev.docker-compose.yml`. -1. Create virtual environment using `python3 -m venv env` -2. Activate virtualenv `source ./env/bin/activate` -3. Install requirements `pip install -r requirements.txt` -4. Add your credential to scrap schedule from SIAK in `scraper/credentials.json` with the following structure: +1. Boot up MongoDB and RabbitMQ: + ```docker-compose -f dev.docker-compose.yaml up``` -``` -{ - "": { - "username": "", - "password": "" - } -} -``` +2. Populate `.env`, using `.env.example` as reference. -You can also see `scraper/credentials.template.json` for example and `sso/additional-info.json` for list of `kd_org`. +3. Connect to MongoDB and create a non-root user. + ``` + mongosh -u + // enter password when prompted + + use ; + db.createUser({user: "", pwd: "", roles: ["readWrite"]}); + // response should be '{ok: 1}', use these credentials in your .env secrets + ``` -5. Start database using `bash start_db.sh` -6. Go to mongo console by running `docker exec -it ristek-mongo mongo -u ` -7. Create database by running `use `. By default, Flask use database named `test` so it becomes `use test` -8. Create user for database: +4. Boot up the Flask server: + ```export PORT=8000 && bash scripts/launch.sh``` -``` -db.createUser( - { - user: "", - pwd: "", - roles:[ - { - role: "readWrite", - db: "" - } - ] - } -); -``` +5. Ping the API: `http://localhost:8000/susunjadwal/api/` -You can quit mongo console now by using Ctrl + D. +### Containerizing the Server -9. Create config file, `instance/config.cfg`. You can see `instance/config.template.cfg` for example and edit db name, username, and password to match the one you created before -10. Run `docker-compose up -d` to start the rabbit mq -11. Create `.env` file from `.env.example` file -12. Finally, run Flask by using `FLASK_ENV="development" flask run` +#### For Local -### Production +1. Populate `.env`. -#### Old +2. Run `docker-compose.yml`. -> We actually have a slightly different setup in the real Ristek server. For future maintainers, you may want to contact past contributors. +#### For Deployment -1. Do everything in development step **except** step no 10, running Flask. Don't forget to modify `instance/config.cfg`, `start_db.sh`, and `scraper/credentials.json` if you want to -2. Run gunicorn using `bash start.sh` -3. Set your Nginx (or other reverse proxy of your choice) to reverse proxy to `sunjad.sock`. For example, to reverse proxy `/susunjadwal/api` you can set +While RISTEK uses a different standardized workflow, here is a general guide on deploying. -``` -location ^~ /susunjadwal/api { - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://unix:/path/to/susunjadwal/backend/sunjad.sock; -} -``` +1. Populate `.env`. + +2. Replace line 23-25 in `docker-compose.yml` to pull from [Docker Hub](https://hub.docker.com/r/ristekoss/): -4. Run the schedule scrapper cron job using `crontab -e` and add the line to run `cron.sh`. For example, to run it every 10 minutes add `*/10 * * * * bash /path/to/susunjadwal/backend/cron.sh` +```image: ristekoss/susunjadwal-backend:stable``` -#### New +3. Modify the credentials in `MONGO_INITDB_XX` environment variables. + +4. `docker compose up` -**notes**: For deployment, SusunJadwal Backend is using **Ubuntu 18.04**. Here's the link to the marketplace https://aws.amazon.com/marketplace/pp/prodview-pkjqrkcfgcaog -1. Do everything in development step **except** step no 5,6,7, and 10 -2. Create `config.cfg` and fill the DB credentials according the given specification in `docker-compose-deploy.yaml` (host must be `mongo`) -3. Run `docker-compose -f docker-compose-deploy.yaml up -d` to execute mongodb, flask, and rabbitmq -4. Run `docker exec -it susunjadwalbackend_mongo mongo -u root-user -p root-user` and create admin in `backend` db, then restart the mongo container ## Dump and Restore Database ### Dump +1. Run the dump script: `bash mongo_dump.sh` +2. The result will be a `.dump` file in the directory `./mongodump`. + +### Restore +1. Copy dump file to MongoDB Container: `docker cp susunjadwalbackend_mongo_1:/` +2. Load dump into DB: `docker exec -it mongorestore -u --archive=` +3. Enter ``'s password when prompted. +4. You should see the success message: `XX documents successfully restored.` -1. Run `bash mongo_dump.sh` -2. The result will be on `./mongodump` +## Deployment -## Restore +We use a standardized pipeline for all our products which we invoke from `.github/workflows/deploy-.yaml` -If you want to restore the database from .dump file +## Legacy Version -1. Copy dump file to mongo container, `docker cp susunjadwalbackend_mongo_1:/` -2. Restore db using command, `docker exec -it susunjadwalbackend_mongo_1 mongorestore -u root-user -p root-user --archive=` +To see the version of `susunjadwal-backend` which was maintained and deployed up until 2023, see [5b8f710](https://github.com/ristekoss/susunjadwal-backend/tree/5b8f71068b62a0f1f684c616cb8e40c087861725). ## License diff --git a/app/__init__.py b/app/__init__.py index 678c693..a75eb38 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -87,11 +87,9 @@ app.register_blueprint(cron) CORS(app) -# print(f'MONGODB_DB:{os.environ.get("MONGODB_DB")}, MONGODB_PORT:{os.environ.get("MONGODB_PORT")}, MONGODB_USERNAME:{os.environ.get("MONGODB_USERNAME")} sad') MongoEngine(app) # Init connection to rabbit mq -# print(f'RABBIT HOST: {os.environ.get("RABBIT_HOST")}, RABBIT USERNAME: {os.environ.get("RABBIT_USERNAME")}') init_pika(app) # Init consumer and create exchange diff --git a/app/views/main.py b/app/views/main.py index cd789a9..7d1545b 100644 --- a/app/views/main.py +++ b/app/views/main.py @@ -16,6 +16,15 @@ router_main = Blueprint('router_sunjad', __name__) +""" +Basic ping / status check +""" +@router_main.route('/', methods=['GET']) +def status(): + return (jsonify({ + "message": "susunjadwal is live!", + }), 200) + """ Provides course list by major kd_org. The kd_org list is provided in sso/additional_info.json @@ -45,7 +54,9 @@ def get_courses_by_kd(major_kd_org): return (jsonify(period.serialize()), 200) - +''' +Provides course list filtered by major ID. +''' @router_main.route('/majors//courses', methods=['GET']) @require_jwt_token def get_courses(major_id): @@ -183,6 +194,27 @@ def scrap_all_schedule(): ) return jsonify(response), status_code +''' +Provides all existing courses, filtered by major ID +regardless active term/period +''' +@router_main.route('/majors//all_courses', methods=['GET']) +def get_all_courses_by_major(major_id): + periods = Period.objects( + major_id=major_id + ).all() + all_courses = [] + for period in periods: + for course in period.courses: + all_courses.append(course.serialize_ulas_kelas()) + return (jsonify({ + 'courses': all_courses + }), 200) + +''' +Provides all existing courses +regardless of major or active term/period +''' @router_main.route('/courses', methods=['GET']) def get_all_courses(): active_period = get_app_config("ACTIVE_PERIOD") diff --git a/dev.docker-compose-rmq.yaml b/dev.docker-compose-rmq.yaml deleted file mode 100644 index c0388f1..0000000 --- a/dev.docker-compose-rmq.yaml +++ /dev/null @@ -1,8 +0,0 @@ -version: "3.7" - -services: - rmq: - image: rabbitmq:management-alpine - ports: - - "15671-15672:15671-15672" - - "5672:5672" diff --git a/dev.docker-compose.yaml b/dev.docker-compose.yaml index 58b551a..63dbff1 100644 --- a/dev.docker-compose.yaml +++ b/dev.docker-compose.yaml @@ -1,4 +1,4 @@ -version: "3.9" +version: "3.7" services: rmq: @@ -6,6 +6,7 @@ services: restart: unless-stopped ports: - "15671-15672:15671-15672" + - "5672:5672" mongo: image: mongo:7 @@ -18,18 +19,8 @@ services: MONGO_INITDB_DATABASE: backend volumes: - sunjad_db:/data/db - - server: - build: - dockerfile: Dockerfile - context: . - restart: unless-stopped - depends_on: - - rmq - - mongo - env_file: - - .env ports: - - 8006:8006 + - 27017:27017 + volumes: sunjad_db: \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..800696e --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,35 @@ +version: "3.9" + +services: + rmq: + image: rabbitmq:management-alpine + restart: unless-stopped + ports: + - "15671-15672:15671-15672" + + mongo: + image: mongo:7 + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: root-user + MONGO_INITDB_ROOT_PASSWORD: root-user + MONGO_INITDB_USERNAME: user + MONGO_INITDB_PASSWORD: user + MONGO_INITDB_DATABASE: backend + volumes: + - sunjad_db:/data/db + + server: + build: + dockerfile: Dockerfile + context: . + restart: unless-stopped + depends_on: + - rmq + - mongo + volumes: + - ./.env:/opt/app/.env:ro + ports: + - 8006:8006 +volumes: + sunjad_db: \ No newline at end of file diff --git a/init-mongo.sh b/scripts/init-mongo.sh similarity index 100% rename from init-mongo.sh rename to scripts/init-mongo.sh diff --git a/launch.sh b/scripts/launch.sh similarity index 100% rename from launch.sh rename to scripts/launch.sh diff --git a/mongo_dump.sh b/scripts/mongo_dump.sh similarity index 100% rename from mongo_dump.sh rename to scripts/mongo_dump.sh diff --git a/start.sh b/scripts/start.sh old mode 100755 new mode 100644 similarity index 100% rename from start.sh rename to scripts/start.sh diff --git a/start_db.sh b/start_db.sh deleted file mode 100755 index bdadfb3..0000000 --- a/start_db.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -MONGO_ADMIN_USER="admin" -MONGO_ADMIN_PASS="admin" - -docker run -d \ - --name ristek-mongo \ - --restart always \ - -p 37017:27017 \ - -e MONGO_INITDB_ROOT_USERNAME=$MONGO_ADMIN_USER \ - -e MONGO_INITDB_ROOT_PASSWORD=$MONGO_ADMIN_PASS \ - mongo