Merge pull request #36 from ristekoss/feat/py-upgrade

feat/py-upgrade: Clean up scripts, configurations, and fix README

.env.example

@@ -1,18 +1,18 @@
+SECRET_KEY=[secret key string]
+ACTIVE_PERIOD=[YYYY-T, for example: 2020-2]
+SSO_UI_FORCE_HTTPS=True
+
 # rabbit mq
 UPDATE_COURSE_LIST_EXCHANGE_NAME=update_course
-RABBIT_HOST=rmq
+RABBIT_HOST=rmq || localhost
 RABBIT_USERNAME=guest
-RABBIT_PASSWORD=[Rabbit MQ Password]
-
-SECRET_KEY=[secret_key_string]
-ACTIVE_PERIOD=[for example: 2020-2]
-SSO_UI_FORCE_HTTPS=True
+RABBIT_PASSWORD=[RabbitMQ password]
 
 # mongo
 MONGODB_DB=backend
-MONGODB_HOST=mongo
+MONGODB_HOST=mongo || localhost
 MONGODB_PORT=27017
-MONGODB_USERNAME=user
+MONGODB_USERNAME=[MongoDB username]
 MONGODB_PASSWORD=[MongoDB password]
 
 # sentry

.github/workflows/deploy-staging.yaml

@@ -2,6 +2,9 @@ name: Deploy (Staging)
 
 on:
   workflow_dispatch:
+  push:
+    branches:
+      - master
 
 jobs:
   deploy_service:

Dockerfile

@@ -6,14 +6,14 @@ ENV APP_ENV="container"
 
 COPY . .
 
-COPY launch.sh /opt/app/launch.sh
+COPY scripts/launch.sh /opt/app/launch.sh
 COPY sso/additional-info.json /opt/app/sso/additional-info.json
 COPY sso/faculty-base-additional-info.json /opt/app/sso/faculty-base-additional-info.json
 COPY sso/faculty_exchange_route.json /opt/app/sso/faculty_exchange_route.json
 
-RUN apk add -u --no-cache tzdata gcc musl-dev libxml2 libxslt-dev && \
-    pip install wheel && \
-    pip install -r requirements.txt
+RUN apk add -u --no-cache tzdata gcc musl-dev libxml2 libxslt-dev
+RUN pip install wheel
+RUN pip install -r requirements.txt
 
 ENV PORT=8006
 

README.md

@@ -1,127 +1,113 @@
 # Susun Jadwal
 
-Susun Jadwal is an open source tool to plan class schedules for university students.
+Susun Jadwal is an open source tool to plan class schedules for university students, 
+developed by Ristek Fasilkom UI. https://susunjadwal.cs.ui.ac.id/
 
-Susun Jadwal by Ristek Fasilkom UI. https://susunjadwal.cs.ui.ac.id/
+In the spirit of Open Source Software, *everyone* is welcome to contribute to Susun Jadwal! 
+See the Contributing Guide below for more.
 
-Monorepo setup with React frontend and Flask backend.
-
-## Structure explained
+## Structure
 
 ```
-app/                // general views
-models/             // mongoDB models
-scraper/            // courses (academic.ui.ac.id) scraper
-sso/                // SSO UI authentication logic
-README.md           // important info
-requirements.txt    // dependency list
-start.sh            // script to start server
-...
-README.md             // workspace-wide information shown in github
+app/                    // general views
+models/                 // mongoDB models
+scraper/                // courses (academic.ui.ac.id) scraper
+sso/                    // SSO UI authentication logic
+requirements.txt        // python dependency list
+scripts/                // (utility scripts)
+├── init-mongo.sh       // script to create non-root mongoDB user
+├── launch.sh           // main script to start flask
+├── mongo_dump.sh       // script to dump mongoDB data to .dump file
+└── start.sh            // alternative script to start flask
+.env.example            // template for .env file
+dev.docker-compose.yml  // docker-compose for mongo and rmq
+docker-compose.yml      // docker-compose for mongo, rmq, & server
 ```
 
 ## Contributing Guide
 
-Feel free to contribute by submitting a pull request.
+*Everyone* is welcome to contribute to Susun Jadwal!
+Feel free to make a contribution by submitting a pull request.
+You can also report bugs and request features / changes by creating a new 
+[Issue](https://github.com/ristekoss/susunjadwal-backend/issues/new).
+
+For in-depth discussion, please join RistekOSS's Discord.
 
-# Susun Jadwal Backend
+## Development
 
-## Requirements
+### Requirements
 
-1. `python 3.6` and `pip using`
+1. `python` (tested on 3.6 and 3.9.18), and `pip`
 2. `docker`
 
-## Configuration
+### Installing
 
-### Development
+The following steps will assume you have already set up a python virtual environment, 
+and will use `dev.docker-compose.yml`. 
 
-1. Create virtual environment using `python3 -m venv env`
-2. Activate virtualenv `source ./env/bin/activate`
-3. Install requirements `pip install -r requirements.txt`
-4. Add your credential to scrap schedule from SIAK in `scraper/credentials.json` with the following structure:
+1. Boot up MongoDB and RabbitMQ: 
+    ```docker-compose -f dev.docker-compose.yaml up```
 
-```
-{
-    "<kd_org>": {
-        "username": "<username>",
-        "password": "<password>"
-    }
-}
-```
+2. Populate `.env`, using `.env.example` as reference.
 
-You can also see `scraper/credentials.template.json` for example and `sso/additional-info.json` for list of `kd_org`.
+3. Connect to MongoDB and create a non-root user.
+    ```
+    mongosh -u <root-username>
+    // enter password when prompted
+    
+    use <db_name>;
+    db.createUser({user: "<MONGODB_USERNAME>", pwd: "<MONGODB_PASSWORD>", roles: ["readWrite"]});
+    // response should be '{ok: 1}', use these credentials in your .env secrets
+    ```
 
-5. Start database using `bash start_db.sh`
-6. Go to mongo console by running `docker exec -it ristek-mongo mongo -u <admin_username>`
-7. Create database by running `use <db_name>`. By default, Flask use database named `test` so it becomes `use test`
-8. Create user for database:
+4. Boot up the Flask server:
+    ```export PORT=8000 && bash scripts/launch.sh```
 
-```
-db.createUser(
-    {
-        user: "<db_user>",
-        pwd: "<db_pwd>",
-        roles:[
-            {
-                role: "readWrite",
-                db: "<db_name>"
-            }
-        ]
-    }
-);
-```
+5. Ping the API: `http://localhost:8000/susunjadwal/api/`
 
-You can quit mongo console now by using Ctrl + D.
+### Containerizing the Server
 
-9. Create config file, `instance/config.cfg`. You can see `instance/config.template.cfg` for example and edit db name, username, and password to match the one you created before
-10. Run `docker-compose up -d` to start the rabbit mq
-11. Create `.env` file from `.env.example` file
-12. Finally, run Flask by using `FLASK_ENV="development" flask run`
+#### For Local
 
-### Production
+1. Populate `.env`.
 
-#### Old
+2. Run `docker-compose.yml`.
 
-> We actually have a slightly different setup in the real Ristek server. For future maintainers, you may want to contact past contributors.
+#### For Deployment
 
-1. Do everything in development step **except** step no 10, running Flask. Don't forget to modify `instance/config.cfg`, `start_db.sh`, and `scraper/credentials.json` if you want to
-2. Run gunicorn using `bash start.sh`
-3. Set your Nginx (or other reverse proxy of your choice) to reverse proxy to `sunjad.sock`. For example, to reverse proxy `/susunjadwal/api` you can set
+While RISTEK uses a different standardized workflow, here is a general guide on deploying.
 
-```
-location ^~ /susunjadwal/api {
-    proxy_set_header Host $http_host;
-    proxy_set_header X-Real-IP $remote_addr;
-    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-    proxy_set_header X-Forwarded-Proto $scheme;
-    proxy_pass http://unix:/path/to/susunjadwal/backend/sunjad.sock;
-}
-```
+1. Populate `.env`.
+
+2. Replace line 23-25 in `docker-compose.yml` to pull from [Docker Hub](https://hub.docker.com/r/ristekoss/):
 
-4. Run the schedule scrapper cron job using `crontab -e` and add the line to run `cron.sh`. For example, to run it every 10 minutes add `*/10 * * * * bash /path/to/susunjadwal/backend/cron.sh`
+```image: ristekoss/susunjadwal-backend:stable```
 
-#### New
+3. Modify the credentials in `MONGO_INITDB_XX` environment variables.
+
+4. `docker compose up`
 
-**notes**: For deployment, SusunJadwal Backend is using **Ubuntu 18.04**. Here's the link to the marketplace https://aws.amazon.com/marketplace/pp/prodview-pkjqrkcfgcaog
 
-1. Do everything in development step **except** step no 5,6,7, and 10
-2. Create `config.cfg` and fill the DB credentials according the given specification in `docker-compose-deploy.yaml` (host must be `mongo`)
-3. Run `docker-compose -f docker-compose-deploy.yaml up -d` to execute mongodb, flask, and rabbitmq
-4. Run `docker exec -it susunjadwalbackend_mongo mongo -u root-user -p root-user` and create admin in `backend` db, then restart the mongo container
 
 ## Dump and Restore Database
 
 ### Dump
+1. Run the dump script: `bash mongo_dump.sh`
+2. The result will be a `.dump` file in the directory `./mongodump`.
+
+### Restore
+1. Copy dump file to MongoDB Container: `docker cp <path_to_dump_file> susunjadwalbackend_mongo_1:/<path_to_dump_file>`
+2. Load dump into DB: `docker exec -it <mongo_container_name> mongorestore -u <root_username> --archive=<file.dump>`
+3. Enter `<root_username>`'s password when prompted.
+4. You should see the success message: `XX documents successfully restored.`
 
-1. Run `bash mongo_dump.sh`
-2. The result will be on `./mongodump`
+## Deployment
 
-## Restore
+We use a standardized pipeline for all our products which we invoke from `.github/workflows/deploy-<env>.yaml`
 
-If you want to restore the database from .dump file
+## Legacy Version
 
-1. Copy dump file to mongo container, `docker cp <path_to_dump_file> susunjadwalbackend_mongo_1:/<path_to_dump_file>`
-2. Restore db using command, `docker exec -it susunjadwalbackend_mongo_1  mongorestore -u root-user -p root-user --archive=<file.dump>`
+To see the version of `susunjadwal-backend` which was maintained and deployed up until 2023, see [5b8f710](https://github.com/ristekoss/susunjadwal-backend/tree/5b8f71068b62a0f1f684c616cb8e40c087861725).
 
 ## License
 

app/__init__.py

@@ -87,11 +87,9 @@
 app.register_blueprint(cron)
 
 CORS(app)
-# print(f'MONGODB_DB:{os.environ.get("MONGODB_DB")}, MONGODB_PORT:{os.environ.get("MONGODB_PORT")}, MONGODB_USERNAME:{os.environ.get("MONGODB_USERNAME")} sad')
 MongoEngine(app)
 
 # Init connection to rabbit mq
-# print(f'RABBIT HOST: {os.environ.get("RABBIT_HOST")}, RABBIT USERNAME: {os.environ.get("RABBIT_USERNAME")}')
 init_pika(app)
 
 # Init consumer and create exchange

app/views/main.py

@@ -16,6 +16,15 @@
 
 router_main = Blueprint('router_sunjad', __name__)
 
+"""
+Basic ping / status check
+"""
+@router_main.route('/', methods=['GET'])
+def status():
+    return (jsonify({
+        "message": "susunjadwal is live!",
+    }), 200)
+
 """
 Provides course list by major kd_org.
 The kd_org list is provided in sso/additional_info.json
@@ -45,7 +54,9 @@ def get_courses_by_kd(major_kd_org):
 
     return (jsonify(period.serialize()), 200)
 
-
+'''
+Provides course list filtered by major ID.
+'''
 @router_main.route('/majors/<major_id>/courses', methods=['GET'])
 @require_jwt_token
 def get_courses(major_id):
@@ -183,6 +194,27 @@ def scrap_all_schedule():
     )
     return jsonify(response), status_code
 
+'''
+Provides all existing courses, filtered by major ID
+regardless active term/period
+'''
+@router_main.route('/majors/<major_id>/all_courses', methods=['GET'])
+def get_all_courses_by_major(major_id):
+    periods = Period.objects(
+        major_id=major_id
+    ).all()
+    all_courses = []
+    for period in periods:
+        for course in period.courses:
+            all_courses.append(course.serialize_ulas_kelas())
+    return (jsonify({
+        'courses': all_courses
+    }), 200)
+
+'''
+Provides all existing courses
+regardless of major or active term/period
+'''
 @router_main.route('/courses', methods=['GET'])
 def get_all_courses():
     active_period = get_app_config("ACTIVE_PERIOD")

dev.docker-compose.yaml

@@ -1,11 +1,12 @@
-version: "3.9"
+version: "3.7"
 
 services:
   rmq:
     image: rabbitmq:management-alpine
     restart: unless-stopped
     ports:
       - "15671-15672:15671-15672"
+      - "5672:5672"
 
   mongo:
     image: mongo:7
@@ -18,18 +19,8 @@ services:
       MONGO_INITDB_DATABASE: backend
     volumes:
       - sunjad_db:/data/db
-
-  server:
-    build:
-      dockerfile: Dockerfile
-      context: .
-    restart: unless-stopped
-    depends_on:
-      - rmq
-      - mongo
-    env_file:
-      - .env
     ports:
-      - 8006:8006
+      - 27017:27017
+
 volumes:
   sunjad_db: