Merge pull request #272 from FIRST-Tech-Challenge/pr_separate_app_engine

Reduced time for deploying to app engine

.github/workflows/prod-deploy.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Write Properties
         run: |
           sha=$(git rev-list --tags --max-count=1)
-          echo "{ \"version\" : \"$(git describe --tags $sha)\" }" > server/app.properties
+          echo "{ \"version\" : \"$(git describe --tags $sha)\" }" > server/app_engine/app.properties
 
       - name: Closure Build
         run: ./gradlew compileJavascript
@@ -81,7 +81,7 @@ jobs:
       - name: Write Properties
         run: |
           sha=$(git rev-list --tags --max-count=1)
-          echo "{ \"version\" : \"$(git describe --tags $sha)\" }" > server/app.properties
+          echo "{ \"version\" : \"$(git describe --tags $sha)\" }" > server/app_engine/app.properties
 
       - name: Closure Build
         run: ./gradlew compileJavascript

.github/workflows/terraform.yml

@@ -29,7 +29,7 @@ jobs:
 
     - name: Write Properties
       run: |
-        echo "{ \"version\" : \"$(git rev-parse --short $GITHUB_SHA) \" }" > server/app.properties
+        echo "{ \"version\" : \"$(git rev-parse --short $GITHUB_SHA) \" }" > server/app_engine/app.properties
 
     - name: Closure Build
       run: ./gradlew compileJavascript

.gitignore

@@ -1,7 +1,6 @@
 .index_updated
 __pycache__/
 compiled/*
-env_variables.yaml
 flask_secret_key.txt
 client_secrets.json
 key.json

README.md

@@ -173,7 +173,7 @@ gcloud auth configure-docker
 Do you want to continue (Y/n)?  y
 ```
 
-## Fill in the values in server/env_variables.yaml
+## Fill in the values in server/app_engine/env_variables.yaml
 
 1. Replace `<YOUR-PROJECT-ID>` with the Google Cloud Project ID for your project.
 1. Replace `<YOUR-ORIGIN>` with the base URL that will serve the website.
@@ -185,11 +185,11 @@ Do you want to continue (Y/n)?  y
 The following command will set the version of fmltc displayed to the user to the current git commit hash.
 
 ```
-echo "{ \"version\": \"$(git rev-parse --short HEAD)\" }" > server/app.properties
+echo "{ \"version\": \"$(git rev-parse --short HEAD)\" }" > server/app_engine/app.properties
 ```
 If you wish the version to be set to something other than the current git commit hash you can use the following code replacing `<VERSION_NAME>` with the desired name of your version
 ```
-echo "{ \"version\": \"<VERSION_NAME>\" }" > server/app.properties
+echo "{ \"version\": \"<VERSION_NAME>\" }" > server/app_engine/app.properties
 ```
 
 ## Setup the environment.
@@ -209,7 +209,7 @@ source env_setup.sh
     scripts/deploy_indexes.sh
     ```
 
-1. Deploy the static content (the CSS styles and the favicon).
+1. Deploy the static content.
     ```
     source env_setup.sh
     scripts/deploy_static.sh

scripts/deploy_cloud_function.sh

@@ -6,8 +6,9 @@ fi
 
 pushd server
 gcloud functions deploy perform_action \
+    --ignore-file=.cloud_function_ignore \
     --runtime=python39 \
-    --set-env-vars PROJECT_ID=${FMLTC_GCLOUD_PROJECT_ID} \
+    --set-env-vars=PROJECT_ID=${FMLTC_GCLOUD_PROJECT_ID} \
     --memory=8192MB \
     --timeout=540 \
     --trigger-resource=${FMLTC_GCLOUD_PROJECT_ID}-action-parameters \

scripts/deploy_gae.sh

@@ -4,6 +4,9 @@ if [[ "${FMLTC_GCLOUD_PROJECT_ID}" == "" ]]; then
 fi
 
 
-pushd server
-gcloud -q app deploy --version v1
-popd
+pushd server/app_engine
+gcloud -q app deploy \
+    --ignore-file=.app_engine_ignore \
+    --appyaml=app.yaml \
+    --version v1
+popd

server/.cloud_function_ignore

@@ -0,0 +1,26 @@
+.cloud_function_ignore
+client_secrets.json
+flask_secret_key.txt
+key.json
+teams
+
+__pycache__/
+app_engine/.app_engine_ignore
+app_engine/announcements.py
+app_engine/app.yaml
+app_engine/app_engine.py
+app_engine/config.py
+app_engine/credentialstore.py
+app_engine/dataset_producer.py
+app_engine/dataset_zipper.py
+app_engine/env_variables.yaml
+app_engine/oidc.py
+app_engine/requirements.txt
+app_engine/roles.py
+app_engine/team_info.py
+app_engine/test_routes.py
+app_engine/tracking.py
+app_engine/wrappers.py
+src/
+static/
+templates/

server/.gcloudignore

@@ -10,7 +10,7 @@
 .git
 .gitignore
 .index_updated
-env_variables.yaml
+app_engine/env_variables.yaml
 flask_secret_key.txt
 client_secrets.json
 key.json

server/app_engine/.app_engine_ignore

@@ -0,0 +1,4 @@
+app.yaml
+env_variables.yaml
+
+__pycache__/

server/app_engine/__init__.py

@@ -0,0 +1,5 @@
+# The python files in this directory are used in our App Engine service as well as our Cloud
+# Functions.
+
+# For relative imports to work.
+import os, sys; sys.path.append(os.path.dirname(os.path.realpath(__file__)))

server/action.py → server/app_engine/action.py

@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2022 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,22 +18,14 @@
 from datetime import datetime, timedelta, timezone
 import json
 import logging
-import time
-import traceback
 import uuid
 
 # Other Modules
 import psutil
 
 # My Modules
 import constants
-import dataset_producer
-import dataset_zipper
-import frame_extractor
-import model_trainer
 import storage
-import tflite_creator
-import tracking
 import util
 
 BUCKET_ACTION_PARAMETERS = ('%s-action-parameters' % constants.PROJECT_ID)
@@ -70,7 +62,8 @@
 
 def create_action_parameters(team_uuid, action_name):
     if (action_name == ACTION_NAME_RESET_REMAINING_TRAINING_MINUTES or
-            action_name == ACTION_NAME_INCREMENT_REMAINING_TRAINING_MINUTES):
+            action_name == ACTION_NAME_INCREMENT_REMAINING_TRAINING_MINUTES or
+            action_name == ACTION_NAME_TEST):
         is_admin_action = True
     else:
         is_admin_action = False
@@ -140,55 +133,6 @@ def trigger_action_via_blob(action_parameters_arg):
     blob.upload_from_string(action_parameters_json, content_type="text/json")
     return action_parameters[ACTION_UUID]
 
-def perform_action_from_blob(action_parameters_blob_name, time_limit):
-    blob = util.storage_client().get_bucket(BUCKET_ACTION_PARAMETERS).blob(action_parameters_blob_name)
-    # If the blob no longer exists, this event is a duplicate and is ignored.
-    if blob.exists():
-        action_parameters_json = blob.download_as_string()
-        blob.delete()
-        action_parameters = json.loads(action_parameters_json)
-        __perform_action(action_parameters, time_limit)
-
-
-def __perform_action(action_parameters, time_limit):
-    action_parameters[ACTION_TIME_LIMIT] = time_limit
-    logging.info('action.__perform_action - %s - start' % action_parameters[ACTION_NAME])
-    storage.action_on_start(action_parameters[ACTION_UUID], action_parameters[ACTION_IS_ADMIN_ACTION])
-
-    action_fns = {
-        ACTION_NAME_TEST: test, # For debugging purposes only
-        ACTION_NAME_WAIT_FOR_VIDEO_UPLOAD: frame_extractor.wait_for_video_upload,
-        ACTION_NAME_FRAME_EXTRACTION: frame_extractor.extract_frames,
-        ACTION_NAME_TRACKING: tracking.start_tracking,
-        ACTION_NAME_DATASET_PRODUCE: dataset_producer.produce_dataset,
-        ACTION_NAME_DATASET_PRODUCE_RECORD: dataset_producer.produce_dataset_record,
-        ACTION_NAME_DELETE_DATASET_RECORD_WRITERS: storage.finish_delete_dataset_record_writers,
-        ACTION_NAME_DATASET_ZIP: dataset_zipper.zip_dataset,
-        ACTION_NAME_DATASET_ZIP_PARTITION: dataset_zipper.zip_dataset_partition,
-        ACTION_NAME_MONITOR_TRAINING: model_trainer.monitor_training,
-        ACTION_NAME_CREATE_TFLITE: tflite_creator.create_tflite,
-        ACTION_NAME_DELETE_MODEL: storage.finish_delete_model,
-        ACTION_NAME_DELETE_DATASET: storage.finish_delete_dataset,
-        ACTION_NAME_DELETE_VIDEO: storage.finish_delete_video,
-        ACTION_NAME_RESET_REMAINING_TRAINING_MINUTES: storage.reset_remaining_training_minutes,
-        ACTION_NAME_INCREMENT_REMAINING_TRAINING_MINUTES: storage.increment_remaining_training_minutes,
-    }
-    action_fn = action_fns.get(action_parameters[ACTION_NAME], None)
-    if action_fn is not None:
-        try:
-            action_fn(action_parameters)
-        except Stop as e:
-            pass
-        except:
-            logging.critical('action.__perform_action - %s exception!!! action_parameters: %s traceback: %s' %
-                (action_parameters[ACTION_NAME], str(action_parameters), traceback.format_exc().replace('\n', ' ... ')))
-    else:
-        logging.warning('action.__perform_action - %s - action_fn is None' % action_parameters[ACTION_NAME])
-
-    if ACTION_RETRIGGERED not in action_parameters:
-        logging.info('action.__perform_action - %s - finish' % action_parameters[ACTION_NAME])
-        storage.action_on_finish(action_parameters[ACTION_UUID], action_parameters[ACTION_IS_ADMIN_ACTION], action_parameters)
-
 
 def retrigger_now(action_parameters):
     if ACTION_RETRIGGERED not in action_parameters:

server/app.properties → server/app_engine/app.properties

@@ -1,3 +1,3 @@
 {
   "version": "development"
-}
+}

server/app.yaml → server/app_engine/app.yaml

@@ -22,12 +22,12 @@ handlers:
 - url: /favicon.ico
   static_files: static/favicon.ico
   upload: static/favicon.ico
-  expiration: 1d
+  expiration: 0s
 
-- url: /WithSupportFromGoogleCloud.png
-  static_files: static/WithSupportFromGoogleCloud.png
-  upload: static/WithSupportFromGoogleCloud.png
-  expiration: 1m
+- url: /robots.txt
+  static_files: static/robots.txt
+  upload: static/robots.txt
+  expiration: 0s
 
 - url: /.*
   script: auto

server/app_engine.py → server/app_engine/app_engine.py

@@ -35,6 +35,11 @@
 import bbox_writer
 import blob_storage
 import cloud_secrets
+import config
+from config import KEY_TRAINING_ENABLED
+from config import KEY_USE_TPU
+from config import KEY_SECURE_SESSION_COOKIES
+from config import KEY_SAMESITE_SESSION_COOKIES
 import constants
 import dataset_producer
 import dataset_zipper
@@ -45,11 +50,6 @@
 import oidc
 import roles
 from roles import Role
-import config
-from config import KEY_TRAINING_ENABLED
-from config import KEY_USE_TPU
-from config import KEY_SECURE_SESSION_COOKIES
-from config import KEY_SAMESITE_SESSION_COOKIES
 import storage
 import team_info
 import test_routes
@@ -583,7 +583,7 @@ def monitor_training():
 @login_required
 @roles_accepted(roles.Role.GLOBAL_ADMIN, roles.Role.ML_DEVELOPER)
 def admin():
-    return flask.render_template('admin.html', config=config.config, max_mins=team_info.TOTAL_TRAINING_MINUTES_PER_TEAM)
+    return flask.render_template('admin.html', config=config.config, max_mins=constants.TOTAL_TRAINING_MINUTES_PER_TEAM)
 
 
 @app.route('/refreshConfig', methods=['POST'])
@@ -1511,7 +1511,7 @@ def resources():
 def reset_remaining_training_minutes():
     data = validate_keys(flask.request.form.to_dict(flat=True),
         ['reset_minutes', 'date_time_string'])
-    reset_minutes = validate_int(data.get('reset_minutes'), min=1, max=team_info.TOTAL_TRAINING_MINUTES_PER_TEAM)
+    reset_minutes = validate_int(data.get('reset_minutes'), min=1, max=constants.TOTAL_TRAINING_MINUTES_PER_TEAM)
     action_parameters = action.create_action_parameters(
         '', action.ACTION_NAME_RESET_REMAINING_TRAINING_MINUTES)
     action_parameters['reset_minutes'] = reset_minutes

server/bbox_writer.py → server/app_engine/bbox_writer.py

@@ -24,8 +24,8 @@
 import numpy as np
 
 # My Modules
-import exceptions
 import constants
+import exceptions
 
 
 def __convert_bbox_to_text(bbox, scale, x_max, y_max):

server/blob_storage.py → server/app_engine/blob_storage.py

@@ -20,7 +20,6 @@
 import re
 import time
 import traceback
-import uuid
 
 # My Modules
 import action

server/config.py → server/app_engine/config.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from google.cloud import datastore
+
 import constants
 import redis
 

server/constants.py → server/app_engine/constants.py

@@ -28,6 +28,10 @@
 # Expects to be 'development' or 'production'
 ENVIRONMENT = os.getenv('ENVIRONMENT')
 
+
+TOTAL_TRAINING_MINUTES_PER_TEAM = 300
+
+
 # Limits
 MAX_DESCRIPTION_LENGTH = 30
 MAX_VIDEOS_PER_TEAM = 50

server/credentialstore.py → server/app_engine/credentialstore.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import redis
+
 import constants
 
 class CredentialStore():

server/app_engine/dataset_producer.py

@@ -0,0 +1,37 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "[email protected] (Liz Looney)"
+
+# My Modules
+import action
+import storage
+
+
+def prepare_to_start_dataset_production(team_uuid, description, video_uuid_list, eval_percent, create_time_ms):
+    # storage.prepare_to_start_dataset_production will raise HttpErrorNotFound
+    # if any of the team_uuid/video_uuids is not found or if none of the videos have labeled frames.
+    dataset_uuid = storage.prepare_to_start_dataset_production(team_uuid, description,
+        video_uuid_list, eval_percent, create_time_ms)
+    return dataset_uuid
+
+def make_action_parameters(team_uuid, dataset_uuid, video_uuid_list, eval_percent, create_time_ms):
+    action_parameters = action.create_action_parameters(
+        team_uuid, action.ACTION_NAME_DATASET_PRODUCE)
+    action_parameters['team_uuid'] = team_uuid
+    action_parameters['dataset_uuid'] = dataset_uuid
+    action_parameters['video_uuid_list'] = video_uuid_list
+    action_parameters['eval_percent'] = eval_percent
+    action_parameters['create_time_ms'] = create_time_ms
+    return action_parameters