From 9e0a3510dd92ca0a5b2053f51bcef03ed372ffb5 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Aug 2023 20:42:18 +0545 Subject: [PATCH] Implemented API for apply/feedback --- backend/core/serializers.py | 28 +++++++++-- backend/core/tasks.py | 94 +++++++++++++------------------------ backend/core/views.py | 22 +++++---- docker-compose.yml | 4 +- 4 files changed, 70 insertions(+), 78 deletions(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 223413e5..9d655aca 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -120,12 +120,18 @@ class Meta: # read_only_fields = ("created_at", "osm_id") -class LabelFileSerializer( - GeoFeatureModelSerializer -): # serializers are used to translate models objects to api +class LabelFileSerializer(GeoFeatureModelSerializer): class Meta: model = Label - geo_field = "geom" # this will be used as geometry in order to create geojson api , geofeatureserializer will let you create api in geojson + geo_field = "geom" + # auto_bbox = True + fields = ("osm_id",) + + +class FeedbackLabelFileSerializer(GeoFeatureModelSerializer): + class Meta: + model = FeedbackLabel + geo_field = "geom" # auto_bbox = True fields = ("osm_id",) @@ -158,7 +164,19 @@ class FeedbackParamSerializer(serializers.Serializer): training_id = serializers.IntegerField(required=True) epochs = serializers.IntegerField(required=False) batch_size = serializers.IntegerField(required=False) - freeze_layers = serializers.BooleanField(required=False) + zoom_level = serializers.ListField(required=False) + + def validate(self, data): + """ + Check supplied data + """ + if "zoom_level" in data: + for i in data["zoom_level"]: + if int(i) < 19 or int(i) > 21: + raise serializers.ValidationError( + "Zoom level Supported between 19-21" + ) + return data class PredictionParamSerializer(serializers.Serializer): diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 8f020b81..4cd34642 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -10,8 +10,12 @@ import ramp.utils import tensorflow as tf from celery import shared_task -from core.models import AOI, Feedback, Label, Training -from core.serializers import FeedbackFileSerializer, LabelFileSerializer +from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training +from core.serializers import ( + FeedbackFileSerializer, + FeedbackLabelFileSerializer, + LabelFileSerializer, +) from core.utils import bbox, download_imagery, get_start_end_download_coords from django.conf import settings from django.contrib.gis.db.models.aggregates import Extent @@ -65,34 +69,32 @@ def train_model( shutil.rmtree(training_input_image_source) os.makedirs(training_input_image_source) if feedback: - feedback_objects = Feedback.objects.filter( - training__id=feedback, - validated=True, - ) - bbox_feedback = feedback_objects.aggregate(Extent("geom"))[ - "geom__extent" - ] - bbox_geo = GEOSGeometry( - f"POLYGON(({bbox_feedback[0]} {bbox_feedback[1]},{bbox_feedback[2]} {bbox_feedback[1]},{bbox_feedback[2]} {bbox_feedback[3]},{bbox_feedback[0]} {bbox_feedback[3]},{bbox_feedback[0]} {bbox_feedback[1]}))" - ) - print(training_input_image_source) - print(bbox_feedback) - with open( - os.path.join(training_input_image_source, "labels_bbox.geojson"), - "w", - encoding="utf-8", - ) as f: - f.write(bbox_geo.geojson) + try: + aois = FeedbackAOI.objects.filter(training=feedback) + except FeedbackAOI.DoesNotExist: + raise ValueError( + f"No Feedback AOI is attached with supplied training id:{dataset_id}, Create AOI first", + ) + + else: + try: + aois = AOI.objects.filter(dataset=dataset_id) + except AOI.DoesNotExist: + raise ValueError( + f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first", + ) + for obj in aois: + bbox_coords = bbox(obj.geom.coords[0]) for z in zoom_level: zm_level = z print( f"""Running Download process for - feedback {training_id} - dataset : {dataset_id} , zoom : {zm_level}""" + aoi : {obj.id} - dataset : {dataset_id} , zoom : {zm_level}""" ) try: tile_size = DEFAULT_TILE_SIZE # by default - bbox_coords = list(bbox_feedback) + start, end = get_start_end_download_coords( bbox_coords, zm_level, tile_size ) @@ -107,49 +109,17 @@ def train_model( except Exception as ex: raise ex - else: - try: - aois = AOI.objects.filter(dataset=dataset_id) - except AOI.DoesNotExist: - raise ValueError( - f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first", - ) - - for obj in aois: - bbox_coords = bbox(obj.geom.coords[0]) - for z in zoom_level: - zm_level = z - print( - f"""Running Download process for - aoi : {obj.id} - dataset : {dataset_id} , zoom : {zm_level}""" - ) - try: - tile_size = DEFAULT_TILE_SIZE # by default - - start, end = get_start_end_download_coords( - bbox_coords, zm_level, tile_size - ) - # start downloading - download_imagery( - start, - end, - zm_level, - base_path=training_input_image_source, - source=source_imagery, - ) - except Exception as ex: - raise ex - ## -----------LABEL GENERATOR--------- - logging.debug("Label Generator started") + logging.info("Label Generator started") + aoi_list = [r.id for r in aois] + logging.info(aoi_list) + if feedback: - feedback_objects = Feedback.objects.filter( - training__id=feedback, - validated=True, - ) - serialized_field = FeedbackFileSerializer(feedback_objects, many=True) + label = FeedbackLabel.objects.filter(feedback_aoi__in=aoi_list) + logging.info(label) + + serialized_field = FeedbackLabelFileSerializer(label, many=True) else: - aoi_list = [r.id for r in aois] label = Label.objects.filter(aoi__in=aoi_list) serialized_field = LabelFileSerializer(label, many=True) diff --git a/backend/core/views.py b/backend/core/views.py index 765f1485..66b3fc63 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -244,8 +244,8 @@ class LabelViewSet(viewsets.ModelViewSet): class RawdataApiFeedbackView(APIView): - # authentication_classes = [OsmAuthentication] - # permission_classes = [IsOsmAuthenticated] + authentication_classes = [OsmAuthentication] + permission_classes = [IsOsmAuthenticated] def post(self, request, feedbackaoi_id, *args, **kwargs): """Downloads available osm data as labels within given feedback aoi @@ -408,6 +408,15 @@ def run_task_status(request, run_id: str): class FeedbackView(APIView): + """Applies Associated feedback to Training Published Checkpoint + + Args: + APIView (_type_): _description_ + + Returns: + _type_: _description_ + """ + authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] @@ -421,12 +430,7 @@ def post(self, request, *args, **kwargs): training_id = deserialized_data["training_id"] training_instance = Training.objects.get(id=training_id) - unique_zoom_levels = ( - Feedback.objects.filter(training__id=training_id, validated=True) - .values("zoom_level") - .distinct() - ) - zoom_level = [z["zoom_level"] for z in unique_zoom_levels] + zoom_level = deserialized_data.get("zoom_level", [19, 20]) epochs = deserialized_data.get("epochs", 20) batch_size = deserialized_data.get("batch_size", 8) instance = Training.objects.create( @@ -448,7 +452,7 @@ def post(self, request, *args, **kwargs): zoom_level=instance.zoom_level, source_imagery=instance.source_imagery, feedback=training_id, - freeze_layers=instance.freeze_layers, + freeze_layers=True, # True by default for feedback ) if not instance.source_imagery: instance.source_imagery = instance.model.dataset.source_imagery diff --git a/docker-compose.yml b/docker-compose.yml index 80dfc78e..263647eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,7 @@ services: volumes: - ./backend:/app - ${RAMP_HOME}:/RAMP_HOME - # - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE depends_on: - redis - postgres @@ -55,7 +55,7 @@ services: volumes: - ./backend:/app - ${RAMP_HOME}:/RAMP_HOME - # - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE depends_on: - backend-api - redis