Skip to content

Commit

Permalink
Implemented API for apply/feedback
Browse files Browse the repository at this point in the history
  • Loading branch information
kshitijrajsharma committed Aug 14, 2023
1 parent ee3bd1c commit 9e0a351
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 78 deletions.
28 changes: 23 additions & 5 deletions backend/core/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,12 +120,18 @@ class Meta:
# read_only_fields = ("created_at", "osm_id")


class LabelFileSerializer(
GeoFeatureModelSerializer
): # serializers are used to translate models objects to api
class LabelFileSerializer(GeoFeatureModelSerializer):
class Meta:
model = Label
geo_field = "geom" # this will be used as geometry in order to create geojson api , geofeatureserializer will let you create api in geojson
geo_field = "geom"
# auto_bbox = True
fields = ("osm_id",)


class FeedbackLabelFileSerializer(GeoFeatureModelSerializer):
class Meta:
model = FeedbackLabel
geo_field = "geom"
# auto_bbox = True
fields = ("osm_id",)

Expand Down Expand Up @@ -158,7 +164,19 @@ class FeedbackParamSerializer(serializers.Serializer):
training_id = serializers.IntegerField(required=True)
epochs = serializers.IntegerField(required=False)
batch_size = serializers.IntegerField(required=False)
freeze_layers = serializers.BooleanField(required=False)
zoom_level = serializers.ListField(required=False)

def validate(self, data):
"""
Check supplied data
"""
if "zoom_level" in data:
for i in data["zoom_level"]:
if int(i) < 19 or int(i) > 21:
raise serializers.ValidationError(
"Zoom level Supported between 19-21"
)
return data


class PredictionParamSerializer(serializers.Serializer):
Expand Down
94 changes: 32 additions & 62 deletions backend/core/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,12 @@
import ramp.utils
import tensorflow as tf
from celery import shared_task
from core.models import AOI, Feedback, Label, Training
from core.serializers import FeedbackFileSerializer, LabelFileSerializer
from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training
from core.serializers import (
FeedbackFileSerializer,
FeedbackLabelFileSerializer,
LabelFileSerializer,
)
from core.utils import bbox, download_imagery, get_start_end_download_coords
from django.conf import settings
from django.contrib.gis.db.models.aggregates import Extent
Expand Down Expand Up @@ -65,34 +69,32 @@ def train_model(
shutil.rmtree(training_input_image_source)
os.makedirs(training_input_image_source)
if feedback:
feedback_objects = Feedback.objects.filter(
training__id=feedback,
validated=True,
)
bbox_feedback = feedback_objects.aggregate(Extent("geom"))[
"geom__extent"
]
bbox_geo = GEOSGeometry(
f"POLYGON(({bbox_feedback[0]} {bbox_feedback[1]},{bbox_feedback[2]} {bbox_feedback[1]},{bbox_feedback[2]} {bbox_feedback[3]},{bbox_feedback[0]} {bbox_feedback[3]},{bbox_feedback[0]} {bbox_feedback[1]}))"
)
print(training_input_image_source)
print(bbox_feedback)
with open(
os.path.join(training_input_image_source, "labels_bbox.geojson"),
"w",
encoding="utf-8",
) as f:
f.write(bbox_geo.geojson)
try:
aois = FeedbackAOI.objects.filter(training=feedback)
except FeedbackAOI.DoesNotExist:
raise ValueError(
f"No Feedback AOI is attached with supplied training id:{dataset_id}, Create AOI first",
)

else:
try:
aois = AOI.objects.filter(dataset=dataset_id)
except AOI.DoesNotExist:
raise ValueError(
f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first",
)

for obj in aois:
bbox_coords = bbox(obj.geom.coords[0])
for z in zoom_level:
zm_level = z
print(
f"""Running Download process for
feedback {training_id} - dataset : {dataset_id} , zoom : {zm_level}"""
aoi : {obj.id} - dataset : {dataset_id} , zoom : {zm_level}"""
)
try:
tile_size = DEFAULT_TILE_SIZE # by default
bbox_coords = list(bbox_feedback)

start, end = get_start_end_download_coords(
bbox_coords, zm_level, tile_size
)
Expand All @@ -107,49 +109,17 @@ def train_model(
except Exception as ex:
raise ex

else:
try:
aois = AOI.objects.filter(dataset=dataset_id)
except AOI.DoesNotExist:
raise ValueError(
f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first",
)

for obj in aois:
bbox_coords = bbox(obj.geom.coords[0])
for z in zoom_level:
zm_level = z
print(
f"""Running Download process for
aoi : {obj.id} - dataset : {dataset_id} , zoom : {zm_level}"""
)
try:
tile_size = DEFAULT_TILE_SIZE # by default

start, end = get_start_end_download_coords(
bbox_coords, zm_level, tile_size
)
# start downloading
download_imagery(
start,
end,
zm_level,
base_path=training_input_image_source,
source=source_imagery,
)
except Exception as ex:
raise ex

## -----------LABEL GENERATOR---------
logging.debug("Label Generator started")
logging.info("Label Generator started")
aoi_list = [r.id for r in aois]
logging.info(aoi_list)

if feedback:
feedback_objects = Feedback.objects.filter(
training__id=feedback,
validated=True,
)
serialized_field = FeedbackFileSerializer(feedback_objects, many=True)
label = FeedbackLabel.objects.filter(feedback_aoi__in=aoi_list)
logging.info(label)

serialized_field = FeedbackLabelFileSerializer(label, many=True)
else:
aoi_list = [r.id for r in aois]
label = Label.objects.filter(aoi__in=aoi_list)
serialized_field = LabelFileSerializer(label, many=True)

Expand Down
22 changes: 13 additions & 9 deletions backend/core/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,8 @@ class LabelViewSet(viewsets.ModelViewSet):


class RawdataApiFeedbackView(APIView):
# authentication_classes = [OsmAuthentication]
# permission_classes = [IsOsmAuthenticated]
authentication_classes = [OsmAuthentication]
permission_classes = [IsOsmAuthenticated]

def post(self, request, feedbackaoi_id, *args, **kwargs):
"""Downloads available osm data as labels within given feedback aoi
Expand Down Expand Up @@ -408,6 +408,15 @@ def run_task_status(request, run_id: str):


class FeedbackView(APIView):
"""Applies Associated feedback to Training Published Checkpoint
Args:
APIView (_type_): _description_
Returns:
_type_: _description_
"""

authentication_classes = [OsmAuthentication]
permission_classes = [IsOsmAuthenticated]

Expand All @@ -421,12 +430,7 @@ def post(self, request, *args, **kwargs):
training_id = deserialized_data["training_id"]
training_instance = Training.objects.get(id=training_id)

unique_zoom_levels = (
Feedback.objects.filter(training__id=training_id, validated=True)
.values("zoom_level")
.distinct()
)
zoom_level = [z["zoom_level"] for z in unique_zoom_levels]
zoom_level = deserialized_data.get("zoom_level", [19, 20])
epochs = deserialized_data.get("epochs", 20)
batch_size = deserialized_data.get("batch_size", 8)
instance = Training.objects.create(
Expand All @@ -448,7 +452,7 @@ def post(self, request, *args, **kwargs):
zoom_level=instance.zoom_level,
source_imagery=instance.source_imagery,
feedback=training_id,
freeze_layers=instance.freeze_layers,
freeze_layers=True, # True by default for feedback
)
if not instance.source_imagery:
instance.source_imagery = instance.model.dataset.source_imagery
Expand Down
4 changes: 2 additions & 2 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ services:
volumes:
- ./backend:/app
- ${RAMP_HOME}:/RAMP_HOME
# - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE
- ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE
depends_on:
- redis
- postgres
Expand All @@ -55,7 +55,7 @@ services:
volumes:
- ./backend:/app
- ${RAMP_HOME}:/RAMP_HOME
# - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE
- ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE
depends_on:
- backend-api
- redis
Expand Down

0 comments on commit 9e0a351

Please sign in to comment.