From 4237aa63f0c89e1eeb8df01bda2524f39682b2b5 Mon Sep 17 00:00:00 2001 From: mark doerr Date: Sat, 27 May 2023 12:01:03 +0200 Subject: [PATCH 1/5] fix: using --directory to set protoc output path --- django_socio_grpc/management/commands/generateproto.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django_socio_grpc/management/commands/generateproto.py b/django_socio_grpc/management/commands/generateproto.py index 351ee4ea..165bd223 100644 --- a/django_socio_grpc/management/commands/generateproto.py +++ b/django_socio_grpc/management/commands/generateproto.py @@ -106,8 +106,10 @@ def handle(self, *args, **options): if self.directory: file_path = self.directory / f"{app_name}.proto" + proto_path = self.directory else: file_path = registry.get_proto_path() + proto_path = registry.get_grpc_folder() file_path.parent.mkdir(parents=True, exist_ok=True) self.check_or_write(file_path, proto, registry.app_name) @@ -115,7 +117,7 @@ def handle(self, *args, **options): if not settings.BASE_DIR: raise ProtobufGenerationException(detail="No BASE_DIR in settings") os.system( - f"python -m grpc_tools.protoc --proto_path={settings.BASE_DIR} --python_out=./ --grpc_python_out=./ {file_path}" + f"python -m grpc_tools.protoc --proto_path={proto_path} --python_out={proto_path} --grpc_python_out={proto_path} {file_path}" ) def check_or_write(self, file: Path, proto, app_name): From 76c5bae0de0b5bbe1f960612bc91c8b4560c7ad0 Mon Sep 17 00:00:00 2001 From: mark doerr Date: Mon, 29 May 2023 21:49:48 +0200 Subject: [PATCH 2/5] feat: first version of gRPC auto interface generation working nicely, protoc import bug needs to be fixed --- .../commands/generategrpcinterface.py | 182 ++++++++++++++++++ .../management/commands/generateproto.py | 13 ++ 2 files changed, 195 insertions(+) create mode 100644 django_socio_grpc/management/commands/generategrpcinterface.py diff --git a/django_socio_grpc/management/commands/generategrpcinterface.py b/django_socio_grpc/management/commands/generategrpcinterface.py new file mode 100644 index 00000000..87011fc7 --- /dev/null +++ b/django_socio_grpc/management/commands/generategrpcinterface.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +""" +The Django socio gRPC interface Generator is a which can automatically generate +(scaffold) a Django grpc interface for you. By doing this it will introspect your +models and automatically generate an table with properties like: + + - `fields` for all local fields + +""" + +import re +import os +import logging + +from django.apps import apps +from django.conf import settings +from django.core.management.base import LabelCommand, CommandError +from django.db import models + + +# Configurable constants +MAX_LINE_WIDTH = getattr(settings, 'MAX_LINE_WIDTH', 120) +INDENT_WIDTH = getattr(settings, 'INDENT_WIDTH', 4) + + +class Command(LabelCommand): + help = '''Generate all required gRPC interface files, like serializers, services and `handlers.py` for the given app (models)''' + # args = "[app_name]" + can_import_settings = True + + def add_arguments(self, parser): + parser.add_argument('app_name', help='Name of the app to generate the gRPC interface for') + #parser.add_argument('model_name', nargs='*') + + #@signalcommand + def handle(self, *args, **options): + self.app_name = options['app_name'] + + logging.warning("!! only a scaffold is generated, please check/add content to the generated files !!!") + + try: + app = apps.get_app_config(self.app_name) + except LookupError: + self.stderr.write('This command requires an existing app name as argument') + self.stderr.write('Available apps:') + app_labels = [app.label for app in apps.get_app_configs()] + for label in sorted(app_labels): + self.stderr.write(' %s' % label) + return + + model_res = [] + # for arg in options['model_name']: + # model_res.append(re.compile(arg, re.IGNORECASE)) + + GRPCInterfaceApp(app, model_res, **options) + + #self.stdout.write() + + +class GRPCInterfaceApp(): + def __init__(self, app_config, model_res, **options): + self.app_config = app_config + self.model_res = model_res + self.options = options + self.app_name = options['app_name'] + + self.serializers_str = "" + self.services_str = "" + self.handler_str = "" + self.model_names = [model.__name__ for model in self.app_config.get_models()] + + self.generate_serializers() + self.generate_services() + self.generate_handlers() + + + def generate_serializer_imports(self): + self.serializers_str += f"""## generated with django-socio-grpc generateprpcinterface {self.app_name} (LARA-version) + +import logging + +from django_socio_grpc import proto_serializers +#import {self.app_name}.grpc.{self.app_name}_pb2 as {self.app_name}_pb2 + +from {self.app_name}.models import {', '.join(self.model_names)}\n\n""" + + def generate_serializers(self): + self.generate_serializer_imports() + + # generate serializer classes + + for model in self.app_config.get_models(): + fields = [field.name for field in model._meta.fields if "_id" not in field.name] + # fields_param_str = ", ".join([f"{field}=None" for field in fields]) + # fields_str = ",".join([f"\n{4 * INDENT_WIDTH * ' '}'{field}'" for field in fields]) + fields_str = ", ".join([f"{field}'" for field in fields]) + + self.serializers_str += f"""class {model.__name__.capitalize()}ProtoSerializer(proto_serializers.ModelProtoSerializer): + class Meta: + model = {model.__name__} + # proto_class = {self.app_name}_pb2.{model.__name__.capitalize()}Response \n + # proto_class_list = {self.app_name}_pb2.{model.__name__.capitalize()}ListResponse \n + + fields = '__all__' # [{fields_str}] \n\n""" + + # check, if serializer.py exists + # then ask, if we should append to file + + if os.path.isfile("serializers.py"): + append = input("serializers.py already exists, append to file? (y/n) ") + if append.lower() == "y": + with open("serializers.py", "a") as f: + f.write(self.serializers_str) + else: + # write sef.serializers_str to file + with open("serializers.py", "w") as f: + f.write(self.serializers_str) + + def generate_services_imports(self): + self.services_str += f"""## generated with django-socio-grpc generateprpcinterface {self.app_name} (LARA-version) + +from django_socio_grpc import generics +from .serializers import {', '.join([model.capitalize() + "ProtoSerializer" for model in self.model_names])}\n\n +from {self.app_name}.models import {', '.join(self.model_names)}\n\n""" + + + def generate_services(self): + self.generate_services_imports() + + # generate service classes + for model in self.model_names: + self.services_str += f"""class {model.capitalize()}Service(generics.ModelService): + queryset = {model}.objects.all() + serializer_class = {model.capitalize()}ProtoSerializer\n\n""" + + # check, if services.py exists + # then ask, if we should append to file + + if os.path.isfile("services.py"): + append = input("services.py already exists, append to file? (y/n) ") + if append.lower() == "y": + with open("services.py", "a") as f: + f.write(self.services_str) + else: + # write self.services_str to file + with open("services.py", "w") as f: + f.write(self.services_str) + + + def generate_handler_imports(self): + self.handler_str += f"""# generated with django-socio-grpc generateprpcinterface {self.app_name} (LARA-version) + +#import logging +from django_socio_grpc.services.app_handler_registry import AppHandlerRegistry +from {self.app_name}.grpc.services import {', '.join([model.capitalize() + "Service" for model in self.model_names])}\n\n""" + + def generate_handlers(self): + self.generate_handler_imports() + + # generate handler functions + self.handler_str += f"""def grpc_handlers(server): + app_registry = AppHandlerRegistry("{self.app_name}", server)\n""" + + for model in self.model_names: + self.handler_str += f""" + app_registry.register({model.capitalize()}Service)\n""" + + # check, if handlers.py exists + # then ask, if we should append to file + + if os.path.isfile("handlers.py"): + append = input("handlers.py already exists, append to file? (y/n) ") + if append.lower() == "y": + with open("handlers.py", "a") as f: + f.write(self.handler_str) + else: + # write self.handler_str to file + with open("handlers.py", "w") as f: + f.write(self.handler_str) + + + diff --git a/django_socio_grpc/management/commands/generateproto.py b/django_socio_grpc/management/commands/generateproto.py index 165bd223..fa6cb217 100644 --- a/django_socio_grpc/management/commands/generateproto.py +++ b/django_socio_grpc/management/commands/generateproto.py @@ -16,6 +16,12 @@ class Command(BaseCommand): help = "Generates proto." def add_arguments(self, parser): + parser.add_argument( + "--build-interface", + "-b", + action="store", + help="build complete default gRPC interface for an apps, please provide app name", + ) parser.add_argument( "--project", "-p", @@ -67,6 +73,12 @@ def handle(self, *args, **options): ) self.project_name = os.environ.get("DJANGO_SETTINGS_MODULE").split(".")[0] + # if app name is provide, we build the default interface for this app with all services + self.app_interface_to_build = options["build_interface"] + # create_handler() + # create_serializers() + # create_services() + self.dry_run = options["dry_run"] self.generate_pb2 = not options["no_generate_pb2"] self.check = options["check"] @@ -76,6 +88,7 @@ def handle(self, *args, **options): self.directory.mkdir(parents=True, exist_ok=True) registry_instance = RegistrySingleton() + # ---------------------------------------------- # --- Proto Generation Process --- From 73e10b05fa97318925048e4063be2f1b20799872 Mon Sep 17 00:00:00 2001 From: mark doerr Date: Mon, 29 May 2023 21:55:36 +0200 Subject: [PATCH 3/5] feat: improved proto compiler added - some parts need to be fixed --- .../management/proto_compiler.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 django_socio_grpc/management/proto_compiler.py diff --git a/django_socio_grpc/management/proto_compiler.py b/django_socio_grpc/management/proto_compiler.py new file mode 100644 index 00000000..791a3c65 --- /dev/null +++ b/django_socio_grpc/management/proto_compiler.py @@ -0,0 +1,135 @@ + +import logging +from typing import List +import os +from grpc_tools import protoc + +def compile_proto_to_python(proto_file: str, + source_dir: str = '.', target_dir: str = '.', + include_dirs: List[str] = None, auto_include_library: bool = True, import_local: bool = False) \ + -> bool: + + if include_dirs is None: + include_dirs = [] + + if auto_include_library: + # add the path the the SilaFramework.proto path as well as the SiLABinaryTransfer.proto + include_dirs.append(os.path.join(os.path.dirname(__file__), '..', 'framework', 'protobuf')) + + # Construct the command + # The main program + command = ['grpc_tools.protoc'] + # All import paths + for path in include_dirs: + command.append('--proto_path={proto_path}'.format(proto_path=path)) + # The source path + command.append('--proto_path={proto_path}'.format(proto_path=source_dir)) + # The target path(s) + command.append('--python_out={output_dir}'.format(output_dir=target_dir)) + command.append('--grpc_python_out={output_dir}'.format(output_dir=target_dir)) + # The proto file + command.append(proto_file) + + if protoc.main(command) != 0: + logging.error( + 'Failed to compile .proto code for from file "{proto_file}" using the command `{command}`'.format( + proto_file=proto_file, + command=command + ) + ) + return False + else: + logging.info( + 'Successfully compiled "{proto_file}"'.format( + proto_file=proto_file + ) + ) + + # correct the SiLAFramework import + # we use absolute imports here, to allow to copy the stubs out of the library and still keep the dependency on + # the SiLAFramework alive + (pb2_files, _) = os.path.splitext(os.path.basename(proto_file)) + pb2_file = pb2_files + '_pb2.py' + pb2_grpc_file = pb2_files + '_pb2_grpc.py' + pb2_file = os.path.join(target_dir, pb2_file) + pb2_grpc_file = os.path.join(target_dir, pb2_grpc_file) + with open(pb2_file, 'r', encoding='utf-8') as file_in: + logging.debug('Correcting {file}'.format(file=pb2_file)) + logging.debug('\t' 'Correcting for import of SiLAFramework') + replaced_text = file_in.read() + if import_local: + replaced_text = replaced_text.replace('import SiLAFramework_pb2', + 'from . import SiLAFramework_pb2') + else: + replaced_text = replaced_text.replace('import SiLAFramework_pb2', + 'import sila2lib.framework.SiLAFramework_pb2') + with open(pb2_file, 'w', encoding='utf-8') as file_out: + file_out.write(replaced_text) + + with open(pb2_grpc_file, 'r', encoding='utf-8') as file_in: + logging.debug('Correcting {file}'.format(file=pb2_grpc_file)) + logging.debug('\t' 'Correcting for import of SiLAFramework') + replaced_text = file_in.read() + if import_local: + replaced_text = replaced_text.replace('import SiLAFramework_pb2', + 'from . import SiLAFramework_pb2') + replaced_text = replaced_text.replace('import SiLABinaryTransfer_pb2', + 'from . import SiLABinaryTransfer_pb2') + else: + replaced_text = replaced_text.replace('import SiLAFramework_pb2', + 'import sila2lib.framework.SiLAFramework_pb2') + replaced_text = replaced_text.replace('import SiLABinaryTransfer_pb2', + 'import sila2lib.framework.SiLABinaryTransfer_pb2') + with open(pb2_grpc_file, 'w', encoding='utf-8') as file_out: + file_out.write(replaced_text) + + return True + +def compile_proto_to_javascript(proto_file: str, + source_dir: str = '.', target_dir: str = '.', + include_dirs: List[str] = None, auto_include_library: bool = True, import_local: bool = False) \ + -> bool: + + try : + protoc_version_output = run_with_return('protoc', ['--version']) + logging.debug(f"protoc version installed on this system: {protoc_version_output}") + protoc_version = protoc_version_output.split()[1] + protoc_ver_maj, protoc_ver_min, protoc_ver_rel = protoc_version.split('.') + + logging.debug(f"protoc maj-version: {protoc_ver_maj} min-ver: {protoc_ver_min} rel-ver: {protoc_ver_rel} ") + + if int(protoc_ver_maj) <= 3 and int(protoc_ver_min) < 6: + logging.warning("JavaScript output only works with protoc >= 3.6.1.\n you have v{protoc_version} installed on your system!" ) + return False + + except Exception as err: + logging.error(f"({err}): protobuf compiler protoc not installed.") + return False + + if include_dirs is None: + include_dirs = [] + + if auto_include_library: + # add the path the the SilaFramework.proto path as well as the SiLABinaryTransfer.proto + include_dirs.append(os.path.join(os.path.dirname(__file__), '..', 'framework', 'protobuf')) + + # Construct the command parameters + # The main program + command_params = [] + # All import paths + for path in include_dirs: + command_params.append('--proto_path={proto_path}'.format(proto_path=path)) + # The source path + command_params.append('--proto_path={proto_path}'.format(proto_path=source_dir)) + # The target path(s) + command_params.append('--js_out=import_style=commonjs:{output_dir}'.format(output_dir=target_dir)) + command_params.append('--grpc-web_out=import_style=commonjs,mode=grpcwebtext:{output_dir}'.format(output_dir=target_dir)) + # The proto file + command_params.append(proto_file) + + # protoc -I="protos" echo.proto --js_out=import_style=commonjs:generated --grpc-web_out=import_style=commonjs,mode=grpcwebtext:generated + run_with_return('protoc', command_params ) + + logging.info(f"Tried to compile proto-file [{proto_file}] to javascript.") + + return True From c7b0dc5cc912f1e32ca8a908f2d3836f6e2d45a5 Mon Sep 17 00:00:00 2001 From: mark doerr Date: Tue, 30 May 2023 11:38:40 +0200 Subject: [PATCH 4/5] feat: protoc rel. import bug fix added --- .../management/commands/generateproto.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/django_socio_grpc/management/commands/generateproto.py b/django_socio_grpc/management/commands/generateproto.py index fa6cb217..f3cfd249 100644 --- a/django_socio_grpc/management/commands/generateproto.py +++ b/django_socio_grpc/management/commands/generateproto.py @@ -1,6 +1,8 @@ +import logging import asyncio import os from pathlib import Path +from grpc_tools import protoc from asgiref.sync import async_to_sync from django.conf import settings @@ -132,6 +134,43 @@ def handle(self, *args, **options): os.system( f"python -m grpc_tools.protoc --proto_path={proto_path} --python_out={proto_path} --grpc_python_out={proto_path} {file_path}" ) + command = ['grpc_tools.protoc'] + command.append(f'--proto_path={str(proto_path)}') + command.append(f'--python_out={str(proto_path)}') + command.append(f'--grpc_python_out={str(proto_path)}') + command.append(str(file_path)) # The proto file + + # if protoc.main(command) != 0: + # logging.error( + # f'Failed to compile .proto code for from file "{file_path}" using the command `{command}`' + # ) + # return False + # else: + # logging.info( + # f'Successfully compiled "{file_path}"' + # ) + # correcting protoc rel. import bug + # + (pb2_files, _) = os.path.splitext(os.path.basename(file_path)) + pb2_file = pb2_files + '_pb2.py' + pb2_module = pb2_files + '_pb2' + + pb2_grpc_file = pb2_files + '_pb2_grpc.py' + + pb2_file_path = os.path.join(proto_path, pb2_file) + pb2_grpc_file_path = os.path.join(proto_path, pb2_grpc_file) + + with open(pb2_grpc_file_path, 'r', encoding='utf-8') as file_in: + print(f'Correcting imports of {pb2_grpc_file_path}') + + replaced_text = file_in.read() + + replaced_text = replaced_text.replace(f'import {pb2_module}', + f'from . import {pb2_module}') + + with open(pb2_grpc_file_path, 'w', encoding='utf-8') as file_out: + file_out.write(replaced_text) + def check_or_write(self, file: Path, proto, app_name): """ From 39d7fb25451d5f4a76de86cc690b8e7a85f0f2a9 Mon Sep 17 00:00:00 2001 From: mark doerr Date: Sat, 17 Jun 2023 22:59:03 +0200 Subject: [PATCH 5/5] feat: allow to select the app name for building the interface --- django_socio_grpc/management/commands/generateproto.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/django_socio_grpc/management/commands/generateproto.py b/django_socio_grpc/management/commands/generateproto.py index f3cfd249..2b08706a 100644 --- a/django_socio_grpc/management/commands/generateproto.py +++ b/django_socio_grpc/management/commands/generateproto.py @@ -29,6 +29,11 @@ def add_arguments(self, parser): "-p", help="specify Django project. Use DJANGO_SETTINGS_MODULE by default", ) + parser.add_argument( + "--app-name", + "-a", + help="specify a Django app for which to create the interface", + ) parser.add_argument( "--dry-run", "-dr", @@ -67,6 +72,7 @@ def handle(self, *args, **options): async_to_sync(grpc_settings.ROOT_HANDLERS_HOOK)(None) else: grpc_settings.ROOT_HANDLERS_HOOK(None) + self.app_name = options["app_name"] self.project_name = options["project"] if not self.project_name: if not os.environ.get("DJANGO_SETTINGS_MODULE"):