From d7905dd6345b530838cdf8f40a39d5ee373f0d00 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 23 May 2022 11:56:48 +1000 Subject: [PATCH 01/11] add dev container --- .devcontainer/Dockerfile | 21 ++++++++++++ .devcontainer/devcontainer.json | 57 +++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..b2c13c80 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,21 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.234.0/containers/python-3/.devcontainer/base.Dockerfile + +# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster +ARG VARIANT="3.9" +FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} + +# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 +ARG NODE_VERSION="none" +RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi + +# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. +# COPY requirements.txt /tmp/pip-tmp/ +# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ +# && rm -rf /tmp/pip-tmp + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + +# [Optional] Uncomment this line to install global node packages. +# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..9cbb6b86 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,57 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.234.0/containers/python-3 +{ + "name": "Python 3", + "build": { + "dockerfile": "Dockerfile", + "context": "..", + "args": { + // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6 + // Append -bullseye or -buster to pin to an OS version. + // Use -bullseye variants on local on arm64/Apple Silicon. + "VARIANT": "3.9", + // Options + "NODE_VERSION": "none" + } + }, + + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", + "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", + "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", + "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", + "python.testing.pytestArgs": [ + "tests/unittests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "sudo python -m pip install -U pip && sudo python -m pip install -U -e .[dev] && sudo python setup.py webhost", + + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode", + + "features": { + "dotnet": "latest" + } +} From 0959c1aaf7b2d2c0ad3b6df1d0802a6a6e6906b1 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 23 May 2022 13:38:03 +1000 Subject: [PATCH 02/11] add benchmark for build_binding_protos --- tests/benchmarks/test_loader_benchmark.py | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/benchmarks/test_loader_benchmark.py diff --git a/tests/benchmarks/test_loader_benchmark.py b/tests/benchmarks/test_loader_benchmark.py new file mode 100644 index 00000000..3584354d --- /dev/null +++ b/tests/benchmarks/test_loader_benchmark.py @@ -0,0 +1,31 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest + +import azure_functions_worker.loader as loader +from azure.functions import Function +from azure.functions.decorators.core import InputBinding + + +def dummy_func(): + ... + + +class FakeInputBinding(InputBinding): + + def __init__(self, + name): + super().__init__(name=name, data_type=None) + + @staticmethod + def get_binding_name() -> str: + return "test_binding" + + +@pytest.mark.parametrize("size", range(10)) +def test_build_binding_protos(benchmark, size): + f = Function(dummy_func, "foo.py") + for i in range(size): + f.add_binding(FakeInputBinding(f"test_binding{i}")) + r = benchmark(loader.build_binding_protos, f) From 8a13fa9ea3f9ea8f6a83869bb23e0cf64048ce18 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 23 May 2022 13:44:06 +1000 Subject: [PATCH 03/11] update requirements --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 16aedff8..e2f0483a 100644 --- a/setup.py +++ b/setup.py @@ -127,6 +127,7 @@ "pytest-randomly", "pytest-instafail", "pytest-rerunfailures", + "pytest-benchmark", "ptvsd" ] } From 699bbbcd013116a272ed46e40a4fb5a7b93c0eaf Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 23 May 2022 13:45:51 +1000 Subject: [PATCH 04/11] refactor build_binding_protos into dict comprehension --- azure_functions_worker/loader.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py index e5120a2b..ed5e9ea1 100644 --- a/azure_functions_worker/loader.py +++ b/azure_functions_worker/loader.py @@ -48,14 +48,13 @@ def uninstall() -> None: def build_binding_protos(indexed_function: List[Function]) -> Dict: - binding_protos = {} - for binding in indexed_function.get_bindings(): - binding_protos[binding.name] = protos.BindingInfo( + return { + binding.name: protos.BindingInfo( type=binding.type, data_type=binding.data_type, direction=binding.direction) - - return binding_protos + for binding in indexed_function.get_bindings() + } def process_indexed_function(functions_registry: functions.Registry, From c84de5cde5a2c005f490ae97cb37ed8a812dd9d1 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 23 May 2022 14:00:22 +1000 Subject: [PATCH 05/11] add process_indexed_function benchmark --- tests/benchmarks/test_loader_benchmark.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/benchmarks/test_loader_benchmark.py b/tests/benchmarks/test_loader_benchmark.py index 3584354d..dd119151 100644 --- a/tests/benchmarks/test_loader_benchmark.py +++ b/tests/benchmarks/test_loader_benchmark.py @@ -4,6 +4,7 @@ import pytest import azure_functions_worker.loader as loader +from azure_functions_worker.functions import Registry from azure.functions import Function from azure.functions.decorators.core import InputBinding @@ -28,4 +29,15 @@ def test_build_binding_protos(benchmark, size): f = Function(dummy_func, "foo.py") for i in range(size): f.add_binding(FakeInputBinding(f"test_binding{i}")) - r = benchmark(loader.build_binding_protos, f) + benchmark(loader.build_binding_protos, f) + + +def test_process_indexed_function(benchmark): + def _test_func(test_binding0, test_binding1, test_binding2, test_binding3, test_binding4): + pass + + f = Function(_test_func, "foo.py") + for i in range(5): # Use 5 bindingss + f.add_binding(FakeInputBinding(f"test_binding{i}")) + reg = Registry() + benchmark(loader.process_indexed_function, reg, [f, f, f, f, f]) From e6252b50e20f02dcbeecca31eea4b0aecdba54ec Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 23 May 2022 15:36:17 +1000 Subject: [PATCH 06/11] optimize loader imports to remove global lookup in loop --- azure_functions_worker/functions.py | 14 +++++++------- azure_functions_worker/loader.py | 11 ++++++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/azure_functions_worker/functions.py b/azure_functions_worker/functions.py index 085fb5de..045dcf59 100644 --- a/azure_functions_worker/functions.py +++ b/azure_functions_worker/functions.py @@ -7,7 +7,7 @@ from azure.functions import DataType, Function -from . import bindings as bindings_utils +from .bindings import has_implicit_output, check_output_type_annotation from . import protos from ._thirdparty import typing_inspect from .protos import BindingInfo @@ -60,7 +60,7 @@ def get_explicit_and_implicit_return(binding_name: str, typing.Tuple[bool, bool]: if binding_name == '$return': explicit_return = True - elif bindings_utils.has_implicit_output( + elif has_implicit_output( binding.type): implicit_return = True bound_params[binding_name] = binding @@ -75,7 +75,7 @@ def get_return_binding(binding_name: str, if binding_name == "$return": return_binding_name = binding_type assert return_binding_name is not None - elif bindings_utils.has_implicit_output(binding_type): + elif has_implicit_output(binding_type): return_binding_name = binding_type return return_binding_name @@ -202,17 +202,17 @@ def validate_function_params(params: dict, bound_params: dict, 'is azure.functions.Out in Python') if param_has_anno and param_py_type in (str, bytes) and ( - not bindings_utils.has_implicit_output(binding.type)): + not has_implicit_output(binding.type)): param_bind_type = 'generic' else: param_bind_type = binding.type if param_has_anno: if is_param_out: - checks_out = bindings_utils.check_output_type_annotation( + checks_out = check_output_type_annotation( param_bind_type, param_py_type) else: - checks_out = bindings_utils.check_input_type_annotation( + checks_out = check_input_type_annotation( param_bind_type, param_py_type) if not checks_out: @@ -263,7 +263,7 @@ def get_function_return_type(annotations: dict, has_explicit_return: bool, if return_pytype is (str, bytes): binding_name = 'generic' - if not bindings_utils.check_output_type_annotation( + if not check_output_type_annotation( binding_name, return_pytype): raise FunctionLoadError( func_name, diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py index ed5e9ea1..a05aa9e4 100644 --- a/azure_functions_worker/loader.py +++ b/azure_functions_worker/loader.py @@ -8,13 +8,14 @@ import os.path import pathlib import sys -import uuid +from uuid import uuid4 from os import PathLike, fspath from typing import List, Optional, Dict from azure.functions import Function, FunctionApp -from . import protos, functions +from . import functions +from .protos import RpcFunctionMetadata, BindingInfo from .constants import MODULE_NOT_FOUND_TS_URL, SCRIPT_FILE_NAME, \ PYTHON_LANGUAGE_RUNTIME from .utils.wrappers import attach_message_to_exception @@ -49,7 +50,7 @@ def uninstall() -> None: def build_binding_protos(indexed_function: List[Function]) -> Dict: return { - binding.name: protos.BindingInfo( + binding.name: BindingInfo( type=binding.type, data_type=binding.data_type, direction=binding.direction) @@ -61,14 +62,14 @@ def process_indexed_function(functions_registry: functions.Registry, indexed_functions: List[Function]): fx_metadata_results = [] for indexed_function in indexed_functions: - function_id = str(uuid.uuid4()) + function_id = str(uuid4()) function_info = functions_registry.add_indexed_function( function_id, function=indexed_function) binding_protos = build_binding_protos(indexed_function) - function_metadata = protos.RpcFunctionMetadata( + function_metadata = RpcFunctionMetadata( name=function_info.name, function_id=function_id, managed_dependency_enabled=False, # only enabled for PowerShell From 3e0dbd4f6588030d8526a3a38e8a24dc1599f9d2 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 23 May 2022 15:36:44 +1000 Subject: [PATCH 07/11] add benchmark functions for all loader functions --- tests/benchmarks/dummy/__init__.py | 26 +++++++++++++++++++++++ tests/benchmarks/test_loader_benchmark.py | 20 +++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tests/benchmarks/dummy/__init__.py diff --git a/tests/benchmarks/dummy/__init__.py b/tests/benchmarks/dummy/__init__.py new file mode 100644 index 00000000..58da4b04 --- /dev/null +++ b/tests/benchmarks/dummy/__init__.py @@ -0,0 +1,26 @@ +import azure.functions as func + +def foo(): + pass + +app = func.FunctionApp() + +@app.route(route="func1") +def func1(req: func.HttpRequest) -> func.HttpResponse: + ... + + +@app.route(route="func1") +def func2(req: func.HttpRequest, arg1) -> func.HttpResponse: + ... + + +@app.route(route="func1") +def func3(req: func.HttpRequest, arg1, arg2) -> func.HttpResponse: + ... + + +@app.route(route="func1") +def func4(req: func.HttpRequest, arg1, arg2, arg3) -> func.HttpResponse: + ... + diff --git a/tests/benchmarks/test_loader_benchmark.py b/tests/benchmarks/test_loader_benchmark.py index dd119151..40f353b8 100644 --- a/tests/benchmarks/test_loader_benchmark.py +++ b/tests/benchmarks/test_loader_benchmark.py @@ -5,6 +5,7 @@ import azure_functions_worker.loader as loader from azure_functions_worker.functions import Registry +from azure_functions_worker.testutils import TESTS_ROOT from azure.functions import Function from azure.functions.decorators.core import InputBinding @@ -41,3 +42,22 @@ def _test_func(test_binding0, test_binding1, test_binding2, test_binding3, test_ f.add_binding(FakeInputBinding(f"test_binding{i}")) reg = Registry() benchmark(loader.process_indexed_function, reg, [f, f, f, f, f]) + + +def test_load_function(benchmark): + loader.install() + benchmark( + loader.load_function, + "http_functions", + TESTS_ROOT / "benchmarks" / "dummy", + TESTS_ROOT / "benchmarks" / "dummy" / "__init__.py", + "foo" + ) + loader.uninstall() + + +def test_index_function_app(benchmark): + benchmark( + loader.index_function_app, + TESTS_ROOT / "benchmarks" / "dummy", + ) From 44314717a6aa172934368bc8bfae1052d845645f Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 23 May 2022 16:08:13 +1000 Subject: [PATCH 08/11] add profiler to benchmark code and fix import error --- .gitignore | 1 + azure_functions_worker/functions.py | 4 +++- setup.py | 1 + tests/.gitignore | 1 + tests/benchmarks/test_loader_benchmark.py | 18 ++++++++++++++++++ 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e8d9736a..cd5f2700 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,4 @@ prof/ tests/**/host.json tests/**/bin tests/**/extensions.csproj +.benchmarks diff --git a/azure_functions_worker/functions.py b/azure_functions_worker/functions.py index 045dcf59..21022cd6 100644 --- a/azure_functions_worker/functions.py +++ b/azure_functions_worker/functions.py @@ -7,7 +7,9 @@ from azure.functions import DataType, Function -from .bindings import has_implicit_output, check_output_type_annotation +from .bindings import (has_implicit_output, + check_output_type_annotation, + check_input_type_annotation) from . import protos from ._thirdparty import typing_inspect from .protos import BindingInfo diff --git a/setup.py b/setup.py index e2f0483a..fb89c7a8 100644 --- a/setup.py +++ b/setup.py @@ -128,6 +128,7 @@ "pytest-instafail", "pytest-rerunfailures", "pytest-benchmark", + "pyinstrument", "ptvsd" ] } diff --git a/tests/.gitignore b/tests/.gitignore index 3e4ede76..c84274a0 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,3 +1,4 @@ *_functions/bin/ *_functions/host.json *_functions/ping/ +benchmarks/.profiles \ No newline at end of file diff --git a/tests/benchmarks/test_loader_benchmark.py b/tests/benchmarks/test_loader_benchmark.py index 40f353b8..53aef134 100644 --- a/tests/benchmarks/test_loader_benchmark.py +++ b/tests/benchmarks/test_loader_benchmark.py @@ -1,7 +1,9 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +from pathlib import Path import pytest +from pyinstrument import Profiler import azure_functions_worker.loader as loader from azure_functions_worker.functions import Registry @@ -10,6 +12,22 @@ from azure.functions.decorators.core import InputBinding +@pytest.fixture(autouse=True) +def profile(request): + # Turn profiling on + profiler = Profiler() + profiler.start() + + yield # Run test + + profiler.stop() + # Uncomment if you want to see on the CLI + # profiler.print(show_all=True) + (TESTS_ROOT / "benchmarks" / ".profiles").mkdir(exist_ok=True) + with open(TESTS_ROOT / "benchmarks" / ".profiles" / f"{request.node.name}.html", "w", encoding="utf-8") as f: + f.write(profiler.output_html()) + + def dummy_func(): ... From 9c88779cb47ae412abd935e7b9e4ecb79b09112b Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 26 May 2022 20:34:17 +1000 Subject: [PATCH 09/11] Add austin and some docker settings to suit. Start profiling the dispatcher --- .devcontainer/Dockerfile | 4 +-- .devcontainer/devcontainer.json | 3 +- setup.py | 1 + tests/benchmarks/conftest.py | 30 +++++++++++++++++++ tests/benchmarks/test_dispatcher_benchmark.py | 22 ++++++++++++++ tests/benchmarks/test_loader_benchmark.py | 20 +++++++------ 6 files changed, 68 insertions(+), 12 deletions(-) create mode 100644 tests/benchmarks/conftest.py create mode 100644 tests/benchmarks/test_dispatcher_benchmark.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b2c13c80..2eecdc87 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -14,8 +14,8 @@ RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/ # && rm -rf /tmp/pip-tmp # [Optional] Uncomment this section to install additional OS packages. -# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ -# && apt-get -y install --no-install-recommends +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends austin # [Optional] Uncomment this line to install global node packages. # RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9cbb6b86..91c3701d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -53,5 +53,6 @@ "features": { "dotnet": "latest" - } + }, + "runArgs": ["--cap-add", "SYS_PTRACE"] } diff --git a/setup.py b/setup.py index fb89c7a8..b9aca1a6 100644 --- a/setup.py +++ b/setup.py @@ -128,6 +128,7 @@ "pytest-instafail", "pytest-rerunfailures", "pytest-benchmark", + "pytest-asyncio", "pyinstrument", "ptvsd" ] diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py new file mode 100644 index 00000000..bf8c6fe8 --- /dev/null +++ b/tests/benchmarks/conftest.py @@ -0,0 +1,30 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +import pytest_asyncio +import pytest +from azure_functions_worker.testutils import TESTS_ROOT + + +@pytest.fixture() +def save_profile(request): + def _save_profile(profiler): + (TESTS_ROOT / "benchmarks" / ".profiles").mkdir(exist_ok=True) + results_file = TESTS_ROOT / "benchmarks" / ".profiles" / f"{request.node.name}.html" + with open(results_file, "w", encoding="utf-8") as f: + f.write(profiler.output_html()) + return _save_profile + + +@pytest_asyncio.fixture +async def aio_benchmark(benchmark, event_loop): + def _wrapper(func, *args, **kwargs): + if asyncio.iscoroutinefunction(func): + @benchmark + def _(): + return event_loop.run_until_complete(func(*args, **kwargs)) + else: + benchmark(func, *args, **kwargs) + + return _wrapper diff --git a/tests/benchmarks/test_dispatcher_benchmark.py b/tests/benchmarks/test_dispatcher_benchmark.py new file mode 100644 index 00000000..43c23210 --- /dev/null +++ b/tests/benchmarks/test_dispatcher_benchmark.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from azure_functions_worker import protos, testutils + + +def test_invoke_function(aio_benchmark): + + async def invoke_function(): + async with testutils.start_mockhost() as host: + await host.load_function('return_str') + + await host.invoke_function( + 'return_str', [ + protos.ParameterBinding( + name='req', + data=protos.TypedData( + http=protos.RpcHttp( + method='GET'))) + ]) + + aio_benchmark(invoke_function) diff --git a/tests/benchmarks/test_loader_benchmark.py b/tests/benchmarks/test_loader_benchmark.py index 53aef134..ed72c3e8 100644 --- a/tests/benchmarks/test_loader_benchmark.py +++ b/tests/benchmarks/test_loader_benchmark.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from pathlib import Path import pytest from pyinstrument import Profiler @@ -13,19 +12,15 @@ @pytest.fixture(autouse=True) -def profile(request): +def profile(save_profile): # Turn profiling on - profiler = Profiler() + profiler = Profiler(async_mode="enabled") profiler.start() yield # Run test profiler.stop() - # Uncomment if you want to see on the CLI - # profiler.print(show_all=True) - (TESTS_ROOT / "benchmarks" / ".profiles").mkdir(exist_ok=True) - with open(TESTS_ROOT / "benchmarks" / ".profiles" / f"{request.node.name}.html", "w", encoding="utf-8") as f: - f.write(profiler.output_html()) + save_profile(profiler) def dummy_func(): @@ -56,7 +51,7 @@ def _test_func(test_binding0, test_binding1, test_binding2, test_binding3, test_ pass f = Function(_test_func, "foo.py") - for i in range(5): # Use 5 bindingss + for i in range(5): # Use 5 bindings f.add_binding(FakeInputBinding(f"test_binding{i}")) reg = Registry() benchmark(loader.process_indexed_function, reg, [f, f, f, f, f]) @@ -79,3 +74,10 @@ def test_index_function_app(benchmark): loader.index_function_app, TESTS_ROOT / "benchmarks" / "dummy", ) + + +def test_str_join(benchmark): + def j(_input): + return ", ".join([x for x in _input]) + + benchmark(j, ["a", "b", "c", "d"]) From 73987cfdd8d95fa524062b8eb3f5d223065e6024 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 27 May 2022 04:53:25 +0000 Subject: [PATCH 10/11] Adding readme on how to run benchmarks --- tests/benchmarks/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/benchmarks/README.md diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md new file mode 100644 index 00000000..da59d7f7 --- /dev/null +++ b/tests/benchmarks/README.md @@ -0,0 +1,26 @@ +This folder contains benchmarks written using `pytest` and profiled using `pyinstrument`. + +# Running the benchmarks + +Open this repository in VSCode and wait for the devcontainer to start up. + +Then you can run all the benchmarks with this command: + +`python -m pytest tests/benchmarks` + +Or use the keyword argument to run just one benchmark test: + +`python -m pytest tests/benchmarks -k test_process_indexed_function` + +When you run that test, a profile will also be saved in the `tests/benchmarks/.profiles` folder, in a file named after the profiled function. Open the file in a browser to see the profile. + +If you run a benchmark test multiple times (either on same code or different versions of the code), you probably want to save it. + +Either pass in `--benchmark-autosave` to save to an auto-generated filename or pass in `--benchmark-save=YOURNAME` to save with your specified name in the filename. All benchmark files will always start with a counter, beginning with 0001. + +Once saved, compare using the `pytest-benchmark` command and the counter numbers: + +`pytest-benchmark compare 0004 0005` + +You can sort the comparison using `--sort`, save it to a CSV using `--csv`, or save it to a histogram with `--histogram`. +More details available in the [pytest-benchmark reference](https://pytest-benchmark.readthedocs.io/en/latest/usage.html#comparison-cli). \ No newline at end of file From 46fb92759188f93adc9363f977898d86b4a7a732 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 31 May 2022 09:55:32 +1000 Subject: [PATCH 11/11] Add additional benchmarks for invoking functions --- .devcontainer/Dockerfile | 4 +- tests/benchmarks/conftest.py | 12 --- tests/benchmarks/test_dispatcher_benchmark.py | 101 ++++++++++++++++-- tests/benchmarks/test_loader_benchmark.py | 10 +- 4 files changed, 100 insertions(+), 27 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 2eecdc87..29080661 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -14,8 +14,8 @@ RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/ # && rm -rf /tmp/pip-tmp # [Optional] Uncomment this section to install additional OS packages. -RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends austin +RUN sudo git clone --depth=1 https://github.com/P403n1x87/austin.git && cd austin \ + sudo autoreconf --install && sudo ./configure && sudo make && sudo make install # [Optional] Uncomment this line to install global node packages. # RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 \ No newline at end of file diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index bf8c6fe8..7e3e86e6 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -3,18 +3,6 @@ import asyncio import pytest_asyncio -import pytest -from azure_functions_worker.testutils import TESTS_ROOT - - -@pytest.fixture() -def save_profile(request): - def _save_profile(profiler): - (TESTS_ROOT / "benchmarks" / ".profiles").mkdir(exist_ok=True) - results_file = TESTS_ROOT / "benchmarks" / ".profiles" / f"{request.node.name}.html" - with open(results_file, "w", encoding="utf-8") as f: - f.write(profiler.output_html()) - return _save_profile @pytest_asyncio.fixture diff --git a/tests/benchmarks/test_dispatcher_benchmark.py b/tests/benchmarks/test_dispatcher_benchmark.py index 43c23210..a3893684 100644 --- a/tests/benchmarks/test_dispatcher_benchmark.py +++ b/tests/benchmarks/test_dispatcher_benchmark.py @@ -1,22 +1,103 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from azure_functions_worker import protos, testutils +import asyncio +import pathlib +import typing +import sys +from azure_functions_worker import protos, testutils, dispatcher -def test_invoke_function(aio_benchmark): - async def invoke_function(): - async with testutils.start_mockhost() as host: - await host.load_function('return_str') +class _MockWebHostWithWorkerController: + + def __init__(self, scripts_dir: pathlib.PurePath, event_loop): + self._event_loop = event_loop + self._host: typing.Optional[testutils._MockWebHost] = None + self._scripts_dir: pathlib.PurePath = scripts_dir + self._worker: typing.Optional[dispatcher.Dispatcher] = None + + async def __aenter__(self) -> typing.Tuple[testutils._MockWebHost, dispatcher.Dispatcher]: + loop = self._event_loop + self._host = testutils._MockWebHost(loop, self._scripts_dir) + + await self._host.start() + + self._worker = await dispatcher.\ + Dispatcher.connect(testutils.LOCALHOST, self._host._port, + self._host.worker_id, self._host.request_id, + connect_timeout=5.0) + + self._worker_task = loop.create_task(self._worker.dispatch_forever()) + + done, pending = await asyncio. \ + wait([self._host._connected_fut, self._worker_task], + return_when=asyncio.FIRST_COMPLETED) + + # noinspection PyBroadException + try: + if self._worker_task in done: + self._worker_task.result() + + if self._host._connected_fut not in done: + raise RuntimeError('could not start a worker thread') + except Exception: + try: + await self._host.close() + self._worker.stop() + finally: + raise - await host.invoke_function( - 'return_str', [ - protos.ParameterBinding( + return self._host, self._worker + + async def __aexit__(self, *exc): + if not self._worker_task.done(): + self._worker_task.cancel() + try: + await self._worker_task + except asyncio.CancelledError: + pass + + self._worker_task = None + self._worker = None + + await self._host.close() + self._host = None + +def start_mockhost_with_worker(event_loop, script_root=testutils.FUNCS_PATH): + scripts_dir = testutils.TESTS_ROOT / script_root + if not (scripts_dir.exists() and scripts_dir.is_dir()): + raise RuntimeError( + f'invalid script_root argument: ' + f'{scripts_dir} directory does not exist') + + sys.path.append(str(scripts_dir)) + + return _MockWebHostWithWorkerController(scripts_dir, event_loop) + +def test_invoke_function_benchmark(aio_benchmark, event_loop): + async def invoke_function(): + wc = start_mockhost_with_worker(event_loop) + async with wc as (host, worker): + await host.load_function('return_http') + + func = host._available_functions['return_http'] + invocation_id = host.make_id() + input_data = [protos.ParameterBinding( name='req', data=protos.TypedData( http=protos.RpcHttp( - method='GET'))) - ]) + method='GET')))] + message = protos.StreamingMessage( + invocation_request=protos.InvocationRequest( + invocation_id=invocation_id, + function_id=func.id, + input_data=input_data, + trigger_metadata={}, + ) + ) + for _ in range(1000): + event_loop.create_task(worker._handle__invocation_request(message)) aio_benchmark(invoke_function) + diff --git a/tests/benchmarks/test_loader_benchmark.py b/tests/benchmarks/test_loader_benchmark.py index ed72c3e8..a5bf6d67 100644 --- a/tests/benchmarks/test_loader_benchmark.py +++ b/tests/benchmarks/test_loader_benchmark.py @@ -12,15 +12,19 @@ @pytest.fixture(autouse=True) -def profile(save_profile): +def auto_profile(request): + PROFILE_ROOT = (TESTS_ROOT / "benchmarks" / ".profiles") # Turn profiling on - profiler = Profiler(async_mode="enabled") + profiler = Profiler() profiler.start() yield # Run test profiler.stop() - save_profile(profiler) + PROFILE_ROOT.mkdir(exist_ok=True) + results_file = PROFILE_ROOT / f"{request.node.name}.html" + with open(results_file, "w", encoding="utf-8") as f_html: + f_html.write(profiler.output_html()) def dummy_func():