From ef20647d0d24d0c75aa586d938b974f631976421 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Fri, 25 Oct 2024 17:42:19 -0700 Subject: [PATCH 01/21] import converter dependencies lazily --- hls4ml/converters/keras/qkeras.py | 4 ++-- hls4ml/converters/keras_to_hls.py | 4 ++-- hls4ml/converters/onnx_to_hls.py | 7 ++++--- hls4ml/converters/pytorch_to_hls.py | 4 ++-- hls4ml/model/quantizers.py | 20 ++++++++++++-------- hls4ml/optimization/__init__.py | 4 +--- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/hls4ml/converters/keras/qkeras.py b/hls4ml/converters/keras/qkeras.py index 7357d95aed..d1910c070d 100644 --- a/hls4ml/converters/keras/qkeras.py +++ b/hls4ml/converters/keras/qkeras.py @@ -1,5 +1,3 @@ -from qkeras.quantizers import get_quantizer - from hls4ml.converters.keras.convolution import parse_conv1d_layer, parse_conv2d_layer from hls4ml.converters.keras.core import parse_batchnorm_layer, parse_dense_layer from hls4ml.converters.keras.recurrent import parse_rnn_layer @@ -88,6 +86,8 @@ def parse_qrnn_layer(keras_layer, input_names, input_shapes, data_reader): @keras_handler('QActivation') def parse_qactivation_layer(keras_layer, input_names, input_shapes, data_reader): + from qkeras.quantizers import get_quantizer + assert keras_layer['class_name'] == 'QActivation' supported_activations = [ 'quantized_relu', diff --git a/hls4ml/converters/keras_to_hls.py b/hls4ml/converters/keras_to_hls.py index e31e2b96a9..9fc63cf398 100644 --- a/hls4ml/converters/keras_to_hls.py +++ b/hls4ml/converters/keras_to_hls.py @@ -160,9 +160,9 @@ def get_model_arch(config): # Model instance passed in config from API keras_model = config['KerasModel'] if isinstance(keras_model, str): - from tensorflow.keras.models import load_model + import keras - keras_model = load_model(keras_model) + keras_model = keras.models.load_model(keras_model) model_arch = json.loads(keras_model.to_json()) reader = KerasModelReader(keras_model) elif 'KerasJson' in config: diff --git a/hls4ml/converters/onnx_to_hls.py b/hls4ml/converters/onnx_to_hls.py index 75850fa93e..99281888f3 100644 --- a/hls4ml/converters/onnx_to_hls.py +++ b/hls4ml/converters/onnx_to_hls.py @@ -1,6 +1,3 @@ -import onnx -from onnx import helper, numpy_helper - from hls4ml.model import ModelGraph @@ -21,6 +18,8 @@ def replace_char_inconsitency(name): def get_onnx_attribute(operation, name, default=None): + from onnx import helper + attr = next((x for x in operation.attribute if x.name == name), None) if attr is None: value = default @@ -76,6 +75,7 @@ def get_input_shape(graph, node): def get_constant_value(graph, constant_name): tensor = next((x for x in graph.initializer if x.name == constant_name), None) + from onnx import numpy_helper return numpy_helper.to_array(tensor) @@ -273,6 +273,7 @@ def onnx_to_hls(config): # Extract model architecture print('Interpreting Model ...') + import onnx onnx_model = onnx.load(config['OnnxModel']) if isinstance(config['OnnxModel'], str) else config['OnnxModel'] layer_list, input_layers, output_layers = parse_onnx_model(onnx_model) diff --git a/hls4ml/converters/pytorch_to_hls.py b/hls4ml/converters/pytorch_to_hls.py index 79ca1fa5c6..3ec5b17691 100644 --- a/hls4ml/converters/pytorch_to_hls.py +++ b/hls4ml/converters/pytorch_to_hls.py @@ -1,5 +1,3 @@ -import torch - from hls4ml.model import ModelGraph @@ -26,6 +24,8 @@ def get_weights_data(self, layer_name, var_name): class PyTorchFileReader(PyTorchModelReader): # Inherit get_weights_data method def __init__(self, config): + import torch + self.config = config if not torch.cuda.is_available(): diff --git a/hls4ml/model/quantizers.py b/hls4ml/model/quantizers.py index a5b9ceb8c4..b445c70af3 100644 --- a/hls4ml/model/quantizers.py +++ b/hls4ml/model/quantizers.py @@ -5,8 +5,6 @@ """ import numpy as np -import tensorflow as tf -from qkeras.quantizers import get_quantizer from hls4ml.model.types import ( ExponentPrecisionType, @@ -87,6 +85,8 @@ class QKerasQuantizer(Quantizer): """ def __init__(self, config): + from qkeras.quantizers import get_quantizer + self.quantizer_fn = get_quantizer(config) self.alpha = config['config'].get('alpha', None) if config['class_name'] == 'quantized_bits': @@ -106,8 +106,8 @@ def __init__(self, config): self.hls_type = FixedPrecisionType(width=16, integer=6, signed=True) def __call__(self, data): - tf_data = tf.convert_to_tensor(data) - return self.quantizer_fn(tf_data).numpy() + data = np.array(data, dtype='float32') + return self.quantizer_fn(data).numpy() # return self.quantizer_fn(data) def _get_type(self, quantizer_config): @@ -132,6 +132,8 @@ class QKerasBinaryQuantizer(Quantizer): """ def __init__(self, config, xnor=False): + from qkeras.quantizers import get_quantizer + self.bits = 1 if xnor else 2 self.hls_type = XnorPrecisionType() if xnor else IntegerPrecisionType(width=2, signed=True) self.alpha = config['config']['alpha'] @@ -141,8 +143,8 @@ def __init__(self, config, xnor=False): self.binary_quantizer = BinaryQuantizer(1) if xnor else BinaryQuantizer(2) def __call__(self, data): - x = tf.convert_to_tensor(data) - y = self.quantizer_fn(x).numpy() + data = np.array(data, dtype='float32') + y = self.quantizer_fn(data).numpy() return self.binary_quantizer(y) @@ -154,14 +156,16 @@ class QKerasPO2Quantizer(Quantizer): """ def __init__(self, config): + from qkeras.quantizers import get_quantizer + self.bits = config['config']['bits'] self.quantizer_fn = get_quantizer(config) self.hls_type = ExponentPrecisionType(width=self.bits, signed=True) def __call__(self, data): # Weights are quantized to nearest power of two - x = tf.convert_to_tensor(data) - y = self.quantizer_fn(x) + data = np.array(data, dtype='float32') + y = self.quantizer_fn(data) if hasattr(y, 'numpy'): y = y.numpy() return y diff --git a/hls4ml/optimization/__init__.py b/hls4ml/optimization/__init__.py index c626b70c2b..2b49886e39 100644 --- a/hls4ml/optimization/__init__.py +++ b/hls4ml/optimization/__init__.py @@ -1,3 +1 @@ -from .dsp_aware_pruning import optimize_keras_model_for_hls4ml # noqa: F401 -from .dsp_aware_pruning.attributes import get_attributes_from_keras_model_and_hls4ml_config # noqa: F401 -from .dsp_aware_pruning.keras import optimize_model # noqa: F401 +# No imports as each of the optimization modules may contain different dependencies. From 028b4d0dd63257c2662d32e818fd606b72c1eca6 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Fri, 25 Oct 2024 20:12:28 -0700 Subject: [PATCH 02/21] make tf and qkeras optionl, stop assuming keras is tf.keras --- hls4ml/converters/__init__.py | 6 ++--- hls4ml/model/optimizer/passes/qkeras.py | 3 ++- hls4ml/model/profiling.py | 26 ++++++++++++++----- .../dsp_aware_pruning/keras/__init__.py | 4 --- hls4ml/utils/config.py | 4 +-- hls4ml/writer/catapult_writer.py | 4 ++- hls4ml/writer/quartus_writer.py | 4 ++- hls4ml/writer/vivado_writer.py | 4 ++- 8 files changed, 35 insertions(+), 20 deletions(-) diff --git a/hls4ml/converters/__init__.py b/hls4ml/converters/__init__.py index 3d7ce1fe56..1343907b54 100644 --- a/hls4ml/converters/__init__.py +++ b/hls4ml/converters/__init__.py @@ -93,10 +93,10 @@ def parse_yaml_config(config_file): """ def construct_keras_model(loader, node): - from tensorflow.keras.models import load_model - model_str = loader.construct_scalar(node) - return load_model(model_str) + import keras + + return keras.models.load_model(model_str) yaml.add_constructor('!keras_model', construct_keras_model, Loader=yaml.SafeLoader) diff --git a/hls4ml/model/optimizer/passes/qkeras.py b/hls4ml/model/optimizer/passes/qkeras.py index 03690bed0d..fb02d4eccf 100644 --- a/hls4ml/model/optimizer/passes/qkeras.py +++ b/hls4ml/model/optimizer/passes/qkeras.py @@ -1,5 +1,4 @@ import numpy as np -import tensorflow as tf from hls4ml.model.layers import ApplyAlpha from hls4ml.model.optimizer import ConfigurableOptimizerPass, OptimizerPass, register_pass @@ -113,6 +112,8 @@ def match(self, node): def transform(self, model, node): # The quantizer has to be applied to set the scale attribute # This must be applied to the _unquantized_ weights to obtain the correct scale + import tensorflow as tf + quantizer = node.weights['weight'].quantizer.quantizer_fn # get QKeras quantizer weights = node.weights['weight'].data_unquantized # get weights qweights = quantizer(tf.convert_to_tensor(weights)) diff --git a/hls4ml/model/profiling.py b/hls4ml/model/profiling.py index 84a83de23e..a7fee506e5 100644 --- a/hls4ml/model/profiling.py +++ b/hls4ml/model/profiling.py @@ -13,12 +13,11 @@ from hls4ml.model.layers import GRU, LSTM, SeparableConv1D, SeparableConv2D try: - import qkeras - from tensorflow import keras + import keras - __tf_profiling_enabled__ = True + __keras_profiling_enabled__ = True except ImportError: - __tf_profiling_enabled__ = False + __keras_profiling_enabled__ = False try: import torch @@ -27,6 +26,19 @@ except ImportError: __torch_profiling_enabled__ = False +try: + import qkeras + + __qkeras_profiling_enabled__ = True +except ImportError: + __qkeras_profiling_enabled__ = False + +_activations = list() +if __keras_profiling_enabled__: + _activations.append(keras.layers.Activation) +if __qkeras_profiling_enabled__: + _activations.append(qkeras.qactivations) + def get_unoptimized_hlsmodel(model): from hls4ml.converters import convert_from_config @@ -482,7 +494,7 @@ def numerical(model=None, hls_model=None, X=None, plot='boxplot'): if hls_model_present: data = weights_hlsmodel(hls_model_unoptimized, fmt='summary', plot=plot) elif model_present: - if __tf_profiling_enabled__ and isinstance(model, keras.Model): + if __keras_profiling_enabled__ and isinstance(model, keras.Model): data = weights_keras(model, fmt='summary', plot=plot) elif __torch_profiling_enabled__ and isinstance(model, torch.nn.Sequential): data = weights_torch(model, fmt='summary', plot=plot) @@ -520,7 +532,7 @@ def numerical(model=None, hls_model=None, X=None, plot='boxplot'): if X is not None: print("Profiling activations" + before) data = None - if __tf_profiling_enabled__ and isinstance(model, keras.Model): + if __keras_profiling_enabled__ and isinstance(model, keras.Model): data = activations_keras(model, X, fmt='summary', plot=plot) elif __torch_profiling_enabled__ and isinstance(model, torch.nn.Sequential): data = activations_torch(model, X, fmt='summary', plot=plot) @@ -590,7 +602,7 @@ def get_ymodel_keras(keras_model, X): if ( hasattr(layer, 'activation') and layer.activation is not None - and not isinstance(layer, (keras.layers.Activation, qkeras.qlayers.QActivation)) + and not isinstance(layer, _activations) and layer.activation.__name__ != 'linear' ): tmp_activation = layer.activation diff --git a/hls4ml/optimization/dsp_aware_pruning/keras/__init__.py b/hls4ml/optimization/dsp_aware_pruning/keras/__init__.py index 29012bd39e..b525f58a33 100644 --- a/hls4ml/optimization/dsp_aware_pruning/keras/__init__.py +++ b/hls4ml/optimization/dsp_aware_pruning/keras/__init__.py @@ -4,9 +4,6 @@ import numpy as np import tensorflow as tf -# Enables printing of loss tensors during custom training loop -from tensorflow.python.ops.numpy_ops import np_config - import hls4ml.optimization.dsp_aware_pruning.keras.utils as utils from hls4ml.optimization.dsp_aware_pruning.config import SUPPORTED_STRUCTURES from hls4ml.optimization.dsp_aware_pruning.keras.builder import build_optimizable_model, remove_custom_regularizers @@ -15,7 +12,6 @@ from hls4ml.optimization.dsp_aware_pruning.keras.reduction import reduce_model from hls4ml.optimization.dsp_aware_pruning.scheduler import OptimizationScheduler -np_config.enable_numpy_behavior() default_regularization_range = np.logspace(-6, -2, num=16).tolist() diff --git a/hls4ml/utils/config.py b/hls4ml/utils/config.py index e450084095..6a356f5f27 100644 --- a/hls4ml/utils/config.py +++ b/hls4ml/utils/config.py @@ -1,7 +1,5 @@ import json -import qkeras - import hls4ml @@ -48,6 +46,8 @@ def create_config(output_dir='my-hls-test', project_name='myproject', backend='V def _get_precision_from_quantizer(quantizer): if isinstance(quantizer, str): + import qkeras + quantizer_obj = qkeras.get_quantizer(quantizer) quantizer = {} # Some activations are classes with get_config method diff --git a/hls4ml/writer/catapult_writer.py b/hls4ml/writer/catapult_writer.py index 7db1063206..9a48460995 100755 --- a/hls4ml/writer/catapult_writer.py +++ b/hls4ml/writer/catapult_writer.py @@ -889,7 +889,9 @@ def keras_model_representer(dumper, keras_model): return dumper.represent_scalar('!keras_model', model_path) try: - from tensorflow.keras import Model as KerasModel + import keras + + KerasModel = keras.models.Model yaml.add_multi_representer(KerasModel, keras_model_representer) except Exception: diff --git a/hls4ml/writer/quartus_writer.py b/hls4ml/writer/quartus_writer.py index 932a8b6a6d..1d61bde1f4 100644 --- a/hls4ml/writer/quartus_writer.py +++ b/hls4ml/writer/quartus_writer.py @@ -1327,7 +1327,9 @@ def keras_model_representer(dumper, keras_model): return dumper.represent_scalar('!keras_model', model_path) try: - from tensorflow.keras import Model as KerasModel + import keras + + KerasModel = keras.models.Model yaml.add_multi_representer(KerasModel, keras_model_representer) except Exception: diff --git a/hls4ml/writer/vivado_writer.py b/hls4ml/writer/vivado_writer.py index 0341959045..6531f9db87 100644 --- a/hls4ml/writer/vivado_writer.py +++ b/hls4ml/writer/vivado_writer.py @@ -817,7 +817,9 @@ def keras_model_representer(dumper, keras_model): return dumper.represent_scalar('!keras_model', model_path) try: - from tensorflow.keras import Model as KerasModel + import keras + + KerasModel = keras.models.Model yaml.add_multi_representer(KerasModel, keras_model_representer) except Exception: From 72eb0531b2a824f437f00d1a4c357c702db5148c Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Fri, 25 Oct 2024 20:34:29 -0700 Subject: [PATCH 03/21] less mandatory dependency --- hls4ml/model/profiling.py | 8 ++++---- hls4ml/report/quartus_report.py | 6 +++--- setup.cfg | 18 +++++++++++++----- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/hls4ml/model/profiling.py b/hls4ml/model/profiling.py index a7fee506e5..6def53f7d1 100644 --- a/hls4ml/model/profiling.py +++ b/hls4ml/model/profiling.py @@ -33,11 +33,11 @@ except ImportError: __qkeras_profiling_enabled__ = False -_activations = list() +__keras_activations = list() if __keras_profiling_enabled__: - _activations.append(keras.layers.Activation) + __keras_activations.append(keras.layers.Activation) if __qkeras_profiling_enabled__: - _activations.append(qkeras.qactivations) + __keras_activations.append(qkeras.QActivation) def get_unoptimized_hlsmodel(model): @@ -602,7 +602,7 @@ def get_ymodel_keras(keras_model, X): if ( hasattr(layer, 'activation') and layer.activation is not None - and not isinstance(layer, _activations) + and not isinstance(layer, tuple(__keras_activations)) and layer.activation.__name__ != 'linear' ): tmp_activation = layer.activation diff --git a/hls4ml/report/quartus_report.py b/hls4ml/report/quartus_report.py index c337e5de10..47fc43c132 100644 --- a/hls4ml/report/quartus_report.py +++ b/hls4ml/report/quartus_report.py @@ -2,9 +2,6 @@ import webbrowser from ast import literal_eval -from calmjs.parse import asttypes, es5 -from tabulate import tabulate - def parse_quartus_report(hls_dir, write_to_file=True): ''' @@ -53,6 +50,8 @@ def read_quartus_report(hls_dir, open_browser=False): Returns: None ''' + from tabulate import tabulate + report = parse_quartus_report(hls_dir) print('HLS Resource Summary\n') @@ -100,6 +99,7 @@ def read_js_object(js_script): Returns: Dictionary of variables defines in script ''' + from calmjs.parse import asttypes, es5 def visit(node): if isinstance(node, asttypes.Program): diff --git a/setup.cfg b/setup.cfg index 0b81e7b592..c987f1c317 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,17 +22,15 @@ description_file = README.md [options] packages = find: install_requires = - calmjs.parse h5py numpy - onnx>=1.4.0 + pyyaml pydigitalwavetools==1.1 pyparsing pyyaml - tabulate - tensorflow>=2.8.0,<=2.14.1 - tensorflow-model-optimization<=0.7.5 + python_requires = >=3.10, <3.12 +python_requires = >=3.10 include_package_data = True scripts = scripts/hls4ml @@ -51,14 +49,24 @@ profiling = matplotlib pandas seaborn +qkeras = + qkeras + tensorflow>=2.8.0,<=2.14.1 + tensorflow-model-optimization<=0.7.5 +quantus_report = + calmjs.parse + tabulate sr = sympy testing = HGQ~=0.2.0 + calmjs.parse + onnx>=1.4.0 pytest pytest-cov pytest-randomly qonnx + tabulate torch [check-manifest] From 63af2acd17abb09a2bcb619c61dc0edaebb85691 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Sat, 26 Oct 2024 09:11:34 -0700 Subject: [PATCH 04/21] fix dsp_aware_pruning test import path --- test/pytest/test_optimization/test_attributes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pytest/test_optimization/test_attributes.py b/test/pytest/test_optimization/test_attributes.py index a42d3a6751..c9e22091f2 100644 --- a/test/pytest/test_optimization/test_attributes.py +++ b/test/pytest/test_optimization/test_attributes.py @@ -1,7 +1,7 @@ from tensorflow.keras.layers import Conv2D, Dense, Flatten, ReLU from tensorflow.keras.models import Sequential -from hls4ml.optimization import get_attributes_from_keras_model_and_hls4ml_config +from hls4ml.optimization.dsp_aware_pruning import get_attributes_from_keras_model_and_hls4ml_config from hls4ml.utils.config import config_from_keras_model From c11dddb59c8666dbd4429ee879f0f0a385630f41 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Sun, 15 Dec 2024 06:31:39 +0000 Subject: [PATCH 05/21] fix broken setup.cfg after rebase, rm pyparsing --- setup.cfg | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index c987f1c317..1d4241f063 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,12 +24,8 @@ packages = find: install_requires = h5py numpy - pyyaml pydigitalwavetools==1.1 - pyparsing pyyaml - -python_requires = >=3.10, <3.12 python_requires = >=3.10 include_package_data = True scripts = scripts/hls4ml From d9aaa1a73273cf25362669aae1972d9b51aaf131 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Sun, 15 Dec 2024 06:31:59 +0000 Subject: [PATCH 06/21] purge qkeras workaround --- hls4ml/__init__.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/hls4ml/__init__.py b/hls4ml/__init__.py index e3a7247b0d..0ff5e52ac9 100644 --- a/hls4ml/__init__.py +++ b/hls4ml/__init__.py @@ -1,33 +1,3 @@ -# Temporary workaround for QKeras installation requirement, will be removed after 1.0.0 -def maybe_install_qkeras(): - import subprocess - import sys - - QKERAS_PKG_NAME = 'QKeras' - # QKERAS_PKG_SOURCE = QKERAS_PKG_NAME - QKERAS_PKG_SOURCE = 'qkeras@git+https://github.com/fastmachinelearning/qkeras.git' - - def pip_list(): - p = subprocess.run([sys.executable, '-m', 'pip', 'list'], check=True, capture_output=True) - return p.stdout.decode() - - def pip_install(package): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', package]) - - all_pkgs = pip_list() - if QKERAS_PKG_NAME not in all_pkgs: - print('QKeras installation not found, installing one...') - pip_install(QKERAS_PKG_SOURCE) - print('QKeras installed.') - - -try: - maybe_install_qkeras() -except Exception: - print('Could not find QKeras installation, make sure you have QKeras installed.') - -# End of workaround - from hls4ml import converters, report, utils # noqa: F401, E402 try: From 485442368b7a376436c09e24076587c15ca8945f Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Sun, 15 Dec 2024 22:57:36 +0000 Subject: [PATCH 07/21] switch to pyproject.toml switch to pyproject.toml include pyproject.toml after install --- .pre-commit-config.yaml | 12 +-- MANIFEST.in | 2 +- scripts/hls4ml => hls4ml/cli/__init__.py | 0 pyproject.toml | 101 ++++++++++++++++++++++- setup.cfg | 74 ----------------- setup.py | 4 - 6 files changed, 104 insertions(+), 89 deletions(-) rename scripts/hls4ml => hls4ml/cli/__init__.py (100%) delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0601a84b2d..9380ac1689 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,11 @@ repos: args: ['--line-length=125', '--skip-string-normalization'] +- repo: https://github.com/tox-dev/pyproject-fmt + rev: v2.5.0 + hooks: + - id: pyproject-fmt + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: @@ -16,6 +21,7 @@ repos: - id: check-case-conflict - id: check-merge-conflict - id: check-symlinks + - id: check-toml - id: check-yaml - id: debug-statements - id: end-of-file-fixer @@ -27,7 +33,6 @@ repos: rev: 5.13.2 hooks: - id: isort - args: ["--profile", "black", --line-length=125] - repo: https://github.com/asottile/pyupgrade rev: v3.19.0 @@ -35,11 +40,6 @@ repos: - id: pyupgrade args: ["--py36-plus"] -- repo: https://github.com/asottile/setup-cfg-fmt - rev: v2.7.0 - hooks: - - id: setup-cfg-fmt - - repo: https://github.com/pycqa/flake8 rev: 7.1.1 hooks: diff --git a/MANIFEST.in b/MANIFEST.in index 549cc6983c..7bcfbfaf6d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include LICENSE README.md CONTRIBUTING.md CITATION.cff pyproject.toml setup.py setup.cfg .clang-format +include LICENSE README.md CONTRIBUTING.md CITATION.cff pyproject.toml setup.py .clang-format graft example-models graft test graft contrib diff --git a/scripts/hls4ml b/hls4ml/cli/__init__.py similarity index 100% rename from scripts/hls4ml rename to hls4ml/cli/__init__.py diff --git a/pyproject.toml b/pyproject.toml index 6402ab0e7a..b713b41d80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,103 @@ [build-system] -# AVOID CHANGING REQUIRES: IT WILL BE UPDATED BY PYSCAFFOLD! -requires = ["setuptools>=46.1.0", "setuptools_scm[toml]>=5", "wheel"] build-backend = "setuptools.build_meta" +requires = [ "setuptools>=61", "setuptools-scm>=8" ] + +[project] +name = "hls4ml" +version = "1.0.0" +description = "Machine learning in FPGAs using HLS" +readme = "README.md" +license = { text = "Apache-2.0" } +authors = [ { name = "hls4ml Team" } ] +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: C++", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = [ "h5py", "numpy", "pydigitalwavetools==1.1", "pyyaml" ] + +optional-dependencies.doc = [ + "sphinx", + "sphinx-contributors", + "sphinx-github-changelog", + "sphinx-rtd-theme", +] +optional-dependencies.HGQ = [ "hgq~=0.2.0" ] +optional-dependencies.optimization = [ + "keras-tuner==1.1.3", + "ortools==9.4.1874", + "packaging", +] +optional-dependencies.profiling = [ "matplotlib", "pandas", "seaborn" ] +optional-dependencies.qkeras = [ + "qkeras", + "tensorflow>=2.8,<=2.14.1", + "tensorflow-model-optimization<=0.7.5", +] +optional-dependencies.quantus_report = [ "calmjs-parse", "tabulate" ] +optional-dependencies.sr = [ "sympy" ] +optional-dependencies.testing = [ + "calmjs-parse", + "hgq~=0.2.0", + "onnx>=1.4", + "pytest", + "pytest-cov", + "pytest-randomly", + "qonnx", + "tabulate", + "torch", +] +urls.Homepage = "https://fastmachinelearning.org/hls4ml" +scripts.hls4ml = "hls4ml.cli:main" +entry-points.pytest_randomly.random_seeder = "hls4ml:reseed" + +[tool.setuptools] +packages = [ "hls4ml" ] +include-package-data = true + [tool.setuptools_scm] -# See configuration details in https://github.com/pypa/setuptools_scm + version_scheme = "release-branch-semver" -git_describe_command = "git describe --dirty --tags --long --match v* --first-parent" +git_describe_command = [ + "git", + "describe", + "--dirty", + "--tags", + "--long", + "--match", + "v*", + "--first-parent", +] write_to = "hls4ml/_version.py" + +[tool.black] +line-length = 125 +skip-string-normalization = true + +[tool.isort] +profile = "black" +line_length = 125 + +[tool.flake8] +max-line-length = 125 +extend-ignore = [ "E203", "T201" ] + +[tool.check-manifest] +ignore = [ + ".github/**", + "docs/**", + ".pre-commit-config.yaml", + "Jenkinsfile", + "hls4ml/_version.py", +] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 1d4241f063..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,74 +0,0 @@ -[metadata] -name = hls4ml -description = Machine learning in FPGAs using HLS -long_description = file: README.md -long_description_content_type = text/markdown -url = https://fastmachinelearning.org/hls4ml -author = hls4ml Team -license = Apache-2.0 -license_files = LICENSE -classifiers = - Development Status :: 4 - Beta - Intended Audience :: Developers - Intended Audience :: Science/Research - License :: OSI Approved :: Apache Software License - Programming Language :: C++ - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only - Topic :: Software Development :: Libraries - Topic :: Software Development :: Libraries :: Python Modules -description_file = README.md - -[options] -packages = find: -install_requires = - h5py - numpy - pydigitalwavetools==1.1 - pyyaml -python_requires = >=3.10 -include_package_data = True -scripts = scripts/hls4ml - -[options.entry_points] -pytest_randomly.random_seeder = - hls4ml = hls4ml:reseed - -[options.extras_require] -HGQ = - HGQ~=0.2.0 -optimization = - keras-tuner==1.1.3 - ortools==9.4.1874 - packaging -profiling = - matplotlib - pandas - seaborn -qkeras = - qkeras - tensorflow>=2.8.0,<=2.14.1 - tensorflow-model-optimization<=0.7.5 -quantus_report = - calmjs.parse - tabulate -sr = - sympy -testing = - HGQ~=0.2.0 - calmjs.parse - onnx>=1.4.0 - pytest - pytest-cov - pytest-randomly - qonnx - tabulate - torch - -[check-manifest] -ignore = - .github/** - docs/** - .pre-commit-config.yaml - Jenkinsfile - hls4ml/_version.py diff --git a/setup.py b/setup.py deleted file mode 100644 index 1abbd068c1..0000000000 --- a/setup.py +++ /dev/null @@ -1,4 +0,0 @@ -import setuptools - -if __name__ == "__main__": - setuptools.setup() From 06f9cda7a705c2c10e83c71c9bc28edc29af644a Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Sun, 15 Dec 2024 22:57:49 +0000 Subject: [PATCH 08/21] format --- hls4ml/converters/onnx_to_hls.py | 2 ++ hls4ml/writer/oneapi_writer.py | 49 ++++++++++++++++++-------------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/hls4ml/converters/onnx_to_hls.py b/hls4ml/converters/onnx_to_hls.py index 99281888f3..f3b6acbaf3 100644 --- a/hls4ml/converters/onnx_to_hls.py +++ b/hls4ml/converters/onnx_to_hls.py @@ -76,6 +76,7 @@ def get_input_shape(graph, node): def get_constant_value(graph, constant_name): tensor = next((x for x in graph.initializer if x.name == constant_name), None) from onnx import numpy_helper + return numpy_helper.to_array(tensor) @@ -274,6 +275,7 @@ def onnx_to_hls(config): print('Interpreting Model ...') import onnx + onnx_model = onnx.load(config['OnnxModel']) if isinstance(config['OnnxModel'], str) else config['OnnxModel'] layer_list, input_layers, output_layers = parse_onnx_model(onnx_model) diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index fe633214f6..c9af2544bd 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -102,9 +102,10 @@ def write_project_cpp(self, model): project_name = model.config.get_project_name() filedir = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(filedir, '../templates/oneapi/firmware/myproject.cpp')) as f, open( - f'{model.config.get_output_dir()}/src/firmware/{project_name}.cpp', 'w' - ) as fout: + with ( + open(os.path.join(filedir, '../templates/oneapi/firmware/myproject.cpp')) as f, + open(f'{model.config.get_output_dir()}/src/firmware/{project_name}.cpp', 'w') as fout, + ): model_inputs = model.get_input_variables() model_outputs = model.get_output_variables() model_brams = [var for var in model.get_weight_variables() if var.storage.lower() == 'bram'] @@ -207,9 +208,10 @@ def write_project_header(self, model): project_name = model.config.get_project_name() filedir = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(filedir, '../templates/oneapi/firmware/myproject.h')) as f, open( - f'{model.config.get_output_dir()}/src/firmware/{project_name}.h', 'w' - ) as fout: + with ( + open(os.path.join(filedir, '../templates/oneapi/firmware/myproject.h')) as f, + open(f'{model.config.get_output_dir()}/src/firmware/{project_name}.h', 'w') as fout, + ): model_inputs = model.get_input_variables() model_outputs = model.get_output_variables() # model_brams = [var for var in model.get_weight_variables() if var.storage.lower() == 'bram'] @@ -254,9 +256,10 @@ def write_defines(self, model): model (ModelGraph): the hls4ml model. """ filedir = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(filedir, '../templates/oneapi/firmware/defines.h')) as f, open( - f'{model.config.get_output_dir()}/src/firmware/defines.h', 'w' - ) as fout: + with ( + open(os.path.join(filedir, '../templates/oneapi/firmware/defines.h')) as f, + open(f'{model.config.get_output_dir()}/src/firmware/defines.h', 'w') as fout, + ): for line in f.readlines(): # Insert numbers if '// hls-fpga-machine-learning insert numbers' in line: @@ -298,9 +301,10 @@ def write_parameters(self, model): model (ModelGraph): the hls4ml model. """ filedir = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(filedir, '../templates/oneapi/firmware/parameters.h')) as f, open( - f'{model.config.get_output_dir()}/src/firmware/parameters.h', 'w' - ) as fout: + with ( + open(os.path.join(filedir, '../templates/oneapi/firmware/parameters.h')) as f, + open(f'{model.config.get_output_dir()}/src/firmware/parameters.h', 'w') as fout, + ): for line in f.readlines(): if '// hls-fpga-machine-learning insert includes' in line: newline = line @@ -376,9 +380,10 @@ def write_test_bench(self, model): output_predictions, f'{model.config.get_output_dir()}/tb_data/tb_output_predictions.dat' ) - with open(os.path.join(filedir, '../templates/oneapi/myproject_test.cpp')) as f, open( - f'{model.config.get_output_dir()}/src/{project_name}_test.cpp', 'w' - ) as fout: + with ( + open(os.path.join(filedir, '../templates/oneapi/myproject_test.cpp')) as f, + open(f'{model.config.get_output_dir()}/src/{project_name}_test.cpp', 'w') as fout, + ): for line in f.readlines(): indent = ' ' * (len(line) - len(line.lstrip(' '))) @@ -434,9 +439,10 @@ def write_bridge(self, model): indent = ' ' filedir = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(filedir, '../templates/oneapi/myproject_bridge.cpp')) as f, open( - f'{model.config.get_output_dir()}/src/{project_name}_bridge.cpp', 'w' - ) as fout: + with ( + open(os.path.join(filedir, '../templates/oneapi/myproject_bridge.cpp')) as f, + open(f'{model.config.get_output_dir()}/src/{project_name}_bridge.cpp', 'w') as fout, + ): for line in f.readlines(): if 'MYPROJECT' in line: newline = line.replace('MYPROJECT', format(project_name.upper())) @@ -511,9 +517,10 @@ def write_build_script(self, model): # Makefile filedir = os.path.dirname(os.path.abspath(__file__)) device = model.config.get_config_value('Part') - with open(os.path.join(filedir, '../templates/oneapi/CMakeLists.txt')) as f, open( - f'{model.config.get_output_dir()}/CMakeLists.txt', 'w' - ) as fout: + with ( + open(os.path.join(filedir, '../templates/oneapi/CMakeLists.txt')) as f, + open(f'{model.config.get_output_dir()}/CMakeLists.txt', 'w') as fout, + ): for line in f.readlines(): line = line.replace('myproject', model.config.get_project_name()) line = line.replace('mystamp', model.config.get_config_value('Stamp')) From 014c1dbc730a57241ca9eab9f402b2758edacd8d Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Sun, 15 Dec 2024 23:12:45 +0000 Subject: [PATCH 09/21] rm useless flake8 config in pyprject.toml --- pyproject.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b713b41d80..756e688d5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,10 +89,6 @@ skip-string-normalization = true profile = "black" line_length = 125 -[tool.flake8] -max-line-length = 125 -extend-ignore = [ "E203", "T201" ] - [tool.check-manifest] ignore = [ ".github/**", From d3c888145910629170bb2399403ce65e65b97ec0 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Mon, 16 Dec 2024 01:20:13 +0000 Subject: [PATCH 10/21] Add hint on import failure --- hls4ml/converters/__init__.py | 47 +++++++++-------------------- hls4ml/converters/onnx_to_hls.py | 4 +++ hls4ml/converters/pytorch_to_hls.py | 4 +++ hls4ml/model/__init__.py | 7 ----- hls4ml/model/quantizers.py | 4 +++ hls4ml/report/quartus_report.py | 4 +++ hls4ml/utils/config.py | 2 ++ pyproject.toml | 1 + 8 files changed, 34 insertions(+), 39 deletions(-) diff --git a/hls4ml/converters/__init__.py b/hls4ml/converters/__init__.py index 1343907b54..693a76f666 100644 --- a/hls4ml/converters/__init__.py +++ b/hls4ml/converters/__init__.py @@ -1,6 +1,5 @@ import importlib import os -import warnings import yaml @@ -10,33 +9,19 @@ from hls4ml.converters.keras_to_hls import get_supported_keras_layers # noqa: F401 from hls4ml.converters.keras_to_hls import parse_keras_model # noqa: F401 from hls4ml.converters.keras_to_hls import keras_to_hls, register_keras_layer_handler +from hls4ml.converters.onnx_to_hls import get_supported_onnx_layers # noqa: F401 from hls4ml.converters.onnx_to_hls import parse_onnx_model # noqa: F401 +from hls4ml.converters.onnx_to_hls import onnx_to_hls, register_onnx_layer_handler +from hls4ml.converters.pytorch_to_hls import ( # noqa: F401 + get_supported_pytorch_layers, + pytorch_to_hls, + register_pytorch_layer_handler, +) from hls4ml.model import ModelGraph from hls4ml.utils.config import create_config +from hls4ml.utils.dependency import requires from hls4ml.utils.symbolic_utils import LUTFunction -# ----------Make converters available if the libraries can be imported----------# -try: - from hls4ml.converters.pytorch_to_hls import ( # noqa: F401 - get_supported_pytorch_layers, - pytorch_to_hls, - register_pytorch_layer_handler, - ) - - __pytorch_enabled__ = True -except ImportError: - warnings.warn("WARNING: Pytorch converter is not enabled!", stacklevel=1) - __pytorch_enabled__ = False - -try: - from hls4ml.converters.onnx_to_hls import get_supported_onnx_layers # noqa: F401 - from hls4ml.converters.onnx_to_hls import onnx_to_hls, register_onnx_layer_handler - - __onnx_enabled__ = True -except ImportError: - warnings.warn("WARNING: ONNX converter is not enabled!", stacklevel=1) - __onnx_enabled__ = False - # ----------Layer handling register----------# model_types = ['keras', 'pytorch', 'onnx'] @@ -51,7 +36,7 @@ # and has 'handles' attribute # and is defined in this module (i.e., not imported) if callable(func) and hasattr(func, 'handles') and func.__module__ == lib.__name__: - for layer in func.handles: + for layer in func.handles: # type: ignore if model_type == 'keras': register_keras_layer_handler(layer, func) elif model_type == 'pytorch': @@ -124,15 +109,9 @@ def convert_from_config(config): model = None if 'OnnxModel' in yamlConfig: - if __onnx_enabled__: - model = onnx_to_hls(yamlConfig) - else: - raise Exception("ONNX not found. Please install ONNX.") + model = onnx_to_hls(yamlConfig) elif 'PytorchModel' in yamlConfig: - if __pytorch_enabled__: - model = pytorch_to_hls(yamlConfig) - else: - raise Exception("PyTorch not found. Please install PyTorch.") + model = pytorch_to_hls(yamlConfig) else: model = keras_to_hls(yamlConfig) @@ -174,6 +153,7 @@ def _check_model_config(model_config): return model_config +@requires('_keras') def convert_from_keras_model( model, output_dir='my-hls-test', @@ -237,6 +217,7 @@ def convert_from_keras_model( return keras_to_hls(config) +@requires('_torch') def convert_from_pytorch_model( model, output_dir='my-hls-test', @@ -308,6 +289,7 @@ def convert_from_pytorch_model( return pytorch_to_hls(config) +@requires('onnx') def convert_from_onnx_model( model, output_dir='my-hls-test', @@ -371,6 +353,7 @@ def convert_from_onnx_model( return onnx_to_hls(config) +@requires('sr') def convert_from_symbolic_expression( expr, n_symbols=None, diff --git a/hls4ml/converters/onnx_to_hls.py b/hls4ml/converters/onnx_to_hls.py index f3b6acbaf3..d51701e726 100644 --- a/hls4ml/converters/onnx_to_hls.py +++ b/hls4ml/converters/onnx_to_hls.py @@ -1,4 +1,5 @@ from hls4ml.model import ModelGraph +from hls4ml.utils.dependency import requires # ----------------------Helpers--------------------- @@ -17,6 +18,7 @@ def replace_char_inconsitency(name): return name.replace('.', '_') +@requires('onnx') def get_onnx_attribute(operation, name, default=None): from onnx import helper @@ -73,6 +75,7 @@ def get_input_shape(graph, node): return rv +@requires('onnx') def get_constant_value(graph, constant_name): tensor = next((x for x in graph.initializer if x.name == constant_name), None) from onnx import numpy_helper @@ -258,6 +261,7 @@ def parse_onnx_model(onnx_model): return layer_list, input_layers, output_layers +@requires('onnx') def onnx_to_hls(config): """Convert onnx model to hls model from configuration. diff --git a/hls4ml/converters/pytorch_to_hls.py b/hls4ml/converters/pytorch_to_hls.py index 3ec5b17691..f279a1970a 100644 --- a/hls4ml/converters/pytorch_to_hls.py +++ b/hls4ml/converters/pytorch_to_hls.py @@ -1,4 +1,5 @@ from hls4ml.model import ModelGraph +from hls4ml.utils.dependency import requires class PyTorchModelReader: @@ -22,6 +23,7 @@ def get_weights_data(self, layer_name, var_name): return data +@requires('_torch') class PyTorchFileReader(PyTorchModelReader): # Inherit get_weights_data method def __init__(self, config): import torch @@ -103,6 +105,7 @@ def decorator(function): # ---------------------------------------------------------------- +@requires('_torch') def parse_pytorch_model(config, verbose=True): """Convert PyTorch model to hls4ml ModelGraph. @@ -368,6 +371,7 @@ def parse_pytorch_model(config, verbose=True): return layer_list, input_layers +@requires('_torch') def pytorch_to_hls(config): layer_list, input_layers = parse_pytorch_model(config) print('Creating HLS model') diff --git a/hls4ml/model/__init__.py b/hls4ml/model/__init__.py index fc504392b6..4ca72e3cd6 100644 --- a/hls4ml/model/__init__.py +++ b/hls4ml/model/__init__.py @@ -1,8 +1 @@ from hls4ml.model.graph import HLSConfig, ModelGraph # noqa: F401 - -try: - from hls4ml.model import profiling # noqa: F401 - - __profiling_enabled__ = True -except ImportError: - __profiling_enabled__ = False diff --git a/hls4ml/model/quantizers.py b/hls4ml/model/quantizers.py index b445c70af3..eb313fc4ea 100644 --- a/hls4ml/model/quantizers.py +++ b/hls4ml/model/quantizers.py @@ -14,6 +14,7 @@ SaturationMode, XnorPrecisionType, ) +from hls4ml.utils.dependency import requires class Quantizer: @@ -84,6 +85,7 @@ class QKerasQuantizer(Quantizer): config (dict): Config of the QKeras quantizer to wrap. """ + @requires('qkeras') def __init__(self, config): from qkeras.quantizers import get_quantizer @@ -131,6 +133,7 @@ class QKerasBinaryQuantizer(Quantizer): config (dict): Config of the QKeras quantizer to wrap. """ + @requires('qkeras') def __init__(self, config, xnor=False): from qkeras.quantizers import get_quantizer @@ -155,6 +158,7 @@ class QKerasPO2Quantizer(Quantizer): config (dict): Config of the QKeras quantizer to wrap. """ + @requires('qkeras') def __init__(self, config): from qkeras.quantizers import get_quantizer diff --git a/hls4ml/report/quartus_report.py b/hls4ml/report/quartus_report.py index 47fc43c132..677a931402 100644 --- a/hls4ml/report/quartus_report.py +++ b/hls4ml/report/quartus_report.py @@ -2,6 +2,8 @@ import webbrowser from ast import literal_eval +from hls4ml.utils.dependency import requires + def parse_quartus_report(hls_dir, write_to_file=True): ''' @@ -39,6 +41,7 @@ def parse_quartus_report(hls_dir, write_to_file=True): return results +@requires('quantus-report') def read_quartus_report(hls_dir, open_browser=False): ''' Parse and print the Quartus report to print the report. Optionally open a browser. @@ -89,6 +92,7 @@ def _find_project_dir(hls_dir): return top_func_name + '-fpga.prj' +@requires('quantus-report') def read_js_object(js_script): ''' Reads the JavaScript file and return a dictionary of variables definded in the script. diff --git a/hls4ml/utils/config.py b/hls4ml/utils/config.py index 6a356f5f27..8c8ff3a069 100644 --- a/hls4ml/utils/config.py +++ b/hls4ml/utils/config.py @@ -1,6 +1,7 @@ import json import hls4ml +from hls4ml.utils.dependency import requires def create_config(output_dir='my-hls-test', project_name='myproject', backend='Vivado', version='1.0.0', **kwargs): @@ -44,6 +45,7 @@ def create_config(output_dir='my-hls-test', project_name='myproject', backend='V return config +@requires('qkeras') def _get_precision_from_quantizer(quantizer): if isinstance(quantizer, str): import qkeras diff --git a/pyproject.toml b/pyproject.toml index 756e688d5c..24175c9612 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ optional-dependencies.doc = [ "sphinx-rtd-theme", ] optional-dependencies.HGQ = [ "hgq~=0.2.0" ] +optional-dependencies.onnx = [ "onnx>=1.4" ] optional-dependencies.optimization = [ "keras-tuner==1.1.3", "ortools==9.4.1874", From 738e5b01ee8b7c8441870d467008d5e011ab14c7 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Mon, 16 Dec 2024 01:32:12 +0000 Subject: [PATCH 11/21] leftover --- hls4ml/utils/dependency.py | 55 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 hls4ml/utils/dependency.py diff --git a/hls4ml/utils/dependency.py b/hls4ml/utils/dependency.py new file mode 100644 index 0000000000..e546dcb8c9 --- /dev/null +++ b/hls4ml/utils/dependency.py @@ -0,0 +1,55 @@ +import sys +from functools import wraps +from importlib.metadata import metadata +from inspect import ismethod + +extra_requires: dict[str, list[str]] = {} +subpackage = None +for k, v in metadata('hls4ml')._headers: # type: ignore + if k != 'Requires-Dist': + continue + if '; extra == ' not in v: + continue + + req, pkg = v.split('; extra == ') + pkg = pkg.strip('"') + + extra_requires.setdefault(pkg, []).append(req) + + +def requires(pkg: str): + """Mark a function or method as requiring a package to be installed. + 'name': requires hls4ml[name] to be installed. + '_name': requires name to be installed. + + Parameters + ---------- + pkg : str + The package to require. + """ + + def deco(f): + if ismethod(f): + qualifier = f"Method {f.__self__.__class__.__name__}.{f.__name__}" + else: + qualifier = f"Function {f.__name__}" + + if not pkg.startswith("_"): + reqs = ", ".join(extra_requires[pkg]) + msg = f"{qualifier} requires {reqs}, but package {{ename}} is missing" + "Please consider install it with `pip install hls4ml[{pkg}]` for full functionality with {pkg}." + else: + msg = f"{qualifier} requires {pkg[1:]}, but package {{ename}} is missing." + "Consider install it with `pip install {pkg}`." + + @wraps(f) + def inner(*args, **kwargs): + try: + return f(*args, **kwargs) + except ImportError as e: + print(msg.format(ename=e.name), file=sys.stderr) + raise e + + return inner + + return deco From bc7778bd13cdd6fc0d7ceb1e00be9bdfc195bcf3 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Mon, 16 Dec 2024 01:32:46 +0000 Subject: [PATCH 12/21] rm setup.py from manifest --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7bcfbfaf6d..708e40c86b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include LICENSE README.md CONTRIBUTING.md CITATION.cff pyproject.toml setup.py .clang-format +include LICENSE README.md CONTRIBUTING.md CITATION.cff pyproject.toml .clang-format graft example-models graft test graft contrib From b76b5cb99e4928ba5f8791f406f5fc89276f5378 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Mon, 16 Dec 2024 01:48:01 +0000 Subject: [PATCH 13/21] manifest fix 2 --- MANIFEST.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 708e40c86b..5bec5fe2a6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,5 +3,6 @@ graft example-models graft test graft contrib recursive-include hls4ml/templates * -global-exclude .git .gitmodules .gitlab-ci.yml +recursive-include hls4ml *.py +global-exclude .git .gitmodules .gitlab-ci.yml *.pyc include hls4ml/backends/vivado_accelerator/supported_boards.json From b7f60f5ae2f895acfe69e283850bc2be4b31db59 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Thu, 7 Nov 2024 05:50:56 +0000 Subject: [PATCH 14/21] keras v3 object based parser --- hls4ml/converters/keras_to_hls.py | 28 ++- hls4ml/converters/keras_v3/__init__.py | 4 + hls4ml/converters/keras_v3/_base.py | 144 +++++++++++++ hls4ml/converters/keras_v3/core.py | 91 +++++++++ hls4ml/converters/keras_v3_to_hls.py | 270 +++++++++++++++++++++++++ 5 files changed, 532 insertions(+), 5 deletions(-) create mode 100644 hls4ml/converters/keras_v3/__init__.py create mode 100644 hls4ml/converters/keras_v3/_base.py create mode 100644 hls4ml/converters/keras_v3/core.py create mode 100644 hls4ml/converters/keras_v3_to_hls.py diff --git a/hls4ml/converters/keras_to_hls.py b/hls4ml/converters/keras_to_hls.py index 9fc63cf398..a206da4da7 100644 --- a/hls4ml/converters/keras_to_hls.py +++ b/hls4ml/converters/keras_to_hls.py @@ -1,9 +1,12 @@ import json +from warnings import warn import h5py from hls4ml.model import ModelGraph +from .keras_v3_to_hls import parse_keras_v3_model + MAXMULT = 4096 @@ -228,8 +231,8 @@ def parse_keras_model(model_arch, reader): layer_config = model_arch['config'] if 'layers' in layer_config: # Newer Keras versions have 'layers' in 'config' key layer_config = layer_config['layers'] - # Sequential doesn't have InputLayer in TF < 2.3 (Keras 2.4.0) if layer_config[0]['class_name'] != 'InputLayer': + warn(DeprecationWarning('keras < 2.4.0 (tf 2.3) is deprecated. Please use a newer version.')) input_layer = {} input_layer['name'] = 'input1' input_layer['class_name'] = 'InputLayer' @@ -241,25 +244,33 @@ def parse_keras_model(model_arch, reader): layer_config = model_arch['config']['layers'] input_layers = [inp[0] for inp in model_arch['config']['input_layers']] output_layers = [out[0] for out in model_arch['config']['output_layers']] + else: + raise Exception(f'ERROR: Model class not supported: {model_arch["class_name"]}') # Get input shape and check for unsupported layer type for keras_layer in layer_config: if keras_layer['class_name'] not in supported_layers: - raise Exception('ERROR: Unsupported layer type: {}'.format(keras_layer['class_name'])) + raise Exception(f'ERROR: Unsupported layer type: {keras_layer["class_name"]}') output_shapes = {} output_shape = None print('Topology:') for keras_layer in layer_config: - if 'batch_input_shape' in keras_layer['config']: + if 'batch_input_shape' in keras_layer['config'] or 'batch_shape' in keras_layer['config']: if 'inbound_nodes' in keras_layer and len(keras_layer['inbound_nodes']) > 0: input_shapes = [output_shapes[inbound_node[0]] for inbound_node in keras_layer['inbound_nodes'][0]] else: - input_shapes = [keras_layer['config']['batch_input_shape']] + _input_shapes = keras_layer['config'].get('batch_input_shape', None) + input_shapes = _input_shapes or keras_layer['config']['batch_shape'] else: if 'inbound_nodes' in keras_layer: - input_shapes = [output_shapes[inbound_node[0]] for inbound_node in keras_layer['inbound_nodes'][0]] + if 'args' in keras_layer['inbound_nodes'][0]: + # keras v3 + input_shapes = [arg['config']['shape'] for arg in keras_layer['inbound_nodes'][0]['args']] + else: + # keras v2 + input_shapes = [output_shapes[inbound_node[0]] for inbound_node in keras_layer['inbound_nodes'][0]] else: # Sequential model, so output_shape from the previous layer is still valid input_shapes = [output_shape] @@ -323,6 +334,13 @@ def parse_keras_model(model_arch, reader): def keras_to_hls(config): + if 'KerasModel' in config: + import keras + + if keras.__version__ >= '3.0': + layer_list, input_layers, output_layers, _ = parse_keras_v3_model(config['KerasModel']) + return ModelGraph(config, layer_list, input_layers, output_layers) + model_arch, reader = get_model_arch(config) layer_list, input_layers, output_layers, _ = parse_keras_model(model_arch, reader) print('Creating HLS model') diff --git a/hls4ml/converters/keras_v3/__init__.py b/hls4ml/converters/keras_v3/__init__.py new file mode 100644 index 0000000000..d064a39cbd --- /dev/null +++ b/hls4ml/converters/keras_v3/__init__.py @@ -0,0 +1,4 @@ +from . import core # noqa: F401 +from ._base import registry as layer_handlers + +__all__ = ['layer_handlers'] diff --git a/hls4ml/converters/keras_v3/_base.py b/hls4ml/converters/keras_v3/_base.py new file mode 100644 index 0000000000..e68db860bc --- /dev/null +++ b/hls4ml/converters/keras_v3/_base.py @@ -0,0 +1,144 @@ +import typing +from typing import Any, Callable, Sequence + +if typing.TYPE_CHECKING: + import keras + from keras.api import KerasTensor + +T_kv3_handler = Callable[ + ['keras.Layer', Sequence['keras.KerasTensor'], Sequence['keras.KerasTensor']], tuple[dict[str, Any], ...] +] + +registry: dict[str, T_kv3_handler] = {} + + +def register(cls: str | type): + """Decorator to register a handler for a specific layer class. Suggested to decorate the `KerasV3LayerHandler` class. + + Parameters + ---------- + cls : str|type + If str, the key to register the handler under. If type, the class to register the handler for. + + Examples + -------- + ```python + @keras_dispatcher.register + class MyLayerHandler(KerasV3LayerHandler): + handles = ('my_package.src.submodule.MyLayer', 'MyLayer2') + + def handle(self, layer, inp_tensors, out_tensors): + # handler code + + + @keras_dispatcher.register('MyLayer3') + def my_layer_handler(layer, inp_tensors, out_tensors): + # handler code + ``` + """ + + def deco(func: T_kv3_handler): + if isinstance(cls, str): + registry[cls] = func + for k in getattr(func, 'handles', ()): + registry[k] = func + return func + + if isinstance(cls, type): + return deco(cls()) + return deco + + +def maybe_add_attrs(config: dict[str, Any], obj: Any, *attrs: str): + for attr in attrs: + if attr not in config and hasattr(obj, attr): + config[attr] = getattr(obj, attr) + + +class KerasV3LayerHandler: + """Base class for keras v3 layer handlers. Subclass this class to create a handler for a specific layer type.""" + + handles = () + + def __call__( + self, + layer: 'keras.Layer', + in_tensors: Sequence['KerasTensor'], + out_tensors: Sequence['KerasTensor'], + ): + """Handle a keras layer. Return a tuple of dictionaries, each + dictionary representing a layer (module) in the HLS model. One + layer may correspond one or more dictionaries (e.g., layers with + activation functions will be split into two layers). + + Some common attributes are automatically added to the dictionary + if the handler returns a single dictionary. If the handler + returns multiple dictionaries, the attributes must be added + manually. Anything returned by the handler will override the + automatic attributes. + + Automatic attributes: - name - class_name - module - + input_keras_tensor_names - input_shape - + output_keras_tensor_names + + If the layer has an activation function, an additional + dictionary will be added to the return value representing the + activation function. + + + Parameters + ---------- + layer : keras.Layer + The layer to be converted to HLS configuration(s). + in_tensors : Sequence[KerasTensor] + The list of input tensors to the layer. + out_tensors : Sequence[KerasTensor] + The list of output tensors from the layer. + + Returns + ------- + dict[str, Any] | tuple[dict[str, Any], ...] + layer configuration(s) for the HLS model to be consumed by + the ModelGraph constructor + """ # noqa: E501 + import keras + + config0 = self.handle(layer, in_tensors, out_tensors) + if isinstance(config0, tuple): + return config0 + + name = layer.name + class_name = layer.__class__.__name__ + module = layer.__module__ + config1 = { + 'name': name, + 'class_name': class_name, + 'module': module, + 'input_keras_tensor_names': [t.name for t in in_tensors], + 'input_shape': [list(t.shape[1:]) for t in in_tensors], + 'output_keras_tensor_names': [t.name for t in out_tensors], + } + + maybe_add_attrs(config1, layer, 'epsilon', 'use_bias', 'data_format') + + config1.update(config0) + ret = (config1,) + + activation = getattr(layer, 'activation', None) + if activation not in (keras.activations.linear, None): + act_cls_name = activation.__class__.__name__ + act_config = { + 'class_name': 'Activation', + 'activation': act_cls_name, + 'name': f'{name}_{act_cls_name}', + } + ret = *ret, act_config + return ret + + def handle( + self, + layer: 'keras.Layer', + in_tensors: Sequence['KerasTensor'], + out_tensors: Sequence['KerasTensor'], + ) -> dict[str, Any] | tuple[dict[str, Any], ...]: + return {} diff --git a/hls4ml/converters/keras_v3/core.py b/hls4ml/converters/keras_v3/core.py new file mode 100644 index 0000000000..e6f1caa881 --- /dev/null +++ b/hls4ml/converters/keras_v3/core.py @@ -0,0 +1,91 @@ +import typing +from typing import Any, Sequence + +import numpy as np + +from ._base import KerasV3LayerHandler, register + +if typing.TYPE_CHECKING: + import keras + from keras.api import KerasTensor + from keras.src.layers.merging.base_merge import Merge + + +@register +class KV3DenseHandler(KerasV3LayerHandler): + handles = ('keras.src.layers.core.dense.Dense',) + + def handle( + self, + layer: 'keras.layers.Dense', + in_tensors: Sequence['KerasTensor'], + out_tensors: Sequence['KerasTensor'], + ): + kernel = np.array(layer.kernel) + assert layer._build_shapes_dict is not None, f"Layer {layer.name} is not built" + # inp_shape = layer._build_shapes_dict['input_shape'][1:] + config = { + 'data_format': 'channels_last', + 'weight_data': kernel, + 'bias_data': np.array(layer.bias) if layer.use_bias else None, + 'n_out': kernel.shape[1], + 'n_in': kernel.shape[0], + } + return config + + +@register +class KV3InputHandler(KerasV3LayerHandler): + handles = ('keras.src.layers.core.input_layer.InputLayer',) + + def handle( + self, + layer: 'keras.layers.InputLayer', + in_tensors: Sequence['KerasTensor'], + out_tensors: Sequence['KerasTensor'], + ): + config = {'input_shape': list(layer._batch_shape[1:])} + return config + + +@register +class KV3MergeHandler(KerasV3LayerHandler): + handles = ( + 'keras.src.layers.merging.add.Add', + 'keras.src.layers.merging.multiply.Multiply', + 'keras.src.layers.merging.average.Average', + 'keras.src.layers.merging.maximum.Maximum', + 'keras.src.layers.merging.minimum.Minimum', + 'keras.src.layers.merging.concatenate.Concatenate', + 'keras.src.layers.merging.subtract.Subtract', + 'keras.src.layers.merging.dot.Dot', + ) + + def handle( + self, + layer: 'Merge', + in_tensors: Sequence['KerasTensor'], + out_tensors: Sequence['KerasTensor'], + ): + assert len(out_tensors) == 1, f"Merge layer {layer.name} has more than one output" + output_shape = list(out_tensors[0].shape[1:]) + + config: dict[str, Any] = { + 'output_shape': output_shape, + 'op': layer.__class__.__name__.lower(), + } + + match layer.__class__.__name__: + case 'Concatenate': + rank = len(output_shape) + class_name = f'Concatenate{rank}d' + config['axis'] = layer.axis + case 'Dot': + class_name = f'Dot{len(output_shape)}d' + rank = len(output_shape) + assert rank == 1, f"Dot product only supported for 1D tensors, got {rank}D on layer {layer.name}" + case _: + class_name = 'Merge' + + config['class_name'] = class_name + return config diff --git a/hls4ml/converters/keras_v3_to_hls.py b/hls4ml/converters/keras_v3_to_hls.py new file mode 100644 index 0000000000..cf5b9c5d25 --- /dev/null +++ b/hls4ml/converters/keras_v3_to_hls.py @@ -0,0 +1,270 @@ +import typing +from itertools import chain +from typing import Any, Callable, Sequence + +if typing.TYPE_CHECKING: + import keras + from keras.api import KerasTensor + +import numpy as np + +from .keras_v3 import layer_handlers as v3_layer_handlers + +T_kv3_handler = Callable[ + ['keras.Layer', Sequence['keras.KerasTensor'], Sequence['keras.KerasTensor']], tuple[dict[str, Any], ...] +] + + +def get_io_tensors(layer: 'keras.Layer', node_whitelist: set[int] | None = None): + """Given a keras layer, return a list of tuples of input and output + tensors. If the layer is called only once (i.e., no shared layers), + the list will contain only one tuple. + + The layer must have been built before calling this function. + + Parameters + ---------- + layer : keras.Layer + The layer to get input and output tensors from. + node_whitelist : set[int]|None, optional + If not None, only return tensors from nodes with ids in this + set, used to filter out nodes that are not part of the model, by + default None + + + Returns + ------- + list[tuple[tuple['KerasTensor', ...], tuple['KerasTensor', ...]]] + A list of tuples of input and output tensors. + """ + in_nodes = layer._inbound_nodes + if node_whitelist is not None: + in_nodes = [node for node in in_nodes if id(node) in node_whitelist] + + ret: list[tuple[tuple['KerasTensor', ...], tuple['KerasTensor', ...]]] = [] + for node in in_nodes: + in_tensors = tuple(node.arguments.keras_tensors) + out_tensors = tuple(node.outputs) + ret.append((in_tensors, out_tensors)) + return ret + + +def resolve_dependency_relation(model: 'keras.Model'): + """Given a keras model, return the following information: + - A list of input tensor names + - A list of output tensor names + - A list of (layer_name, input_tensor_names, output_tensor_names) tuples + - A dictionary of tensor_name -> KerasTensor + + Parameters + ---------- + model : keras.Model + The keras model to analyze. + + Returns + ------- + tuple[tuple[str, ...], tuple[str, ...], list[tuple[str, tuple[str, ...], tuple[str, ...]]], dict[str, KerasTensor]] + inp_tensor_names, out_tensor_names, layer_io, tensors + """ + tensors: dict[str, 'KerasTensor'] = {} + "tensor_name -> KerasTensor" + depends_on: dict[str, tuple[str, ...]] = {} + "tensor_name -> {tensor_name}" + layer_io: list[tuple[str, tuple[str, ...], tuple[str, ...]]] = [] + "layer_name -> ((input_tensor_names), (output_tensor_names))" + + inputs = tuple(t.name for t in model.inputs) + outputs = tuple(t.name for t in model.outputs) + node_whitelist = {id(node) for v in model._nodes_by_depth.values() for node in v} + + for layer in model.layers: + for in_tensors, out_tensors in get_io_tensors(layer, node_whitelist): + in_tensor_names = tuple(t.name for t in in_tensors) + out_tensor_names = tuple(t.name for t in out_tensors) + for t in chain(in_tensors, out_tensors): + tensors[t.name] = t + for o_name in out_tensor_names: + depends_on[o_name] = in_tensor_names + layer_io.append((layer.name, in_tensor_names, out_tensor_names)) + + return inputs, outputs, layer_io, tensors + + +class UniqueName: + """Helper class to generate unique names for layers, if one being used multiple times.""" + + def __init__(self): + self.used_names: set[str] = set() + + def next_name(self, name: str): + i = 0 + if name in self.used_names: + while f'{name}_{i}' in self.used_names: + i += 1 + name = f'{name}_{i}' + self.used_names.add(name) + return name + + def __call__(self, name: str): + return self.next_name(name) + + def reset(self): + self.used_names.clear() + + +class KerasV3HandlerDispatcher: + """Dispatcher class to handle different types of keras v3 layers.""" + + def __init__(self, layer_handlers: dict[str, T_kv3_handler], v2_layer_handlers=None): + self.registry = layer_handlers + self.v2_layer_handlers = v2_layer_handlers or {} + + def __call__( + self, layer: 'keras.Layer', in_tensors: Sequence['keras.KerasTensor'], out_tensors: Sequence['keras.KerasTensor'] + ) -> tuple[dict[str, Any], ...]: + assert layer.built, f"Layer {layer.name} is not built" + + ret = self.v3_call(layer, in_tensors, out_tensors) + if ret is not None: + return ret + ret = self.v2_call(layer, in_tensors, out_tensors) + if ret is not None: + return ret + + raise ValueError( + f"Layer {layer.__class__.__module__}.{layer.__class__.__name__} not found in either v3 or v2 handlers" + ) + + def v3_call( + self, layer: 'keras.layers.Layer', inp_tensors: Sequence['KerasTensor'], out_tensors: Sequence['KerasTensor'] + ): + cls_name = layer.__class__.__name__ + module = layer.__module__ + key = f"{module}.{cls_name}" + + # keras v3 handlers + handler = self.registry.get(key, None) + handler = handler or self.registry.get(cls_name, None) + + if handler is None: + return None + return handler(layer, inp_tensors, out_tensors) + + def v2_call( + self, layer: 'keras.layers.Layer', inp_tensors: Sequence['KerasTensor'], out_tensors: Sequence['KerasTensor'] + ): + # keras v2 handlers fallback + print("v2 handler") + config = layer.get_config() + layer_dict = {'config': config, 'class_name': layer.__class__.__name__} + + class DummyReader: + def get_weights_data(self, layer_name, var_name): + assert layer_name == layer.name, f"Processing {layer.name}, but handler tried to read {layer_name}" + for w in layer.weights: + if var_name in w.name: + return np.array(w) + raise ValueError(f"Variable {var_name} not found in layer {layer.name}") + + reader = DummyReader() + input_shapes = [list(t.shape) for t in inp_tensors] + input_names = [t.name for t in inp_tensors] + output_names = [t.name for t in out_tensors] + key = layer.__class__.__name__ + handler = self.v2_layer_handlers.get(key, None) + if handler is None: + return None + + ret, _ = handler(layer_dict, input_names, input_shapes, reader) + ret['outputs'] = output_names + ret = (ret,) + + activation = getattr(layer, 'activation', None) + if activation not in (keras.activations.linear, None): + act_cls_name = activation.__class__.__name__ + act_config = { + 'class_name': 'Activation', + 'activation': act_cls_name, + 'name': f'{layer.name}_{act_cls_name}', + } + ret = *ret, act_config + return ret + + +def parse_keras_v3_model(model: 'keras.Model'): + """Parse a keras model into a list of dictionaries, each + representing a layer in the HLS model, and a list of input and + output layer names. + + Parameters + ---------- + model : keras.Model + + Returns + ------- + tuple[list[dict[str, Any]], list[str], list[str], list[list[int]]] + layer_list, input_layer_names, output_layer_names, + batch_output_shapes + + Raises + ------ + ValueError + If a circular dependency is detected. + """ + + from .keras_to_hls import layer_handlers as v2_layer_handlers # Delayed import to avoid circular import + + keras_v3_dispatcher = KerasV3HandlerDispatcher(v3_layer_handlers, v2_layer_handlers) + + model_inputs, model_outputs, dependency, tensors = resolve_dependency_relation(model) + + satisfied = set() + total = len(tensors) + + unique_name = UniqueName() + + layer_list: list[dict[str, Any]] = [] + while len(satisfied) < total: + # Until all tensors in the model are satisfied + for i, (layer_name, in_tensor_names, out_tensor_names) in enumerate(dependency): + if not all(t in satisfied for t in in_tensor_names): + continue # Skip layer if some inputs are not ready + if all(t in satisfied for t in out_tensor_names): + continue # Skip layer if the outputs are already satisfied + + layer: 'keras.Layer' = model.get_layer(layer_name) + inp_tensors = [tensors[t] for t in in_tensor_names] + out_tensors = [tensors[t] for t in out_tensor_names] + + _configs = keras_v3_dispatcher(layer, inp_tensors, out_tensors) + # Dispatch to v3 handler if available, else fallback to v2 + # handler + + # Prevent name conflicts. If a layer is used multiple times, + # add a suffix to the name At this stage, connections + # between modules are recorded by i/o keras tensor names + # (guaranteed unique), thus we can safely rename the layers + for _conf in _configs: + _conf['name'] = unique_name(_conf['name']) + + layer_list.extend(_configs) # Add the layer to the list + satisfied.update(out_tensor_names) # Mark the outputs as satisfied + dependency.pop(i) + break # Restart the loop to add another layer + else: + # If no layer was added in the loop, then there is a circular dependency + raise ValueError("Circular dependency detected") + + # Mark inputs[inp layer name] for ModelGraph to parse from i/o keras tensor names + provides: dict[str, str] = {} # tensor_name -> src_layer_name + for conf in layer_list: + for out_name in conf['output_keras_tensor_names']: + provides[out_name] = conf['name'] + inputs = [provides[tname] for tname in conf['input_keras_tensor_names']] + conf['inputs'] = inputs + + input_layer_names = [provides[tname] for tname in model_inputs] + output_layer_names = [provides[tname] for tname in model_outputs] + batch_output_shapes = [list(tensors[tname].shape) for tname in model_outputs] + + return layer_list, input_layer_names, output_layer_names, batch_output_shapes From a7206b433a031afc578be2cf41885423442a63cd Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Fri, 8 Nov 2024 02:56:59 +0000 Subject: [PATCH 15/21] sequential and i/o tensor name parsing fix --- hls4ml/converters/keras_v3/_base.py | 57 ++++++++++++++++++++++------ hls4ml/converters/keras_v3_to_hls.py | 36 ++++++++++++------ hls4ml/utils/config.py | 14 +++++-- 3 files changed, 80 insertions(+), 27 deletions(-) diff --git a/hls4ml/converters/keras_v3/_base.py b/hls4ml/converters/keras_v3/_base.py index e68db860bc..6f69473073 100644 --- a/hls4ml/converters/keras_v3/_base.py +++ b/hls4ml/converters/keras_v3/_base.py @@ -1,5 +1,19 @@ import typing -from typing import Any, Callable, Sequence +from types import FunctionType +from typing import Any, Callable, Sequence, TypedDict + + +class DefaultConfig(TypedDict, total=False): + name: str + class_name: str + module: str + input_keras_tensor_names: list[str] + input_shape: list[list[int]] + output_keras_tensor_names: list[str] + epsilon: float + use_bias: bool + data_format: str + if typing.TYPE_CHECKING: import keras @@ -49,7 +63,7 @@ def deco(func: T_kv3_handler): return deco -def maybe_add_attrs(config: dict[str, Any], obj: Any, *attrs: str): +def maybe_add_attrs(config: dict[str, Any] | DefaultConfig, obj: Any, *attrs: str): for attr in attrs: if attr not in config and hasattr(obj, attr): config[attr] = getattr(obj, attr) @@ -103,36 +117,55 @@ def __call__( """ # noqa: E501 import keras - config0 = self.handle(layer, in_tensors, out_tensors) - if isinstance(config0, tuple): - return config0 - name = layer.name class_name = layer.__class__.__name__ module = layer.__module__ - config1 = { + + default_config: DefaultConfig = { 'name': name, 'class_name': class_name, 'module': module, 'input_keras_tensor_names': [t.name for t in in_tensors], - 'input_shape': [list(t.shape[1:]) for t in in_tensors], + 'input_shape': [list(t.shape[1:]) for t in in_tensors], # type: ignore 'output_keras_tensor_names': [t.name for t in out_tensors], } - maybe_add_attrs(config1, layer, 'epsilon', 'use_bias', 'data_format') + maybe_add_attrs(default_config, layer, 'epsilon', 'use_bias', 'data_format') - config1.update(config0) - ret = (config1,) + mandatory_keys = ['name', 'class_name', 'output_keras_tensor_names', 'input_keras_tensor_names'] + self.default_config = default_config + config0 = self.handle(layer, in_tensors, out_tensors) + del self.default_config + + if isinstance(config0, tuple): + for conf in config0: + for key in mandatory_keys: + assert key in conf, f"Key {key} missing from layer {name} handled by {self.__class__.__name__}" + return config0 + + config = {} + config.update(default_config) + config.update(config0) + ret = (config,) + + # If activation exists, append it activation = getattr(layer, 'activation', None) if activation not in (keras.activations.linear, None): - act_cls_name = activation.__class__.__name__ + assert len(out_tensors) == 1, f"Layer {name} has more than one output, but has an activation function" + assert isinstance(activation, FunctionType), f"Activation function for layer {name} is not a function" + intermediate_tensor_name = f'{out_tensors[0].name}_activation' + ret[0]['output_keras_tensor_names'] = [intermediate_tensor_name] + act_cls_name = activation.__name__ act_config = { 'class_name': 'Activation', 'activation': act_cls_name, 'name': f'{name}_{act_cls_name}', + 'input_keras_tensor_names': [intermediate_tensor_name], + 'output_keras_tensor_names': [out_tensors[0].name], } ret = *ret, act_config + return ret def handle( diff --git a/hls4ml/converters/keras_v3_to_hls.py b/hls4ml/converters/keras_v3_to_hls.py index cf5b9c5d25..d602dcf5f3 100644 --- a/hls4ml/converters/keras_v3_to_hls.py +++ b/hls4ml/converters/keras_v3_to_hls.py @@ -1,5 +1,6 @@ import typing from itertools import chain +from types import FunctionType from typing import Any, Callable, Sequence if typing.TYPE_CHECKING: @@ -154,7 +155,10 @@ def v2_call( self, layer: 'keras.layers.Layer', inp_tensors: Sequence['KerasTensor'], out_tensors: Sequence['KerasTensor'] ): # keras v2 handlers fallback - print("v2 handler") + print(f"v2 handler used for layer {layer.name}") + + import keras + config = layer.get_config() layer_dict = {'config': config, 'class_name': layer.__class__.__name__} @@ -176,16 +180,22 @@ def get_weights_data(self, layer_name, var_name): return None ret, _ = handler(layer_dict, input_names, input_shapes, reader) - ret['outputs'] = output_names + ret['output_keras_tensor_names'] = output_names + ret['input_keras_tensor_names'] = input_names ret = (ret,) activation = getattr(layer, 'activation', None) if activation not in (keras.activations.linear, None): - act_cls_name = activation.__class__.__name__ + assert isinstance(activation, FunctionType), f"Activation function for layer {layer.name} is not a function" + intermediate_tensor_name = f'{output_names[0]}_activation' + ret[0]['output_keras_tensor_names'] = (intermediate_tensor_name,) + act_cls_name = activation.__name__ act_config = { 'class_name': 'Activation', 'activation': act_cls_name, 'name': f'{layer.name}_{act_cls_name}', + 'input_keras_tensor_names': (intermediate_tensor_name,), + 'output_keras_tensor_names': output_names, } ret = *ret, act_config return ret @@ -212,6 +222,13 @@ def parse_keras_v3_model(model: 'keras.Model'): If a circular dependency is detected. """ + assert model.built, "Model must be built before parsing" + + import keras + + if isinstance(model, keras.Sequential): + model = model._functional # everything is functional under the hood lol + from .keras_to_hls import layer_handlers as v2_layer_handlers # Delayed import to avoid circular import keras_v3_dispatcher = KerasV3HandlerDispatcher(v3_layer_handlers, v2_layer_handlers) @@ -219,12 +236,12 @@ def parse_keras_v3_model(model: 'keras.Model'): model_inputs, model_outputs, dependency, tensors = resolve_dependency_relation(model) satisfied = set() - total = len(tensors) unique_name = UniqueName() layer_list: list[dict[str, Any]] = [] - while len(satisfied) < total: + + while any(t not in satisfied for t in model_outputs): # Until all tensors in the model are satisfied for i, (layer_name, in_tensor_names, out_tensor_names) in enumerate(dependency): if not all(t in satisfied for t in in_tensor_names): @@ -237,13 +254,10 @@ def parse_keras_v3_model(model: 'keras.Model'): out_tensors = [tensors[t] for t in out_tensor_names] _configs = keras_v3_dispatcher(layer, inp_tensors, out_tensors) - # Dispatch to v3 handler if available, else fallback to v2 - # handler + # Dispatch to v3 handler if available, else fallback to v2 handler - # Prevent name conflicts. If a layer is used multiple times, - # add a suffix to the name At this stage, connections - # between modules are recorded by i/o keras tensor names - # (guaranteed unique), thus we can safely rename the layers + # Prevent name conflicts. If a layer is used multiple times, add a suffix to the name. + # At this stage connections between modules are recorded by i/o keras tensor names for _conf in _configs: _conf['name'] = unique_name(_conf['name']) diff --git a/hls4ml/utils/config.py b/hls4ml/utils/config.py index 8c8ff3a069..f20aa49835 100644 --- a/hls4ml/utils/config.py +++ b/hls4ml/utils/config.py @@ -1,6 +1,7 @@ import json import hls4ml +import hls4ml.converters.keras_v3_to_hls from hls4ml.utils.dependency import requires @@ -159,12 +160,17 @@ def config_from_keras_model( if isinstance(model, dict): model_arch = model + reader = hls4ml.converters.KerasModelReader(model) + layer_list, _, _, _ = hls4ml.converters.parse_keras_model(model_arch, reader) else: - model_arch = json.loads(model.to_json()) + import keras - reader = hls4ml.converters.KerasModelReader(model) - - layer_list, _, _, _ = hls4ml.converters.parse_keras_model(model_arch, reader) + if keras.__version__ > '3.0': + layer_list, *_ = hls4ml.converters.parse_keras_v3_model(model) + else: + model_arch = json.loads(model.to_json()) + reader = hls4ml.converters.KerasModelReader(model) + layer_list, _, _, _ = hls4ml.converters.parse_keras_model(model_arch, reader) def make_layer_config(layer): cls_name = layer['class_name'] From 1605f96050350fca1592c763943dfd1445eaae64 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Fri, 8 Nov 2024 03:04:02 +0000 Subject: [PATCH 16/21] support activation layers --- hls4ml/converters/__init__.py | 3 + hls4ml/converters/keras_v3/core.py | 113 +++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/hls4ml/converters/__init__.py b/hls4ml/converters/__init__.py index 693a76f666..47569b1ad9 100644 --- a/hls4ml/converters/__init__.py +++ b/hls4ml/converters/__init__.py @@ -9,6 +9,7 @@ from hls4ml.converters.keras_to_hls import get_supported_keras_layers # noqa: F401 from hls4ml.converters.keras_to_hls import parse_keras_model # noqa: F401 from hls4ml.converters.keras_to_hls import keras_to_hls, register_keras_layer_handler +from hls4ml.converters.keras_v3_to_hls import parse_keras_v3_model # noqa: F401 from hls4ml.converters.onnx_to_hls import get_supported_onnx_layers # noqa: F401 from hls4ml.converters.onnx_to_hls import parse_onnx_model # noqa: F401 from hls4ml.converters.onnx_to_hls import onnx_to_hls, register_onnx_layer_handler @@ -17,6 +18,8 @@ pytorch_to_hls, register_pytorch_layer_handler, ) + +# from hls4ml.converters.pytorch_to_hls import parse_pytorch_model # noqa: F401 from hls4ml.model import ModelGraph from hls4ml.utils.config import create_config from hls4ml.utils.dependency import requires diff --git a/hls4ml/converters/keras_v3/core.py b/hls4ml/converters/keras_v3/core.py index e6f1caa881..ea63f97095 100644 --- a/hls4ml/converters/keras_v3/core.py +++ b/hls4ml/converters/keras_v3/core.py @@ -1,3 +1,4 @@ +import inspect import typing from typing import Any, Sequence @@ -89,3 +90,115 @@ def handle( config['class_name'] = class_name return config + + +@register +class KV3ActivationHandler(KerasV3LayerHandler): + handles = ('keras.src.layers.activations.activation.Activation',) + + def handle( + self, + layer: 'keras.layers.Activation', + in_tensors: Sequence['KerasTensor'], + out_tensors: Sequence['KerasTensor'], + ): + import keras + + config = {} + config.update(self.default_config) + + activation = getattr(layer, 'activation', keras.activations.linear) + match activation: + case keras.activations.softmax: + class_name = 'Softmax' + config['axis'] = -1 + case keras.activations.hard_sigmoid: + class_name = 'HardActivation' + case keras.activations.leaky_relu: + class_name = 'LeakyReLU' + signature = inspect.signature(keras.activations.leaky_relu) + config['activ_param'] = signature.parameters['negative_slope'].default + case keras.activations.elu: + class_name = 'ELU' + signature = inspect.signature(keras.activations.elu) + config['activ_param'] = signature.parameters['alpha'].default + case _: + class_name = 'Activation' + + config['activation'] = activation.__name__ + config['class_name'] = class_name + return (config,) + + +@register +class KV3ReLUHandler(KerasV3LayerHandler): + handles = ( + 'keras.src.layers.activations.leaky_relu.LeakyReLU', + 'keras.src.layers.activations.prelu.PReLU', + 'keras.src.layers.activations.relu.ReLU', + ) + + def handle( + self, + layer: 'keras.layers.ReLU', + in_tensors: Sequence['KerasTensor'], + out_tensors: Sequence['KerasTensor'], + ): + config = {} + config.update(self.default_config) + + if layer.__class__.__name__ == 'ReLU': + config['class_name'] = 'Activation' + config['activation'] = 'relu' + return config + + if layer.__class__.__name__ == 'PReLU': + config['class_name'] = 'PReLU' + config['param_data'] = np.array(layer.alpha) + config['activation'] = 'prelu' + else: + config['class_name'] = 'LeakyReLU' + config['activ_param'] = float(layer.negative_slope) + config['activation'] = 'leaky_relu' + + return (config,) + + +@register +class KV3SoftmaxHandler(KerasV3LayerHandler): + handles = ('keras.src.layers.activations.softmax.Softmax',) + + def handle( + self, + layer: 'keras.layers.Softmax', + in_tensors: Sequence['KerasTensor'], + out_tensors: Sequence['KerasTensor'], + ): + config = {} + config.update(self.default_config) + + config['class_name'] = 'Softmax' + config['axis'] = layer.axis + config['activation'] = 'softmax' + + return (config,) + + +@register +class KV3HardActivationHandler(KerasV3LayerHandler): + handles = ('keras.src.layers.activations.elu.ELU',) + + def handle( + self, + layer: 'keras.layers.ELU', + in_tensors: Sequence['KerasTensor'], + out_tensors: Sequence['KerasTensor'], + ): + config = {} + config.update(self.default_config) + + config['class_name'] = 'ELU' + config['activ_param'] = float(layer.alpha) + config['activation'] = 'elu' + + return (config,) From a8aa48967558aff62c0a074311aae27eece1bad8 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Fri, 8 Nov 2024 03:29:00 +0000 Subject: [PATCH 17/21] consistent v2 weight reader behavior --- hls4ml/converters/keras_v3_to_hls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hls4ml/converters/keras_v3_to_hls.py b/hls4ml/converters/keras_v3_to_hls.py index d602dcf5f3..5c0168cc1e 100644 --- a/hls4ml/converters/keras_v3_to_hls.py +++ b/hls4ml/converters/keras_v3_to_hls.py @@ -168,7 +168,7 @@ def get_weights_data(self, layer_name, var_name): for w in layer.weights: if var_name in w.name: return np.array(w) - raise ValueError(f"Variable {var_name} not found in layer {layer.name}") + return None reader = DummyReader() input_shapes = [list(t.shape) for t in inp_tensors] From eafe8b989e3208d35f7ffcc8af98972f1cf60cc6 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Fri, 8 Nov 2024 05:17:30 +0000 Subject: [PATCH 18/21] add v3 conv handlers --- hls4ml/converters/keras_v3/__init__.py | 1 + hls4ml/converters/keras_v3/conv.py | 122 +++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 hls4ml/converters/keras_v3/conv.py diff --git a/hls4ml/converters/keras_v3/__init__.py b/hls4ml/converters/keras_v3/__init__.py index d064a39cbd..f658faa1fb 100644 --- a/hls4ml/converters/keras_v3/__init__.py +++ b/hls4ml/converters/keras_v3/__init__.py @@ -1,3 +1,4 @@ +from . import conv # noqa: F401 from . import core # noqa: F401 from ._base import registry as layer_handlers diff --git a/hls4ml/converters/keras_v3/conv.py b/hls4ml/converters/keras_v3/conv.py new file mode 100644 index 0000000000..871bcb942d --- /dev/null +++ b/hls4ml/converters/keras_v3/conv.py @@ -0,0 +1,122 @@ +import typing +from math import ceil +from typing import Sequence + +import numpy as np + +from ._base import KerasV3LayerHandler, register + +if typing.TYPE_CHECKING: + import keras + from keras.api import KerasTensor + + +@register +class KV3ConvHandler(KerasV3LayerHandler): + handles = ( + 'keras.src.layers.convolutional.conv1d.Conv1D', + 'keras.src.layers.convolutional.conv2d.Conv2D', + 'keras.src.layers.convolutional.depthwise_conv1d.DepthwiseConv1D', + 'keras.src.layers.convolutional.depthwise_conv2d.DepthwiseConv2D', + 'keras.src.layers.convolutional.separable_conv1d.SeparableConv1D', + 'keras.src.layers.convolutional.separable_conv2d.SeparableConv2D', + ) + + def handle( + self, + layer: 'keras.layers.Conv1D|keras.layers.Conv2D|keras.layers.DepthwiseConv1D|keras.layers.DepthwiseConv2D', + in_tensors: Sequence['KerasTensor'], + out_tensors: Sequence['KerasTensor'], + ): + from keras.src.layers.convolutional.base_conv import BaseConv + from keras.src.layers.convolutional.base_depthwise_conv import BaseDepthwiseConv + from keras.src.layers.convolutional.base_separable_conv import BaseSeparableConv + + assert len(in_tensors) == 1, f"Layer {layer.name} has more than one input" + assert len(out_tensors) == 1, f"Layer {layer.name} has more than one output" + + in_shape: tuple[int, ...] = in_tensors[0].shape[1:] # type: ignore + out_shape: tuple[int, ...] = out_tensors[0].shape[1:] # type: ignore + assert all(isinstance(x, int) for x in in_shape), f"Layer {layer.name} has non-fixed size input: {in_shape}" + assert all(isinstance(x, int) for x in out_shape), f"Layer {layer.name} has non-fixed size output: {out_shape}" + + kernel = np.array(layer.kernel) + if layer.use_bias: + bias = np.array(layer.bias) + else: + bias = None + + ker_px_shape: tuple[int, ...] = layer.kernel_size + data_format = layer.data_format + + if data_format == 'channels_last': + *px_in_shape, ch_in = in_shape + *px_out_shape, ch_out = out_shape + else: + ch_in, *px_in_shape = in_shape + ch_out, *px_out_shape = out_shape + + if layer.padding == 'same': + n_padding = [ceil(N / n) * n - N for N, n in zip(px_in_shape, ker_px_shape)] + n_padding0 = [p // 2 for p in n_padding] + n_padding1 = [p - p0 for p, p0 in zip(n_padding, n_padding0)] + elif layer.padding == 'valid': + n_padding0 = [0] * len(px_in_shape) + n_padding1 = [0] * len(px_in_shape) + elif layer.padding == 'causal': + n_padding0 = [ker_px_shape[0] - 1] + [0] * (len(px_in_shape) - 1) + n_padding1 = [0] * len(px_in_shape) + else: + raise ValueError(f"Invalid padding mode {layer.padding} for layer {layer.name}") + + config = { + 'bias_data': bias, + 'data_format': data_format, + 'weight_data': kernel, + 'bias_data': bias, + 'n_filt': ch_out, + 'n_chan': ch_in, + } + + if layer.rank == 1: + config.update( + { + 'filt_width': ker_px_shape[0], + 'stride_width': layer.strides[0], + 'pad_left': n_padding0[0], + 'pad_right': n_padding1[0], + 'in_width': px_in_shape[0], + 'out_width': px_out_shape[0], + } + ) + elif layer.rank == 2: + config.update( + { + 'filt_height': ker_px_shape[0], + 'filt_width': ker_px_shape[1], + 'stride_height': layer.strides[0], + 'stride_width': layer.strides[1], + 'pad_top': n_padding0[0], + 'pad_bottom': n_padding1[0], + 'pad_left': n_padding0[1], + 'pad_right': n_padding1[1], + 'in_height': px_in_shape[0], + 'in_width': px_in_shape[1], + 'out_height': px_out_shape[0], + 'out_width': px_out_shape[1], + } + ) + else: + _cls = f"{layer.__class__.__module__}.{layer.__class__.__name__}" + raise ValueError(f"Only 1D and 2D conv layers are supported, got {_cls} (rank={layer.rank})") + if isinstance(layer, BaseDepthwiseConv): + config['depthwise_data'] = kernel + config['depth_multiplier'] = layer.depth_multiplier + elif isinstance(layer, BaseSeparableConv): + config['depthwise_data'] = kernel + config['pointwise_data'] = np.array(layer.pointwise_kernel) + config['depth_multiplier'] = layer.depth_multiplier + elif isinstance(layer, BaseConv): + config['weight_data'] = kernel + + return config From 6b8a44cdccef561eeca7c87ff9ce77427008faca Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Fri, 8 Nov 2024 14:01:18 +0000 Subject: [PATCH 19/21] add test --- test/pytest/test_keras_v3_api.py | 516 +++++++++++++++++++++++++++++++ 1 file changed, 516 insertions(+) create mode 100644 test/pytest/test_keras_v3_api.py diff --git a/test/pytest/test_keras_v3_api.py b/test/pytest/test_keras_v3_api.py new file mode 100644 index 0000000000..81ac5c240c --- /dev/null +++ b/test/pytest/test_keras_v3_api.py @@ -0,0 +1,516 @@ +import math +from pathlib import Path + +import keras +import numpy as np +import pytest + +if keras.__version__ < '3.0': + pytest.skip('Keras API tests are only for Keras 3.0 and above', allow_module_level=True) + +from keras.api.layers import ( + ELU, + Activation, + AveragePooling1D, + AveragePooling2D, + Conv1D, + Conv2D, + Dense, + DepthwiseConv1D, + DepthwiseConv2D, + LeakyReLU, + MaxPooling1D, + MaxPooling2D, + PReLU, +) + +import hls4ml + +test_root_path = Path('/tmp/tests') + + +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI', 'Catapult']) +@pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) +def test_dense(backend, io_type): + model = keras.Sequential( + [ + Dense( + 2, + input_shape=(1,), + name='Dense', + use_bias=True, + kernel_initializer=keras.initializers.RandomUniform(minval=1, maxval=10), # type: ignore + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + ), + Activation(activation='elu', name='Activation'), + ] + ) + model.compile(optimizer='adam', loss='mse') + + X_input = np.random.rand(1000, 1) + + keras_prediction = model.predict(X_input, verbose=0) # type: ignore + + config = hls4ml.utils.config_from_keras_model(model) + output_dir = str(test_root_path / f'hls4mlprj_keras_api_dense_{backend}_{io_type}') + + hls_model = hls4ml.converters.convert_from_keras_model( + model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type + ) + + hls_model.compile() + + hls_prediction = hls_model.predict(X_input) + + np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=0, atol=0.02) + + assert len(model.layers) + 1 == len(hls_model.get_layers()) + assert list(hls_model.get_layers())[0].attributes['class_name'] == "InputLayer" + assert list(hls_model.get_layers())[1].attributes["class_name"] == model.layers[0].name + assert list(hls_model.get_layers())[2].attributes['class_name'] == 'ELU' + + +# TODO: add ThresholdedReLU test when it can be made to pass +# https://github.com/fastmachinelearning/hls4ml/issues/376 + + +@pytest.mark.parametrize( + "activation_function", + [ + Activation(activation='relu', name='relu'), + LeakyReLU(negative_slope=0.5), + ELU(alpha=1.0), + PReLU( + alpha_initializer="zeros", + ), + Activation(activation='sigmoid', name='sigmoid'), + ], +) +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) +@pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) +def test_activations(activation_function, backend, io_type): + model = keras.models.Sequential() + model.add(Dense(64, input_shape=(1,), name='Dense', kernel_initializer='lecun_uniform', kernel_regularizer=None)) + model.add(activation_function) + + model.compile(optimizer='adam', loss='mse') + + model.summary() + + X_input = np.random.rand(1000, 1) + keras_prediction = model.predict(X_input, verbose=0) # type: ignore + config = hls4ml.utils.config_from_keras_model(model) + output_dir = str(test_root_path / f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}') + hls_model = hls4ml.converters.convert_from_keras_model( + model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type + ) + hls_model.compile() + hls_prediction = hls_model.predict(X_input) + + np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=0, atol=0.02) + + for layer in hls_model.get_layers(): + print(layer.attributes.attributes['class_name']) + assert len(model.layers) + 1 == len(hls_model.get_layers()) + + assert list(hls_model.get_layers())[2].attributes['class_name'] == activation_function.__class__.__name__ + + +padds_options = ['same', 'valid'] + + +@pytest.mark.parametrize('padds', padds_options) +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI', 'Catapult']) +@pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) +def test_conv1d(padds, backend, io_type): + model = keras.models.Sequential() + input_shape = (10, 128, 4) + model.add( + Conv1D( + filters=32, + kernel_size=3, + strides=2, + padding=padds, + activation='relu', + input_shape=input_shape[1:], + kernel_initializer='normal', + use_bias=False, + data_format='channels_last', + name='conv', + ) + ) + model.add(Activation(activation='relu')) + model.compile(optimizer='adam', loss='mse') + + X_input = np.random.rand(10, 128, 4) + keras_prediction = model.predict(X_input, verbose=0) # type: ignore + + config = hls4ml.utils.config_from_keras_model(model) + output_dir = str(test_root_path / f'hls4mlprj_keras_api_conv1d_{padds}_{backend}_{io_type}') + hls_model = hls4ml.converters.convert_from_keras_model( + model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type + ) + hls_model.compile() + hls_prediction = hls_model.predict(X_input).reshape(keras_prediction.shape) # type: ignore + + # 5e-2 might be too high + np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=0, atol=5e-2) + + if backend in ('Vivado', 'Vitis', 'Catapult') and io_type == 'io_stream' and padds == 'same': + # Vivado/Vitis inserts and additional layer for 'same' padding in io_stream + return + + conv: keras.layers.Conv1D = model.layers[0] + ker_w, ch_in, ch_out = conv.kernel.shape + inp_shape = model.inputs[0].shape[1:] + out_shape = model.outputs[0].shape[1:] + hls_attr = hls_model.graph['conv'].attributes + _stride = conv.strides[0] + + assert len(model.layers) + 2 == len(hls_model.get_layers()) + + assert hls_attr['name'] == model.layers[0].name + assert hls_attr['class_name'] == 'Conv1D' + assert hls_attr["in_width"] == inp_shape[0] + assert hls_attr['filt_width'] == ker_w + assert hls_attr['n_chan'] == ch_in + assert hls_attr['n_filt'] == ch_out + assert hls_attr['stride_width'] == _stride + assert hls_attr['data_format'] == conv.data_format + assert hls_attr["out_width"] == out_shape[0] + + w_pad = math.ceil(inp_shape[0] / ker_w) * ker_w - inp_shape[0] + + pad_left = w_pad // 2 + pad_right = w_pad - pad_left + + if model.layers[0].padding == 'same': + assert hls_attr['pad_left'] == pad_left + assert hls_attr['pad_right'] == pad_right + elif model.layers[0].padding == 'valid': + assert hls_attr['pad_left'] == 0 + assert hls_attr['pad_right'] == 0 + + +chans_options = ['channels_last'] +padds_options = ['same', 'valid'] + + +@pytest.mark.parametrize('chans', chans_options) +@pytest.mark.parametrize('padds', padds_options) +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI', 'Catapult']) +@pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) +def test_conv2d(chans, padds, backend, io_type): + input_shape = (32, 32, 3) + model = keras.Sequential( + [ + keras.layers.InputLayer(input_shape), + Conv2D( + filters=32, + kernel_size=(2, 3), + strides=(4, 5), + padding=padds, + kernel_initializer='normal', + use_bias=False, + data_format=chans, + name='conv', + ), + ] + ) + model.compile(optimizer='adam', loss='mse') + + X_input = np.random.rand(1000, *input_shape) + keras_prediction = model.predict(X_input) + + config = hls4ml.utils.config_from_keras_model(model) + output_dir = str(test_root_path / f'hls4ml_project_keras_api_conv2d_{backend}_{chans}_{padds}_{io_type}') + hls_model = hls4ml.converters.convert_from_keras_model( + model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type + ) + hls_model.compile() + hls_prediction = hls_model.predict(X_input).reshape(keras_prediction.shape) # type: ignore + + # A high tolerance, simply to verify correct functionality + np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=0, atol=5e-2) + + hls_conv_attr = hls_model.graph['conv'].attributes + + conv: keras.layers.Conv2D = model.get_layer('conv') + + kh, kw, ch_in, ch_out = conv.kernel.shape # type: ignore + _stride = conv.strides + inp_shape = model.inputs[0].shape[1:] + out_shape = model.outputs[0].shape[1:] + + if io_type == 'io_stream' and padds == 'same' and backend in ('Vivado', 'Vitis', 'Catapult'): + return + + assert len(model.layers) + 1 == len(hls_model.get_layers()) + assert hls_conv_attr['name'] == conv.name + assert hls_conv_attr['class_name'] == 'Conv2D' + assert hls_conv_attr['filt_width'] == kw + assert hls_conv_attr['filt_height'] == kh + assert hls_conv_attr['n_filt'] == ch_out + assert hls_conv_attr['stride_width'] == _stride[1] + assert hls_conv_attr['stride_height'] == _stride[0] + assert hls_conv_attr['data_format'] == conv.data_format + + if conv.data_format == 'channels_first': + assert hls_conv_attr['n_chan'] == inp_shape[0] + assert hls_conv_attr['in_height'] == inp_shape[1] + assert hls_conv_attr['in_width'] == inp_shape[2] + assert hls_conv_attr['out_height'] == out_shape[1] + assert hls_conv_attr['out_width'] == out_shape[2] + elif model.layers[0].data_format == 'channels_last': + assert hls_conv_attr['n_chan'] == inp_shape[2] + assert hls_conv_attr['in_height'] == inp_shape[0] + assert hls_conv_attr['in_width'] == inp_shape[1] + assert hls_conv_attr['out_height'] == out_shape[0] + assert hls_conv_attr['out_width'] == out_shape[1] + + if conv.padding == 'same': + if conv.data_format == 'channels_first': + h_pad = math.ceil(inp_shape[1] / kh) * kh - inp_shape[1] + w_pad = math.ceil(inp_shape[2] / kw) * kw - inp_shape[2] + elif model.layers[0].data_format == 'channels_last': + h_pad = math.ceil(inp_shape[0] / kh) * kh - inp_shape[0] + w_pad = math.ceil(inp_shape[1] / kw) * kw - inp_shape[1] + else: + raise ValueError('Invalid data_format') + pad_top = h_pad // 2 + pad_bottom = h_pad - pad_top + pad_left = w_pad // 2 + pad_right = w_pad - pad_left + assert hls_conv_attr['pad_top'] == pad_top + assert hls_conv_attr['pad_bottom'] == pad_bottom + assert hls_conv_attr['pad_left'] == pad_left + assert hls_conv_attr['pad_right'] == pad_right + elif model.layers[0].padding == 'valid': + assert hls_conv_attr['pad_top'] == 0 + assert hls_conv_attr['pad_bottom'] == 0 + assert hls_conv_attr['pad_left'] == 0 + assert hls_conv_attr['pad_right'] == 0 + + +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Catapult']) +@pytest.mark.parametrize('io_type', ['io_stream', 'io_parallel']) +def test_depthwise2d(backend, io_type): + ''' + Test proper handling of DepthwiseConv2D + ''' + X = np.random.rand(10, 32, 32, 3) + X = np.round(X * 2**10) * 2**-10 # make it an exact ap_fixed<16,6> + model = keras.models.Sequential([keras.layers.Input((32, 32, 3)), DepthwiseConv2D(kernel_size=(3, 3))]) + model.compile() + + config = hls4ml.utils.config_from_keras_model( + model, granularity='name', default_precision='fixed<32,12>', backend=backend + ) + output_dir = str(test_root_path / f'hls4mlprj_keras_api_depthwiseconv2d_{backend}_{io_type}') + hls_model = hls4ml.converters.convert_from_keras_model( + model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type + ) + hls_model.compile() + + y_qkeras = model.predict(X) + y_hls4ml = hls_model.predict(X) + + np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01) # type: ignore + + +# Currently only Vivado and Vitis is supported for io_stream. +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis']) +@pytest.mark.parametrize('io_type', ['io_stream']) +def test_depthwise1d(backend, io_type): + ''' + Test proper handling of DepthwiseConv1D. + ''' + X = np.random.rand(10, 32, 3) + X = np.round(X * 2**10) * 2**-10 # make it an exact ap_fixed<16,6> + model = keras.Sequential([DepthwiseConv1D(kernel_size=3, input_shape=(32, 3))]) + model.compile() + + config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend=backend) + output_dir = str(test_root_path / f'hls4mlprj_keras_api_depthwiseconv1d_{backend}_{io_type}') + hls_model = hls4ml.converters.convert_from_keras_model( + model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type + ) + hls_model.compile() + + y_qkeras = model.predict(X) + y_hls4ml = hls_model.predict(X) + + np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01) # type: ignore + + +pooling_layers = [MaxPooling1D, MaxPooling2D, AveragePooling1D, AveragePooling2D] + + +@pytest.mark.parametrize('pooling', pooling_layers) +@pytest.mark.parametrize('padds', padds_options) +@pytest.mark.parametrize('chans', chans_options) +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI', 'Catapult']) +def test_pooling(pooling, padds, chans, backend): + assert '1D' in pooling.__name__ or '2D' in pooling.__name__ + + input_shape = (18, 15, 3) if '2D' in pooling.__name__ else (121, 3) + pool_size = (4, 2) if '2D' in pooling.__name__ else 2 + + X_input = np.random.rand(100, *input_shape) + + keras_model = keras.Sequential([pooling(pool_size, padding=padds, input_shape=input_shape)]) + keras_model.compile() + + hls_cfg = hls4ml.utils.config_from_keras_model(keras_model) + output_dir = str( + test_root_path / f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}' + ) + hls_model = hls4ml.converters.convert_from_keras_model( + keras_model, hls_config=hls_cfg, output_dir=output_dir, backend=backend + ) + hls_model.compile() + + # Verify accuracy + keras_prediction = keras_model.predict(X_input) + hls_prediction = hls_model.predict(X_input).reshape(keras_prediction.shape) # type: ignore + np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=0, atol=3e-2) + + # # Verify correct parsing of layer + # hls_pool = list(hls_model.get_layers())[-1] + # ker_pool = keras_model.layers[-1] + # if '2D' in pooling.__name__: + # assert hls_pool.attributes['name'] == ker_pool._name + # assert hls_pool.attributes['class_name'][-2] == str(2) + # assert hls_pool.attributes['stride_height'] == ker_pool.strides[0] + # assert hls_pool.attributes['stride_width'] == ker_pool.strides[1] + # assert hls_pool.attributes['pool_height'] == ker_pool.pool_size[1] + # assert hls_pool.attributes['pool_width'] == ker_pool.pool_size[0] + + # if hls_pool.attributes['data_format'] == 'channels_last': + # assert hls_pool.attributes['in_height'] == ker_pool.input_shape[1] + # assert hls_pool.attributes['in_width'] == ker_pool.input_shape[2] + # assert hls_pool.attributes['n_filt'] == ker_pool.input_shape[3] + # elif hls_pool.attributes['data_format'] == 'channels_first': + # assert hls_pool.attributes['in_height'] == ker_pool.input_shape[2] + # assert hls_pool.attributes['in_width'] == ker_pool.input_shape[3] + # assert hls_pool.attributes['n_filt'] == ker_pool.input_shape[1] + + # if ker_pool.padding == 'same': + # # Height + # in_height = ker_pool.input_shape[1] + # if ker_pool.data_format == 'channels_first': + # in_height = ker_pool.input_shape[2] + # out_height = int(math.ceil(float(in_height) / float(ker_pool.strides[0]))) + # assert out_height == hls_pool.attributes['out_height'] + # if in_height % ker_pool.strides[0] == 0: + # pad_along_height = max(ker_pool.pool_size[1] - ker_pool.strides[0], 0) + # else: + # pad_along_height = max(ker_pool.pool_size[1] - (in_height % ker_pool.strides[0]), 0) + # pad_top = pad_along_height // 2 + # pad_bottom = pad_along_height - pad_top + # assert pad_bottom == hls_pool.attributes['pad_bottom'] + # assert pad_top == hls_pool.attributes['pad_top'] + + # # Width + # in_width = ker_pool.input_shape[2] + # if ker_pool.data_format == 'channels_first': + # in_height = keras_model.layers[1].input_shape[-1] + # out_width = int(math.ceil(float(in_width) / float(ker_pool.strides[1]))) + # assert out_width == hls_pool.attributes['out_width'] + # if in_width % ker_pool.strides[1] == 0: + # pad_along_width = max(ker_pool.pool_size[0] - ker_pool.strides[1], 0) + # else: + # pad_along_width = max(ker_pool.pool_size[0] - (in_width % ker_pool.strides[1]), 0) + # pad_left = pad_along_width // 2 + # pad_right = pad_along_width - pad_left + # assert pad_left == hls_pool.attributes['pad_left'] + # assert pad_right == hls_pool.attributes['pad_right'] + + # elif ker_pool.padding == 'valid': + # if hls_pool.attributes['data_format'] == 'channels_first': + # in_height = ker_pool.input_shape[2] + # in_width = ker_pool.input_shape[3] + # elif hls_pool.attributes['data_format'] == 'channels_last': + # in_height = ker_pool.input_shape[1] + # in_width = ker_pool.input_shape[2] + # else: + # raise ValueError('Invalid data_format') + + # out_width = int(math.ceil(float(in_width - ker_pool.pool_size[0] + 1) / float(ker_pool.strides[1]))) + # out_height = int(math.ceil(float(in_height - ker_pool.pool_size[1] + 1) / float(ker_pool.strides[0]))) + + # assert hls_pool.attributes['out_height'] == out_height + # assert hls_pool.attributes['out_width'] == out_width + # assert hls_pool.attributes['pad_top'] == 0 + # assert hls_pool.attributes['pad_bottom'] == 0 + # assert hls_pool.attributes['pad_left'] == 0 + # assert hls_pool.attributes['pad_right'] == 0 + + # elif '1D' in pooling.__name__: + # assert hls_pool.attributes['name'] == ker_pool._name + # assert hls_pool.attributes['class_name'][-2] == str(1) + # assert hls_pool.attributes['n_in'] == ker_pool.input_shape[1] + # assert hls_pool.attributes['n_filt'] == ker_pool.input_shape[2] + # assert hls_pool.attributes['pool_width'] == ker_pool.pool_size[0] + # assert hls_pool.attributes['stride_width'] == ker_pool.strides[0] + + # out_same = math.ceil(float(ker_pool.input_shape[1]) / float(ker_pool.strides[0])) + # out_valid = math.ceil(float(ker_pool.input_shape[1] - ker_pool.pool_size[0] + 1) / ker_pool.strides[0]) + + # if ker_pool.padding == 'same': + # assert hls_pool.attributes['n_out'] == out_same + # if ker_pool.input_shape[1] % ker_pool.strides[0] == 0: + # pad_along_width = max(ker_pool.pool_size[0] - ker_pool.strides[0], 0) + # else: + # pad_along_width = max(ker_pool.pool_size[0] - (ker_pool.input_shape[1] % ker_pool.strides[0]), 0) + # assert hls_pool.attributes['pad_left'] == pad_along_width // 2 + # assert hls_pool.attributes['pad_right'] == pad_along_width - pad_along_width // 2 + + # elif ker_pool.padding == 'valid': + # assert hls_pool.attributes['n_out'] == out_valid + # assert hls_pool.attributes['pad_left'] == 0 + # assert hls_pool.attributes['pad_right'] == 0 + + +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'Catapult', 'oneAPI']) +@pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) +def test_reused_layer(backend, io_type): + + inp1 = keras.layers.Input(shape=(10, 10)) + inp2 = keras.layers.Input(shape=(10, 10)) + + conv = keras.layers.Conv1D(2, 3, activation='relu') + + o1 = conv(inp1) + o2 = conv(inp2) + o3 = keras.layers.Add()([o1, o2]) + o4 = keras.layers.Dense(5)(o3) + + _ = keras.layers.Dense(5)(o3) + + model = keras.models.Model(inputs=[inp1, inp2], outputs=[o1, o2, o3, o4]) + + _ = model([inp1, inp1]) + + hls_config = {'Model': {'Precision': 'ap_fixed<32,8>', 'ReuseFactor': 1}} + output_dir = str(test_root_path / f'hls4mlprj_keras_api_conv1d_{backend}_{io_type}') + + model_hls = hls4ml.converters.convert_from_keras_model( + model, backend=backend, io_type=io_type, hls_config=hls_config, output_dir=output_dir + ) + + model_hls.compile() + + data = [np.random.rand(1000, 10, 10).astype(np.float32), np.random.rand(1000, 10, 10).astype(np.float32)] + keras_pred = model.predict(data) + hls_pred = model_hls.predict(data) + + np.testing.assert_allclose(keras_pred[0].reshape(hls_pred[0].shape), hls_pred[0], rtol=0, atol=1e-5) + np.testing.assert_allclose(keras_pred[1].reshape(hls_pred[1].shape), hls_pred[1], rtol=0, atol=1e-5) + np.testing.assert_allclose(keras_pred[2].reshape(hls_pred[2].shape), hls_pred[2], rtol=0, atol=1e-5) + np.testing.assert_allclose(keras_pred[3].reshape(hls_pred[3].shape), hls_pred[3], rtol=0, atol=1e-2) From 3f8acb5d7187be1347734e62ed96595d4449cdec Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Tue, 17 Dec 2024 08:57:49 +0000 Subject: [PATCH 20/21] pre-commit fix --- hls4ml/converters/keras_v3/conv.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hls4ml/converters/keras_v3/conv.py b/hls4ml/converters/keras_v3/conv.py index 871bcb942d..df226fc6b5 100644 --- a/hls4ml/converters/keras_v3/conv.py +++ b/hls4ml/converters/keras_v3/conv.py @@ -73,7 +73,6 @@ def handle( 'bias_data': bias, 'data_format': data_format, 'weight_data': kernel, - 'bias_data': bias, 'n_filt': ch_out, 'n_chan': ch_in, } From d2ccfb4be90994bf66a01480d45094f4828c48e6 Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Fri, 6 Dec 2024 06:16:21 +0000 Subject: [PATCH 21/21] revert keras v2 converter --- hls4ml/converters/keras_to_hls.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/hls4ml/converters/keras_to_hls.py b/hls4ml/converters/keras_to_hls.py index a206da4da7..aa7bfe8862 100644 --- a/hls4ml/converters/keras_to_hls.py +++ b/hls4ml/converters/keras_to_hls.py @@ -1,5 +1,4 @@ import json -from warnings import warn import h5py @@ -231,8 +230,8 @@ def parse_keras_model(model_arch, reader): layer_config = model_arch['config'] if 'layers' in layer_config: # Newer Keras versions have 'layers' in 'config' key layer_config = layer_config['layers'] + # Sequential doesn't have InputLayer in TF < 2.3 (Keras 2.4.0) if layer_config[0]['class_name'] != 'InputLayer': - warn(DeprecationWarning('keras < 2.4.0 (tf 2.3) is deprecated. Please use a newer version.')) input_layer = {} input_layer['name'] = 'input1' input_layer['class_name'] = 'InputLayer' @@ -244,33 +243,25 @@ def parse_keras_model(model_arch, reader): layer_config = model_arch['config']['layers'] input_layers = [inp[0] for inp in model_arch['config']['input_layers']] output_layers = [out[0] for out in model_arch['config']['output_layers']] - else: - raise Exception(f'ERROR: Model class not supported: {model_arch["class_name"]}') # Get input shape and check for unsupported layer type for keras_layer in layer_config: if keras_layer['class_name'] not in supported_layers: - raise Exception(f'ERROR: Unsupported layer type: {keras_layer["class_name"]}') + raise Exception('ERROR: Unsupported layer type: {}'.format(keras_layer['class_name'])) output_shapes = {} output_shape = None print('Topology:') for keras_layer in layer_config: - if 'batch_input_shape' in keras_layer['config'] or 'batch_shape' in keras_layer['config']: + if 'batch_input_shape' in keras_layer['config']: if 'inbound_nodes' in keras_layer and len(keras_layer['inbound_nodes']) > 0: input_shapes = [output_shapes[inbound_node[0]] for inbound_node in keras_layer['inbound_nodes'][0]] else: - _input_shapes = keras_layer['config'].get('batch_input_shape', None) - input_shapes = _input_shapes or keras_layer['config']['batch_shape'] + input_shapes = [keras_layer['config']['batch_input_shape']] else: if 'inbound_nodes' in keras_layer: - if 'args' in keras_layer['inbound_nodes'][0]: - # keras v3 - input_shapes = [arg['config']['shape'] for arg in keras_layer['inbound_nodes'][0]['args']] - else: - # keras v2 - input_shapes = [output_shapes[inbound_node[0]] for inbound_node in keras_layer['inbound_nodes'][0]] + input_shapes = [output_shapes[inbound_node[0]] for inbound_node in keras_layer['inbound_nodes'][0]] else: # Sequential model, so output_shape from the previous layer is still valid input_shapes = [output_shape]