From 7449afff5100c1d7e75726c302bd2d3e9f425676 Mon Sep 17 00:00:00 2001 From: Yashvardhan Nanavati Date: Wed, 24 Jul 2024 23:23:57 -0700 Subject: [PATCH] Add add-deprecations API endpoint Refers to CLOUDDST-23446 --- iib/web/api_v1.py | 26 +++ iib/web/iib_static_types.py | 15 ++ ...4b328_add_add_deprecations_api_endpoint.py | 161 +++++++++++++++++ iib/web/models.py | 170 ++++++++++++++++++ tests/test_web/test_api_v1.py | 48 ++++- tests/test_web/test_models.py | 1 + 6 files changed, 420 insertions(+), 1 deletion(-) create mode 100644 iib/web/migrations/versions/49d13af4b328_add_add_deprecations_api_endpoint.py diff --git a/iib/web/api_v1.py b/iib/web/api_v1.py index 0841e1262..b8b706661 100644 --- a/iib/web/api_v1.py +++ b/iib/web/api_v1.py @@ -25,6 +25,7 @@ Operator, Request, RequestAdd, + RequestAddDeprecations, RequestFbcOperations, RequestMergeIndexImage, RequestRecursiveRelatedBundles, @@ -53,6 +54,7 @@ from iib.workers.tasks.build_create_empty_index import handle_create_empty_index_request from iib.workers.tasks.general import failed_request_callback from iib.web.iib_static_types import ( + AddDeprecationRequestPayload, AddRequestPayload, AddRmBatchPayload, CreateEmptyIndexPayload, @@ -1321,3 +1323,27 @@ def fbc_operations() -> Tuple[flask.Response, int]: flask.current_app.logger.debug('Successfully scheduled request %d', request.id) return flask.jsonify(request.to_json()), 201 + + +@api_v1.route('/builds/add-deprecations', methods=['POST']) +@login_required +@instrument_tracing(span_name="web.api_v1.add_deprecations") +def add_deprecations() -> Tuple[flask.Response, int]: + """ + Submit a request to add operator deprecations to an index image. + + :rtype: flask.Response + :raise ValidationError: if required parameters are not supplied + """ + payload: AddDeprecationRequestPayload = cast( + AddDeprecationRequestPayload, flask.request.get_json() + ) + if not isinstance(payload, dict): + raise ValidationError('The input data must be a JSON object') + + request = RequestAddDeprecations.from_json(payload) + db.session.add(request) + db.session.commit() + + flask.current_app.logger.debug('Successfully validated request %d', request.id) + return flask.jsonify({'msg': 'This API endpoint hasn not been implemented yet'}), 501 diff --git a/iib/web/iib_static_types.py b/iib/web/iib_static_types.py index b72520d88..f27c74fa7 100644 --- a/iib/web/iib_static_types.py +++ b/iib/web/iib_static_types.py @@ -40,6 +40,7 @@ class RelatedBundlesMetadata(TypedDict): # try inheritance from other payloads PayloadTags = Literal[ + 'AddDeprecationRequestPayload', 'AddRequestPayload', 'RmRequestPayload', 'RegenerateBundlePayload', @@ -63,12 +64,14 @@ class RelatedBundlesMetadata(TypedDict): 'cnr_token', 'check_related_images', 'deprecation_list', + 'deprecation_schema', 'distribution_scope', 'force_backport', 'from_bundle_image', 'from_index', 'graph_update_mode', 'labels', + 'operator_package', 'operators', 'organization', 'output_fbc', @@ -83,6 +86,17 @@ class RelatedBundlesMetadata(TypedDict): ] +class AddDeprecationRequestPayload(TypedDict): + """Data structure of the request to /builds/add-deprecations API endpoint.""" + + binary_image: NotRequired[str] + deprecation_schema: str + from_index: str + operator_package: str + overwrite_from_index: NotRequired[bool] + overwrite_from_index_token: NotRequired[str] + + class AddRequestPayload(TypedDict): """Datastructure of the request to /builds/add API point.""" @@ -226,6 +240,7 @@ class RequestPayload(TypedDict): PayloadTypesUnion = Union[ + AddDeprecationRequestPayload, AddRequestPayload, CreateEmptyIndexPayload, FbcOperationRequestPayload, diff --git a/iib/web/migrations/versions/49d13af4b328_add_add_deprecations_api_endpoint.py b/iib/web/migrations/versions/49d13af4b328_add_add_deprecations_api_endpoint.py new file mode 100644 index 000000000..065913db1 --- /dev/null +++ b/iib/web/migrations/versions/49d13af4b328_add_add_deprecations_api_endpoint.py @@ -0,0 +1,161 @@ +"""Add add-deprecations API endpoint. + +Revision ID: 49d13af4b328 +Revises: 1920ad83d0ab +Create Date: 2024-07-26 00:17:44.283197 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '49d13af4b328' +down_revision = '1920ad83d0ab' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'deprecation_schema', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('schema', sa.Text(), nullable=False), + sa.PrimaryKeyConstraint('id'), + ) + with op.batch_alter_table('deprecation_schema', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_deprecation_schema_schema'), ['schema'], unique=True) + + op.create_table( + 'request_add_deprecations', + sa.Column('id', sa.Integer(), autoincrement=False, nullable=False), + sa.Column('binary_image_id', sa.Integer(), nullable=True), + sa.Column('binary_image_resolved_id', sa.Integer(), nullable=True), + sa.Column('from_index_id', sa.Integer(), nullable=True), + sa.Column('from_index_resolved_id', sa.Integer(), nullable=True), + sa.Column('index_image_id', sa.Integer(), nullable=True), + sa.Column('index_image_resolved_id', sa.Integer(), nullable=True), + sa.Column('internal_index_image_copy_id', sa.Integer(), nullable=True), + sa.Column('internal_index_image_copy_resolved_id', sa.Integer(), nullable=True), + sa.Column('distribution_scope', sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ['binary_image_id'], + ['image.id'], + ), + sa.ForeignKeyConstraint( + ['binary_image_resolved_id'], + ['image.id'], + ), + sa.ForeignKeyConstraint( + ['from_index_id'], + ['image.id'], + ), + sa.ForeignKeyConstraint( + ['from_index_resolved_id'], + ['image.id'], + ), + sa.ForeignKeyConstraint( + ['id'], + ['request.id'], + ), + sa.ForeignKeyConstraint( + ['index_image_id'], + ['image.id'], + ), + sa.ForeignKeyConstraint( + ['index_image_resolved_id'], + ['image.id'], + ), + sa.ForeignKeyConstraint( + ['internal_index_image_copy_id'], + ['image.id'], + ), + sa.ForeignKeyConstraint( + ['internal_index_image_copy_resolved_id'], + ['image.id'], + ), + sa.PrimaryKeyConstraint('id'), + ) + op.create_table( + 'request_add_deprecations_deprecation_schema', + sa.Column('request_add_deprecations_id', sa.Integer(), autoincrement=False, nullable=False), + sa.Column('deprecation_schema_id', sa.Integer(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ['deprecation_schema_id'], + ['deprecation_schema.id'], + ), + sa.ForeignKeyConstraint( + ['request_add_deprecations_id'], + ['request_add_deprecations.id'], + ), + sa.PrimaryKeyConstraint('request_add_deprecations_id', 'deprecation_schema_id'), + sa.UniqueConstraint('request_add_deprecations_id', 'deprecation_schema_id'), + ) + with op.batch_alter_table( + 'request_add_deprecations_deprecation_schema', schema=None + ) as batch_op: + batch_op.create_index( + batch_op.f('ix_request_add_deprecations_deprecation_schema_deprecation_schema_id'), + ['deprecation_schema_id'], + unique=False, + ) + batch_op.create_index( + batch_op.f( + 'ix_request_add_deprecations_deprecation_schema_request_add_deprecations_id' + ), + ['request_add_deprecations_id'], + unique=False, + ) + + op.create_table( + 'request_add_deprecations_operator', + sa.Column('request_add_deprecations_id', sa.Integer(), autoincrement=False, nullable=False), + sa.Column('operator_id', sa.Integer(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ['operator_id'], + ['operator.id'], + ), + sa.ForeignKeyConstraint( + ['request_add_deprecations_id'], + ['request_add_deprecations.id'], + ), + sa.PrimaryKeyConstraint('request_add_deprecations_id', 'operator_id'), + sa.UniqueConstraint('request_add_deprecations_id', 'operator_id'), + ) + with op.batch_alter_table('request_add_deprecations_operator', schema=None) as batch_op: + batch_op.create_index( + batch_op.f('ix_request_add_deprecations_operator_operator_id'), + ['operator_id'], + unique=False, + ) + batch_op.create_index( + batch_op.f('ix_request_add_deprecations_operator_request_add_deprecations_id'), + ['request_add_deprecations_id'], + unique=False, + ) + + +def downgrade(): + with op.batch_alter_table('request_add_deprecations_operator', schema=None) as batch_op: + batch_op.drop_index( + batch_op.f('ix_request_add_deprecations_operator_request_add_deprecations_id') + ) + batch_op.drop_index(batch_op.f('ix_request_add_deprecations_operator_operator_id')) + + op.drop_table('request_add_deprecations_operator') + with op.batch_alter_table( + 'request_add_deprecations_deprecation_schema', schema=None + ) as batch_op: + batch_op.drop_index( + batch_op.f('ix_request_add_deprecations_deprecation_schema_request_add_deprecations_id') + ) + batch_op.drop_index( + batch_op.f('ix_request_add_deprecations_deprecation_schema_deprecation_schema_id') + ) + + op.drop_table('request_add_deprecations_deprecation_schema') + op.drop_table('request_add_deprecations') + with op.batch_alter_table('deprecation_schema', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_deprecation_schema_schema')) + + op.drop_table('deprecation_schema') diff --git a/iib/web/models.py b/iib/web/models.py index 44a006260..c87414e12 100644 --- a/iib/web/models.py +++ b/iib/web/models.py @@ -10,6 +10,7 @@ from flask import current_app, url_for from flask_login import UserMixin, current_user from flask_sqlalchemy.model import DefaultMeta +import ruamel.yaml import sqlalchemy from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import joinedload, load_only, Mapped, validates @@ -21,6 +22,7 @@ from iib.web.iib_static_types import ( + AddDeprecationRequestPayload, AddRequestPayload, AddRequestResponse, AddRmBatchPayload, @@ -44,6 +46,17 @@ FbcOperationRequestResponse, ) +yaml = ruamel.yaml.YAML() +# IMPORTANT: ruamel will introduce a line break if the yaml line is longer than yaml.width. +# Unfortunately, this causes issues for JSON values nested within a YAML file, e.g. +# metadata.annotations."alm-examples" in a CSV file. +# The default value is 80. Set it to a more forgiving higher number to avoid issues +yaml.width = 200 +# ruamel will also cause issues when normalizing a YAML object that contains +# a nested JSON object when it does not preserve quotes. Thus, it produces +# invalid YAML. Let's prevent this from happening at all. +yaml.preserve_quotes = True + class BaseEnum(Enum): """A base class for IIB enums.""" @@ -103,6 +116,7 @@ class RequestTypeMapping(BaseEnum): create_empty_index: int = 5 recursive_related_bundles: int = 6 fbc_operations: int = 7 + add_deprecations: int = 8 @classmethod def pretty(cls, num: int) -> str: @@ -2200,3 +2214,159 @@ def get_mutable_keys(self) -> Set[str]: rv.update(self.get_index_image_mutable_keys()) rv.add('fbc_fragment_resolved') return rv + + +class RequestAddDeprecationsOperator(db.Model): + """An association table between add-deprecations requests and the operator they contain.""" + + # A primary key is required by SQLAlchemy when using declaritive style tables, so a composite + # primary key is used on the two required columns + request_add_deprecations_id: Mapped[int] = db.mapped_column( + db.ForeignKey('request_add_deprecations.id'), + autoincrement=False, + index=True, + primary_key=True, + ) + operator_id: Mapped[int] = db.mapped_column( + db.ForeignKey('operator.id'), autoincrement=False, index=True, primary_key=True + ) + + __table_args__ = (db.UniqueConstraint('request_add_deprecations_id', 'operator_id'),) + + +class RequestAddDeprecationsDeprecationSchema(db.Model): + """An association table between add-deprecations requests and the deprecation schema.""" + + # A primary key is required by SQLAlchemy when using declaritive style tables, so a composite + # primary key is used on the two required columns + request_add_deprecations_id: Mapped[int] = db.mapped_column( + db.ForeignKey('request_add_deprecations.id'), + autoincrement=False, + index=True, + primary_key=True, + ) + deprecation_schema_id: Mapped[int] = db.mapped_column( + db.ForeignKey('deprecation_schema.id'), autoincrement=False, index=True, primary_key=True + ) + + __table_args__ = (db.UniqueConstraint('request_add_deprecations_id', 'deprecation_schema_id'),) + + +class DeprecationSchema(db.Model): + """A DeprecationSchema that has been handled by IIB.""" + + id: Mapped[int] = db.mapped_column(primary_key=True) + schema: Mapped[str] = db.mapped_column('schema', db.Text, index=True, unique=True) + + def __repr__(self) -> str: + return ''.format(self.schema) + + @classmethod + def get_or_create(cls, deprecation_schema: str) -> DeprecationSchema: + """ + Get the deprecation schema from the database and create it if it doesn't exist. + + :param str deprecation_schema: the deprecation schema for an operator + :return: a DeprecationSchema object based on the input name; the DeprecationSchema object + will be added to the database session, but not committed, if it was created + :rtype: DeprecationSchema + """ + # cls.query triggers an auto-flush of the session by default. So if there are + # multiple requests with same parameters submitted to IIB, call to query pre-maturely + # flushes the contents of the session not allowing our handlers to resolve conflicts. + # https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session.params.autoflush + with db.session.no_autoflush: + schema = cls.query.filter_by(schema=deprecation_schema).first() + if not schema: + schema = DeprecationSchema(schema=deprecation_schema) + try: + # This is a SAVEPOINT so that the rest of the session is not rolled back when + # adding the image conflicts with an already existing row added by another request + # with similar pullspecs is submitted at the same time. When the context manager + # completes, the objects local to it are committed. If an error is raised, it + # rolls back objects local to it while keeping the parent session unaffected. + # https://docs.sqlalchemy.org/en/20/orm/session_transaction.html#using-savepoint + with db.session.begin_nested(): + db.session.add(schema) + except sqlalchemy.exc.IntegrityError: + current_app.logger.info('Schema is already in database. "%s"', deprecation_schema) + schema = cls.query.filter_by(schema=deprecation_schema).first() + + return schema + + +class RequestAddDeprecations(Request, RequestIndexImageMixin): + """An "add-deprecations" index image build request.""" + + __tablename__ = 'request_add_deprecations' + + id: Mapped[int] = db.mapped_column( + db.ForeignKey('request.id'), autoincrement=False, primary_key=True + ) + operator_package: Mapped[Operator] = db.relationship( + 'Operator', secondary=RequestAddDeprecationsOperator.__table__ + ) + deprecation_schema: Mapped[DeprecationSchema] = db.relationship( + 'DeprecationSchema', secondary=RequestAddDeprecationsDeprecationSchema.__table__ + ) + + __mapper_args__ = { + 'polymorphic_identity': RequestTypeMapping.__members__['add_deprecations'].value + } + + @classmethod + def from_json( # type: ignore[override] # noqa: F821 + cls, + kwargs: AddDeprecationRequestPayload, + batch: Optional[Batch] = None, + ) -> RequestAddDeprecations: + """ + Handle JSON requests for the Add API endpoint. + + :param dict kwargs: the JSON payload of the request. + :param Batch batch: the batch to specify with the request. + """ + request_kwargs = deepcopy(kwargs) + + _operator_package = request_kwargs.get('operator_package') + if not _operator_package or not isinstance(_operator_package, str): + raise ValidationError('"operator_package" should be a non-empty string') + + _deprecation_schema = request_kwargs.get('deprecation_schema') + if not _deprecation_schema or not isinstance(_deprecation_schema, str): + raise ValidationError('"deprecation_schema" should be a non-empty string') + try: + yaml.load(_deprecation_schema) + except ruamel.yaml.YAMLError: + raise ValidationError('"deprecation_schema" string should be valid YAML') + + # cast to more wider type, see _from_json method + cls._from_json( + cast(RequestPayload, request_kwargs), + additional_required_params=[ + 'deprecation_schema', + 'from_index', + 'operator_package', + ], + batch=batch, + ) + + request_kwargs['operator_package'] = Operator.get_or_create(name=_operator_package) + request_kwargs['deprecation_schema'] = DeprecationSchema.get_or_create( + deprecation_schema=_deprecation_schema + ) + + request = cls(**request_kwargs) + request.add_state('failed', 'The API endpoint has not been implemented yet') + return request + + def get_mutable_keys(self) -> Set[str]: + """ + Return the set of keys representing the attributes that can be modified. + + :return: a set of key names + :rtype: set + """ + rv = super().get_mutable_keys() + rv.update(self.get_index_image_mutable_keys()) + return rv diff --git a/tests/test_web/test_api_v1.py b/tests/test_web/test_api_v1.py index 56a657b02..6347e7b14 100644 --- a/tests/test_web/test_api_v1.py +++ b/tests/test_web/test_api_v1.py @@ -300,7 +300,7 @@ def test_get_builds_invalid_type(app, client, db): 'error': ( 'wrong-type is not a valid build request type. Valid request_types are: ' 'generic, add, rm, regenerate-bundle, merge-index-image, ' - 'create-empty-index, recursive-related-bundles, fbc-operations' + 'create-empty-index, recursive-related-bundles, fbc-operations, add-deprecations' ) } @@ -2853,3 +2853,49 @@ def test_fbc_operations( assert 'The "binary_image" value must be a non-empty string' == rv.json['error'] mock_smfc.assert_not_called() mock_hfor.assert_not_called() + + +@pytest.mark.parametrize( + 'data, error_msg', + ( + ( + {'from_index': 'pull:spec', 'binary_image': 'binary:image'}, + '"operator_package" should be a non-empty string', + ), + ( + { + 'from_index': 'pull:spec', + 'binary_image': 'binary:image', + 'operator_package': 'my-package', + }, + '"deprecation_schema" should be a non-empty string', + ), + ( + { + 'from_index': 'pull:spec', + 'binary_image': 'binary:image', + 'operator_package': 'my-package', + 'deprecation_schema': '{invalid_yaml', + }, + '"deprecation_schema" string should be valid YAML', + ), + ), +) +@mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') +def test_add_deprecations_invalid_params_format(mock_smfsc, db, auth_env, client, data, error_msg): + rv = client.post(f'/api/v1/builds/add-deprecations', json=data, environ_base=auth_env) + assert rv.status_code == 400 + assert error_msg == rv.json['error'] + mock_smfsc.assert_not_called() + + +def test_add_deprecations_success(db, auth_env, client): + data = { + 'from_index': 'pull:spec', + 'binary_image': 'binary:image', + 'operator_package': 'my-package', + 'deprecation_schema': 'valid_yaml', + } + rv = client.post(f'/api/v1/builds/add-deprecations', json=data, environ_base=auth_env) + # TODO: Change this status code to 201 once the endpoint functionality is implemented + assert rv.status_code == 501 diff --git a/tests/test_web/test_models.py b/tests/test_web/test_models.py index 948e13a09..6ea9dcd34 100644 --- a/tests/test_web/test_models.py +++ b/tests/test_web/test_models.py @@ -82,6 +82,7 @@ def test_get_state_names(): def test_get_type_names(): assert models.RequestTypeMapping.get_names() == [ 'add', + 'add_deprecations', 'create_empty_index', 'fbc_operations', 'generic',