Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Disable Sending of ML Events by Default #923

Merged
merged 8 commits into from
Sep 20, 2023
Merged

Disable Sending of ML Events by Default #923

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7a6a667
Disable sending of ML events by default.
umaannamalai Sep 18, 2023
68963d5
Merge branch 'develop-scikitlearn' into disable-ml-events-default
umaannamalai Sep 18, 2023
e19e4fe
Override settings for ML event testing.
umaannamalai Sep 18, 2023
1226b96
Merge branch 'disable-ml-events-default' of github.com:newrelic/newre…
umaannamalai Sep 18, 2023
89b3c7d
Update scikit-learn tests with settings overrides.
umaannamalai Sep 19, 2023
ca4d532
Update all scikitlearn tests with enabled settings override.
umaannamalai Sep 19, 2023
152807e
Revert Update all scikitlearn tests with enabled settings override.
umaannamalai Sep 19, 2023
8e7dda3
Formatting.
umaannamalai Sep 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions newrelic/core/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -717,7 +717,7 @@ def default_otlp_host(host):
_settings.transaction_events.attributes.include = []

_settings.custom_insights_events.enabled = True
_settings.ml_insights_events.enabled = True
_settings.ml_insights_events.enabled = False
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this supposed to stay false too? Or just the global override setting?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This one was supposed to stay False since we check this as a standalone setting in calls to record_ml_event and not in conjunction with machine_learning.enabled.


_settings.distributed_tracing.enabled = _environ_as_bool("NEW_RELIC_DISTRIBUTED_TRACING_ENABLED", default=True)
_settings.distributed_tracing.exclude_newrelic_header = False
Expand Down Expand Up @@ -894,7 +894,7 @@ def default_otlp_host(host):
_settings.application_logging.local_decorating.enabled = _environ_as_bool(
"NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False
)
_settings.machine_learning.enabled = _environ_as_bool("NEW_RELIC_MACHINE_LEARNING_ENABLED", default=True)
_settings.machine_learning.enabled = _environ_as_bool("NEW_RELIC_MACHINE_LEARNING_ENABLED", default=False)
_settings.machine_learning.inference_events_value.enabled = _environ_as_bool(
"NEW_RELIC_MACHINE_LEARNING_INFERENCE_EVENT_VALUE_ENABLED", default=True
)
Expand Down
14 changes: 10 additions & 4 deletions tests/agent_features/test_ml_events.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ def core_app(collector_agent_registration):
return app._agent.application(app.name)


@override_application_settings({"ml_insights_events.enabled": True})
@validate_ml_event_payload(
[{"foo": "bar", "real_agent_id": "1234567", "event.domain": "newrelic.ml_events", "event.name": "InferenceEvent"}]
)
Expand All @@ -70,6 +71,7 @@ def _test():
core_app.harvest()


@override_application_settings({"ml_insights_events.enabled": True})
@validate_ml_event_payload(
[{"foo": "bar", "real_agent_id": "1234567", "event.domain": "newrelic.ml_events", "event.name": "InferenceEvent"}]
)
Expand All @@ -83,6 +85,7 @@ def _test():
core_app.harvest()


@override_application_settings({"ml_insights_events.enabled": True})
@pytest.mark.parametrize(
"params,expected",
[
Expand All @@ -102,6 +105,7 @@ def _test():
_test()


@override_application_settings({"ml_insights_events.enabled": True})
@pytest.mark.parametrize(
"params,expected",
[
Expand All @@ -121,27 +125,31 @@ def _test():
_test()


@override_application_settings({"ml_insights_events.enabled": True})
@reset_core_stats_engine()
@validate_ml_event_count(count=0)
@background_task()
def test_record_ml_event_inside_transaction_bad_event_type():
record_ml_event("!@#$%^&*()", {"foo": "bar"})


@override_application_settings({"ml_insights_events.enabled": True})
@reset_core_stats_engine()
@validate_ml_event_count(count=0)
def test_record_ml_event_outside_transaction_bad_event_type():
app = application()
record_ml_event("!@#$%^&*()", {"foo": "bar"}, application=app)


@override_application_settings({"ml_insights_events.enabled": True})
@reset_core_stats_engine()
@validate_ml_event_count(count=0)
@background_task()
def test_record_ml_event_inside_transaction_params_not_a_dict():
record_ml_event("ParamsListEvent", ["not", "a", "dict"])


@override_application_settings({"ml_insights_events.enabled": True})
umaannamalai marked this conversation as resolved.
Show resolved Hide resolved
@reset_core_stats_engine()
@validate_ml_event_count(count=0)
def test_record_ml_event_outside_transaction_params_not_a_dict():
Expand All @@ -152,11 +160,10 @@ def test_record_ml_event_outside_transaction_params_not_a_dict():
# Tests for ML Events configuration settings


@override_application_settings({"ml_insights_events.enabled": False})
@reset_core_stats_engine()
@validate_ml_event_count(count=0)
@background_task()
def test_ml_event_settings_check_ml_insights_enabled():
def test_ml_event_settings_check_ml_insights_disabled():
record_ml_event("FooEvent", {"foo": "bar"})


Expand All @@ -167,15 +174,13 @@ def test_ml_event_settings_check_ml_insights_enabled():
# event_type and attribute processing.


@override_application_settings({"ml_insights_events.enabled": False})
@reset_core_stats_engine()
@function_not_called("newrelic.api.transaction", "create_custom_event")
@background_task()
def test_transaction_create_ml_event_not_called():
record_ml_event("FooEvent", {"foo": "bar"})


@override_application_settings({"ml_insights_events.enabled": False})
@reset_core_stats_engine()
@function_not_called("newrelic.core.application", "create_custom_event")
@background_task()
Expand All @@ -184,6 +189,7 @@ def test_application_create_ml_event_not_called():
record_ml_event("FooEvent", {"foo": "bar"}, application=app)


@override_application_settings({"ml_insights_events.enabled": True})
@pytest.fixture(scope="module", autouse=True, params=["protobuf", "json"])
def otlp_content_encoding(request):
if six.PY2 and request.param == "protobuf":
Expand Down
8 changes: 8 additions & 0 deletions tests/mlmodel_sklearn/test_calibration_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,22 @@
# limitations under the License.

import pytest
from testing_support.fixtures import override_application_settings
from testing_support.validators.validate_transaction_metrics import (
validate_transaction_metrics,
)

from newrelic.api.background_task import background_task
from newrelic.packages import six

enabled_settings = {
"machine_learning.enabled": True,
"machine_learning.inference_events_value.enabled": True,
"ml_insights_events.enabled": True
}


@override_application_settings(enabled_settings)
def test_model_methods_wrapped_in_function_trace(calibration_model_name, run_calibration_model):
expected_scoped_metrics = {
"CalibratedClassifierCV": [
Expand Down
9 changes: 9 additions & 0 deletions tests/mlmodel_sklearn/test_cluster_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import pytest
from sklearn import __version__ # noqa: this is needed for get_package_version
from testing_support.fixtures import override_application_settings
from testing_support.validators.validate_transaction_metrics import (
validate_transaction_metrics,
)
Expand All @@ -24,7 +25,14 @@

SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split(".")))

enabled_settings = {
"machine_learning.enabled": True,
"machine_learning.inference_events_value.enabled": True,
"ml_insights_events.enabled": True
}


@override_application_settings(enabled_settings)
@pytest.mark.parametrize(
"cluster_model_name",
[
Expand Down Expand Up @@ -115,6 +123,7 @@ def _test():
_test()


@override_application_settings(enabled_settings)
@pytest.mark.skipif(SKLEARN_VERSION < (1, 1, 0), reason="Requires sklearn > 1.1")
@pytest.mark.parametrize(
"cluster_model_name",
Expand Down
8 changes: 8 additions & 0 deletions tests/mlmodel_sklearn/test_compose_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,22 @@
import pytest
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import Normalizer
from testing_support.fixtures import override_application_settings
from testing_support.validators.validate_transaction_metrics import (
validate_transaction_metrics,
)

from newrelic.api.background_task import background_task
from newrelic.packages import six

enabled_settings = {
"machine_learning.enabled": True,
"machine_learning.inference_events_value.enabled": True,
"ml_insights_events.enabled": True
}


@override_application_settings(enabled_settings)
@pytest.mark.parametrize(
"compose_model_name",
[
Expand Down
8 changes: 8 additions & 0 deletions tests/mlmodel_sklearn/test_covariance_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,22 @@
# limitations under the License.

import pytest
from testing_support.fixtures import override_application_settings
from testing_support.validators.validate_transaction_metrics import (
validate_transaction_metrics,
)

from newrelic.api.background_task import background_task
from newrelic.packages import six

enabled_settings = {
"machine_learning.enabled": True,
"machine_learning.inference_events_value.enabled": True,
"ml_insights_events.enabled": True
}


@override_application_settings(enabled_settings)
@pytest.mark.parametrize(
"covariance_model_name",
[
Expand Down
8 changes: 8 additions & 0 deletions tests/mlmodel_sklearn/test_cross_decomposition_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,22 @@
# limitations under the License.

import pytest
from testing_support.fixtures import override_application_settings
from testing_support.validators.validate_transaction_metrics import (
validate_transaction_metrics,
)

from newrelic.api.background_task import background_task
from newrelic.packages import six

enabled_settings = {
"machine_learning.enabled": True,
"machine_learning.inference_events_value.enabled": True,
"ml_insights_events.enabled": True
}


@override_application_settings(enabled_settings)
@pytest.mark.parametrize(
"cross_decomposition_model_name",
[
Expand Down
8 changes: 8 additions & 0 deletions tests/mlmodel_sklearn/test_discriminant_analysis_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,22 @@
# limitations under the License.

import pytest
from testing_support.fixtures import override_application_settings
from testing_support.validators.validate_transaction_metrics import (
validate_transaction_metrics,
)

from newrelic.api.background_task import background_task
from newrelic.packages import six

enabled_settings = {
"machine_learning.enabled": True,
"machine_learning.inference_events_value.enabled": True,
"ml_insights_events.enabled": True
}


@override_application_settings(enabled_settings)
@pytest.mark.parametrize(
"discriminant_analysis_model_name",
[
Expand Down
8 changes: 8 additions & 0 deletions tests/mlmodel_sklearn/test_dummy_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import pytest
from sklearn import __init__ # noqa: needed for get_package_version
from testing_support.fixtures import override_application_settings
from testing_support.validators.validate_transaction_metrics import (
validate_transaction_metrics,
)
Expand All @@ -24,7 +25,14 @@

SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split(".")))

enabled_settings = {
"machine_learning.enabled": True,
"machine_learning.inference_events_value.enabled": True,
"ml_insights_events.enabled": True
}


@override_application_settings(enabled_settings)
@pytest.mark.parametrize(
"dummy_model_name",
[
Expand Down
10 changes: 10 additions & 0 deletions tests/mlmodel_sklearn/test_ensemble_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import pytest
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from testing_support.fixtures import override_application_settings
from testing_support.validators.validate_transaction_metrics import (
validate_transaction_metrics,
)
Expand All @@ -24,7 +25,14 @@

SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split(".")))

enabled_settings = {
"machine_learning.enabled": True,
"machine_learning.inference_events_value.enabled": True,
"ml_insights_events.enabled": True
}


@override_application_settings(enabled_settings)
@pytest.mark.parametrize(
"ensemble_model_name",
[
Expand Down Expand Up @@ -141,6 +149,7 @@ def _test():
_test()


@override_application_settings(enabled_settings)
@pytest.mark.skipif(SKLEARN_VERSION < (1, 0, 0) or SKLEARN_VERSION >= (1, 1, 0), reason="Requires 1.0 <= sklearn < 1.1")
@pytest.mark.parametrize(
"ensemble_model_name",
Expand Down Expand Up @@ -197,6 +206,7 @@ def _test():
_test()


@override_application_settings(enabled_settings)
@pytest.mark.skipif(SKLEARN_VERSION < (1, 1, 0), reason="Requires sklearn >= 1.1")
@pytest.mark.parametrize(
"ensemble_model_name",
Expand Down
9 changes: 9 additions & 0 deletions tests/mlmodel_sklearn/test_feature_selection_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import pytest
from sklearn.ensemble import AdaBoostClassifier
from testing_support.fixtures import override_application_settings
from testing_support.validators.validate_transaction_metrics import (
validate_transaction_metrics,
)
Expand All @@ -24,7 +25,14 @@

SKLEARN_VERSION = tuple(map(int, get_package_version("sklearn").split(".")))

enabled_settings = {
"machine_learning.enabled": True,
"machine_learning.inference_events_value.enabled": True,
"ml_insights_events.enabled": True
}


@override_application_settings(enabled_settings)
@pytest.mark.parametrize(
"feature_selection_model_name",
[
Expand Down Expand Up @@ -73,6 +81,7 @@ def _test():
_test()


@override_application_settings(enabled_settings)
@pytest.mark.skipif(SKLEARN_VERSION < (1, 0, 0), reason="Requires sklearn >= 1.0")
@pytest.mark.parametrize(
"feature_selection_model_name",
Expand Down
8 changes: 8 additions & 0 deletions tests/mlmodel_sklearn/test_gaussian_process_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,22 @@
# limitations under the License.

import pytest
from testing_support.fixtures import override_application_settings
from testing_support.validators.validate_transaction_metrics import (
validate_transaction_metrics,
)

from newrelic.api.background_task import background_task
from newrelic.packages import six

enabled_settings = {
"machine_learning.enabled": True,
"machine_learning.inference_events_value.enabled": True,
"ml_insights_events.enabled": True
}


@override_application_settings(enabled_settings)
@pytest.mark.parametrize(
"gaussian_process_model_name",
[
Expand Down
Loading
Loading