From 9ea466ae8d696d731745a0549f9d7e7dfab0d7bd Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 9 Nov 2022 16:13:26 -0800 Subject: [PATCH 01/20] Add sklearn to tox --- newrelic/config.py | 131 +++++++++++++++++++++++++++++++++++++++++++++ tox.ini | 3 ++ 2 files changed, 134 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index f0b638cd4..335339d86 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2790,6 +2790,137 @@ def _process_module_builtin_defaults(): ) _process_module_definition("tastypie.api", "newrelic.hooks.component_tastypie", "instrument_tastypie_api") + _process_module_definition( + "sklearn.base", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_base_models", + ) + _process_module_definition( + "sklearn.calibration", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_calibration_models", + ) + _process_module_definition( + "sklearn.cluster", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_cluster_models", + ) + _process_module_definition( + "sklearn.compose", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_compose_models", + ) + _process_module_definition( + "sklearn.covariance", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_covariance_models", + ) + _process_module_definition( + "sklearn.cross_decomposition", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_cross_decomposition_models", + ) + _process_module_definition( + "sklearn.discriminant_analysis", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_discriminant_analysis_models", + ) + _process_module_definition( + "sklearn.dummy", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_dummy_models", + ) + _process_module_definition( + "sklearn.ensemble", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_ensemble_models", + ) + _process_module_definition( + "sklearn.feature_selection", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_feature_selection_models", + ) + _process_module_definition( + "sklearn.gaussian_process", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_gaussian_process_models", + ) + _process_module_definition( + "sklearn.isotonic", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_isotonic_models", + ) + _process_module_definition( + "sklearn.kernel_ridge", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_kernel_ridge_models", + ) + _process_module_definition( + "sklearn.linear_model", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_linear_model_models", + ) + _process_module_definition( + "sklearn.metrics", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_metrics_models", + ) + _process_module_definition( + "sklearn.mixture", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_mixture_models", + ) + _process_module_definition( + "sklearn.model_selection", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_model_selection_models", + ) + _process_module_definition( + "sklearn.multiclass", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_multiclass_models", + ) + _process_module_definition( + "sklearn.multioutput", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_multioutput_models", + ) + _process_module_definition( + "sklearn.naive_bayes", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_naive_bayes_models", + ) + _process_module_definition( + "sklearn.neighbors", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_neighbors_models", + ) + _process_module_definition( + "sklearn.neural_network", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_neural_network_models", + ) + _process_module_definition( + "sklearn.pipeline", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_pipeline_models", + ) + _process_module_definition( + "sklearn.semi_supervised", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_semi_supervised_models", + ) + _process_module_definition( + "sklearn.svm", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_svm_models", + ) + _process_module_definition( + "sklearn.tree._classes", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_tree_models", + ) + _process_module_definition( "rest_framework.views", "newrelic.hooks.component_djangorestframework", diff --git a/tox.ini b/tox.ini index f8e2b2e0c..db0bba091 100644 --- a/tox.ini +++ b/tox.ini @@ -60,6 +60,7 @@ envlist = python-agent_unittests-{pypy,pypy37}-without_extensions, python-application_celery-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, gearman-application_gearman-{py27,pypy}, + python-component_sklearn-{py38,py39,py310,py311}-scikitlearnlatest, python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, @@ -203,6 +204,7 @@ deps = application_celery: celery<6.0 application_celery-py{py37,37}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 + component_sklearn-scikitlearnlatest: scikit-learn component_djangorestframework-djangorestframework0300: Django < 1.9 component_djangorestframework-djangorestframework0300: djangorestframework < 3.1 component_djangorestframework-djangorestframeworklatest: Django @@ -431,6 +433,7 @@ changedir = agent_unittests: tests/agent_unittests application_celery: tests/application_celery application_gearman: tests/application_gearman + component_sklearn: tests/component_sklearn component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest component_graphqlserver: tests/component_graphqlserver From 07fef71b35b6b87024d9c34dbba32075b4b89ba0 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 9 Nov 2022 16:14:03 -0800 Subject: [PATCH 02/20] Add function traces around model methods --- newrelic/config.py | 125 -------------------- newrelic/hooks/component_sklearn.py | 67 +++++++++++ tests/component_sklearn/conftest.py | 38 ++++++ tests/component_sklearn/test_tree_models.py | 92 ++++++++++++++ 4 files changed, 197 insertions(+), 125 deletions(-) create mode 100644 newrelic/hooks/component_sklearn.py create mode 100644 tests/component_sklearn/conftest.py create mode 100644 tests/component_sklearn/test_tree_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 335339d86..e7eb69e98 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2790,131 +2790,6 @@ def _process_module_builtin_defaults(): ) _process_module_definition("tastypie.api", "newrelic.hooks.component_tastypie", "instrument_tastypie_api") - _process_module_definition( - "sklearn.base", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_base_models", - ) - _process_module_definition( - "sklearn.calibration", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_calibration_models", - ) - _process_module_definition( - "sklearn.cluster", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_cluster_models", - ) - _process_module_definition( - "sklearn.compose", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_compose_models", - ) - _process_module_definition( - "sklearn.covariance", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_covariance_models", - ) - _process_module_definition( - "sklearn.cross_decomposition", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_cross_decomposition_models", - ) - _process_module_definition( - "sklearn.discriminant_analysis", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_discriminant_analysis_models", - ) - _process_module_definition( - "sklearn.dummy", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_dummy_models", - ) - _process_module_definition( - "sklearn.ensemble", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_ensemble_models", - ) - _process_module_definition( - "sklearn.feature_selection", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_feature_selection_models", - ) - _process_module_definition( - "sklearn.gaussian_process", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_gaussian_process_models", - ) - _process_module_definition( - "sklearn.isotonic", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_isotonic_models", - ) - _process_module_definition( - "sklearn.kernel_ridge", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_kernel_ridge_models", - ) - _process_module_definition( - "sklearn.linear_model", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_linear_model_models", - ) - _process_module_definition( - "sklearn.metrics", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_metrics_models", - ) - _process_module_definition( - "sklearn.mixture", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_mixture_models", - ) - _process_module_definition( - "sklearn.model_selection", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_model_selection_models", - ) - _process_module_definition( - "sklearn.multiclass", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_multiclass_models", - ) - _process_module_definition( - "sklearn.multioutput", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_multioutput_models", - ) - _process_module_definition( - "sklearn.naive_bayes", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_naive_bayes_models", - ) - _process_module_definition( - "sklearn.neighbors", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_neighbors_models", - ) - _process_module_definition( - "sklearn.neural_network", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_neural_network_models", - ) - _process_module_definition( - "sklearn.pipeline", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_pipeline_models", - ) - _process_module_definition( - "sklearn.semi_supervised", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_semi_supervised_models", - ) - _process_module_definition( - "sklearn.svm", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_svm_models", - ) _process_module_definition( "sklearn.tree._classes", "newrelic.hooks.component_sklearn", diff --git a/newrelic/hooks/component_sklearn.py b/newrelic/hooks/component_sklearn.py new file mode 100644 index 000000000..03ab6eb53 --- /dev/null +++ b/newrelic/hooks/component_sklearn.py @@ -0,0 +1,67 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.api.function_trace import FunctionTraceWrapper +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper + + +def wrap_model_method(method_name): + @function_wrapper + def _wrap_method(wrapped, instance, args, kwargs): + # If there is no transaction, do not wrap anything. + if not current_transaction(): + return wrapped(*args, **kwargs) + + # If the method has already been wrapped do not wrap it again. This happens + # when one model inherits from another and they both implement the method. + if getattr(instance, "_nr_wrapped_%s" % method_name, False): + return wrapped(*args, **kwargs) + + # Set the _nr_wrapped attribute to denote that this method has now been wrapped. + setattr(instance, "_nr_wrapped_%s" % method_name, True) + + # MLModel/Sklearn/Named/. + func_name = wrapped.__name__ + name = "%s.%s" % (wrapped.__self__.__class__.__name__, func_name) + return FunctionTraceWrapper(wrapped, name=name, group="MLModel/Sklearn/Named")(*args, **kwargs) + + return _wrap_method + + +def wrap_model_init(wrapped, instance, args, kwargs): + return_val = wrapped(*args, **kwargs) + + methods_to_wrap = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") + for method_name in methods_to_wrap: + if hasattr(instance, method_name): + setattr(instance, method_name, wrap_model_method(method_name)(getattr(instance, method_name))) + + return return_val + + +def _nr_instrument_model(module, model_class): + wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_model_init) + + +def instrument_sklearn_tree_models(module): + model_classes = ( + "DecisionTreeClassifier", + "DecisionTreeRegressor", + "ExtraTreeClassifier", + "ExtraTreeRegressor", + ) + for model_cls in model_classes: + if hasattr(module, model_cls): + _nr_instrument_model(module, model_cls) diff --git a/tests/component_sklearn/conftest.py b/tests/component_sklearn/conftest.py new file mode 100644 index 000000000..e251d91bb --- /dev/null +++ b/tests/component_sklearn/conftest.py @@ -0,0 +1,38 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 + code_coverage_fixture, + collector_agent_registration_fixture, + collector_available_fixture, +) + +_coverage_source = [ + "newrelic.hooks.component_sklearn", +] + +code_coverage = code_coverage_fixture(source=_coverage_source) + +_default_settings = { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (component_sklearn)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (component_sklearn)"], +) diff --git a/tests/component_sklearn/test_tree_models.py b/tests/component_sklearn/test_tree_models.py new file mode 100644 index 000000000..235e161e0 --- /dev/null +++ b/tests/component_sklearn/test_tree_models.py @@ -0,0 +1,92 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model): + expected_scoped_metrics = { + "ExtraTreeRegressor": [ + ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), + ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 1), + ("MLModel/Sklearn/Named/ExtraTreeRegressor.score", 1), + ], + "DecisionTreeClassifier": [ + ("MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.score", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 1), + ], + "ExtraTreeClassifier": [ + ("MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.score", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 1), + ], + "DecisionTreeRegressor": [ + ("MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), + ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 1), + ("MLModel/Sklearn/Named/DecisionTreeRegressor.score", 1), + ], + } + expected_transaction_name = "test_tree_models:_test" + if six.PY3: + expected_transaction_name = "test_tree_models:test_model_methods_wrapped_in_function_trace.._test" + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[tree_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_tree_model() + + _test() + + +@pytest.fixture(params=["ExtraTreeRegressor", "DecisionTreeClassifier", "ExtraTreeClassifier", "DecisionTreeRegressor"]) +def tree_model_name(request): + return request.param + + +@pytest.fixture +def run_tree_model(tree_model_name): + def _run(): + import sklearn.tree + + x_train = [[0, 0], [1, 1]] + y_train = [0, 1] + x_test = [[2.0, 2.0], [2.0, 1.0]] + y_test = [1, 1] + + clf = getattr(sklearn.tree, tree_model_name)(random_state=0) + model = clf.fit(x_train, y_train) + + labels = model.predict(x_test) + model.score(x_test, y_test) + # Only classifier models have proba methods. + if tree_model_name in ("DecisionTreeClassifier", "ExtraTreeClassifier"): + model.predict_log_proba(x_test) + model.predict_proba(x_test) + + return _run From 8a3986242fff19b6949d1b7f2b23c93e5bb56661 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Tue, 15 Nov 2022 16:36:55 -0800 Subject: [PATCH 03/20] Support Python 2.7 & 3.7 sklearn --- newrelic/config.py | 5 +++++ tox.ini | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index e7eb69e98..439305a64 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2795,6 +2795,11 @@ def _process_module_builtin_defaults(): "newrelic.hooks.component_sklearn", "instrument_sklearn_tree_models", ) + _process_module_definition( + "sklearn.tree.tree", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_tree_models", + ) _process_module_definition( "rest_framework.views", diff --git a/tox.ini b/tox.ini index db0bba091..ea05efa2f 100644 --- a/tox.ini +++ b/tox.ini @@ -61,6 +61,8 @@ envlist = python-application_celery-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, gearman-application_gearman-{py27,pypy}, python-component_sklearn-{py38,py39,py310,py311}-scikitlearnlatest, + python-component_sklearn-{py37,pypy37}-scikitlearn101, + python-component_sklearn-{py27,pypy27}-scikitlearn020, python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, @@ -205,6 +207,8 @@ deps = application_celery-py{py37,37}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 component_sklearn-scikitlearnlatest: scikit-learn + component_sklearn-scikitlearn020: scikit-learn < 0.21 + component_sklearn-scikitlearn101: scikit-learn < 1.1 component_djangorestframework-djangorestframework0300: Django < 1.9 component_djangorestframework-djangorestframework0300: djangorestframework < 3.1 component_djangorestframework-djangorestframeworklatest: Django From c72d9df6992cf7a408062e13a2ef2fd735f03d88 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 16 Nov 2022 15:24:58 -0800 Subject: [PATCH 04/20] Add test for multiple calls to model method --- newrelic/hooks/component_sklearn.py | 9 ++- tests/component_sklearn/test_tree_models.py | 68 +++++++++++++++++++-- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/newrelic/hooks/component_sklearn.py b/newrelic/hooks/component_sklearn.py index 03ab6eb53..fd740ef87 100644 --- a/newrelic/hooks/component_sklearn.py +++ b/newrelic/hooks/component_sklearn.py @@ -29,13 +29,18 @@ def _wrap_method(wrapped, instance, args, kwargs): if getattr(instance, "_nr_wrapped_%s" % method_name, False): return wrapped(*args, **kwargs) - # Set the _nr_wrapped attribute to denote that this method has now been wrapped. + # Set the _nr_wrapped attribute to denote that this method is being wrapped. setattr(instance, "_nr_wrapped_%s" % method_name, True) # MLModel/Sklearn/Named/. func_name = wrapped.__name__ name = "%s.%s" % (wrapped.__self__.__class__.__name__, func_name) - return FunctionTraceWrapper(wrapped, name=name, group="MLModel/Sklearn/Named")(*args, **kwargs) + return_val = FunctionTraceWrapper(wrapped, name=name, group="MLModel/Sklearn/Named")(*args, **kwargs) + + # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. + setattr(instance, "_nr_wrapped_%s" % method_name, False) + + return return_val return _wrap_method diff --git a/tests/component_sklearn/test_tree_models.py b/tests/component_sklearn/test_tree_models.py index 235e161e0..2ba2e8862 100644 --- a/tests/component_sklearn/test_tree_models.py +++ b/tests/component_sklearn/test_tree_models.py @@ -25,26 +25,26 @@ def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model expected_scoped_metrics = { "ExtraTreeRegressor": [ ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 1), + ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 2), ("MLModel/Sklearn/Named/ExtraTreeRegressor.score", 1), ], "DecisionTreeClassifier": [ ("MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 2), ("MLModel/Sklearn/Named/DecisionTreeClassifier.score", 1), ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 2), ], "ExtraTreeClassifier": [ ("MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 2), ("MLModel/Sklearn/Named/ExtraTreeClassifier.score", 1), ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 2), ], "DecisionTreeRegressor": [ ("MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 1), + ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 2), ("MLModel/Sklearn/Named/DecisionTreeRegressor.score", 1), ], } @@ -55,6 +55,7 @@ def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model @validate_transaction_metrics( expected_transaction_name, scoped_metrics=expected_scoped_metrics[tree_model_name], + rollup_metrics=expected_scoped_metrics[tree_model_name], background_task=True, ) @background_task() @@ -64,6 +65,60 @@ def _test(): _test() +def test_multiple_calls_to_model_methods(tree_model_name, run_tree_model): + expected_scoped_metrics = { + "ExtraTreeRegressor": [ + ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), + ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 4), + ("MLModel/Sklearn/Named/ExtraTreeRegressor.score", 2), + ], + "DecisionTreeClassifier": [ + ("MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 4), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.score", 2), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 2), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 4), + ], + "ExtraTreeClassifier": [ + ("MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 4), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.score", 2), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 2), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 4), + ], + "DecisionTreeRegressor": [ + ("MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), + ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 4), + ("MLModel/Sklearn/Named/DecisionTreeRegressor.score", 2), + ], + } + expected_transaction_name = "test_tree_models:_test" + if six.PY3: + expected_transaction_name = "test_tree_models:test_model_methods_wrapped_in_function_trace.._test" + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[tree_model_name], + rollup_metrics=expected_scoped_metrics[tree_model_name], + background_task=True, + ) + @background_task() + def _test(): + x_test = [[2.0, 2.0], [2.0, 1.0]] + y_test = [1, 1] + + model = run_tree_model() + + model.predict(x_test) + model.score(x_test, y_test) + # Only classifier models have proba methods. + if tree_model_name in ("DecisionTreeClassifier", "ExtraTreeClassifier"): + model.predict_log_proba(x_test) + model.predict_proba(x_test) + + _test() + + @pytest.fixture(params=["ExtraTreeRegressor", "DecisionTreeClassifier", "ExtraTreeClassifier", "DecisionTreeRegressor"]) def tree_model_name(request): return request.param @@ -88,5 +143,6 @@ def _run(): if tree_model_name in ("DecisionTreeClassifier", "ExtraTreeClassifier"): model.predict_log_proba(x_test) model.predict_proba(x_test) + return model return _run From fc1f179b130d394f231c2fa8daf9f7f1ce4e753b Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 16 Nov 2022 17:57:52 -0800 Subject: [PATCH 05/20] Fixup: add comments & organize --- newrelic/config.py | 1 + tests/component_sklearn/test_tree_models.py | 8 +++++++- tox.ini | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 439305a64..c8661055b 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2795,6 +2795,7 @@ def _process_module_builtin_defaults(): "newrelic.hooks.component_sklearn", "instrument_sklearn_tree_models", ) + # In scikit-learn < 0.21 the model classes are in tree.py instead of _classes.py. _process_module_definition( "sklearn.tree.tree", "newrelic.hooks.component_sklearn", diff --git a/tests/component_sklearn/test_tree_models.py b/tests/component_sklearn/test_tree_models.py index 2ba2e8862..8499f5177 100644 --- a/tests/component_sklearn/test_tree_models.py +++ b/tests/component_sklearn/test_tree_models.py @@ -22,6 +22,9 @@ def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model): + # Note: in the following expected metrics, predict and predict_proba are called by + # score and predict_log_proba so they are expected to be called twice instead of + # once like the rest of the methods. expected_scoped_metrics = { "ExtraTreeRegressor": [ ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), @@ -66,6 +69,9 @@ def _test(): def test_multiple_calls_to_model_methods(tree_model_name, run_tree_model): + # Note: in the following expected metrics, predict and predict_proba are called by + # score and predict_log_proba so they are expected to be called twice as often as + # the other methods. expected_scoped_metrics = { "ExtraTreeRegressor": [ ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), @@ -94,7 +100,7 @@ def test_multiple_calls_to_model_methods(tree_model_name, run_tree_model): } expected_transaction_name = "test_tree_models:_test" if six.PY3: - expected_transaction_name = "test_tree_models:test_model_methods_wrapped_in_function_trace.._test" + expected_transaction_name = "test_tree_models:test_multiple_calls_to_model_methods.._test" @validate_transaction_metrics( expected_transaction_name, diff --git a/tox.ini b/tox.ini index ea05efa2f..23e0dcafb 100644 --- a/tox.ini +++ b/tox.ini @@ -207,8 +207,8 @@ deps = application_celery-py{py37,37}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 component_sklearn-scikitlearnlatest: scikit-learn - component_sklearn-scikitlearn020: scikit-learn < 0.21 component_sklearn-scikitlearn101: scikit-learn < 1.1 + component_sklearn-scikitlearn020: scikit-learn < 0.21 component_djangorestframework-djangorestframework0300: Django < 1.9 component_djangorestframework-djangorestframework0300: djangorestframework < 3.1 component_djangorestframework-djangorestframeworklatest: Django From 59a951178392e0743b3086b81ddf2e108608bced Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Thu, 17 Nov 2022 11:55:01 -0800 Subject: [PATCH 06/20] Add ensemble models --- newrelic/config.py | 10 ++++-- newrelic/hooks/component_sklearn.py | 54 ++++++++++++++++++++++++----- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index c8661055b..af78cac74 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2793,13 +2793,19 @@ def _process_module_builtin_defaults(): _process_module_definition( "sklearn.tree._classes", "newrelic.hooks.component_sklearn", - "instrument_sklearn_tree_models", + "instrument_sklearn_models", ) # In scikit-learn < 0.21 the model classes are in tree.py instead of _classes.py. _process_module_definition( "sklearn.tree.tree", "newrelic.hooks.component_sklearn", - "instrument_sklearn_tree_models", + "instrument_sklearn_models", + ) + + _process_module_definition( + "sklearn.ensemble._bagging", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", ) _process_module_definition( diff --git a/newrelic/hooks/component_sklearn.py b/newrelic/hooks/component_sklearn.py index fd740ef87..c914c4519 100644 --- a/newrelic/hooks/component_sklearn.py +++ b/newrelic/hooks/component_sklearn.py @@ -45,7 +45,7 @@ def _wrap_method(wrapped, instance, args, kwargs): return _wrap_method -def wrap_model_init(wrapped, instance, args, kwargs): +def wrap_tree_model_init(wrapped, instance, args, kwargs): return_val = wrapped(*args, **kwargs) methods_to_wrap = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") @@ -56,17 +56,55 @@ def wrap_model_init(wrapped, instance, args, kwargs): return return_val -def _nr_instrument_model(module, model_class): - wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_model_init) +def wrap_ensemble_model_init(wrapped, instance, args, kwargs): + return_val = wrapped(*args, **kwargs) + methods_to_wrap = ( + "predict", + "predict_proba", + "predict_log_proba", + "fit", + "fit_predict", + "staged_predict", + "staged_predict_proba", + ) + for method_name in methods_to_wrap: + if hasattr(instance, method_name): + setattr(instance, method_name, wrap_model_method(method_name)(getattr(instance, method_name))) -def instrument_sklearn_tree_models(module): - model_classes = ( + return return_val + + +# def _nr_instrument_tree_model(module, model_class): +# wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_tree_model_init) + + +def instrument_sklearn_models(module): + tree_model_classes = ( "DecisionTreeClassifier", "DecisionTreeRegressor", "ExtraTreeClassifier", "ExtraTreeRegressor", ) - for model_cls in model_classes: - if hasattr(module, model_cls): - _nr_instrument_model(module, model_cls) + ensemble_model_classes = ( + "AdaBoostClassifier", + "AdaBoostRegressor" "BaggingClassifier", + "BaggingRegressor", + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "GradientBoostingClassifier", + "GradientBoostingRegressor", + "HistGradientBoostingClassifier", + "HistGradientBoostingRegressor", + "IsolationForest", + "RandomForestClassifier", + "StackingClassifier", + "VotingClassifier", + "VotingRegressor", + ) + for model_class in tree_model_classes: + if hasattr(module, model_class): + wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_tree_model_init) + for model_class in ensemble_model_classes: + if hasattr(module, model_class): + wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_ensemble_model_init) From b26fa84951b1ff8cb720f4ac5aa6d9633962321b Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Thu, 17 Nov 2022 17:27:17 -0800 Subject: [PATCH 07/20] Add ensemble model tests --- newrelic/config.py | 42 +++++ newrelic/hooks/component_sklearn.py | 7 +- .../component_sklearn/test_ensemble_models.py | 172 ++++++++++++++++++ tests/component_sklearn/test_tree_models.py | 2 +- 4 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 tests/component_sklearn/test_ensemble_models.py diff --git a/newrelic/config.py b/newrelic/config.py index af78cac74..edbae57ef 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2808,6 +2808,48 @@ def _process_module_builtin_defaults(): "instrument_sklearn_models", ) + _process_module_definition( + "sklearn.ensemble._forest", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + + _process_module_definition( + "sklearn.ensemble._iforest", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + + _process_module_definition( + "sklearn.ensemble._weight_boosting", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + + _process_module_definition( + "sklearn.ensemble._gb", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + + _process_module_definition( + "sklearn.ensemble._voting", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + + _process_module_definition( + "sklearn.ensemble._stacking", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + + _process_module_definition( + "sklearn.ensemble._hist_gradient_boosting.gradient_boosting", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + _process_module_definition( "rest_framework.views", "newrelic.hooks.component_djangorestframework", diff --git a/newrelic/hooks/component_sklearn.py b/newrelic/hooks/component_sklearn.py index c914c4519..0136db361 100644 --- a/newrelic/hooks/component_sklearn.py +++ b/newrelic/hooks/component_sklearn.py @@ -75,10 +75,6 @@ def wrap_ensemble_model_init(wrapped, instance, args, kwargs): return return_val -# def _nr_instrument_tree_model(module, model_class): -# wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_tree_model_init) - - def instrument_sklearn_models(module): tree_model_classes = ( "DecisionTreeClassifier", @@ -88,7 +84,8 @@ def instrument_sklearn_models(module): ) ensemble_model_classes = ( "AdaBoostClassifier", - "AdaBoostRegressor" "BaggingClassifier", + "AdaBoostRegressor", + "BaggingClassifier", "BaggingRegressor", "ExtraTreesClassifier", "ExtraTreesRegressor", diff --git a/tests/component_sklearn/test_ensemble_models.py b/tests/component_sklearn/test_ensemble_models.py new file mode 100644 index 000000000..0ee18298b --- /dev/null +++ b/tests/component_sklearn/test_ensemble_models.py @@ -0,0 +1,172 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +def test_model_methods_wrapped_in_function_trace(ensemble_model_name, run_ensemble_model): + expected_scoped_metrics = { + "AdaBoostClassifier": [ + ("MLModel/Sklearn/Named/AdaBoostClassifier.predict", 1), + ("MLModel/Sklearn/Named/AdaBoostClassifier.predict_log_proba", 1), + ("MLModel/Sklearn/Named/AdaBoostClassifier.predict_proba", 1), + ("MLModel/Sklearn/Named/AdaBoostClassifier.staged_predict", 1), + ("MLModel/Sklearn/Named/AdaBoostClassifier.staged_predict_proba", 1), + ], + "AdaBoostRegressor": [ + ("MLModel/Sklearn/Named/AdaBoostRegressor.predict", 1), + ("MLModel/Sklearn/Named/AdaBoostRegressor.staged_predict", 1), + ], + "BaggingClassifier": [ + ("MLModel/Sklearn/Named/BaggingClassifier.predict", 1), + ("MLModel/Sklearn/Named/BaggingClassifier.predict_log_proba", 1), + ("MLModel/Sklearn/Named/BaggingClassifier.predict_proba", 1), + ], + "BaggingRegressor": [ + ("MLModel/Sklearn/Named/BaggingRegressor.predict", 1), + ], + "ExtraTreesClassifier": [ + ("MLModel/Sklearn/Named/ExtraTreesClassifier.predict", 1), + ("MLModel/Sklearn/Named/ExtraTreesClassifier.predict_log_proba", 1), + ("MLModel/Sklearn/Named/ExtraTreesClassifier.predict_proba", 1), + ], + "ExtraTreesRegressor": [ + ("MLModel/Sklearn/Named/ExtraTreesRegressor.predict", 1), + ], + "GradientBoostingClassifier": [ + ("MLModel/Sklearn/Named/GradientBoostingClassifier.predict", 1), + ("MLModel/Sklearn/Named/GradientBoostingClassifier.predict_log_proba", 1), + ("MLModel/Sklearn/Named/GradientBoostingClassifier.predict_proba", 1), + ("MLModel/Sklearn/Named/GradientBoostingClassifier.staged_predict", 1), + ("MLModel/Sklearn/Named/GradientBoostingClassifier.staged_predict_proba", 1), + ], + "GradientBoostingRegressor": [ + ("MLModel/Sklearn/Named/GradientBoostingRegressor.predict", 1), + ("MLModel/Sklearn/Named/GradientBoostingRegressor.staged_predict", 1), + ], + "HistGradientBoostingClassifier": [ + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.staged_predict", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.staged_predict_proba", 1), + ], + "HistGradientBoostingRegressor": [ + ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.staged_predict", 1), + ], + "IsolationForest": [ + ("MLModel/Sklearn/Named/IsolationForest.fit_predict", 1), + ("MLModel/Sklearn/Named/IsolationForest.predict", 1), + ], + "RandomForestClassifier": [ + ("MLModel/Sklearn/Named/RandomForestClassifier.predict", 1), + ("MLModel/Sklearn/Named/RandomForestClassifier.predict_log_proba", 1), + ("MLModel/Sklearn/Named/RandomForestClassifier.predict_proba", 1), + ], + "RandomForestRegressor": [ + ("MLModel/Sklearn/Named/RandomForestRegressor.predict", 1), + ], + "StackingClassifier": [ + ("MLModel/Sklearn/Named/StackingClassifier.predict", 1), + ("MLModel/Sklearn/Named/StackingClassifier.predict_proba", 1), + ], + "StackingRegressor": [ + ("MLModel/Sklearn/Named/StackingRegressor.predict", 1), + ], + "VotingClassifier": [ + ("MLModel/Sklearn/Named/VotingClassifier.predict", 1), + ("MLModel/Sklearn/Named/VotingClassifier.predict_proba", 1), + ], + "VotingRegressor": [ + ("MLModel/Sklearn/Named/VotingRegressor.predict", 1), + ], + } + expected_transaction_name = "test_ensemble_models:_test" + if six.PY3: + expected_transaction_name = "test_ensemble_models:test_model_methods_wrapped_in_function_trace.._test" + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[ensemble_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_ensemble_model() + + _test() + + +@pytest.fixture( + params=[ + "AdaBoostClassifier", + "AdaBoostRegressor", + "BaggingClassifier", + "BaggingRegressor", + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "GradientBoostingClassifier", + "GradientBoostingRegressor", + "HistGradientBoostingClassifier", + "HistGradientBoostingRegressor", + "IsolationForest", + "RandomForestClassifier", + "StackingClassifier", + "VotingClassifier", + "VotingRegressor", + ] +) +def ensemble_model_name(request): + return request.param + + +@pytest.fixture +def run_ensemble_model(ensemble_model_name): + def _run(): + import sklearn + + x_train = [[0, 0], [1, 1]] + y_train = [0, 1] + x_test = [[2.0, 2.0], [2.0, 1.0]] + # y_test = [1, 1] + + clf = getattr(sklearn.ensemble, ensemble_model_name)(random_state=0) + model = clf.fit(x_train, y_train) + + # model.staged_predict(x_train) + # model.staged_predict_proba(x_train) + # labels = model.predict(x_test) + + # Only classifier models have proba methods. + classifier_models = ( + "AdaBoostClassifier", + "BaggingClassifier", + "ExtraTreesClassifier", + "GradientBoostingClassifier", + "HistGradientBoostingClassifier", + "RandomForestClassifier", + "StackingClassifier", + "VotingClassifier", + ) + if ensemble_model_name in classifier_models: + model.predict_log_proba(x_test) + model.predict_proba(x_test) + + return _run diff --git a/tests/component_sklearn/test_tree_models.py b/tests/component_sklearn/test_tree_models.py index 8499f5177..aece048a0 100644 --- a/tests/component_sklearn/test_tree_models.py +++ b/tests/component_sklearn/test_tree_models.py @@ -133,7 +133,7 @@ def tree_model_name(request): @pytest.fixture def run_tree_model(tree_model_name): def _run(): - import sklearn.tree + import sklearn x_train = [[0, 0], [1, 1]] y_train = [0, 1] From fba42c86d04d82317dc9561fe5dfa11827629f3b Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Fri, 18 Nov 2022 10:39:45 -0800 Subject: [PATCH 08/20] Edit tests --- tests/component_sklearn/test_ensemble_models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/component_sklearn/test_ensemble_models.py b/tests/component_sklearn/test_ensemble_models.py index 0ee18298b..958327f02 100644 --- a/tests/component_sklearn/test_ensemble_models.py +++ b/tests/component_sklearn/test_ensemble_models.py @@ -150,9 +150,9 @@ def _run(): clf = getattr(sklearn.ensemble, ensemble_model_name)(random_state=0) model = clf.fit(x_train, y_train) - # model.staged_predict(x_train) - # model.staged_predict_proba(x_train) - # labels = model.predict(x_test) + model.staged_predict(x_train) + model.staged_predict_proba(x_train) + model.predict(x_test) # Only classifier models have proba methods. classifier_models = ( From 80ad2c0ff287ada74ceabcd12370682a53d9ddbf Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Wed, 23 Nov 2022 13:31:43 -0800 Subject: [PATCH 09/20] Add ensemble library models from sklearn --- newrelic/hooks/component_sklearn.py | 27 +--- .../component_sklearn/test_ensemble_models.py | 142 +++++++++++------- 2 files changed, 93 insertions(+), 76 deletions(-) diff --git a/newrelic/hooks/component_sklearn.py b/newrelic/hooks/component_sklearn.py index 0136db361..5083b71b8 100644 --- a/newrelic/hooks/component_sklearn.py +++ b/newrelic/hooks/component_sklearn.py @@ -45,7 +45,7 @@ def _wrap_method(wrapped, instance, args, kwargs): return _wrap_method -def wrap_tree_model_init(wrapped, instance, args, kwargs): +def wrap_model_init(wrapped, instance, args, kwargs): return_val = wrapped(*args, **kwargs) methods_to_wrap = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") @@ -56,25 +56,6 @@ def wrap_tree_model_init(wrapped, instance, args, kwargs): return return_val -def wrap_ensemble_model_init(wrapped, instance, args, kwargs): - return_val = wrapped(*args, **kwargs) - - methods_to_wrap = ( - "predict", - "predict_proba", - "predict_log_proba", - "fit", - "fit_predict", - "staged_predict", - "staged_predict_proba", - ) - for method_name in methods_to_wrap: - if hasattr(instance, method_name): - setattr(instance, method_name, wrap_model_method(method_name)(getattr(instance, method_name))) - - return return_val - - def instrument_sklearn_models(module): tree_model_classes = ( "DecisionTreeClassifier", @@ -95,13 +76,15 @@ def instrument_sklearn_models(module): "HistGradientBoostingRegressor", "IsolationForest", "RandomForestClassifier", + "RandomForestRegressor", "StackingClassifier", + "StackingRegressor", "VotingClassifier", "VotingRegressor", ) for model_class in tree_model_classes: if hasattr(module, model_class): - wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_tree_model_init) + wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_model_init) for model_class in ensemble_model_classes: if hasattr(module, model_class): - wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_ensemble_model_init) + wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_model_init) diff --git a/tests/component_sklearn/test_ensemble_models.py b/tests/component_sklearn/test_ensemble_models.py index 958327f02..39bd35dae 100644 --- a/tests/component_sklearn/test_ensemble_models.py +++ b/tests/component_sklearn/test_ensemble_models.py @@ -13,6 +13,7 @@ # limitations under the License. import pytest +from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) @@ -24,78 +25,101 @@ def test_model_methods_wrapped_in_function_trace(ensemble_model_name, run_ensemble_model): expected_scoped_metrics = { "AdaBoostClassifier": [ - ("MLModel/Sklearn/Named/AdaBoostClassifier.predict", 1), + ("MLModel/Sklearn/Named/AdaBoostClassifier.fit", 1), + ("MLModel/Sklearn/Named/AdaBoostClassifier.predict", 2), ("MLModel/Sklearn/Named/AdaBoostClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/AdaBoostClassifier.predict_proba", 1), - ("MLModel/Sklearn/Named/AdaBoostClassifier.staged_predict", 1), - ("MLModel/Sklearn/Named/AdaBoostClassifier.staged_predict_proba", 1), + ("MLModel/Sklearn/Named/AdaBoostClassifier.predict_proba", 2), + ("MLModel/Sklearn/Named/AdaBoostClassifier.score", 1), ], "AdaBoostRegressor": [ - ("MLModel/Sklearn/Named/AdaBoostRegressor.predict", 1), - ("MLModel/Sklearn/Named/AdaBoostRegressor.staged_predict", 1), + ("MLModel/Sklearn/Named/AdaBoostRegressor.fit", 1), + ("MLModel/Sklearn/Named/AdaBoostRegressor.predict", 2), + ("MLModel/Sklearn/Named/AdaBoostRegressor.score", 1), ], "BaggingClassifier": [ - ("MLModel/Sklearn/Named/BaggingClassifier.predict", 1), + ("MLModel/Sklearn/Named/BaggingClassifier.fit", 1), + ("MLModel/Sklearn/Named/BaggingClassifier.predict", 2), + ("MLModel/Sklearn/Named/BaggingClassifier.score", 1), ("MLModel/Sklearn/Named/BaggingClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/BaggingClassifier.predict_proba", 1), + ("MLModel/Sklearn/Named/BaggingClassifier.predict_proba", 3), ], "BaggingRegressor": [ - ("MLModel/Sklearn/Named/BaggingRegressor.predict", 1), + ("MLModel/Sklearn/Named/BaggingRegressor.fit", 1), + ("MLModel/Sklearn/Named/BaggingRegressor.predict", 2), + ("MLModel/Sklearn/Named/BaggingRegressor.score", 1), ], "ExtraTreesClassifier": [ - ("MLModel/Sklearn/Named/ExtraTreesClassifier.predict", 1), + ("MLModel/Sklearn/Named/ExtraTreesClassifier.fit", 1), + ("MLModel/Sklearn/Named/ExtraTreesClassifier.predict", 2), + ("MLModel/Sklearn/Named/ExtraTreesClassifier.score", 1), ("MLModel/Sklearn/Named/ExtraTreesClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/ExtraTreesClassifier.predict_proba", 1), + ("MLModel/Sklearn/Named/ExtraTreesClassifier.predict_proba", 4), ], "ExtraTreesRegressor": [ - ("MLModel/Sklearn/Named/ExtraTreesRegressor.predict", 1), + ("MLModel/Sklearn/Named/ExtraTreesRegressor.fit", 1), + ("MLModel/Sklearn/Named/ExtraTreesRegressor.predict", 2), + ("MLModel/Sklearn/Named/ExtraTreesRegressor.score", 1), ], "GradientBoostingClassifier": [ - ("MLModel/Sklearn/Named/GradientBoostingClassifier.predict", 1), + ("MLModel/Sklearn/Named/GradientBoostingClassifier.fit", 1), + ("MLModel/Sklearn/Named/GradientBoostingClassifier.predict", 2), + ("MLModel/Sklearn/Named/GradientBoostingClassifier.score", 1), ("MLModel/Sklearn/Named/GradientBoostingClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/GradientBoostingClassifier.predict_proba", 1), - ("MLModel/Sklearn/Named/GradientBoostingClassifier.staged_predict", 1), - ("MLModel/Sklearn/Named/GradientBoostingClassifier.staged_predict_proba", 1), + ("MLModel/Sklearn/Named/GradientBoostingClassifier.predict_proba", 2), ], "GradientBoostingRegressor": [ - ("MLModel/Sklearn/Named/GradientBoostingRegressor.predict", 1), - ("MLModel/Sklearn/Named/GradientBoostingRegressor.staged_predict", 1), + ("MLModel/Sklearn/Named/GradientBoostingRegressor.fit", 1), + ("MLModel/Sklearn/Named/GradientBoostingRegressor.predict", 2), + ("MLModel/Sklearn/Named/GradientBoostingRegressor.score", 1), ], "HistGradientBoostingClassifier": [ - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.staged_predict", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.staged_predict_proba", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), ], "HistGradientBoostingRegressor": [ - ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.staged_predict", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.fit", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 2), + ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.score", 1), ], "IsolationForest": [ - ("MLModel/Sklearn/Named/IsolationForest.fit_predict", 1), + ("MLModel/Sklearn/Named/IsolationForest.fit", 1), ("MLModel/Sklearn/Named/IsolationForest.predict", 1), ], "RandomForestClassifier": [ - ("MLModel/Sklearn/Named/RandomForestClassifier.predict", 1), + ("MLModel/Sklearn/Named/RandomForestClassifier.fit", 1), + ("MLModel/Sklearn/Named/RandomForestClassifier.predict", 2), + ("MLModel/Sklearn/Named/RandomForestClassifier.score", 1), ("MLModel/Sklearn/Named/RandomForestClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/RandomForestClassifier.predict_proba", 1), + ("MLModel/Sklearn/Named/RandomForestClassifier.predict_proba", 4), ], "RandomForestRegressor": [ - ("MLModel/Sklearn/Named/RandomForestRegressor.predict", 1), + ("MLModel/Sklearn/Named/RandomForestRegressor.fit", 1), + ("MLModel/Sklearn/Named/RandomForestRegressor.predict", 2), + ("MLModel/Sklearn/Named/RandomForestRegressor.score", 1), ], "StackingClassifier": [ - ("MLModel/Sklearn/Named/StackingClassifier.predict", 1), + ("MLModel/Sklearn/Named/StackingClassifier.fit", 1), + ("MLModel/Sklearn/Named/StackingClassifier.predict", 2), + ("MLModel/Sklearn/Named/StackingClassifier.score", 1), ("MLModel/Sklearn/Named/StackingClassifier.predict_proba", 1), ], "StackingRegressor": [ - ("MLModel/Sklearn/Named/StackingRegressor.predict", 1), + ("MLModel/Sklearn/Named/StackingRegressor.fit", 1), + ("MLModel/Sklearn/Named/StackingRegressor.predict", 2), + ("MLModel/Sklearn/Named/StackingRegressor.score", 1), ], "VotingClassifier": [ - ("MLModel/Sklearn/Named/VotingClassifier.predict", 1), - ("MLModel/Sklearn/Named/VotingClassifier.predict_proba", 1), + ("MLModel/Sklearn/Named/VotingClassifier.fit", 1), + ("MLModel/Sklearn/Named/VotingClassifier.predict", 2), + ("MLModel/Sklearn/Named/VotingClassifier.score", 1), + ("MLModel/Sklearn/Named/VotingClassifier.predict_proba", 3), ], "VotingRegressor": [ - ("MLModel/Sklearn/Named/VotingRegressor.predict", 1), + ("MLModel/Sklearn/Named/VotingRegressor.fit", 1), + ("MLModel/Sklearn/Named/VotingRegressor.predict", 2), + ("MLModel/Sklearn/Named/VotingRegressor.score", 1), ], } expected_transaction_name = "test_ensemble_models:_test" @@ -105,6 +129,7 @@ def test_model_methods_wrapped_in_function_trace(ensemble_model_name, run_ensemb @validate_transaction_metrics( expected_transaction_name, scoped_metrics=expected_scoped_metrics[ensemble_model_name], + rollup_metrics=expected_scoped_metrics[ensemble_model_name], background_task=True, ) @background_task() @@ -128,7 +153,9 @@ def _test(): "HistGradientBoostingRegressor", "IsolationForest", "RandomForestClassifier", + "RandomForestRegressor", "StackingClassifier", + "StackingRegressor", "VotingClassifier", "VotingRegressor", ] @@ -140,33 +167,40 @@ def ensemble_model_name(request): @pytest.fixture def run_ensemble_model(ensemble_model_name): def _run(): - import sklearn + import sklearn.ensemble + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split - x_train = [[0, 0], [1, 1]] - y_train = [0, 1] - x_test = [[2.0, 2.0], [2.0, 1.0]] - # y_test = [1, 1] + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) - clf = getattr(sklearn.ensemble, ensemble_model_name)(random_state=0) - model = clf.fit(x_train, y_train) + if ensemble_model_name in ["StackingClassifier"]: + clf = getattr(sklearn.ensemble, ensemble_model_name)( + estimators=[("rf", RandomForestClassifier())], final_estimator=RandomForestClassifier() + ) + elif ensemble_model_name in ["VotingClassifier"]: + clf = getattr(sklearn.ensemble, ensemble_model_name)( + estimators=[("rf", RandomForestClassifier())], voting="soft" + ) + elif ensemble_model_name in ["StackingRegressor"]: + clf = getattr(sklearn.ensemble, ensemble_model_name)( + estimators=[("rf", RandomForestRegressor())], final_estimator=RandomForestRegressor() + ) + elif ensemble_model_name in ["VotingRegressor"]: + clf = getattr(sklearn.ensemble, ensemble_model_name)([("rf", RandomForestRegressor())]) + else: + clf = getattr(sklearn.ensemble, ensemble_model_name)(random_state=0) - model.staged_predict(x_train) - model.staged_predict_proba(x_train) + model = clf.fit(x_train, y_train) model.predict(x_test) - # Only classifier models have proba methods. - classifier_models = ( - "AdaBoostClassifier", - "BaggingClassifier", - "ExtraTreesClassifier", - "GradientBoostingClassifier", - "HistGradientBoostingClassifier", - "RandomForestClassifier", - "StackingClassifier", - "VotingClassifier", - ) - if ensemble_model_name in classifier_models: + if hasattr(model, "score"): + model.score(x_test, y_test) + if hasattr(model, "predict_log_proba"): model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): model.predict_proba(x_test) + return model + return _run From 531b12a8cad640fe9f032ac6de8dc68080b15912 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Wed, 23 Nov 2022 13:37:35 -0800 Subject: [PATCH 10/20] Start tests with empty commit From 94cb0c27534bb82f1f0dae844e0c2e81c71997a6 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Wed, 23 Nov 2022 14:53:26 -0800 Subject: [PATCH 11/20] Clean up tests --- tests/component_sklearn/test_ensemble_models.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/component_sklearn/test_ensemble_models.py b/tests/component_sklearn/test_ensemble_models.py index 39bd35dae..251bb5b2e 100644 --- a/tests/component_sklearn/test_ensemble_models.py +++ b/tests/component_sklearn/test_ensemble_models.py @@ -171,22 +171,23 @@ def _run(): from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split + # This works better with StackingClassifier and StackingRegressor models X, y = load_iris(return_X_y=True) x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) - if ensemble_model_name in ["StackingClassifier"]: + if ensemble_model_name == "StackingClassifier": clf = getattr(sklearn.ensemble, ensemble_model_name)( estimators=[("rf", RandomForestClassifier())], final_estimator=RandomForestClassifier() ) - elif ensemble_model_name in ["VotingClassifier"]: + elif ensemble_model_name == "VotingClassifier": clf = getattr(sklearn.ensemble, ensemble_model_name)( estimators=[("rf", RandomForestClassifier())], voting="soft" ) - elif ensemble_model_name in ["StackingRegressor"]: + elif ensemble_model_name == "StackingRegressor": clf = getattr(sklearn.ensemble, ensemble_model_name)( estimators=[("rf", RandomForestRegressor())], final_estimator=RandomForestRegressor() ) - elif ensemble_model_name in ["VotingRegressor"]: + elif ensemble_model_name == "VotingRegressor": clf = getattr(sklearn.ensemble, ensemble_model_name)([("rf", RandomForestRegressor())]) else: clf = getattr(sklearn.ensemble, ensemble_model_name)(random_state=0) From 36115e5ff42cf84fca3e7dbe84257ff56d42208f Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Thu, 1 Dec 2022 23:01:02 -0800 Subject: [PATCH 12/20] Fix tests for various versions of sklearn --- newrelic/config.py | 36 +++++ .../component_sklearn/test_ensemble_models.py | 148 ++++++++++++------ 2 files changed, 138 insertions(+), 46 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index edbae57ef..2040e1ecd 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2808,36 +2808,72 @@ def _process_module_builtin_defaults(): "instrument_sklearn_models", ) + _process_module_definition( + "sklearn.ensemble.bagging", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + _process_module_definition( "sklearn.ensemble._forest", "newrelic.hooks.component_sklearn", "instrument_sklearn_models", ) + _process_module_definition( + "sklearn.ensemble.forest", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + _process_module_definition( "sklearn.ensemble._iforest", "newrelic.hooks.component_sklearn", "instrument_sklearn_models", ) + _process_module_definition( + "sklearn.ensemble.iforest", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + _process_module_definition( "sklearn.ensemble._weight_boosting", "newrelic.hooks.component_sklearn", "instrument_sklearn_models", ) + _process_module_definition( + "sklearn.ensemble.weight_boosting", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + _process_module_definition( "sklearn.ensemble._gb", "newrelic.hooks.component_sklearn", "instrument_sklearn_models", ) + _process_module_definition( + "sklearn.ensemble.gradient_boosting", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + _process_module_definition( "sklearn.ensemble._voting", "newrelic.hooks.component_sklearn", "instrument_sklearn_models", ) + _process_module_definition( + "sklearn.ensemble.voting_classifier", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_models", + ) + _process_module_definition( "sklearn.ensemble._stacking", "newrelic.hooks.component_sklearn", diff --git a/tests/component_sklearn/test_ensemble_models.py b/tests/component_sklearn/test_ensemble_models.py index 251bb5b2e..ff3f672a4 100644 --- a/tests/component_sklearn/test_ensemble_models.py +++ b/tests/component_sklearn/test_ensemble_models.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys + import pytest from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor from testing_support.validators.validate_transaction_metrics import ( @@ -19,8 +21,15 @@ ) from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version from newrelic.packages import six +SKLEARN_VERSION = get_package_version("sklearn") + +SKLEARN_BELOW_v1_0 = SKLEARN_VERSION < "1.0" +SKLEARN_v1_0_TO_v1_1 = SKLEARN_VERSION >= "1.0" and SKLEARN_VERSION < "1.1" +SKLEARN_v1_1_AND_ABOVE = SKLEARN_VERSION >= "1.1" + def test_model_methods_wrapped_in_function_trace(ensemble_model_name, run_ensemble_model): expected_scoped_metrics = { @@ -72,17 +81,6 @@ def test_model_methods_wrapped_in_function_trace(ensemble_model_name, run_ensemb ("MLModel/Sklearn/Named/GradientBoostingRegressor.predict", 2), ("MLModel/Sklearn/Named/GradientBoostingRegressor.score", 1), ], - "HistGradientBoostingClassifier": [ - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), - ], - "HistGradientBoostingRegressor": [ - ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.fit", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 2), - ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.score", 1), - ], "IsolationForest": [ ("MLModel/Sklearn/Named/IsolationForest.fit", 1), ("MLModel/Sklearn/Named/IsolationForest.predict", 1), @@ -99,29 +97,73 @@ def test_model_methods_wrapped_in_function_trace(ensemble_model_name, run_ensemb ("MLModel/Sklearn/Named/RandomForestRegressor.predict", 2), ("MLModel/Sklearn/Named/RandomForestRegressor.score", 1), ], - "StackingClassifier": [ + "VotingClassifier": [ + ("MLModel/Sklearn/Named/VotingClassifier.fit", 1), + ("MLModel/Sklearn/Named/VotingClassifier.predict", 2), + ("MLModel/Sklearn/Named/VotingClassifier.score", 1), + ("MLModel/Sklearn/Named/VotingClassifier.transform", 1), + ], + } + + if SKLEARN_v1_0_TO_v1_1: + expected_scoped_metrics["HistGradientBoostingClassifier"] = [ + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), + ] + expected_scoped_metrics["HistGradientBoostingRegressor"] = [ + ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.fit", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 2), + ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.score", 1), + ] + expected_scoped_metrics["StackingClassifier"] = [ + ("MLModel/Sklearn/Named/StackingClassifier.fit", 1), + ] + expected_scoped_metrics["StackingRegressor"] = [ + ("MLModel/Sklearn/Named/StackingRegressor.fit", 1), + ] + expected_scoped_metrics["VotingRegressor"] = [ + ("MLModel/Sklearn/Named/VotingRegressor.fit", 1), + ("MLModel/Sklearn/Named/VotingRegressor.predict", 2), + ("MLModel/Sklearn/Named/VotingRegressor.score", 1), + ("MLModel/Sklearn/Named/VotingRegressor.transform", 1), + ] + elif SKLEARN_v1_1_AND_ABOVE: + if sys.version_info[:2] > (3, 7): + expected_scoped_metrics["VotingClassifier"].append( + ("MLModel/Sklearn/Named/VotingClassifier.predict_proba", 3) + ) + expected_scoped_metrics["StackingClassifier"] = [ ("MLModel/Sklearn/Named/StackingClassifier.fit", 1), ("MLModel/Sklearn/Named/StackingClassifier.predict", 2), ("MLModel/Sklearn/Named/StackingClassifier.score", 1), ("MLModel/Sklearn/Named/StackingClassifier.predict_proba", 1), - ], - "StackingRegressor": [ + ("MLModel/Sklearn/Named/StackingClassifier.transform", 4), + ] + expected_scoped_metrics["StackingRegressor"] = [ ("MLModel/Sklearn/Named/StackingRegressor.fit", 1), ("MLModel/Sklearn/Named/StackingRegressor.predict", 2), ("MLModel/Sklearn/Named/StackingRegressor.score", 1), - ], - "VotingClassifier": [ - ("MLModel/Sklearn/Named/VotingClassifier.fit", 1), - ("MLModel/Sklearn/Named/VotingClassifier.predict", 2), - ("MLModel/Sklearn/Named/VotingClassifier.score", 1), - ("MLModel/Sklearn/Named/VotingClassifier.predict_proba", 3), - ], - "VotingRegressor": [ + ] + expected_scoped_metrics["VotingRegressor"] = [ ("MLModel/Sklearn/Named/VotingRegressor.fit", 1), ("MLModel/Sklearn/Named/VotingRegressor.predict", 2), ("MLModel/Sklearn/Named/VotingRegressor.score", 1), - ], - } + ("MLModel/Sklearn/Named/VotingRegressor.transform", 1), + ] + expected_scoped_metrics["HistGradientBoostingClassifier"] = [ + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), + ] + expected_scoped_metrics["HistGradientBoostingRegressor"] = [ + ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.fit", 1), + ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 2), + ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.score", 1), + ] + expected_transaction_name = "test_ensemble_models:_test" if six.PY3: expected_transaction_name = "test_ensemble_models:test_model_methods_wrapped_in_function_trace.._test" @@ -139,27 +181,33 @@ def _test(): _test() -@pytest.fixture( - params=[ - "AdaBoostClassifier", - "AdaBoostRegressor", - "BaggingClassifier", - "BaggingRegressor", - "ExtraTreesClassifier", - "ExtraTreesRegressor", - "GradientBoostingClassifier", - "GradientBoostingRegressor", - "HistGradientBoostingClassifier", - "HistGradientBoostingRegressor", - "IsolationForest", - "RandomForestClassifier", - "RandomForestRegressor", - "StackingClassifier", - "StackingRegressor", - "VotingClassifier", - "VotingRegressor", - ] -) +class_params = [ + "AdaBoostClassifier", + "AdaBoostRegressor", + "BaggingClassifier", + "BaggingRegressor", + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "GradientBoostingClassifier", + "GradientBoostingRegressor", + "IsolationForest", + "RandomForestClassifier", + "RandomForestRegressor", +] +if SKLEARN_v1_0_TO_v1_1 or SKLEARN_v1_1_AND_ABOVE: + class_params.extend( + ( + "HistGradientBoostingClassifier", + "HistGradientBoostingRegressor", + "StackingClassifier", + "StackingRegressor", + "VotingClassifier", + "VotingRegressor", + ) + ) + + +@pytest.fixture(params=class_params) def ensemble_model_name(request): return request.param @@ -180,8 +228,14 @@ def _run(): estimators=[("rf", RandomForestClassifier())], final_estimator=RandomForestClassifier() ) elif ensemble_model_name == "VotingClassifier": + # Voting=soft is needed to also be able to test for "predict_proba" + # However, predict_proba is not supported in versions less than v1.1 + if SKLEARN_BELOW_v1_0 or SKLEARN_v1_0_TO_v1_1: + voting_flag = "hard" + else: + voting_flag = "soft" clf = getattr(sklearn.ensemble, ensemble_model_name)( - estimators=[("rf", RandomForestClassifier())], voting="soft" + estimators=[("rf", RandomForestClassifier())], voting=voting_flag ) elif ensemble_model_name == "StackingRegressor": clf = getattr(sklearn.ensemble, ensemble_model_name)( @@ -201,6 +255,8 @@ def _run(): model.predict_log_proba(x_test) if hasattr(model, "predict_proba"): model.predict_proba(x_test) + if hasattr(model, "transform"): + model.transform(x_test) return model From ad14a001c841ece971fcd3e7491e494d031397ae Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Tue, 6 Dec 2022 16:08:24 -0800 Subject: [PATCH 13/20] Fix ensemble tests with changes from tree PR --- .pre-commit-config.yaml | 25 +++ .python-version | 8 + newrelic/config.py | 30 ++-- newrelic/hooks/component_sklearn.py | 90 ---------- newrelic/hooks/mlmodel_sklearn.py | 27 +++ tests/component_sklearn/conftest.py | 38 ----- tests/component_sklearn/test_tree_models.py | 154 ----------------- .../test_ensemble_models.py | 158 +++++++++--------- 8 files changed, 154 insertions(+), 376 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 .python-version delete mode 100644 newrelic/hooks/component_sklearn.py delete mode 100644 tests/component_sklearn/conftest.py delete mode 100644 tests/component_sklearn/test_tree_models.py rename tests/{component_sklearn => mlmodel_sklearn}/test_ensemble_models.py (50%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..3ddd0fc54 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +repos: +- repo: https://github.com/psf/black + rev: 22.6.0 + hooks: + - id: black + language_version: python3 +- repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + args: ["--skip", "migrations", "--profile", "black"] +- repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 + hooks: + - id: flake8 +- repo: https://github.com/PyCQA/bandit + rev: 1.7.4 + hooks: + - id: bandit + args: ["-x", "./tests", "--skip", "B101,B110,B404,B608"] +- repo: https://github.com/PyCQA/pylint + rev: v2.15.5 + hooks: + - id: pylint + args: ["--fail-under=0", "--fail-on=E"] diff --git a/.python-version b/.python-version new file mode 100644 index 000000000..b3a7ca181 --- /dev/null +++ b/.python-version @@ -0,0 +1,8 @@ +3.10.0 +3.9.5 +3.8.10 +3.7.10 +pypy3.7-7.3.9 +pypy2.7-7.3.4 +2.7.18 +3.11.0rc2 diff --git a/newrelic/config.py b/newrelic/config.py index 1100a8f4a..7ddf882d9 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2799,91 +2799,91 @@ def _process_module_builtin_defaults(): _process_module_definition( "sklearn.tree.tree", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_tree_models", ) _process_module_definition( "sklearn.ensemble._bagging", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble.bagging", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble._forest", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble.forest", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble._iforest", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble.iforest", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble._weight_boosting", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble.weight_boosting", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble._gb", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble.gradient_boosting", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble._voting", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble.voting_classifier", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble._stacking", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( "sklearn.ensemble._hist_gradient_boosting.gradient_boosting", "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_models", + "instrument_sklearn_ensemble_models", ) _process_module_definition( diff --git a/newrelic/hooks/component_sklearn.py b/newrelic/hooks/component_sklearn.py deleted file mode 100644 index 5083b71b8..000000000 --- a/newrelic/hooks/component_sklearn.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from newrelic.api.function_trace import FunctionTraceWrapper -from newrelic.api.transaction import current_transaction -from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper - - -def wrap_model_method(method_name): - @function_wrapper - def _wrap_method(wrapped, instance, args, kwargs): - # If there is no transaction, do not wrap anything. - if not current_transaction(): - return wrapped(*args, **kwargs) - - # If the method has already been wrapped do not wrap it again. This happens - # when one model inherits from another and they both implement the method. - if getattr(instance, "_nr_wrapped_%s" % method_name, False): - return wrapped(*args, **kwargs) - - # Set the _nr_wrapped attribute to denote that this method is being wrapped. - setattr(instance, "_nr_wrapped_%s" % method_name, True) - - # MLModel/Sklearn/Named/. - func_name = wrapped.__name__ - name = "%s.%s" % (wrapped.__self__.__class__.__name__, func_name) - return_val = FunctionTraceWrapper(wrapped, name=name, group="MLModel/Sklearn/Named")(*args, **kwargs) - - # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. - setattr(instance, "_nr_wrapped_%s" % method_name, False) - - return return_val - - return _wrap_method - - -def wrap_model_init(wrapped, instance, args, kwargs): - return_val = wrapped(*args, **kwargs) - - methods_to_wrap = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") - for method_name in methods_to_wrap: - if hasattr(instance, method_name): - setattr(instance, method_name, wrap_model_method(method_name)(getattr(instance, method_name))) - - return return_val - - -def instrument_sklearn_models(module): - tree_model_classes = ( - "DecisionTreeClassifier", - "DecisionTreeRegressor", - "ExtraTreeClassifier", - "ExtraTreeRegressor", - ) - ensemble_model_classes = ( - "AdaBoostClassifier", - "AdaBoostRegressor", - "BaggingClassifier", - "BaggingRegressor", - "ExtraTreesClassifier", - "ExtraTreesRegressor", - "GradientBoostingClassifier", - "GradientBoostingRegressor", - "HistGradientBoostingClassifier", - "HistGradientBoostingRegressor", - "IsolationForest", - "RandomForestClassifier", - "RandomForestRegressor", - "StackingClassifier", - "StackingRegressor", - "VotingClassifier", - "VotingRegressor", - ) - for model_class in tree_model_classes: - if hasattr(module, model_class): - wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_model_init) - for model_class in ensemble_model_classes: - if hasattr(module, model_class): - wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_model_init) diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 4034b5dfd..ce96079ba 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -47,6 +47,7 @@ def _nr_wrapper_method(wrapped, instance, args, kwargs): # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. setattr(trace, wrapped_attr_name, False) + # breakpoint() return return_val wrap_function_wrapper(module, "%s.%s" % (_class, method), _nr_wrapper_method) @@ -56,6 +57,7 @@ def _nr_instrument_model(module, model_class): for method_name in METHODS_TO_WRAP: if hasattr(getattr(module, model_class), method_name): # Function/MLModel/Sklearn/Named/. + # breakpoint() name = "MLModel/Sklearn/Named/%s.%s" % (model_class, method_name) _wrap_method_trace(module, model_class, method_name, name=name) @@ -63,6 +65,7 @@ def _nr_instrument_model(module, model_class): def _instrument_sklearn_models(module, model_classes): for model_cls in model_classes: if hasattr(module, model_cls): + # breakpoint() _nr_instrument_model(module, model_cls) @@ -74,3 +77,27 @@ def instrument_sklearn_tree_models(module): "ExtraTreeRegressor", ) _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_models(module): + model_classes = ( + "AdaBoostClassifier", + "AdaBoostRegressor", + "BaggingClassifier", + "BaggingRegressor", + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "GradientBoostingClassifier", + "GradientBoostingRegressor", + "HistGradientBoostingClassifier", + "HistGradientBoostingRegressor", + "IsolationForest", + "RandomForestClassifier", + "RandomForestRegressor", + "StackingClassifier", + "StackingRegressor", + "VotingClassifier", + "VotingRegressor", + ) + # breakpoint() + _instrument_sklearn_models(module, model_classes) diff --git a/tests/component_sklearn/conftest.py b/tests/component_sklearn/conftest.py deleted file mode 100644 index e251d91bb..000000000 --- a/tests/component_sklearn/conftest.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 - code_coverage_fixture, - collector_agent_registration_fixture, - collector_available_fixture, -) - -_coverage_source = [ - "newrelic.hooks.component_sklearn", -] - -code_coverage = code_coverage_fixture(source=_coverage_source) - -_default_settings = { - "transaction_tracer.explain_threshold": 0.0, - "transaction_tracer.transaction_threshold": 0.0, - "transaction_tracer.stack_trace_threshold": 0.0, - "debug.log_data_collector_payloads": True, - "debug.record_transaction_failure": True, -} -collector_agent_registration = collector_agent_registration_fixture( - app_name="Python Agent Test (component_sklearn)", - default_settings=_default_settings, - linked_applications=["Python Agent Test (component_sklearn)"], -) diff --git a/tests/component_sklearn/test_tree_models.py b/tests/component_sklearn/test_tree_models.py deleted file mode 100644 index aece048a0..000000000 --- a/tests/component_sklearn/test_tree_models.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from testing_support.validators.validate_transaction_metrics import ( - validate_transaction_metrics, -) - -from newrelic.api.background_task import background_task -from newrelic.packages import six - - -def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model): - # Note: in the following expected metrics, predict and predict_proba are called by - # score and predict_log_proba so they are expected to be called twice instead of - # once like the rest of the methods. - expected_scoped_metrics = { - "ExtraTreeRegressor": [ - ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 2), - ("MLModel/Sklearn/Named/ExtraTreeRegressor.score", 1), - ], - "DecisionTreeClassifier": [ - ("MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 2), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.score", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 2), - ], - "ExtraTreeClassifier": [ - ("MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 2), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.score", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 2), - ], - "DecisionTreeRegressor": [ - ("MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 2), - ("MLModel/Sklearn/Named/DecisionTreeRegressor.score", 1), - ], - } - expected_transaction_name = "test_tree_models:_test" - if six.PY3: - expected_transaction_name = "test_tree_models:test_model_methods_wrapped_in_function_trace.._test" - - @validate_transaction_metrics( - expected_transaction_name, - scoped_metrics=expected_scoped_metrics[tree_model_name], - rollup_metrics=expected_scoped_metrics[tree_model_name], - background_task=True, - ) - @background_task() - def _test(): - run_tree_model() - - _test() - - -def test_multiple_calls_to_model_methods(tree_model_name, run_tree_model): - # Note: in the following expected metrics, predict and predict_proba are called by - # score and predict_log_proba so they are expected to be called twice as often as - # the other methods. - expected_scoped_metrics = { - "ExtraTreeRegressor": [ - ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 4), - ("MLModel/Sklearn/Named/ExtraTreeRegressor.score", 2), - ], - "DecisionTreeClassifier": [ - ("MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 4), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.score", 2), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 2), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 4), - ], - "ExtraTreeClassifier": [ - ("MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 4), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.score", 2), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 2), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 4), - ], - "DecisionTreeRegressor": [ - ("MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 4), - ("MLModel/Sklearn/Named/DecisionTreeRegressor.score", 2), - ], - } - expected_transaction_name = "test_tree_models:_test" - if six.PY3: - expected_transaction_name = "test_tree_models:test_multiple_calls_to_model_methods.._test" - - @validate_transaction_metrics( - expected_transaction_name, - scoped_metrics=expected_scoped_metrics[tree_model_name], - rollup_metrics=expected_scoped_metrics[tree_model_name], - background_task=True, - ) - @background_task() - def _test(): - x_test = [[2.0, 2.0], [2.0, 1.0]] - y_test = [1, 1] - - model = run_tree_model() - - model.predict(x_test) - model.score(x_test, y_test) - # Only classifier models have proba methods. - if tree_model_name in ("DecisionTreeClassifier", "ExtraTreeClassifier"): - model.predict_log_proba(x_test) - model.predict_proba(x_test) - - _test() - - -@pytest.fixture(params=["ExtraTreeRegressor", "DecisionTreeClassifier", "ExtraTreeClassifier", "DecisionTreeRegressor"]) -def tree_model_name(request): - return request.param - - -@pytest.fixture -def run_tree_model(tree_model_name): - def _run(): - import sklearn - - x_train = [[0, 0], [1, 1]] - y_train = [0, 1] - x_test = [[2.0, 2.0], [2.0, 1.0]] - y_test = [1, 1] - - clf = getattr(sklearn.tree, tree_model_name)(random_state=0) - model = clf.fit(x_train, y_train) - - labels = model.predict(x_test) - model.score(x_test, y_test) - # Only classifier models have proba methods. - if tree_model_name in ("DecisionTreeClassifier", "ExtraTreeClassifier"): - model.predict_log_proba(x_test) - model.predict_proba(x_test) - return model - - return _run diff --git a/tests/component_sklearn/test_ensemble_models.py b/tests/mlmodel_sklearn/test_ensemble_models.py similarity index 50% rename from tests/component_sklearn/test_ensemble_models.py rename to tests/mlmodel_sklearn/test_ensemble_models.py index ff3f672a4..a2e071135 100644 --- a/tests/component_sklearn/test_ensemble_models.py +++ b/tests/mlmodel_sklearn/test_ensemble_models.py @@ -34,134 +34,134 @@ def test_model_methods_wrapped_in_function_trace(ensemble_model_name, run_ensemble_model): expected_scoped_metrics = { "AdaBoostClassifier": [ - ("MLModel/Sklearn/Named/AdaBoostClassifier.fit", 1), - ("MLModel/Sklearn/Named/AdaBoostClassifier.predict", 2), - ("MLModel/Sklearn/Named/AdaBoostClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/AdaBoostClassifier.predict_proba", 2), - ("MLModel/Sklearn/Named/AdaBoostClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/AdaBoostClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/AdaBoostClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/AdaBoostClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/AdaBoostClassifier.predict_proba", 2), + ("Function/MLModel/Sklearn/Named/AdaBoostClassifier.score", 1), ], "AdaBoostRegressor": [ - ("MLModel/Sklearn/Named/AdaBoostRegressor.fit", 1), - ("MLModel/Sklearn/Named/AdaBoostRegressor.predict", 2), - ("MLModel/Sklearn/Named/AdaBoostRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/AdaBoostRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/AdaBoostRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/AdaBoostRegressor.score", 1), ], "BaggingClassifier": [ - ("MLModel/Sklearn/Named/BaggingClassifier.fit", 1), - ("MLModel/Sklearn/Named/BaggingClassifier.predict", 2), - ("MLModel/Sklearn/Named/BaggingClassifier.score", 1), - ("MLModel/Sklearn/Named/BaggingClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/BaggingClassifier.predict_proba", 3), + ("Function/MLModel/Sklearn/Named/BaggingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/BaggingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/BaggingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/BaggingClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/BaggingClassifier.predict_proba", 3), ], "BaggingRegressor": [ - ("MLModel/Sklearn/Named/BaggingRegressor.fit", 1), - ("MLModel/Sklearn/Named/BaggingRegressor.predict", 2), - ("MLModel/Sklearn/Named/BaggingRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/BaggingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/BaggingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/BaggingRegressor.score", 1), ], "ExtraTreesClassifier": [ - ("MLModel/Sklearn/Named/ExtraTreesClassifier.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreesClassifier.predict", 2), - ("MLModel/Sklearn/Named/ExtraTreesClassifier.score", 1), - ("MLModel/Sklearn/Named/ExtraTreesClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/ExtraTreesClassifier.predict_proba", 4), + ("Function/MLModel/Sklearn/Named/ExtraTreesClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreesClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/ExtraTreesClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreesClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreesClassifier.predict_proba", 4), ], "ExtraTreesRegressor": [ - ("MLModel/Sklearn/Named/ExtraTreesRegressor.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreesRegressor.predict", 2), - ("MLModel/Sklearn/Named/ExtraTreesRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreesRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/ExtraTreesRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/ExtraTreesRegressor.score", 1), ], "GradientBoostingClassifier": [ - ("MLModel/Sklearn/Named/GradientBoostingClassifier.fit", 1), - ("MLModel/Sklearn/Named/GradientBoostingClassifier.predict", 2), - ("MLModel/Sklearn/Named/GradientBoostingClassifier.score", 1), - ("MLModel/Sklearn/Named/GradientBoostingClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/GradientBoostingClassifier.predict_proba", 2), + ("Function/MLModel/Sklearn/Named/GradientBoostingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/GradientBoostingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/GradientBoostingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/GradientBoostingClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/GradientBoostingClassifier.predict_proba", 2), ], "GradientBoostingRegressor": [ - ("MLModel/Sklearn/Named/GradientBoostingRegressor.fit", 1), - ("MLModel/Sklearn/Named/GradientBoostingRegressor.predict", 2), - ("MLModel/Sklearn/Named/GradientBoostingRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/GradientBoostingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/GradientBoostingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/GradientBoostingRegressor.score", 1), ], "IsolationForest": [ - ("MLModel/Sklearn/Named/IsolationForest.fit", 1), - ("MLModel/Sklearn/Named/IsolationForest.predict", 1), + ("Function/MLModel/Sklearn/Named/IsolationForest.fit", 1), + ("Function/MLModel/Sklearn/Named/IsolationForest.predict", 1), ], "RandomForestClassifier": [ - ("MLModel/Sklearn/Named/RandomForestClassifier.fit", 1), - ("MLModel/Sklearn/Named/RandomForestClassifier.predict", 2), - ("MLModel/Sklearn/Named/RandomForestClassifier.score", 1), - ("MLModel/Sklearn/Named/RandomForestClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/RandomForestClassifier.predict_proba", 4), + ("Function/MLModel/Sklearn/Named/RandomForestClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/RandomForestClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/RandomForestClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/RandomForestClassifier.predict_log_proba", 1), + ("Function/MLModel/Sklearn/Named/RandomForestClassifier.predict_proba", 4), ], "RandomForestRegressor": [ - ("MLModel/Sklearn/Named/RandomForestRegressor.fit", 1), - ("MLModel/Sklearn/Named/RandomForestRegressor.predict", 2), - ("MLModel/Sklearn/Named/RandomForestRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/RandomForestRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/RandomForestRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/RandomForestRegressor.score", 1), ], "VotingClassifier": [ - ("MLModel/Sklearn/Named/VotingClassifier.fit", 1), - ("MLModel/Sklearn/Named/VotingClassifier.predict", 2), - ("MLModel/Sklearn/Named/VotingClassifier.score", 1), - ("MLModel/Sklearn/Named/VotingClassifier.transform", 1), + ("Function/MLModel/Sklearn/Named/VotingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/VotingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/VotingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/VotingClassifier.transform", 1), ], } if SKLEARN_v1_0_TO_v1_1: expected_scoped_metrics["HistGradientBoostingClassifier"] = [ - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), ] expected_scoped_metrics["HistGradientBoostingRegressor"] = [ - ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.fit", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 2), - ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.score", 1), ] expected_scoped_metrics["StackingClassifier"] = [ - ("MLModel/Sklearn/Named/StackingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/StackingClassifier.fit", 1), ] expected_scoped_metrics["StackingRegressor"] = [ - ("MLModel/Sklearn/Named/StackingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/StackingRegressor.fit", 1), ] expected_scoped_metrics["VotingRegressor"] = [ - ("MLModel/Sklearn/Named/VotingRegressor.fit", 1), - ("MLModel/Sklearn/Named/VotingRegressor.predict", 2), - ("MLModel/Sklearn/Named/VotingRegressor.score", 1), - ("MLModel/Sklearn/Named/VotingRegressor.transform", 1), + ("Function/MLModel/Sklearn/Named/VotingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/VotingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/VotingRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/VotingRegressor.transform", 1), ] elif SKLEARN_v1_1_AND_ABOVE: if sys.version_info[:2] > (3, 7): expected_scoped_metrics["VotingClassifier"].append( - ("MLModel/Sklearn/Named/VotingClassifier.predict_proba", 3) + ("Function/MLModel/Sklearn/Named/VotingClassifier.predict_proba", 3) ) expected_scoped_metrics["StackingClassifier"] = [ - ("MLModel/Sklearn/Named/StackingClassifier.fit", 1), - ("MLModel/Sklearn/Named/StackingClassifier.predict", 2), - ("MLModel/Sklearn/Named/StackingClassifier.score", 1), - ("MLModel/Sklearn/Named/StackingClassifier.predict_proba", 1), - ("MLModel/Sklearn/Named/StackingClassifier.transform", 4), + ("Function/MLModel/Sklearn/Named/StackingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/StackingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/StackingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/StackingClassifier.predict_proba", 1), + ("Function/MLModel/Sklearn/Named/StackingClassifier.transform", 4), ] expected_scoped_metrics["StackingRegressor"] = [ - ("MLModel/Sklearn/Named/StackingRegressor.fit", 1), - ("MLModel/Sklearn/Named/StackingRegressor.predict", 2), - ("MLModel/Sklearn/Named/StackingRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/StackingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/StackingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/StackingRegressor.score", 1), ] expected_scoped_metrics["VotingRegressor"] = [ - ("MLModel/Sklearn/Named/VotingRegressor.fit", 1), - ("MLModel/Sklearn/Named/VotingRegressor.predict", 2), - ("MLModel/Sklearn/Named/VotingRegressor.score", 1), - ("MLModel/Sklearn/Named/VotingRegressor.transform", 1), + ("Function/MLModel/Sklearn/Named/VotingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/VotingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/VotingRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/VotingRegressor.transform", 1), ] expected_scoped_metrics["HistGradientBoostingClassifier"] = [ - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), ] expected_scoped_metrics["HistGradientBoostingRegressor"] = [ - ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.fit", 1), - ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 2), - ("MLModel/Sklearn/Named/HistGradientBoostingRegressor.score", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.fit", 1), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.predict", 2), + ("Function/MLModel/Sklearn/Named/HistGradientBoostingRegressor.score", 1), ] expected_transaction_name = "test_ensemble_models:_test" From 3d553483fba6463588912635605f2bdf12cca059 Mon Sep 17 00:00:00 2001 From: lrafeei Date: Wed, 7 Dec 2022 00:10:21 +0000 Subject: [PATCH 14/20] [Mega-Linter] Apply linters fixes --- .pre-commit-config.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ddd0fc54..256ff94b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,25 +1,25 @@ repos: -- repo: https://github.com/psf/black + - repo: https://github.com/psf/black rev: 22.6.0 hooks: - - id: black - language_version: python3 -- repo: https://github.com/PyCQA/isort + - id: black + language_version: python3 + - repo: https://github.com/PyCQA/isort rev: 5.10.1 hooks: - - id: isort - args: ["--skip", "migrations", "--profile", "black"] -- repo: https://github.com/PyCQA/flake8 + - id: isort + args: ["--skip", "migrations", "--profile", "black"] + - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - - id: flake8 -- repo: https://github.com/PyCQA/bandit + - id: flake8 + - repo: https://github.com/PyCQA/bandit rev: 1.7.4 hooks: - - id: bandit - args: ["-x", "./tests", "--skip", "B101,B110,B404,B608"] -- repo: https://github.com/PyCQA/pylint + - id: bandit + args: ["-x", "./tests", "--skip", "B101,B110,B404,B608"] + - repo: https://github.com/PyCQA/pylint rev: v2.15.5 hooks: - - id: pylint - args: ["--fail-under=0", "--fail-on=E"] + - id: pylint + args: ["--fail-under=0", "--fail-on=E"] From 1202ff7efbc368d002292302e62fd5d2d1d86827 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Tue, 6 Dec 2022 17:03:42 -0800 Subject: [PATCH 15/20] Remove breakpoints --- newrelic/hooks/mlmodel_sklearn.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index ce96079ba..500ff2fb4 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -47,7 +47,6 @@ def _nr_wrapper_method(wrapped, instance, args, kwargs): # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. setattr(trace, wrapped_attr_name, False) - # breakpoint() return return_val wrap_function_wrapper(module, "%s.%s" % (_class, method), _nr_wrapper_method) @@ -57,7 +56,6 @@ def _nr_instrument_model(module, model_class): for method_name in METHODS_TO_WRAP: if hasattr(getattr(module, model_class), method_name): # Function/MLModel/Sklearn/Named/. - # breakpoint() name = "MLModel/Sklearn/Named/%s.%s" % (model_class, method_name) _wrap_method_trace(module, model_class, method_name, name=name) @@ -65,7 +63,6 @@ def _nr_instrument_model(module, model_class): def _instrument_sklearn_models(module, model_classes): for model_cls in model_classes: if hasattr(module, model_cls): - # breakpoint() _nr_instrument_model(module, model_cls) @@ -99,5 +96,4 @@ def instrument_sklearn_ensemble_models(module): "VotingClassifier", "VotingRegressor", ) - # breakpoint() _instrument_sklearn_models(module, model_classes) From f38410d53c62dc7f2e26252b43321131a2608b3e Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Wed, 7 Dec 2022 12:58:47 -0800 Subject: [PATCH 16/20] Create tests/instrumentation for calibration models --- newrelic/config.py | 6 + newrelic/hooks/mlmodel_sklearn.py | 8 + .../test_calibration_models.py | 163 ++++++++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 tests/mlmodel_sklearn/test_calibration_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 4514b7c29..9e06ca999 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2891,6 +2891,12 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_models", ) + _process_module_definition( + "sklearn.calibration", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_calibration_models", + ) + _process_module_definition( "rest_framework.views", "newrelic.hooks.component_djangorestframework", diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 500ff2fb4..ca2040c6e 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -97,3 +97,11 @@ def instrument_sklearn_ensemble_models(module): "VotingRegressor", ) _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_calibration_models(module): + model_classes = ( + "CalibratedClassififerCV", + # "CalibrationDisplay" + ) + _instrument_sklearn_models(module, model_classes) diff --git a/tests/mlmodel_sklearn/test_calibration_models.py b/tests/mlmodel_sklearn/test_calibration_models.py new file mode 100644 index 000000000..9f4181f46 --- /dev/null +++ b/tests/mlmodel_sklearn/test_calibration_models.py @@ -0,0 +1,163 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from sklearn import __version__ # noqa: needed for get_package_version +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version +from newrelic.packages import six + +SKLEARN_VERSION = get_package_version("sklearn") + +SKLEARN_BELOW_v1_0 = SKLEARN_VERSION < "1.0" +SKLEARN_v1_0_TO_v1_1 = SKLEARN_VERSION >= "1.0" and SKLEARN_VERSION < "1.1" +SKLEARN_v1_1_AND_ABOVE = SKLEARN_VERSION >= "1.1" + + +def test_model_methods_wrapped_in_function_trace(calibration_model_name, run_calibration_model): + expected_scoped_metrics = { + "CalibratedClassifierCV": [ + ("Function/MLModel/Sklearn/Named/CalibratedClassifierCV.fit", 1), + ("Function/MLModel/Sklearn/Named/CalibratedClassifierCV.predict", 2), + ("Function/MLModel/Sklearn/Named/CalibratedClassifierCV.predict_proba", 2), + ], + # "CalibrationDisplay": [ + # ("Function/MLModel/Sklearn/Named/CalibrationDisplay.from_predictions", 2), + # ], + } + + # if SKLEARN_v1_0_TO_v1_1: + # expected_scoped_metrics["HistGradientBoostingClassifier"] = [ + # ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), + # ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), + # ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), + # ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), + # ] + # elif SKLEARN_v1_1_AND_ABOVE: + # if sys.version_info[:2] > (3, 7): + # expected_scoped_metrics["VotingClassifier"].append( + # ("Function/MLModel/Sklearn/Named/VotingClassifier.predict_proba", 3) + # ) + # expected_scoped_metrics["StackingClassifier"] = [ + # ("Function/MLModel/Sklearn/Named/StackingClassifier.fit", 1), + # ("Function/MLModel/Sklearn/Named/StackingClassifier.predict", 2), + # ("Function/MLModel/Sklearn/Named/StackingClassifier.score", 1), + # ("Function/MLModel/Sklearn/Named/StackingClassifier.predict_proba", 1), + # ("Function/MLModel/Sklearn/Named/StackingClassifier.transform", 4), + # ] + + expected_transaction_name = "test_calibration_models:_test" + if six.PY3: + expected_transaction_name = ( + "test_calibration_models:test_model_methods_wrapped_in_function_trace.._test" + ) + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[calibration_model_name], + rollup_metrics=expected_scoped_metrics[calibration_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_calibration_model() + + _test() + + +# class_params = [ +# "AdaBoostClassifier", +# "AdaBoostRegressor", +# "BaggingClassifier", +# "BaggingRegressor", +# "ExtraTreesClassifier", +# "ExtraTreesRegressor", +# "GradientBoostingClassifier", +# "GradientBoostingRegressor", +# "IsolationForest", +# "RandomForestClassifier", +# "RandomForestRegressor", +# ] +# if SKLEARN_v1_0_TO_v1_1 or SKLEARN_v1_1_AND_ABOVE: +# class_params.extend( +# ( +# "HistGradientBoostingClassifier", +# "HistGradientBoostingRegressor", +# "StackingClassifier", +# "StackingRegressor", +# "VotingClassifier", +# "VotingRegressor", +# ) +# ) + + +@pytest.fixture(params=["CalibratedClassifierCV"]) +def calibration_model_name(request): + return request.param + + +@pytest.fixture +def run_calibration_model(calibration_model_name): + def _run(): + from sklearn import calibration as calibration + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + + # This works better with StackingClassifier and StackingRegressor models + X, y = load_iris(return_X_y=True) + x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) + + # if calibration_model_name == "StackingClassifier": + # clf = getattr(sklearn.calibration, calibration_model_name)( + # estimators=[("rf", RandomForestClassifier())], final_estimator=RandomForestClassifier() + # ) + # elif calibration_model_name == "VotingClassifier": + # # Voting=soft is needed to also be able to test for "predict_proba" + # # However, predict_proba is not supported in versions less than v1.1 + # if SKLEARN_BELOW_v1_0 or SKLEARN_v1_0_TO_v1_1: + # voting_flag = "hard" + # else: + # voting_flag = "soft" + # clf = getattr(sklearn.calibration, calibration_model_name)( + # estimators=[("rf", RandomForestClassifier())], voting=voting_flag + # ) + # elif calibration_model_name == "StackingRegressor": + # clf = getattr(sklearn.calibration, calibration_model_name)( + # estimators=[("rf", RandomForestRegressor())], final_estimator=RandomForestRegressor() + # ) + # elif calibration_model_name == "VotingRegressor": + # clf = getattr(sklearn.calibration, calibration_model_name)([("rf", RandomForestRegressor())]) + # else: + # clf = getattr(calibration, calibration_model_name)(random_state=0) + clf = getattr(calibration, calibration_model_name)() + + model = clf.fit(x_train, y_train) + model.predict(x_test) + + if hasattr(model, "score"): + model.score(x_test, y_test) + if hasattr(model, "predict_log_proba"): + model.predict_log_proba(x_test) + if hasattr(model, "predict_proba"): + model.predict_proba(x_test) + if hasattr(model, "transform"): + model.transform(x_test) + + return model + + return _run From f24c8d40359d077f1f45c9e29928aa84edfa2bfd Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Wed, 7 Dec 2022 16:06:17 -0800 Subject: [PATCH 17/20] Fix calibration tests --- newrelic/hooks/mlmodel_sklearn.py | 3 +- .../test_calibration_models.py | 95 +------------------ 2 files changed, 6 insertions(+), 92 deletions(-) diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index ca2040c6e..5052a6c6b 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -101,7 +101,8 @@ def instrument_sklearn_ensemble_models(module): def instrument_sklearn_calibration_models(module): model_classes = ( - "CalibratedClassififerCV", + "CalibratedClassifierCV", # "CalibrationDisplay" ) + # breakpoint() _instrument_sklearn_models(module, model_classes) diff --git a/tests/mlmodel_sklearn/test_calibration_models.py b/tests/mlmodel_sklearn/test_calibration_models.py index 9f4181f46..39ac34cb2 100644 --- a/tests/mlmodel_sklearn/test_calibration_models.py +++ b/tests/mlmodel_sklearn/test_calibration_models.py @@ -13,54 +13,23 @@ # limitations under the License. import pytest -from sklearn import __version__ # noqa: needed for get_package_version from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) from newrelic.api.background_task import background_task -from newrelic.common.package_version_utils import get_package_version from newrelic.packages import six -SKLEARN_VERSION = get_package_version("sklearn") - -SKLEARN_BELOW_v1_0 = SKLEARN_VERSION < "1.0" -SKLEARN_v1_0_TO_v1_1 = SKLEARN_VERSION >= "1.0" and SKLEARN_VERSION < "1.1" -SKLEARN_v1_1_AND_ABOVE = SKLEARN_VERSION >= "1.1" - def test_model_methods_wrapped_in_function_trace(calibration_model_name, run_calibration_model): expected_scoped_metrics = { "CalibratedClassifierCV": [ ("Function/MLModel/Sklearn/Named/CalibratedClassifierCV.fit", 1), - ("Function/MLModel/Sklearn/Named/CalibratedClassifierCV.predict", 2), + ("Function/MLModel/Sklearn/Named/CalibratedClassifierCV.predict", 1), ("Function/MLModel/Sklearn/Named/CalibratedClassifierCV.predict_proba", 2), ], - # "CalibrationDisplay": [ - # ("Function/MLModel/Sklearn/Named/CalibrationDisplay.from_predictions", 2), - # ], } - # if SKLEARN_v1_0_TO_v1_1: - # expected_scoped_metrics["HistGradientBoostingClassifier"] = [ - # ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.fit", 1), - # ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict", 2), - # ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.score", 1), - # ("Function/MLModel/Sklearn/Named/HistGradientBoostingClassifier.predict_proba", 3), - # ] - # elif SKLEARN_v1_1_AND_ABOVE: - # if sys.version_info[:2] > (3, 7): - # expected_scoped_metrics["VotingClassifier"].append( - # ("Function/MLModel/Sklearn/Named/VotingClassifier.predict_proba", 3) - # ) - # expected_scoped_metrics["StackingClassifier"] = [ - # ("Function/MLModel/Sklearn/Named/StackingClassifier.fit", 1), - # ("Function/MLModel/Sklearn/Named/StackingClassifier.predict", 2), - # ("Function/MLModel/Sklearn/Named/StackingClassifier.score", 1), - # ("Function/MLModel/Sklearn/Named/StackingClassifier.predict_proba", 1), - # ("Function/MLModel/Sklearn/Named/StackingClassifier.transform", 4), - # ] - expected_transaction_name = "test_calibration_models:_test" if six.PY3: expected_transaction_name = ( @@ -80,32 +49,6 @@ def _test(): _test() -# class_params = [ -# "AdaBoostClassifier", -# "AdaBoostRegressor", -# "BaggingClassifier", -# "BaggingRegressor", -# "ExtraTreesClassifier", -# "ExtraTreesRegressor", -# "GradientBoostingClassifier", -# "GradientBoostingRegressor", -# "IsolationForest", -# "RandomForestClassifier", -# "RandomForestRegressor", -# ] -# if SKLEARN_v1_0_TO_v1_1 or SKLEARN_v1_1_AND_ABOVE: -# class_params.extend( -# ( -# "HistGradientBoostingClassifier", -# "HistGradientBoostingRegressor", -# "StackingClassifier", -# "StackingRegressor", -# "VotingClassifier", -# "VotingRegressor", -# ) -# ) - - @pytest.fixture(params=["CalibratedClassifierCV"]) def calibration_model_name(request): return request.param @@ -114,49 +57,19 @@ def calibration_model_name(request): @pytest.fixture def run_calibration_model(calibration_model_name): def _run(): - from sklearn import calibration as calibration + import sklearn.calibration from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split - # This works better with StackingClassifier and StackingRegressor models X, y = load_iris(return_X_y=True) x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) - # if calibration_model_name == "StackingClassifier": - # clf = getattr(sklearn.calibration, calibration_model_name)( - # estimators=[("rf", RandomForestClassifier())], final_estimator=RandomForestClassifier() - # ) - # elif calibration_model_name == "VotingClassifier": - # # Voting=soft is needed to also be able to test for "predict_proba" - # # However, predict_proba is not supported in versions less than v1.1 - # if SKLEARN_BELOW_v1_0 or SKLEARN_v1_0_TO_v1_1: - # voting_flag = "hard" - # else: - # voting_flag = "soft" - # clf = getattr(sklearn.calibration, calibration_model_name)( - # estimators=[("rf", RandomForestClassifier())], voting=voting_flag - # ) - # elif calibration_model_name == "StackingRegressor": - # clf = getattr(sklearn.calibration, calibration_model_name)( - # estimators=[("rf", RandomForestRegressor())], final_estimator=RandomForestRegressor() - # ) - # elif calibration_model_name == "VotingRegressor": - # clf = getattr(sklearn.calibration, calibration_model_name)([("rf", RandomForestRegressor())]) - # else: - # clf = getattr(calibration, calibration_model_name)(random_state=0) - clf = getattr(calibration, calibration_model_name)() + clf = getattr(sklearn.calibration, calibration_model_name)() model = clf.fit(x_train, y_train) model.predict(x_test) - if hasattr(model, "score"): - model.score(x_test, y_test) - if hasattr(model, "predict_log_proba"): - model.predict_log_proba(x_test) - if hasattr(model, "predict_proba"): - model.predict_proba(x_test) - if hasattr(model, "transform"): - model.transform(x_test) + model.predict_proba(x_test) return model From bda993c8d175541103cf86bd0729577503807e23 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Wed, 14 Dec 2022 15:06:00 -0800 Subject: [PATCH 18/20] Remove commented out code --- newrelic/hooks/mlmodel_sklearn.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py index 90cfa5025..8e691dccf 100644 --- a/newrelic/hooks/mlmodel_sklearn.py +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -190,11 +190,7 @@ def instrument_sklearn_ensemble_hist_models(module): def instrument_sklearn_calibration_models(module): - model_classes = ( - "CalibratedClassifierCV", - # "CalibrationDisplay" - ) - # breakpoint() + model_classes = ("CalibratedClassifierCV",) _instrument_sklearn_models(module, model_classes) From 03951d594c9164f301e023e605dcd9bfb4c748c6 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Wed, 14 Dec 2022 15:14:35 -0800 Subject: [PATCH 19/20] Remove yaml file in commit --- .pre-commit-config.yaml | 25 ------------------------- .python-version | 8 -------- 2 files changed, 33 deletions(-) delete mode 100644 .pre-commit-config.yaml delete mode 100644 .python-version diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 256ff94b7..000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -repos: - - repo: https://github.com/psf/black - rev: 22.6.0 - hooks: - - id: black - language_version: python3 - - repo: https://github.com/PyCQA/isort - rev: 5.10.1 - hooks: - - id: isort - args: ["--skip", "migrations", "--profile", "black"] - - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 - hooks: - - id: flake8 - - repo: https://github.com/PyCQA/bandit - rev: 1.7.4 - hooks: - - id: bandit - args: ["-x", "./tests", "--skip", "B101,B110,B404,B608"] - - repo: https://github.com/PyCQA/pylint - rev: v2.15.5 - hooks: - - id: pylint - args: ["--fail-under=0", "--fail-on=E"] diff --git a/.python-version b/.python-version deleted file mode 100644 index b3a7ca181..000000000 --- a/.python-version +++ /dev/null @@ -1,8 +0,0 @@ -3.10.0 -3.9.5 -3.8.10 -3.7.10 -pypy3.7-7.3.9 -pypy2.7-7.3.4 -2.7.18 -3.11.0rc2 From e1dac0099e52c48edc381a4bf7a683186e13adae Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Thu, 15 Dec 2022 11:31:50 -0800 Subject: [PATCH 20/20] Remove duplicate ensemble module defs --- newrelic/config.py | 84 ---------------------------------------------- 1 file changed, 84 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 1af14a361..47a502cd3 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2902,90 +2902,6 @@ def _process_module_builtin_defaults(): "instrument_sklearn_ensemble_hist_models", ) - _process_module_definition( - "sklearn.ensemble._bagging", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble.bagging", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble._forest", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble.forest", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble._iforest", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble.iforest", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble._weight_boosting", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble.weight_boosting", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble._gb", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble.gradient_boosting", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble._voting", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble.voting_classifier", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble._stacking", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - - _process_module_definition( - "sklearn.ensemble._hist_gradient_boosting.gradient_boosting", - "newrelic.hooks.mlmodel_sklearn", - "instrument_sklearn_ensemble_models", - ) - _process_module_definition( "sklearn.calibration", "newrelic.hooks.mlmodel_sklearn",