You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
import json
import logging
from google.cloud import aiplatform as aip
aip.init(project=project, staging_bucket=staging_bucket, location=location)
def retrieve_existing_model_metrics() -> int:
"""
Retrieve the auRoc_existing metric from the evaluation of the existing model.
"""
# Retrieve the existing model and its evaluation
models = aip.Model.list(filter=f'labels.notebook={NOTEBOOK}')
model = models[1]
evaluation = model.get_model_evaluation().to_dict()
# Extract the log loss metric
auRoc_existing = evaluation['metrics']['auRoc']
return auRoc_existing
# Fetch model eval info
def get_eval_info(model):
response = model.list_model_evaluations()
metrics_list = []
metrics_string_list = []
for evaluation in response:
evaluation = evaluation.to_dict()
print("model_evaluation")
print(" name:", evaluation["name"])
print(" metrics_schema_uri:", evaluation["metricsSchemaUri"])
metrics = evaluation["metrics"]
for metric in metrics.keys():
logging.info("metric: %s, value: %s", metric, metrics[metric])
metrics_str = json.dumps(metrics)
metrics_list.append(metrics)
metrics_string_list.append(metrics_str)
return (
evaluation["name"],
metrics_list,
metrics_string_list,
)
# Use the given metrics threshold(s) to determine whether the model is
# accurate enough to deploy.
def classification_thresholds_check(metrics_dict, thresholds_dict):
for k, v in thresholds_dict.items():
logging.info("k {}, v {}".format(k, v))
if k in ["auRoc", "auPrc"]: # higher is better
if metrics_dict[k] > v and metrics_dict[k] > retrieve_existing_model_metrics(): # if under threshold, and under previous model aurRoc don't deploy
logging.info("AucRoc model trained: {} ; threshold: {}; AucRoc existing model: {}; returning True, Deploy the model".format(metrics_dict[k], v, retrieve_existing_model_metrics()))
return True
logging.info("threshold checks not passed, don't deploy.")
return False
def log_metrics(metrics_list, metricsc):
test_confusion_matrix = metrics_list[0]["confusionMatrix"]
logging.info("rows: %s", test_confusion_matrix["rows"])
# log the ROC curve
fpr = []
tpr = []
thresholds = []
for item in metrics_list[0]["confidenceMetrics"]:
fpr.append(item.get("falsePositiveRate", 0.0))
tpr.append(item.get("recall", 0.0))
thresholds.append(item.get("confidenceThreshold", 0.0))
print(f"fpr: {fpr}")
print(f"tpr: {tpr}")
print(f"thresholds: {thresholds}")
metricsc.log_roc_curve(fpr, tpr, thresholds)
# log the confusion matrix
annotations = []
for item in test_confusion_matrix["annotationSpecs"]:
annotations.append(item["displayName"])
logging.info("confusion matrix annotations: %s", annotations)
metricsc.log_confusion_matrix(
annotations,
test_confusion_matrix["rows"],
)
# log textual metrics info as well
for metric in metrics_list[0].keys():
if metric != "confidenceMetrics":
val_string = json.dumps(metrics_list[0][metric])
metrics.log_metric(metric, val_string)
logging.getLogger().setLevel(logging.INFO)
# extract the model resource name from the input Model Artifact
model_resource_path = model.metadata["resourceName"]
logging.info("model path: %s", model_resource_path)
# Get the trained model resource
model = aip.Model(model_resource_path)
# Get model evaluation metrics from the the trained model
eval_name, metrics_list, metrics_str_list = get_eval_info(model)
logging.info("got evaluation name: %s", eval_name)
logging.info("got metrics list: %s", metrics_list)
log_metrics(metrics_list, metricsc)
thresholds_dict = json.loads(thresholds_dict_str)
deploy = classification_thresholds_check(metrics_list[0], thresholds_dict)
if deploy:
dep_decision = "true"
else:
dep_decision = "false"
logging.info("deployment decision is %s", dep_decision)
return (dep_decision,)
Then I checked that if deploy is false I get an error:
ERROR 2024-04-10T11:30:01.606136975Z [resource.labels.taskName: workerpool0-0] /opt/conda/lib/python3.7/runpy.py:109: FutureWarning: Python 3.7 has reached end-of-life. KFP will drop support for Python 3.7 on April 23, 2024. To use new versions of the KFP SDK after that date, you will need to upgrade to Python >= 3.8. See https://devguide.python.org/versions/ for more details. ERROR 2024-04-10T11:30:01.606150371Z [resource.labels.taskName: workerpool0-0] __import__(pkg_name) ERROR 2024-04-10T11:30:01.606162923Z [resource.labels.taskName: workerpool0-0] Traceback (most recent call last): ERROR 2024-04-10T11:30:01.606177750Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/runpy.py", line 193, in _run_module_as_main ERROR 2024-04-10T11:30:01.606191014Z [resource.labels.taskName: workerpool0-0] "__main__", mod_spec) ERROR 2024-04-10T11:30:01.606203029Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/runpy.py", line 85, in _run_code ERROR 2024-04-10T11:30:01.606214414Z [resource.labels.taskName: workerpool0-0] exec(code, run_globals) ERROR 2024-04-10T11:30:01.606226670Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor_main.py", line 109, in <module> ERROR 2024-04-10T11:30:01.606238530Z [resource.labels.taskName: workerpool0-0] executor_main() ERROR 2024-04-10T11:30:01.606249872Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor_main.py", line 101, in executor_main ERROR 2024-04-10T11:30:01.606266251Z [resource.labels.taskName: workerpool0-0] output_file = executor.execute() ERROR 2024-04-10T11:30:01.606277959Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor.py", line 362, in execute ERROR 2024-04-10T11:30:01.606289417Z [resource.labels.taskName: workerpool0-0] return self.write_executor_output(result) ERROR 2024-04-10T11:30:01.606300357Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor.py", line 254, in write_executor_output ERROR 2024-04-10T11:30:01.606311640Z [resource.labels.taskName: workerpool0-0] if len(self.return_annotation._fields) != len(func_output): ERROR 2024-04-10T11:30:01.606323337Z [resource.labels.taskName: workerpool0-0] TypeError: object of type 'bool' has no len()
The text was updated successfully, but these errors were encountered:
Hello,
Thank you for your support, I am talking about this script
vertex-ai-samples/notebooks/official/pipelines/automl_tabular_classification_beans.ipynb
Is there a way to list just the deployed models and match the previous AucRoc, because I can list just the models in the registry.
`@component(
base_image="gcr.io/deeplearning-platform-release/tf2-cpu.2-6:latest",
packages_to_install=["google-cloud-aiplatform"],
)
def classification_model_eval_metrics(
project: str,
location: str,
staging_bucket: str,
thresholds_dict_str: str,
model: Input[Artifact],
metrics: Output[Metrics],
metricsc: Output[ClassificationMetrics],
) -> NamedTuple("Outputs", [("dep_decision", str)]): # Return parameter.
compiler.Compiler().compile(
classification_model_eval_metrics, "tabular_eval_component.yaml"
)`
Then I checked that if deploy is false I get an error:
ERROR 2024-04-10T11:30:01.606136975Z [resource.labels.taskName: workerpool0-0] /opt/conda/lib/python3.7/runpy.py:109: FutureWarning: Python 3.7 has reached end-of-life. KFP will drop support for Python 3.7 on April 23, 2024. To use new versions of the KFP SDK after that date, you will need to upgrade to Python >= 3.8. See https://devguide.python.org/versions/ for more details. ERROR 2024-04-10T11:30:01.606150371Z [resource.labels.taskName: workerpool0-0] __import__(pkg_name) ERROR 2024-04-10T11:30:01.606162923Z [resource.labels.taskName: workerpool0-0] Traceback (most recent call last): ERROR 2024-04-10T11:30:01.606177750Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/runpy.py", line 193, in _run_module_as_main ERROR 2024-04-10T11:30:01.606191014Z [resource.labels.taskName: workerpool0-0] "__main__", mod_spec) ERROR 2024-04-10T11:30:01.606203029Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/runpy.py", line 85, in _run_code ERROR 2024-04-10T11:30:01.606214414Z [resource.labels.taskName: workerpool0-0] exec(code, run_globals) ERROR 2024-04-10T11:30:01.606226670Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor_main.py", line 109, in <module> ERROR 2024-04-10T11:30:01.606238530Z [resource.labels.taskName: workerpool0-0] executor_main() ERROR 2024-04-10T11:30:01.606249872Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor_main.py", line 101, in executor_main ERROR 2024-04-10T11:30:01.606266251Z [resource.labels.taskName: workerpool0-0] output_file = executor.execute() ERROR 2024-04-10T11:30:01.606277959Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor.py", line 362, in execute ERROR 2024-04-10T11:30:01.606289417Z [resource.labels.taskName: workerpool0-0] return self.write_executor_output(result) ERROR 2024-04-10T11:30:01.606300357Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor.py", line 254, in write_executor_output ERROR 2024-04-10T11:30:01.606311640Z [resource.labels.taskName: workerpool0-0] if len(self.return_annotation._fields) != len(func_output): ERROR 2024-04-10T11:30:01.606323337Z [resource.labels.taskName: workerpool0-0] TypeError: object of type 'bool' has no len()
The text was updated successfully, but these errors were encountered: