From e806ba02e0591a9cbee16512acc099a8bd859fbe Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Tue, 20 Oct 2020 22:42:54 -0700 Subject: [PATCH 1/4] [metricstransformprocessor] add new operation- convert_resource_attributes_to_metric_labels Signed-off-by: Rayhan Hossain --- processor/metricstransformprocessor/README.md | 16 ++++- processor/metricstransformprocessor/config.go | 5 ++ .../metricstransformprocessor/config_test.go | 9 +++ .../metrics_testcase_builder_test.go | 18 ++++++ .../metrics_transform_processor.go | 30 +++++++-- ...rics_transform_processor_testcases_test.go | 62 +++++++++++++++++++ ...n_convert_resource_attributes_to_labels.go | 35 +++++++++++ .../testdata/config_full.yaml | 4 ++ 8 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 processor/metricstransformprocessor/operation_convert_resource_attributes_to_labels.go diff --git a/processor/metricstransformprocessor/README.md b/processor/metricstransformprocessor/README.md index ea6cd5e87b40..473b80f27ad6 100644 --- a/processor/metricstransformprocessor/README.md +++ b/processor/metricstransformprocessor/README.md @@ -5,7 +5,7 @@ Supported pipeline types: metrics - e.g. If want to rename a metric or label to `new_name` while there is already a metric or label called `new_name`, this operation will not take any effect. There will also be an error logged ## Description -The metrics transform processor can be used to rename metrics, labels, or label values. It can also be used to perform aggregations on metrics across labels or label values. +The metrics transform processor can be used to rename metrics, labels, or label values. It can also be used to perform aggregations on metrics across labels or label values. Additionally, it supports converting resource attributes to metric labels. ## Capabilities - Rename metrics (e.g. rename `cpu/usage` to `cpu/usage_time`) @@ -17,12 +17,15 @@ The metrics transform processor can be used to rename metrics, labels, or label - Aggregation_type: sum, mean, max - Add label to an existing metric - When adding or updating a label value, specify `{{version}}` to include the application version number +- Convert resource attributes to metric labels ## Configuration ```yaml # transforms is a list of transformations with each element transforming a metric selected by metric name transforms: # name is used to match with the metric to operate on. This implementation doesn’t utilize the filtermetric’s MatchProperties struct because it doesn’t match well with what I need at this phase. All is needed for this processor at this stage is a single name string that can be used to match with selected metrics. The list of metric names and the match type in the filtermetric’s MatchProperties struct are unnecessary. Also, based on the issue about improving filtering configuration, it seems like this struct is subject to be slightly modified. + + # new addition: now it accepts `all_metrics` as the value of 'metric_name' field. This batch operation applies changes to all the metrics. However, currently, it only works and is tested for the 'convert_resource_attributes_to_labels' operation. For other operations (e.g. update_label, add_label), it may not work properly. But, it shouldn't break existing user experience. - metric_name: # action specifies if the operations are performed on the current copy of the metric or on a newly created metric that will be inserted @@ -55,6 +58,9 @@ transforms: aggregated_values: [values...] new_value: aggregation_type: {sum, mean, max} + + # convert_resource_attributes_to_labels action converts all the resourde attributes to metric labels by default. Right now, it doesn't support selecting specific resource attributes as metric labels. + - action: convert_resource_attributes_to_labels ``` ## Examples @@ -148,3 +154,11 @@ operation: label: label label_value: value ``` + +### Convert Resource Attributes to Metric Labels +```yaml +# converts resource attributes to metric labels. +... +operation: + - action: convert_resource_attributes_to_labels +``` diff --git a/processor/metricstransformprocessor/config.go b/processor/metricstransformprocessor/config.go index a21dd936a9a9..092555813ecd 100644 --- a/processor/metricstransformprocessor/config.go +++ b/processor/metricstransformprocessor/config.go @@ -138,6 +138,11 @@ const ( // DeleteLabelValue deletes a label value by also removing all the points associated with this label value DeleteLabelValue OperationAction = "delete_label_value" + // ConvertResourceAttributesToLabels converts resource attributes to metric labels. + // TODO: right now it converts all the attributes to metric labels by default. Make it + // configurable to select specific resource attributes as metric labels. + ConvertResourceAttributesToLabels OperationAction = "convert_resource_attributes_to_labels" + // Mean indicates taking the mean of the aggregated data. Mean AggregationType = "mean" diff --git a/processor/metricstransformprocessor/config_test.go b/processor/metricstransformprocessor/config_test.go index 301ea3c643e0..810260258a7e 100644 --- a/processor/metricstransformprocessor/config_test.go +++ b/processor/metricstransformprocessor/config_test.go @@ -76,6 +76,15 @@ var ( NewName: "new_name", Operations: testDataOperations, }, + { + MetricName: "all_metrics", + Action: Update, + Operations: []Operation{ + { + Action: ConvertResourceAttributesToLabels, + }, + }, + }, }, }, }, diff --git a/processor/metricstransformprocessor/metrics_testcase_builder_test.go b/processor/metricstransformprocessor/metrics_testcase_builder_test.go index bb9d4a6d9252..1465267028d3 100644 --- a/processor/metricstransformprocessor/metrics_testcase_builder_test.go +++ b/processor/metricstransformprocessor/metrics_testcase_builder_test.go @@ -16,6 +16,7 @@ package metricstransformprocessor import ( metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + resourcepb "github.com/census-instrumentation/opencensus-proto/gen-go/resource/v1" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -33,6 +34,17 @@ func metricBuilder() builder { } } +// metricBuilderWithResource is used to build metrics with Resource for testing +func metricBuilderWithResource() builder { + return builder{ + metric: &metricspb.Metric{ + MetricDescriptor: &metricspb.MetricDescriptor{}, + Timeseries: make([]*metricspb.TimeSeries, 0), + Resource: &resourcepb.Resource{}, + }, + } +} + // setName sets the name of the metric func (b builder) setName(name string) builder { b.metric.MetricDescriptor.Name = name @@ -51,6 +63,12 @@ func (b builder) setLabels(labels []string) builder { return b } +// setResourceAttributes sets resource attributes +func (b builder) setResourceAttributes(attributes map[string]string) builder { + b.metric.Resource.Labels = attributes + return b +} + // addTimeseries adds new timeseries with the labelValuesVal and startTimestamp func (b builder) addTimeseries(startTimestampSeconds int64, labelValuesVal []string) builder { labelValues := make([]*metricspb.LabelValue, len(labelValuesVal)) diff --git a/processor/metricstransformprocessor/metrics_transform_processor.go b/processor/metricstransformprocessor/metrics_transform_processor.go index 5426ee63db05..a9ff248f1fa0 100644 --- a/processor/metricstransformprocessor/metrics_transform_processor.go +++ b/processor/metricstransformprocessor/metrics_transform_processor.go @@ -27,10 +27,11 @@ import ( ) type internalTransform struct { - MetricName string - Action ConfigAction - NewName string - Operations []internalOperation + MetricName string + Action ConfigAction + NewName string + Operations []internalOperation + resourceAttributeMap map[string]string } type internalOperation struct { @@ -65,7 +66,26 @@ func (mtp *metricsTransformProcessor) ProcessMetrics(_ context.Context, md pdata nameToMetricMapping[metric.MetricDescriptor.Name] = metric } + resourceAttributes := make(map[string]string) + if data.Resource != nil { + resourceAttributes = data.Resource.Labels + } for _, transform := range mtp.transforms { + transform.resourceAttributeMap = resourceAttributes + + // When user sets "metric_name=all_metrics", these operations are going to be applied + // to all metric data points. Right now, this batch operation only works and is tested for + // `convert_resource_attributes_to_labels` operation, and does not break existing experience + // for other operations. After updating the metrics, it will continue for the next iteration. + // TODO: add more unit tests and update README to pubicly anounce this feature for other + // operations as well. + if transform.MetricName == "all_metrics" { + for _, metric := range data.Metrics { + mtp.update(metric, transform) + } + continue + } + metric, ok := nameToMetricMapping[transform.MetricName] if !ok { continue @@ -110,6 +130,8 @@ func (mtp *metricsTransformProcessor) update(metric *metricspb.Metric, transform mtp.addLabelOp(metric, op) case DeleteLabelValue: mtp.deleteLabelValueOp(metric, op) + case ConvertResourceAttributesToLabels: + mtp.convertResourceAttributesToLabels(metric, transform.resourceAttributeMap, op) } } } diff --git a/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go b/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go index f9e1ae5cb07d..f40e17ec1b3b 100644 --- a/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go +++ b/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go @@ -570,6 +570,68 @@ var ( build(), }, }, + { + name: "all_metrics_batch_operation_test", + transforms: []internalTransform{ + { + MetricName: "all_metrics", + Action: Update, + Operations: []internalOperation{ + { + configOperation: Operation{ + Action: UpdateLabel, + Label: "label1", + NewLabel: "new/label1", + }, + }, + }, + }, + }, + in: []*metricspb.Metric{ + metricBuilder().setName("metric1"). + setDataType(metricspb.MetricDescriptor_CUMULATIVE_INT64). + setLabels([]string{"label1"}). + addTimeseries(1, []string{"value1"}). + addInt64Point(0, 3, 2).build(), + }, + out: []*metricspb.Metric{ + metricBuilder().setName("metric1"). + setDataType(metricspb.MetricDescriptor_CUMULATIVE_INT64). + setLabels([]string{"new/label1"}). + addTimeseries(1, []string{"value1"}). + addInt64Point(0, 3, 2).build(), + }, + }, + { + name: "convert_resource_attributes_to_labels", + transforms: []internalTransform{ + { + MetricName: "all_metrics", + Action: Update, + Operations: []internalOperation{ + { + configOperation: Operation{ + Action: ConvertResourceAttributesToLabels, + }, + }, + }, + }, + }, + in: []*metricspb.Metric{ + metricBuilderWithResource().setName("metric1"). + setDataType(metricspb.MetricDescriptor_CUMULATIVE_INT64). + setResourceAttributes(map[string]string{"label1": "value1"}). + addTimeseries(1, []string{}). + addInt64Point(0, 3, 2).build(), + }, + out: []*metricspb.Metric{ + metricBuilder().setName("metric1"). + setDataType(metricspb.MetricDescriptor_CUMULATIVE_INT64). + setLabels([]string{"label1"}). + addTimeseries(1, []string{"value1"}). + addInt64Point(0, 3, 2).build(), + }, + }, // INSERT { name: "metric_name_insert", diff --git a/processor/metricstransformprocessor/operation_convert_resource_attributes_to_labels.go b/processor/metricstransformprocessor/operation_convert_resource_attributes_to_labels.go new file mode 100644 index 000000000000..98a10891c08b --- /dev/null +++ b/processor/metricstransformprocessor/operation_convert_resource_attributes_to_labels.go @@ -0,0 +1,35 @@ +// Copyright 2020 OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metricstransformprocessor + +import metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + +func (mtp *metricsTransformProcessor) convertResourceAttributesToLabels(metric *metricspb.Metric, resourceAttributes map[string]string, op internalOperation) { + + for key, value := range resourceAttributes { + lablelKey := &metricspb.LabelKey{ + Key: key, + } + labelValue := &metricspb.LabelValue{ + Value: value, + HasValue: true, + } + + metric.MetricDescriptor.LabelKeys = append(metric.MetricDescriptor.LabelKeys, lablelKey) + for _, ts := range metric.Timeseries { + ts.LabelValues = append(ts.LabelValues, labelValue) + } + } +} diff --git a/processor/metricstransformprocessor/testdata/config_full.yaml b/processor/metricstransformprocessor/testdata/config_full.yaml index f6dd8e5b20d7..816d0f37fc25 100644 --- a/processor/metricstransformprocessor/testdata/config_full.yaml +++ b/processor/metricstransformprocessor/testdata/config_full.yaml @@ -22,6 +22,10 @@ processors: aggregated_values: [value1, value2] new_value: new_value aggregation_type: sum + - metric_name: all_metrics + action: update + operations: + - action: convert_resource_attributes_to_labels metricstransform/addlabel: transforms: - metric_name: some_name From bca271211aa3c15e041d5ca2a14bbdc5ad551893 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Wed, 21 Oct 2020 12:14:46 -0700 Subject: [PATCH 2/4] Update README and add a new unit test --- processor/metricstransformprocessor/README.md | 4 +-- ...rics_transform_processor_testcases_test.go | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/processor/metricstransformprocessor/README.md b/processor/metricstransformprocessor/README.md index 473b80f27ad6..c17180124358 100644 --- a/processor/metricstransformprocessor/README.md +++ b/processor/metricstransformprocessor/README.md @@ -25,7 +25,7 @@ The metrics transform processor can be used to rename metrics, labels, or label transforms: # name is used to match with the metric to operate on. This implementation doesn’t utilize the filtermetric’s MatchProperties struct because it doesn’t match well with what I need at this phase. All is needed for this processor at this stage is a single name string that can be used to match with selected metrics. The list of metric names and the match type in the filtermetric’s MatchProperties struct are unnecessary. Also, based on the issue about improving filtering configuration, it seems like this struct is subject to be slightly modified. - # new addition: now it accepts `all_metrics` as the value of 'metric_name' field. This batch operation applies changes to all the metrics. However, currently, it only works and is tested for the 'convert_resource_attributes_to_labels' operation. For other operations (e.g. update_label, add_label), it may not work properly. But, it shouldn't break existing user experience. + # it also accepts `all_metrics` as the value of 'metric_name' field. This batch operation applies changes to all the metrics. However, currently, it only works and is tested for the 'convert_resource_attributes_to_labels' operation. - metric_name: # action specifies if the operations are performed on the current copy of the metric or on a newly created metric that will be inserted @@ -59,7 +59,7 @@ transforms: new_value: aggregation_type: {sum, mean, max} - # convert_resource_attributes_to_labels action converts all the resourde attributes to metric labels by default. Right now, it doesn't support selecting specific resource attributes as metric labels. + # convert_resource_attributes_to_labels action converts all the resourde attributes to metric labels by default. - action: convert_resource_attributes_to_labels ``` diff --git a/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go b/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go index f40e17ec1b3b..7ffb608fa163 100644 --- a/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go +++ b/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go @@ -602,6 +602,36 @@ var ( addInt64Point(0, 3, 2).build(), }, }, + { + name: "convert_resource_attributes_to_labels_with_nil_resource", + transforms: []internalTransform{ + { + MetricName: "all_metrics", + Action: Update, + Operations: []internalOperation{ + { + configOperation: Operation{ + Action: ConvertResourceAttributesToLabels, + }, + }, + }, + }, + }, + in: []*metricspb.Metric{ + metricBuilder().setName("metric1"). + setDataType(metricspb.MetricDescriptor_CUMULATIVE_INT64). + setLabels([]string{"label1"}). + addTimeseries(1, []string{"value1"}). + addInt64Point(0, 3, 2).build(), + }, + out: []*metricspb.Metric{ + metricBuilder().setName("metric1"). + setDataType(metricspb.MetricDescriptor_CUMULATIVE_INT64). + setLabels([]string{"label1"}). + addTimeseries(1, []string{"value1"}). + addInt64Point(0, 3, 2).build(), + }, + }, { name: "convert_resource_attributes_to_labels", transforms: []internalTransform{ From 72328180066bd54881628ca7986e041ec1a38e39 Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Wed, 21 Oct 2020 16:40:02 -0700 Subject: [PATCH 3/4] Select specific resource attributes to convert them to metric labels Signed-off-by: Rayhan Hossain --- processor/metricstransformprocessor/README.md | 12 +++++-- processor/metricstransformprocessor/config.go | 3 ++ .../metricstransformprocessor/config_test.go | 3 +- .../metrics_transform_processor.go | 2 -- ...rics_transform_processor_testcases_test.go | 33 ++++++++++++++++++- ...n_convert_resource_attributes_to_labels.go | 18 ++++++++-- .../testdata/config_full.yaml | 3 ++ 7 files changed, 66 insertions(+), 8 deletions(-) diff --git a/processor/metricstransformprocessor/README.md b/processor/metricstransformprocessor/README.md index c17180124358..7ee6e0513775 100644 --- a/processor/metricstransformprocessor/README.md +++ b/processor/metricstransformprocessor/README.md @@ -5,7 +5,7 @@ Supported pipeline types: metrics - e.g. If want to rename a metric or label to `new_name` while there is already a metric or label called `new_name`, this operation will not take any effect. There will also be an error logged ## Description -The metrics transform processor can be used to rename metrics, labels, or label values. It can also be used to perform aggregations on metrics across labels or label values. Additionally, it supports converting resource attributes to metric labels. +The metrics transform processor can be used to rename metrics, labels, or label values. It can also be used to perform aggregations on metrics across labels or label values. Additionally, it supports converting resource attributes to metric labels. Customers can select which resource attributes they want to set as metric labels. ## Capabilities - Rename metrics (e.g. rename `cpu/usage` to `cpu/usage_time`) @@ -59,8 +59,13 @@ transforms: new_value: aggregation_type: {sum, mean, max} - # convert_resource_attributes_to_labels action converts all the resourde attributes to metric labels by default. + # convert_resource_attributes_to_labels action converts the resourde attributes to metric labels. - action: convert_resource_attributes_to_labels + # customers can select which resource attributes they want to set as metric labels. If `resource_attributes` are not set or the list is empty, + # it convert all the resource attributes to metric labels by default. + resource_attributes: + - resource_attribute_1 + - resource_attribute_2 ``` ## Examples @@ -161,4 +166,7 @@ operation: ... operation: - action: convert_resource_attributes_to_labels + resource_attributes: + - ecs.cluster + - ecs.task_id ``` diff --git a/processor/metricstransformprocessor/config.go b/processor/metricstransformprocessor/config.go index 092555813ecd..1b2654375603 100644 --- a/processor/metricstransformprocessor/config.go +++ b/processor/metricstransformprocessor/config.go @@ -91,6 +91,9 @@ type Operation struct { // LabelValue identifies the exact label value to operate on LabelValue string `mapstructure:"label_value"` + + // ResourceAttributes identifies the resource attributes to convert them to metric labels + ResourceAttributes []string `mapstructure:"resource_attributes"` } // ValueAction renames label values. diff --git a/processor/metricstransformprocessor/config_test.go b/processor/metricstransformprocessor/config_test.go index 810260258a7e..dc5d625ea700 100644 --- a/processor/metricstransformprocessor/config_test.go +++ b/processor/metricstransformprocessor/config_test.go @@ -81,7 +81,8 @@ var ( Action: Update, Operations: []Operation{ { - Action: ConvertResourceAttributesToLabels, + Action: ConvertResourceAttributesToLabels, + ResourceAttributes: []string{"ecs.cluster"}, }, }, }, diff --git a/processor/metricstransformprocessor/metrics_transform_processor.go b/processor/metricstransformprocessor/metrics_transform_processor.go index a9ff248f1fa0..2e77adf9be59 100644 --- a/processor/metricstransformprocessor/metrics_transform_processor.go +++ b/processor/metricstransformprocessor/metrics_transform_processor.go @@ -77,8 +77,6 @@ func (mtp *metricsTransformProcessor) ProcessMetrics(_ context.Context, md pdata // to all metric data points. Right now, this batch operation only works and is tested for // `convert_resource_attributes_to_labels` operation, and does not break existing experience // for other operations. After updating the metrics, it will continue for the next iteration. - // TODO: add more unit tests and update README to pubicly anounce this feature for other - // operations as well. if transform.MetricName == "all_metrics" { for _, metric := range data.Metrics { mtp.update(metric, transform) diff --git a/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go b/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go index 7ffb608fa163..85f6153cd139 100644 --- a/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go +++ b/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go @@ -633,7 +633,7 @@ var ( }, }, { - name: "convert_resource_attributes_to_labels", + name: "convert_resource_attributes_to_labels_empty_list_config", transforms: []internalTransform{ { MetricName: "all_metrics", @@ -662,6 +662,37 @@ var ( addInt64Point(0, 3, 2).build(), }, }, + { + name: "convert_resource_attributes_to_labels_with_config_value", + transforms: []internalTransform{ + { + MetricName: "all_metrics", + Action: Update, + Operations: []internalOperation{ + { + configOperation: Operation{ + Action: ConvertResourceAttributesToLabels, + ResourceAttributes: []string{"label1"}, + }, + }, + }, + }, + }, + in: []*metricspb.Metric{ + metricBuilderWithResource().setName("metric1"). + setDataType(metricspb.MetricDescriptor_CUMULATIVE_INT64). + setResourceAttributes(map[string]string{"label1": "value1"}). + addTimeseries(1, []string{}). + addInt64Point(0, 3, 2).build(), + }, + out: []*metricspb.Metric{ + metricBuilder().setName("metric1"). + setDataType(metricspb.MetricDescriptor_CUMULATIVE_INT64). + setLabels([]string{"label1"}). + addTimeseries(1, []string{"value1"}). + addInt64Point(0, 3, 2).build(), + }, + }, // INSERT { name: "metric_name_insert", diff --git a/processor/metricstransformprocessor/operation_convert_resource_attributes_to_labels.go b/processor/metricstransformprocessor/operation_convert_resource_attributes_to_labels.go index 98a10891c08b..3d840bc9fcda 100644 --- a/processor/metricstransformprocessor/operation_convert_resource_attributes_to_labels.go +++ b/processor/metricstransformprocessor/operation_convert_resource_attributes_to_labels.go @@ -14,11 +14,25 @@ package metricstransformprocessor -import metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" +import ( + metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" +) func (mtp *metricsTransformProcessor) convertResourceAttributesToLabels(metric *metricspb.Metric, resourceAttributes map[string]string, op internalOperation) { + expectedLabelMap := make(map[string]string) - for key, value := range resourceAttributes { + if len(op.configOperation.ResourceAttributes) > 0 { + for _, attribute := range op.configOperation.ResourceAttributes { + attributeValue, ok := resourceAttributes[attribute] + if ok { + expectedLabelMap[attribute] = attributeValue + } + } + } else { + expectedLabelMap = resourceAttributes + } + + for key, value := range expectedLabelMap { lablelKey := &metricspb.LabelKey{ Key: key, } diff --git a/processor/metricstransformprocessor/testdata/config_full.yaml b/processor/metricstransformprocessor/testdata/config_full.yaml index 816d0f37fc25..769b9688b4a8 100644 --- a/processor/metricstransformprocessor/testdata/config_full.yaml +++ b/processor/metricstransformprocessor/testdata/config_full.yaml @@ -26,6 +26,9 @@ processors: action: update operations: - action: convert_resource_attributes_to_labels + resource_attributes: + - ecs.cluster + metricstransform/addlabel: transforms: - metric_name: some_name From 4dbdafa26dec7e3c102afa647034f64943616d3f Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Wed, 21 Oct 2020 17:25:45 -0700 Subject: [PATCH 4/4] Make metric_name field optional and empty metric name means batch operation for all metrics Signed-off-by: Rayhan Hossain --- processor/metricstransformprocessor/README.md | 2 +- processor/metricstransformprocessor/config_test.go | 2 +- processor/metricstransformprocessor/factory.go | 4 ---- processor/metricstransformprocessor/factory_test.go | 5 ++--- .../metrics_transform_processor.go | 4 ++-- .../metrics_transform_processor_testcases_test.go | 10 +++++----- .../testdata/config_full.yaml | 2 +- ..._metricname.yaml => config_missing_metricname.yaml} | 0 8 files changed, 12 insertions(+), 17 deletions(-) rename processor/metricstransformprocessor/testdata/{config_invalid_metricname.yaml => config_missing_metricname.yaml} (100%) diff --git a/processor/metricstransformprocessor/README.md b/processor/metricstransformprocessor/README.md index 7ee6e0513775..3b0f3a5235dc 100644 --- a/processor/metricstransformprocessor/README.md +++ b/processor/metricstransformprocessor/README.md @@ -25,7 +25,7 @@ The metrics transform processor can be used to rename metrics, labels, or label transforms: # name is used to match with the metric to operate on. This implementation doesn’t utilize the filtermetric’s MatchProperties struct because it doesn’t match well with what I need at this phase. All is needed for this processor at this stage is a single name string that can be used to match with selected metrics. The list of metric names and the match type in the filtermetric’s MatchProperties struct are unnecessary. Also, based on the issue about improving filtering configuration, it seems like this struct is subject to be slightly modified. - # it also accepts `all_metrics` as the value of 'metric_name' field. This batch operation applies changes to all the metrics. However, currently, it only works and is tested for the 'convert_resource_attributes_to_labels' operation. + # if the value for `metric_name` is not set (empty string), the operations will be applied to all the metrics. However, currently, it only works and is tested for the 'convert_resource_attributes_to_labels' operation. Existing experience works fine with correct `metric_name`. - metric_name: # action specifies if the operations are performed on the current copy of the metric or on a newly created metric that will be inserted diff --git a/processor/metricstransformprocessor/config_test.go b/processor/metricstransformprocessor/config_test.go index dc5d625ea700..8b6c1d2323e0 100644 --- a/processor/metricstransformprocessor/config_test.go +++ b/processor/metricstransformprocessor/config_test.go @@ -77,7 +77,7 @@ var ( Operations: testDataOperations, }, { - MetricName: "all_metrics", + MetricName: "", Action: Update, Operations: []Operation{ { diff --git a/processor/metricstransformprocessor/factory.go b/processor/metricstransformprocessor/factory.go index bf14cac45cd7..6848753e0d01 100644 --- a/processor/metricstransformprocessor/factory.go +++ b/processor/metricstransformprocessor/factory.go @@ -73,10 +73,6 @@ func createMetricsProcessor( // An error is returned if there are any invalid inputs. func validateConfiguration(config *Config) error { for _, transform := range config.Transforms { - if transform.MetricName == "" { - return fmt.Errorf("missing required field %q", MetricNameFieldName) - } - if transform.Action != Update && transform.Action != Insert { return fmt.Errorf("unsupported %q: %v, the supported actions are %q and %q", ActionFieldName, transform.Action, Insert, Update) } diff --git a/processor/metricstransformprocessor/factory_test.go b/processor/metricstransformprocessor/factory_test.go index 5458fe75a2f3..61a302b7231a 100644 --- a/processor/metricstransformprocessor/factory_test.go +++ b/processor/metricstransformprocessor/factory_test.go @@ -67,9 +67,8 @@ func TestCreateProcessors(t *testing.T) { succeed: false, errorMessage: fmt.Sprintf("unsupported %q: %v, the supported actions are %q and %q", ActionFieldName, "invalid", Insert, Update), }, { - configName: "config_invalid_metricname.yaml", - succeed: false, - errorMessage: fmt.Sprintf("missing required field %q", MetricNameFieldName), + configName: "config_missing_metricname.yaml", + succeed: true, }, { configName: "config_invalid_label.yaml", succeed: false, diff --git a/processor/metricstransformprocessor/metrics_transform_processor.go b/processor/metricstransformprocessor/metrics_transform_processor.go index 2e77adf9be59..3b0a2627a3aa 100644 --- a/processor/metricstransformprocessor/metrics_transform_processor.go +++ b/processor/metricstransformprocessor/metrics_transform_processor.go @@ -73,11 +73,11 @@ func (mtp *metricsTransformProcessor) ProcessMetrics(_ context.Context, md pdata for _, transform := range mtp.transforms { transform.resourceAttributeMap = resourceAttributes - // When user sets "metric_name=all_metrics", these operations are going to be applied + // When user does not set the value for `metric_name` (empty string), these operations are going to be applied // to all metric data points. Right now, this batch operation only works and is tested for // `convert_resource_attributes_to_labels` operation, and does not break existing experience // for other operations. After updating the metrics, it will continue for the next iteration. - if transform.MetricName == "all_metrics" { + if transform.MetricName == "" { for _, metric := range data.Metrics { mtp.update(metric, transform) } diff --git a/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go b/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go index 85f6153cd139..3777d475d487 100644 --- a/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go +++ b/processor/metricstransformprocessor/metrics_transform_processor_testcases_test.go @@ -571,10 +571,10 @@ var ( }, }, { - name: "all_metrics_batch_operation_test", + name: "empty_metric_name_batch_operation_test", transforms: []internalTransform{ { - MetricName: "all_metrics", + MetricName: "", Action: Update, Operations: []internalOperation{ { @@ -606,7 +606,7 @@ var ( name: "convert_resource_attributes_to_labels_with_nil_resource", transforms: []internalTransform{ { - MetricName: "all_metrics", + MetricName: "", Action: Update, Operations: []internalOperation{ { @@ -636,7 +636,7 @@ var ( name: "convert_resource_attributes_to_labels_empty_list_config", transforms: []internalTransform{ { - MetricName: "all_metrics", + MetricName: "", Action: Update, Operations: []internalOperation{ { @@ -666,7 +666,7 @@ var ( name: "convert_resource_attributes_to_labels_with_config_value", transforms: []internalTransform{ { - MetricName: "all_metrics", + MetricName: "", Action: Update, Operations: []internalOperation{ { diff --git a/processor/metricstransformprocessor/testdata/config_full.yaml b/processor/metricstransformprocessor/testdata/config_full.yaml index 769b9688b4a8..f3efec09a005 100644 --- a/processor/metricstransformprocessor/testdata/config_full.yaml +++ b/processor/metricstransformprocessor/testdata/config_full.yaml @@ -22,7 +22,7 @@ processors: aggregated_values: [value1, value2] new_value: new_value aggregation_type: sum - - metric_name: all_metrics + - metric_name: # empty metric_name means a batch operation to all metrics action: update operations: - action: convert_resource_attributes_to_labels diff --git a/processor/metricstransformprocessor/testdata/config_invalid_metricname.yaml b/processor/metricstransformprocessor/testdata/config_missing_metricname.yaml similarity index 100% rename from processor/metricstransformprocessor/testdata/config_invalid_metricname.yaml rename to processor/metricstransformprocessor/testdata/config_missing_metricname.yaml