[TEP-0076]Support Array Results substitution

This is part of work in TEP-0076.
This commit provides the support to apply array results replacements.
Previous this commit we support emitting array results so users can
write array results to task level, but we cannot pass array results from
tasks within one pipeline. This commit adds the support for this.

docs/pipelines.md

@@ -908,7 +908,10 @@ Tasks can emit [`Results`](tasks.md#emitting-results) when they execute. A Pipel
 Sharing `Results` between `Tasks` in a `Pipeline` happens via
 [variable substitution](variables.md#variables-available-in-a-pipeline) - one `Task` emits
 a `Result` and another receives it as a `Parameter` with a variable such as
-`$(tasks.<task-name>.results.<result-name>)`.
+`$(tasks.<task-name>.results.<result-name>)`. Array `Results` is supported as alpha feature and
+can be referer as `$(tasks.<task-name>.results.<result-name>[*])`.
+
+**Note:** Array `Result` cannot be used in `script`.
 
 When one `Task` receives the `Results` of another, there is a dependency created between those
 two `Tasks`. In order for the receiving `Task` to get data from another `Task's` `Result`,
@@ -923,6 +926,8 @@ before this one.
 params:
  - name: foo
  value: "$(tasks.checkout-source.results.commit)"
+ - name: array-params
+ value: "$(tasks.checkout-source.results.array-results[*])"
 ```
 
 **Note:** If `checkout-source` exits successfully without initializing `commit` `Result`,

docs/variables.md

@@ -25,6 +25,9 @@ For instructions on using variable substitutions see the relevant section of [th
 | `tasks.<taskName>.results.<resultName>[i]` | The ith value of the `Task's` array result. Can alter `Task` execution order within a `Pipeline`.) |
 | `tasks.<taskName>.results['<resultName>'][i]` | (see above)) |
 | `tasks.<taskName>.results["<resultName>"][i]` | (see above)) |
+| `tasks.<taskName>.results.<resultName>[*]` | The array value of the `Task's` result. Can alter `Task` execution order within a `Pipeline`. Cannot be used in `script`.) |
+| `tasks.<taskName>.results['<resultName>'][*]` | (see above)) |
+| `tasks.<taskName>.results["<resultName>"][*]` | (see above)) |
 | `workspaces.<workspaceName>.bound` | Whether a `Workspace` has been bound or not. "false" if the `Workspace` declaration has `optional: true` and the Workspace binding was omitted by the PipelineRun. |
 | `context.pipelineRun.name` | The name of the `PipelineRun` that this `Pipeline` is running in. |
 | `context.pipelineRun.namespace` | The namespace of the `PipelineRun` that this `Pipeline` is running in. |

examples/v1beta1/pipelineruns/alpha/pipelinerun-array-results-substitution.yaml

@@ -0,0 +1,56 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+ name: pipelinerun-array-results
+spec:
+ pipelineSpec:
+ tasks:
+ - name: task1
+ taskSpec:
+ results:
+ - name: array-results
+ type: array
+ description: The array results
+ steps:
+ - name: write-array
+ image: bash:latest
+ script: |
+ #!/usr/bin/env bash
+ echo -n "[\"1\",\"2\",\"3\"]" | tee $(results.array-results.path)
+ - name: task2
+ params:
+ - name: foo
+ value: "$(tasks.task1.results.array-results[*])"
+ - name: bar
+ value: "$(tasks.task1.results.array-results[2])"
+ taskSpec:
+ params:
+ - name: foo
+ type: array
+ default:
+ - "defaultparam1"
+ - "defaultparam2"
+ - name: bar
+ type: string
+ default: "defaultparam1"
+ steps:
+ - name: print-foo
+ image: bash:latest
+ args: [
+ "echo",
+ "$(params.foo[*])"
+ ]
+ - name: print-bar
+ image: ubuntu
+ script: |
+ #!/bin/bash
+ VALUE=$(params.bar)
+ EXPECTED=3
+ diff=$(diff <(printf "%s\n" "${VALUE[@]}") <(printf "%s\n" "${EXPECTED[@]}"))
+ if [[ -z "$diff" ]]; then
+ echo "Get expected: ${VALUE}"
+ exit 0
+ else
+ echo "Want: ${EXPECTED} Got: ${VALUE}"
+ exit 1
+ fi

pkg/apis/pipeline/v1beta1/result_types.go

@@ -13,6 +13,8 @@ limitations under the License.
 
 package v1beta1
 
+import "strings"
+
 // TaskResult used to describe the results of a task
 type TaskResult struct {
  // Name the given name
@@ -64,3 +66,8 @@ const (
 
 // AllResultsTypes can be used for ResultsTypes validation.
 var AllResultsTypes = []ResultsType{ResultsTypeString, ResultsTypeArray, ResultsTypeObject}
+
+// ResultsArrayReference returns the reference of the result. e.g. results.resultname from $(results.resultname[*])
+func ResultsArrayReference(a string) string {
+ return strings.TrimSuffix(strings.TrimSuffix(strings.TrimPrefix(a, "$("), ")"), "[*]")
+}

pkg/apis/pipeline/v1beta1/resultref.go

@@ -53,7 +53,8 @@ const (
  ResultNameFormat = `^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$`
 )
 
-var variableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
+// VariableSubstitutionRegex is a regex to find all result matching substitutions
+var VariableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
 var exactVariableSubstitutionRegex = regexp.MustCompile(exactVariableSubstitutionFormat)
 var resultNameFormatRegex = regexp.MustCompile(ResultNameFormat)
 
@@ -131,7 +132,7 @@ func GetVarSubstitutionExpressionsForPipelineResult(result PipelineResult) ([]st
 }
 
 func validateString(value string) []string {
- expressions := variableSubstitutionRegex.FindAllString(value, -1)
+ expressions := VariableSubstitutionRegex.FindAllString(value, -1)
  if expressions == nil {
  return nil
  }

pkg/apis/pipeline/v1beta1/when_types.go

@@ -62,9 +62,12 @@ func (we *WhenExpression) applyReplacements(replacements map[string]string, arra
  for _, val := range we.Values {
  // arrayReplacements holds a list of array parameters with a pattern - params.arrayParam1
  // array params are referenced using $(params.arrayParam1[*])
+ // array results are referenced using $(results.resultname[*])
  // check if the param exist in the arrayReplacements to replace it with a list of values
  if _, ok := arrayReplacements[fmt.Sprintf("%s.%s", ParamsPrefix, ArrayReference(val))]; ok {
  replacedValues = append(replacedValues, substitution.ApplyArrayReplacements(val, replacements, arrayReplacements)...)
+ } else if _, ok := arrayReplacements[ResultsArrayReference(val)]; ok {
+ replacedValues = append(replacedValues, substitution.ApplyArrayReplacements(val, replacements, arrayReplacements)...)
  } else {
  replacedValues = append(replacedValues, substitution.ApplyReplacements(val, replacements))
  }

pkg/apis/pipeline/v1beta1/when_types_test.go

@@ -248,6 +248,43 @@ func TestApplyReplacements(t *testing.T) {
  Operator: selection.In,
  Values: []string{"barfoo"},
  },
+ }, {
+ name: "replace array results variables",
+ original: &WhenExpression{
+ Input: "$(tasks.foo.results.bar)",
+ Operator: selection.In,
+ Values: []string{"$(tasks.aTask.results.aResult[*])"},
+ },
+ replacements: map[string]string{
+ "tasks.foo.results.bar": "foobar",
+ },
+ arrayReplacements: map[string][]string{
+ "tasks.aTask.results.aResult": {"dev", "stage"},
+ },
+ expected: &WhenExpression{
+ Input: "foobar",
+ Operator: selection.In,
+ Values: []string{"dev", "stage"},
+ },
+ }, {
+ name: "invaliad array results replacements",
+ original: &WhenExpression{
+ Input: "$(tasks.foo.results.bar)",
+ Operator: selection.In,
+ Values: []string{"$(tasks.aTask.results.aResult[invalid])"},
+ },
+ replacements: map[string]string{
+ "tasks.foo.results.bar": "foobar",
+ "tasks.aTask.results.aResult[*]": "barfoo",
+ },
+ arrayReplacements: map[string][]string{
+ "tasks.aTask.results.aResult[*]": {"dev", "stage"},
+ },
+ expected: &WhenExpression{
+ Input: "foobar",
+ Operator: selection.In,
+ Values: []string{"$(tasks.aTask.results.aResult[invalid])"},
+ },
  }, {
  name: "replace array params",
  original: &WhenExpression{

pkg/reconciler/pipeline/dag/dag_test.go

@@ -292,6 +292,7 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
  c := v1beta1.PipelineTask{Name: "c"}
  d := v1beta1.PipelineTask{Name: "d"}
  e := v1beta1.PipelineTask{Name: "e"}
+ f := v1beta1.PipelineTask{Name: "f"}
  xDependsOnA := v1beta1.PipelineTask{
  Name: "x",
  Params: []v1beta1.Param{{
@@ -314,27 +315,38 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
  Value: *v1beta1.NewArrayOrString("$(tasks.d.results.resultD) $(tasks.e.results.resultE)"),
  }},
  }
+ wDependsOnF := v1beta1.PipelineTask{
+ Name: "w",
+ Params: []v1beta1.Param{{
+ Name: "paramw",
+ Value: *v1beta1.NewArrayOrString("$(tasks.f.results.resultF[*])"),
+ }},
+ }
 
- // a b c d e
- // | \ / \ /
- // x y z
+ // a b c d e f
+ // | \ / \ / |
+ // x y z w
  nodeA := &dag.Node{Task: a}
  nodeB := &dag.Node{Task: b}
  nodeC := &dag.Node{Task: c}
  nodeD := &dag.Node{Task: d}
  nodeE := &dag.Node{Task: e}
+ nodeF := &dag.Node{Task: f}
  nodeX := &dag.Node{Task: xDependsOnA}
  nodeY := &dag.Node{Task: yDependsOnBRunsAfterC}
  nodeZ := &dag.Node{Task: zDependsOnDAndE}
+ nodeW := &dag.Node{Task: wDependsOnF}
 
  nodeA.Next = []*dag.Node{nodeX}
  nodeB.Next = []*dag.Node{nodeY}
  nodeC.Next = []*dag.Node{nodeY}
  nodeD.Next = []*dag.Node{nodeZ}
  nodeE.Next = []*dag.Node{nodeZ}
+ nodeF.Next = []*dag.Node{nodeW}
  nodeX.Prev = []*dag.Node{nodeA}
  nodeY.Prev = []*dag.Node{nodeB, nodeC}
  nodeZ.Prev = []*dag.Node{nodeD, nodeE}
+ nodeW.Prev = []*dag.Node{nodeF}
 
  expectedDAG := &dag.Graph{
  Nodes: map[string]*dag.Node{
@@ -343,15 +355,17 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
  "c": nodeC,
  "d": nodeD,
  "e": nodeE,
+ "f": nodeF,
  "x": nodeX,
  "y": nodeY,
  "z": nodeZ,
+ "w": nodeW,
  },
  }
  p := &v1beta1.Pipeline{
  ObjectMeta: metav1.ObjectMeta{Name: "pipeline"},
  Spec: v1beta1.PipelineSpec{
- Tasks: []v1beta1.PipelineTask{a, b, c, d, e, xDependsOnA, yDependsOnBRunsAfterC, zDependsOnDAndE},
+ Tasks: []v1beta1.PipelineTask{a, b, c, d, e, f, xDependsOnA, yDependsOnBRunsAfterC, zDependsOnDAndE, wDependsOnF},
  },
  }
  tasks := v1beta1.PipelineTaskList(p.Spec.Tasks)

pkg/reconciler/pipelinerun/resources/apply.go

@@ -122,12 +122,13 @@ func ApplyPipelineTaskContexts(pt *v1beta1.PipelineTask) *v1beta1.PipelineTask {
 // ApplyTaskResults applies the ResolvedResultRef to each PipelineTask.Params and Pipeline.WhenExpressions in targets
 func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResultRefs) {
  stringReplacements := resolvedResultRefs.getStringReplacements()
+ arrayReplacements := resolvedResultRefs.getArrayReplacements()
  for _, resolvedPipelineRunTask := range targets {
  if resolvedPipelineRunTask.PipelineTask != nil {
  pipelineTask := resolvedPipelineRunTask.PipelineTask.DeepCopy()
- pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, nil, nil)
- pipelineTask.Matrix = replaceParamValues(pipelineTask.Matrix, stringReplacements, nil, nil)
- pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, nil)
+ pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, arrayReplacements, nil)
+ pipelineTask.Matrix = replaceParamValues(pipelineTask.Matrix, stringReplacements, arrayReplacements, nil)
+ pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, arrayReplacements)
  resolvedPipelineRunTask.PipelineTask = pipelineTask
  }
  }

pkg/reconciler/pipelinerun/resources/apply_test.go

@@ -1051,6 +1051,38 @@ func TestApplyTaskResults_MinimalExpression(t *testing.T) {
  }},
  },
  }},
+ }, {
+ name: "Test array result substitution on minimal variable substitution expression - params",
+ resolvedResultRefs: ResolvedResultRefs{{
+ Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
+ ResultReference: v1beta1.ResultRef{
+ PipelineTask: "aTask",
+ Result: "a.Result",
+ },
+ FromTaskRun: "aTaskRun",
+ }},
+ targets: PipelineRunState{{
+ PipelineTask: &v1beta1.PipelineTask{
+ Name: "bTask",
+ TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+ Params: []v1beta1.Param{{
+ Name: "bParam",
+ Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeArray,
+ ArrayVal: []string{`$(tasks.aTask.results["a.Result"][*])`},
+ },
+ }},
+ },
+ }},
+ want: PipelineRunState{{
+ PipelineTask: &v1beta1.PipelineTask{
+ Name: "bTask",
+ TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+ Params: []v1beta1.Param{{
+ Name: "bParam",
+ Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
+ }},
+ },
+ }},
  }, {
  name: "Test result substitution on minimal variable substitution expression - matrix",
  resolvedResultRefs: ResolvedResultRefs{{
@@ -1141,6 +1173,39 @@ func TestApplyTaskResults_MinimalExpression(t *testing.T) {
  }},
  },
  }},
+ }, {
+ name: "Test array result substitution on minimal variable substitution expression - when expressions",
+ resolvedResultRefs: ResolvedResultRefs{{
+ Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
+ ResultReference: v1beta1.ResultRef{
+ PipelineTask: "aTask",
+ Result: "aResult",
+ },
+ FromTaskRun: "aTaskRun",
+ }},
+ targets: PipelineRunState{{
+ PipelineTask: &v1beta1.PipelineTask{
+ Name: "bTask",
+ TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+ WhenExpressions: []v1beta1.WhenExpression{{
+ // Note that Input doesn't support array replacement.
+ Input: "anInput",
+ Operator: selection.In,
+ Values: []string{"$(tasks.aTask.results.aResult[*])"},
+ }},
+ },
+ }},
+ want: PipelineRunState{{
+ PipelineTask: &v1beta1.PipelineTask{
+ Name: "bTask",
+ TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+ WhenExpressions: []v1beta1.WhenExpression{{
+ Input: "anInput",
+ Operator: selection.In,
+ Values: []string{"arrayResultValueOne", "arrayResultValueTwo"},
+ }},
+ },
+ }},
  }, {
  name: "Test result substitution on minimal variable substitution expression - when expressions",
  resolvedResultRefs: ResolvedResultRefs{{

pkg/reconciler/pipelinerun/resources/resultrefresolution.go

@@ -215,6 +215,18 @@ func (rs ResolvedResultRefs) getStringReplacements() map[string]string {
  return replacements
 }
 
+func (rs ResolvedResultRefs) getArrayReplacements() map[string][]string {
+ replacements := map[string][]string{}
+ for _, r := range rs {
+ if r.Value.Type == v1beta1.ParamType(v1beta1.ResultsTypeArray) {
+ for _, target := range r.getReplaceTarget() {
+ replacements[target] = r.Value.ArrayVal
+ }
+ }
+ }
+ return replacements
+}
+
 func (r *ResolvedResultRef) getReplaceTarget() []string {
  return []string{
  fmt.Sprintf("%s.%s.%s.%s", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),

pkg/reconciler/taskrun/validate_resources.go

@@ -133,13 +133,22 @@ func extraParamsNames(ctx context.Context, neededParams []string, providedParams
 }
 
 func wrongTypeParamsNames(params []v1beta1.Param, matrix []v1beta1.Param, neededParamsTypes map[string]v1beta1.ParamType) []string {
+ // TODO(#4723): validate that $(task.taskname.result.resultname) is invalid for array and object type.
+ // It should be used to refer string and need to add [*] to refer to array or object.
  var wrongTypeParamNames []string
  for _, param := range params {
  if _, ok := neededParamsTypes[param.Name]; !ok {
  // Ignore any missing params - this happens when extra params were
  // passed to the task that aren't being used.
  continue
  }
+ // This is needed to support array replacements in params. Users want to use $(tasks.taskName.results.resultname[*])
+ // to pass array result to array param, yet in yaml format this will be
+ // unmarshalled to string for ArrayOrString. So we need to check and skip this validation.
+ // Please refer issue #4879 for more details and examples.
+ if param.Value.Type == v1beta1.ParamTypeString && (neededParamsTypes[param.Name] == v1beta1.ParamTypeArray || (neededParamsTypes[param.Name] == v1beta1.ParamTypeObject)) && v1beta1.VariableSubstitutionRegex.MatchString(param.Value.StringVal) {
+ continue
+ }
  if param.Value.Type != neededParamsTypes[param.Name] {
  wrongTypeParamNames = append(wrongTypeParamNames, param.Name)
  }