[TEP-0076]Support Array Results substitution

This is part of work in TEP-0076.
This commit provides the support to apply array results replacements.
Previous this commit we support emitting array results so users can
write array results to task level, but we cannot pass array results from
tasks within one pipeline. This commit adds the support for this.

docs/pipelines.md

@@ -908,7 +908,10 @@ Tasks can emit [`Results`](tasks.md#emitting-results) when they execute. A Pipel
 Sharing `Results` between `Tasks` in a `Pipeline` happens via
 [variable substitution](variables.md#variables-available-in-a-pipeline) - one `Task` emits
 a `Result` and another receives it as a `Parameter` with a variable such as
-`$(tasks.<task-name>.results.<result-name>)`.
+`$(tasks.<task-name>.results.<result-name>)`. Array `Results` is supported as alpha feature and
+can be referer as `$(tasks.<task-name>.results.<result-name>[*])`.
+
+**Note:** Array `Result` cannot be used in `script`.
 
 When one `Task` receives the `Results` of another, there is a dependency created between those
 two `Tasks`. In order for the receiving `Task` to get data from another `Task's` `Result`,
@@ -923,6 +926,8 @@ before this one.
 params:
  - name: foo
  value: "$(tasks.checkout-source.results.commit)"
+ - name: array-params
+ value: "$(tasks.checkout-source.results.array-results[*])"
 ```
 
 **Note:** If `checkout-source` exits successfully without initializing `commit` `Result`,

docs/variables.md

@@ -25,6 +25,9 @@ For instructions on using variable substitutions see the relevant section of [th
 | `tasks.<taskName>.results.<resultName>[i]` | The ith value of the `Task's` array result. Can alter `Task` execution order within a `Pipeline`.) |
 | `tasks.<taskName>.results['<resultName>'][i]` | (see above)) |
 | `tasks.<taskName>.results["<resultName>"][i]` | (see above)) |
+| `tasks.<taskName>.results.<resultName>[*]` | The array value of the `Task's` result. Can alter `Task` execution order within a `Pipeline`. Cannot be used in `script`.) |
+| `tasks.<taskName>.results['<resultName>'][*]` | (see above)) |
+| `tasks.<taskName>.results["<resultName>"][*]` | (see above)) |
 | `workspaces.<workspaceName>.bound` | Whether a `Workspace` has been bound or not. "false" if the `Workspace` declaration has `optional: true` and the Workspace binding was omitted by the PipelineRun. |
 | `context.pipelineRun.name` | The name of the `PipelineRun` that this `Pipeline` is running in. |
 | `context.pipelineRun.namespace` | The namespace of the `PipelineRun` that this `Pipeline` is running in. |

examples/v1beta1/pipelineruns/alpha/pipelinerun-array-results.yaml

@@ -0,0 +1,56 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+ name: pipelinerun-array-results
+spec:
+ pipelineSpec:
+ tasks:
+ - name: task1
+ taskSpec:
+ results:
+ - name: array-results
+ type: array
+ description: The array results
+ steps:
+ - name: write-array
+ image: bash:latest
+ script: |
+ #!/usr/bin/env bash
+ echo -n "[\"1\",\"2\",\"3\"]" | tee $(results.array-results.path)
+ - name: task2
+ params:
+ - name: foo
+ value: "$(tasks.task1.results.array-results[*])"
+ - name: bar
+ value: "$(tasks.task1.results.array-results[2])"
+ taskSpec:
+ params:
+ - name: foo
+ type: array
+ default:
+ - "defaultparam1"
+ - "defaultparam2"
+ - name: bar
+ type: string
+ default: "defaultparam1"
+ steps:
+ - name: print-foo
+ image: bash:latest
+ args: [
+ "echo",
+ "$(params.foo[*])"
+ ]
+ - name: print-bar
+ image: ubuntu
+ script: |
+ #!/bin/bash
+ VALUE=$(params.bar)
+ EXPECTED=3
+ diff=$(diff <(printf "%s\n" "${VALUE[@]}") <(printf "%s\n" "${EXPECTED[@]}"))
+ if [[ -z "$diff" ]]; then
+ echo "Get expected: ${VALUE}"
+ exit 0
+ else
+ echo "Want: ${EXPECTED} Got: ${VALUE}"
+ exit 1
+ fi

pkg/apis/pipeline/v1beta1/param_types.go

@@ -234,7 +234,6 @@ func (arrayOrString *ArrayOrString) applyOrCorrect(stringReplacements map[string
  if _, ok := stringReplacements[trimedStringVal]; ok {
  arrayOrString.StringVal = substitution.ApplyReplacements(arrayOrString.StringVal, stringReplacements)
  }
-
  // if the stringVal is a reference to an array param, we need to change the type other than apply replacement
  if _, ok := arrayReplacements[trimedStringVal]; ok {
  arrayOrString.StringVal = ""

pkg/apis/pipeline/v1beta1/result_types.go

@@ -13,6 +13,8 @@ limitations under the License.
 
 package v1beta1
 
+import "strings"
+
 // TaskResult used to describe the results of a task
 type TaskResult struct {
  // Name the given name
@@ -64,3 +66,8 @@ const (
 
 // AllResultsTypes can be used for ResultsTypes validation.
 var AllResultsTypes = []ResultsType{ResultsTypeString, ResultsTypeArray, ResultsTypeObject}
+
+// ResultsArrayReference returns the reference of the result. e.g. results.resultname from $(results.resultname[*])
+func ResultsArrayReference(a string) string {
+ return strings.TrimSuffix(strings.TrimSuffix(strings.TrimPrefix(a, "$("), ")"), "[*]")
+}

pkg/apis/pipeline/v1beta1/resultref.go

@@ -53,7 +53,8 @@ const (
  ResultNameFormat = `^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$`
 )
 
-var variableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
+// VariableSubstitutionRegex is a regex to find all result matching substitutions
+var VariableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
 var exactVariableSubstitutionRegex = regexp.MustCompile(exactVariableSubstitutionFormat)
 var resultNameFormatRegex = regexp.MustCompile(ResultNameFormat)
 
@@ -131,7 +132,7 @@ func GetVarSubstitutionExpressionsForPipelineResult(result PipelineResult) ([]st
 }
 
 func validateString(value string) []string {
- expressions := variableSubstitutionRegex.FindAllString(value, -1)
+ expressions := VariableSubstitutionRegex.FindAllString(value, -1)
  if expressions == nil {
  return nil
  }

pkg/apis/pipeline/v1beta1/when_types.go

@@ -62,9 +62,12 @@ func (we *WhenExpression) applyReplacements(replacements map[string]string, arra
  for _, val := range we.Values {
  // arrayReplacements holds a list of array parameters with a pattern - params.arrayParam1
  // array params are referenced using $(params.arrayParam1[*])
+ // array results are referenced using $(results.resultname[*])
  // check if the param exist in the arrayReplacements to replace it with a list of values
  if _, ok := arrayReplacements[fmt.Sprintf("%s.%s", ParamsPrefix, ArrayReference(val))]; ok {
  replacedValues = append(replacedValues, substitution.ApplyArrayReplacements(val, replacements, arrayReplacements)...)
+ } else if _, ok := arrayReplacements[ResultsArrayReference(val)]; ok {
+ replacedValues = append(replacedValues, substitution.ApplyArrayReplacements(val, replacements, arrayReplacements)...)
  } else {
  replacedValues = append(replacedValues, substitution.ApplyReplacements(val, replacements))
  }

pkg/apis/pipeline/v1beta1/when_types_test.go

@@ -248,6 +248,43 @@ func TestApplyReplacements(t *testing.T) {
  Operator: selection.In,
  Values: []string{"barfoo"},
  },
+ }, {
+ name: "replace array results variables",
+ original: &WhenExpression{
+ Input: "$(tasks.foo.results.bar)",
+ Operator: selection.In,
+ Values: []string{"$(tasks.aTask.results.aResult[*])"},
+ },
+ replacements: map[string]string{
+ "tasks.foo.results.bar": "foobar",
+ },
+ arrayReplacements: map[string][]string{
+ "tasks.aTask.results.aResult": {"dev", "stage"},
+ },
+ expected: &WhenExpression{
+ Input: "foobar",
+ Operator: selection.In,
+ Values: []string{"dev", "stage"},
+ },
+ }, {
+ name: "invaliad array results replacements",
+ original: &WhenExpression{
+ Input: "$(tasks.foo.results.bar)",
+ Operator: selection.In,
+ Values: []string{"$(tasks.aTask.results.aResult[invalid])"},
+ },
+ replacements: map[string]string{
+ "tasks.foo.results.bar": "foobar",
+ "tasks.aTask.results.aResult[*]": "barfoo",
+ },
+ arrayReplacements: map[string][]string{
+ "tasks.aTask.results.aResult[*]": {"dev", "stage"},
+ },
+ expected: &WhenExpression{
+ Input: "foobar",
+ Operator: selection.In,
+ Values: []string{"$(tasks.aTask.results.aResult[invalid])"},
+ },
  }, {
  name: "replace array params",
  original: &WhenExpression{

pkg/reconciler/pipeline/dag/dag_test.go

@@ -292,6 +292,7 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
  c := v1beta1.PipelineTask{Name: "c"}
  d := v1beta1.PipelineTask{Name: "d"}
  e := v1beta1.PipelineTask{Name: "e"}
+ f := v1beta1.PipelineTask{Name: "f"}
  xDependsOnA := v1beta1.PipelineTask{
  Name: "x",
  Params: []v1beta1.Param{{
@@ -314,27 +315,38 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
  Value: *v1beta1.NewArrayOrString("$(tasks.d.results.resultD) $(tasks.e.results.resultE)"),
  }},
  }
+ wDependsOnF := v1beta1.PipelineTask{
+ Name: "w",
+ Params: []v1beta1.Param{{
+ Name: "paramw",
+ Value: *v1beta1.NewArrayOrString("$(tasks.f.results.resultF[*])"),
+ }},
+ }
 
- // a b c d e
- // | \ / \ /
- // x y z
+ // a b c d e f
+ // | \ / \ / |
+ // x y z w
  nodeA := &dag.Node{Task: a}
  nodeB := &dag.Node{Task: b}
  nodeC := &dag.Node{Task: c}
  nodeD := &dag.Node{Task: d}
  nodeE := &dag.Node{Task: e}
+ nodeF := &dag.Node{Task: f}
  nodeX := &dag.Node{Task: xDependsOnA}
  nodeY := &dag.Node{Task: yDependsOnBRunsAfterC}
  nodeZ := &dag.Node{Task: zDependsOnDAndE}
+ nodeW := &dag.Node{Task: wDependsOnF}
 
  nodeA.Next = []*dag.Node{nodeX}
  nodeB.Next = []*dag.Node{nodeY}
  nodeC.Next = []*dag.Node{nodeY}
  nodeD.Next = []*dag.Node{nodeZ}
  nodeE.Next = []*dag.Node{nodeZ}
+ nodeF.Next = []*dag.Node{nodeW}
  nodeX.Prev = []*dag.Node{nodeA}
  nodeY.Prev = []*dag.Node{nodeB, nodeC}
  nodeZ.Prev = []*dag.Node{nodeD, nodeE}
+ nodeW.Prev = []*dag.Node{nodeF}
 
  expectedDAG := &dag.Graph{
  Nodes: map[string]*dag.Node{
@@ -343,15 +355,17 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
  "c": nodeC,
  "d": nodeD,
  "e": nodeE,
+ "f": nodeF,
  "x": nodeX,
  "y": nodeY,
  "z": nodeZ,
+ "w": nodeW,
  },
  }
  p := &v1beta1.Pipeline{
  ObjectMeta: metav1.ObjectMeta{Name: "pipeline"},
  Spec: v1beta1.PipelineSpec{
- Tasks: []v1beta1.PipelineTask{a, b, c, d, e, xDependsOnA, yDependsOnBRunsAfterC, zDependsOnDAndE},
+ Tasks: []v1beta1.PipelineTask{a, b, c, d, e, f, xDependsOnA, yDependsOnBRunsAfterC, zDependsOnDAndE, wDependsOnF},
  },
  }
  tasks := v1beta1.PipelineTaskList(p.Spec.Tasks)

pkg/reconciler/pipelinerun/resources/apply.go

@@ -122,12 +122,13 @@ func ApplyPipelineTaskContexts(pt *v1beta1.PipelineTask) *v1beta1.PipelineTask {
 // ApplyTaskResults applies the ResolvedResultRef to each PipelineTask.Params and Pipeline.WhenExpressions in targets
 func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResultRefs) {
  stringReplacements := resolvedResultRefs.getStringReplacements()
+ arrayReplacements := resolvedResultRefs.getArrayReplacements()
  for _, resolvedPipelineRunTask := range targets {
  if resolvedPipelineRunTask.PipelineTask != nil {
  pipelineTask := resolvedPipelineRunTask.PipelineTask.DeepCopy()
- pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, nil, nil)
- pipelineTask.Matrix = replaceParamValues(pipelineTask.Matrix, stringReplacements, nil, nil)
- pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, nil)
+ pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, arrayReplacements, nil)
+ pipelineTask.Matrix = replaceParamValues(pipelineTask.Matrix, stringReplacements, arrayReplacements, nil)
+ pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, arrayReplacements)
  resolvedPipelineRunTask.PipelineTask = pipelineTask
  }
  }

pkg/reconciler/pipelinerun/resources/apply_test.go

@@ -1051,6 +1051,38 @@ func TestApplyTaskResults_MinimalExpression(t *testing.T) {
  }},
  },
  }},
+ }, {
+ name: "Test array result substitution on minimal variable substitution expression - params",
+ resolvedResultRefs: ResolvedResultRefs{{
+ Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
+ ResultReference: v1beta1.ResultRef{
+ PipelineTask: "aTask",
+ Result: "a.Result",
+ },
+ FromTaskRun: "aTaskRun",
+ }},
+ targets: PipelineRunState{{
+ PipelineTask: &v1beta1.PipelineTask{
+ Name: "bTask",
+ TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+ Params: []v1beta1.Param{{
+ Name: "bParam",
+ Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeArray,
+ ArrayVal: []string{`$(tasks.aTask.results["a.Result"][*])`},
+ },
+ }},
+ },
+ }},
+ want: PipelineRunState{{
+ PipelineTask: &v1beta1.PipelineTask{
+ Name: "bTask",
+ TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+ Params: []v1beta1.Param{{
+ Name: "bParam",
+ Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
+ }},
+ },
+ }},
  }, {
  name: "Test result substitution on minimal variable substitution expression - matrix",
  resolvedResultRefs: ResolvedResultRefs{{
@@ -1141,6 +1173,39 @@ func TestApplyTaskResults_MinimalExpression(t *testing.T) {
  }},
  },
  }},
+ }, {
+ name: "Test array result substitution on minimal variable substitution expression - when expressions",
+ resolvedResultRefs: ResolvedResultRefs{{
+ Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
+ ResultReference: v1beta1.ResultRef{
+ PipelineTask: "aTask",
+ Result: "aResult",
+ },
+ FromTaskRun: "aTaskRun",
+ }},
+ targets: PipelineRunState{{
+ PipelineTask: &v1beta1.PipelineTask{
+ Name: "bTask",
+ TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+ WhenExpressions: []v1beta1.WhenExpression{{
+ // Note that Input doesn't support array replacement.
+ Input: "anInput",
+ Operator: selection.In,
+ Values: []string{"$(tasks.aTask.results.aResult[*])"},
+ }},
+ },
+ }},
+ want: PipelineRunState{{
+ PipelineTask: &v1beta1.PipelineTask{
+ Name: "bTask",
+ TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+ WhenExpressions: []v1beta1.WhenExpression{{
+ Input: "anInput",
+ Operator: selection.In,
+ Values: []string{"arrayResultValueOne", "arrayResultValueTwo"},
+ }},
+ },
+ }},
  }, {
  name: "Test result substitution on minimal variable substitution expression - when expressions",
  resolvedResultRefs: ResolvedResultRefs{{

pkg/reconciler/pipelinerun/resources/resultrefresolution.go

@@ -215,6 +215,18 @@ func (rs ResolvedResultRefs) getStringReplacements() map[string]string {
  return replacements
 }
 
+func (rs ResolvedResultRefs) getArrayReplacements() map[string][]string {
+ replacements := map[string][]string{}
+ for _, r := range rs {
+ if r.Value.Type == v1beta1.ParamType(v1beta1.ResultsTypeArray) {
+ for _, target := range r.getArrayReplaceTarget() {
+ replacements[target] = r.Value.ArrayVal
+ }
+ }
+ }
+ return replacements
+}
+
 func (r *ResolvedResultRef) getReplaceTarget() []string {
  return []string{
  fmt.Sprintf("%s.%s.%s.%s", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
@@ -230,3 +242,10 @@ func (r *ResolvedResultRef) getReplaceTargetfromArrayIndex(idx int) []string {
  fmt.Sprintf("%s.%s.%s['%s'][%d]", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result, idx),
  }
 }
+func (r *ResolvedResultRef) getArrayReplaceTarget() []string {
+ return []string{
+ fmt.Sprintf("%s.%s.%s.%s", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
+ fmt.Sprintf("%s.%s.%s[%q]", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
+ fmt.Sprintf("%s.%s.%s['%s']", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
+ }
+}