fix: add a simple way to reuse steps from a Task

so we can easily share, say, a git-clone Task inside a Task without requiring 2 Tasks, 2 Pods and forcing a PVC to be used

for background see: tektoncd/pipeline#3476

pkg/triggerconfig/inrepo/load_pipelinerun.go

@@ -2,6 +2,8 @@ package inrepo
 
 import (
  "fmt"
+ "io/ioutil"
+ "net/http"
  "os"
  "path/filepath"
  "regexp"
@@ -20,6 +22,12 @@ const (
 
  // LoadFileRefPattern the regular expression to match which Pipeline/Task references to load via files
  LoadFileRefPattern = "lighthouse.jenkins-x.io/loadFileRefs"
+
+ // PrependStepURL loads the steps from the given URL and prepends them to the given Task
+ PrependStepURL = "lighthouse.jenkins-x.io/prependStepsURL"
+
+ // AppendStepURL loads the steps from the given URL and appends them to the end of the Task steps
+ AppendStepURL = "lighthouse.jenkins-x.io/appendStepsURL"
 )
 
 // DefaultValues default values applied to a PipelineRun if wrapping a Pipeline/Task/TaskRun as a PipelineRun
@@ -92,6 +100,10 @@ func LoadTektonResourceAsPipelineRun(data []byte, dir, message string, getData f
  return prs, err
  }
  }
+ prs, err = inheritTaskSteps(prs)
+ if err != nil {
+ return prs, errors.Wrapf(err, "failed to inherit steps")
+ }
  return DefaultPipelineParameters(prs)
 
  case "PipelineRun":
@@ -110,6 +122,10 @@ func LoadTektonResourceAsPipelineRun(data []byte, dir, message string, getData f
  return prs, err
  }
  }
+ prs, err = inheritTaskSteps(prs)
+ if err != nil {
+ return prs, errors.Wrapf(err, "failed to inherit steps")
+ }
  return DefaultPipelineParameters(prs)
 
  case "Task":
@@ -132,6 +148,10 @@ func LoadTektonResourceAsPipelineRun(data []byte, dir, message string, getData f
  return prs, err
  }
  }
+ prs, err = inheritTaskSteps(prs)
+ if err != nil {
+ return prs, errors.Wrapf(err, "failed to inherit steps")
+ }
  return DefaultPipelineParameters(prs)
 
  case "TaskRun":
@@ -154,15 +174,80 @@ func LoadTektonResourceAsPipelineRun(data []byte, dir, message string, getData f
  return prs, err
  }
  }
+ prs, err = inheritTaskSteps(prs)
+ if err != nil {
+ return prs, errors.Wrapf(err, "failed to inherit steps")
+ }
  return DefaultPipelineParameters(prs)
 
  default:
  return nil, errors.Errorf("kind %s is not supported for %s", kind, message)
  }
 }
 
-// loadTektonRefsFromFilesPattern returns a regular expression matching the Pipeline/Task references we should load via the file system
-// as separate local files
+// inheritTaskSteps allows Task steps to be prepended or appended if the annotations are present
+func inheritTaskSteps(prs *tektonv1beta1.PipelineRun) (*tektonv1beta1.PipelineRun, error) {
+ ps := prs.Spec.PipelineSpec
+ if ps == nil || len(ps.Tasks) == 0 {
+ return prs, nil
+ }
+ if prs.Annotations == nil {
+ return prs, nil
+ }
+ appendURL := prs.Annotations[AppendStepURL]
+ prependURL := prs.Annotations[PrependStepURL]
+
+ var appendTask *tektonv1beta1.Task
+ var prependTask *tektonv1beta1.Task
+ var err error
+
+ if appendURL != "" {
+ appendTask, err = loadTaskByURL(appendURL)
+ if err != nil {
+ return prs, errors.Wrapf(err, "failed to load append steps Task")
+ }
+ }
+ if prependURL != "" {
+ prependTask, err = loadTaskByURL(prependURL)
+ if err != nil {
+ return prs, errors.Wrapf(err, "failed to load prepend steps Task")
+ }
+ }
+ if prependTask != nil {
+ firstTask := &ps.Tasks[0]
+ if firstTask.TaskSpec != nil {
+ firstTask.TaskSpec.Steps = append(prependTask.Spec.Steps, firstTask.TaskSpec.Steps...)
+ }
+ }
+ if appendTask != nil {
+ lastTask := &ps.Tasks[len(ps.Tasks)-1]
+ lastTask.TaskSpec.Steps = append(lastTask.TaskSpec.Steps, appendTask.Spec.Steps...)
+ }
+ return prs, nil
+}
+
+func loadTaskByURL(uri string) (*tektonv1beta1.Task, error) {
+ resp, err := http.Get(uri) // #nosec
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to read URL %s", uri)
+ }
+ defer resp.Body.Close()
+
+ data, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to read body from URL %s", uri)
+ }
+
+ task := &tektonv1beta1.Task{}
+ err = yaml.Unmarshal(data, &task)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to unmarshall YAML from URL %s", uri)
+ }
+ return task, nil
+}
+
+// loadTektonRefsFromFilesPattern returns a regular expression matching the Pipeline/Task references we should load
+// via the file system as separate local files
 func loadTektonRefsFromFilesPattern(prs *tektonv1beta1.PipelineRun) (*regexp.Regexp, error) {
  if prs.Annotations == nil {
  return nil, nil

pkg/triggerconfig/inrepo/test_data/load_pipelinerun/task-append-steps/expected.yaml

@@ -0,0 +1,170 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+ annotations:
+ lighthouse.jenkins-x.io/appendStepsURL: https://gist.githubusercontent.com/jstrachan/1937a809fd4223f3588db32cbb3a034f/raw/3bb56dd20cae2795e0ec4788d79e18d9486da9d9/sample-git-clone-task.yaml
+ creationTimestamp: null
+ name: cheese
+spec:
+ pipelineSpec:
+ params:
+ - description: the unique build number
+ name: BUILD_ID
+ type: string
+ - description: the name of the job which is the trigger context name
+ name: JOB_NAME
+ type: string
+ - description: the specification of the job
+ name: JOB_SPEC
+ type: string
+ - description: '''the kind of job: postsubmit or presubmit'''
+ name: JOB_TYPE
+ type: string
+ - description: the base git reference of the pull request
+ name: PULL_BASE_REF
+ type: string
+ - description: the git sha of the base of the pull request
+ name: PULL_BASE_SHA
+ type: string
+ - description: git pull request number
+ name: PULL_NUMBER
+ type: string
+ - description: git pull request ref in the form 'refs/pull/$PULL_NUMBER/head'
+ name: PULL_PULL_REF
+ type: string
+ - description: git revision to checkout (branch, tag, sha, ref…)
+ name: PULL_PULL_SHA
+ type: string
+ - description: git pull reference strings of base and latest in the form 'master:$PULL_BASE_SHA,$PULL_NUMBER:$PULL_PULL_SHA:refs/pull/$PULL_NUMBER/head'
+ name: PULL_REFS
+ type: string
+ - description: git repository name
+ name: REPO_NAME
+ type: string
+ - description: git repository owner (user or organisation)
+ name: REPO_OWNER
+ type: string
+ - description: git url to clone
+ name: REPO_URL
+ type: string
+ tasks:
+ - name: cheese
+ params:
+ - name: BUILD_ID
+ value: $(params.BUILD_ID)
+ - name: JOB_NAME
+ value: $(params.JOB_NAME)
+ - name: JOB_SPEC
+ value: $(params.JOB_SPEC)
+ - name: JOB_TYPE
+ value: $(params.JOB_TYPE)
+ - name: PULL_BASE_REF
+ value: $(params.PULL_BASE_REF)
+ - name: PULL_BASE_SHA
+ value: $(params.PULL_BASE_SHA)
+ - name: PULL_NUMBER
+ value: $(params.PULL_NUMBER)
+ - name: PULL_PULL_REF
+ value: $(params.PULL_PULL_REF)
+ - name: PULL_PULL_SHA
+ value: $(params.PULL_PULL_SHA)
+ - name: PULL_REFS
+ value: $(params.PULL_REFS)
+ - name: REPO_NAME
+ value: $(params.REPO_NAME)
+ - name: REPO_OWNER
+ value: $(params.REPO_OWNER)
+ - name: REPO_URL
+ value: $(params.REPO_URL)
+ taskSpec:
+ params:
+ - description: the unique build number
+ name: BUILD_ID
+ type: string
+ - description: the name of the job which is the trigger context name
+ name: JOB_NAME
+ type: string
+ - description: the specification of the job
+ name: JOB_SPEC
+ type: string
+ - description: '''the kind of job: postsubmit or presubmit'''
+ name: JOB_TYPE
+ type: string
+ - description: the base git reference of the pull request
+ name: PULL_BASE_REF
+ type: string
+ - description: the git sha of the base of the pull request
+ name: PULL_BASE_SHA
+ type: string
+ - description: git pull request number
+ name: PULL_NUMBER
+ type: string
+ - description: git pull request ref in the form 'refs/pull/$PULL_NUMBER/head'
+ name: PULL_PULL_REF
+ type: string
+ - description: git revision to checkout (branch, tag, sha, ref…)
+ name: PULL_PULL_SHA
+ type: string
+ - description: git pull reference strings of base and latest in the form 'master:$PULL_BASE_SHA,$PULL_NUMBER:$PULL_PULL_SHA:refs/pull/$PULL_NUMBER/head'
+ name: PULL_REFS
+ type: string
+ - description: git repository name
+ name: REPO_NAME
+ type: string
+ - description: git repository owner (user or organisation)
+ name: REPO_OWNER
+ type: string
+ - description: git url to clone
+ name: REPO_URL
+ type: string
+ stepTemplate:
+ env:
+ - name: BUILD_ID
+ value: $(params.BUILD_ID)
+ - name: JOB_NAME
+ value: $(params.JOB_NAME)
+ - name: JOB_SPEC
+ value: $(params.JOB_SPEC)
+ - name: JOB_TYPE
+ value: $(params.JOB_TYPE)
+ - name: PULL_BASE_REF
+ value: $(params.PULL_BASE_REF)
+ - name: PULL_BASE_SHA
+ value: $(params.PULL_BASE_SHA)
+ - name: PULL_NUMBER
+ value: $(params.PULL_NUMBER)
+ - name: PULL_PULL_REF
+ value: $(params.PULL_PULL_REF)
+ - name: PULL_PULL_SHA
+ value: $(params.PULL_PULL_SHA)
+ - name: PULL_REFS
+ value: $(params.PULL_REFS)
+ - name: REPO_NAME
+ value: $(params.REPO_NAME)
+ - name: REPO_OWNER
+ value: $(params.REPO_OWNER)
+ - name: REPO_URL
+ value: $(params.REPO_URL)
+ name: ""
+ resources: {}
+ steps:
+ - image: some/linter:1.2.3
+ name: lint
+ resources: {}
+ workingDir: /workspace/source
+ - image: gcr.io/jenkinsxio/builder-jx:2.1.142-761
+ name: git-clone
+ resources: {}
+ script: |
+ #!/usr/bin/env bash
+ export SUBDIR="source"
+ echo "git cloning url: $REPO_URL version $PULL_PULL_SHA to dir: $SUBDIR"
+ git config --global --add user.name ${GIT_AUTHOR_NAME:-jenkins-x-bot}
+ git config --global --add user.email ${GIT_AUTHOR_EMAIL:[email protected]}
+ git config --global credential.helper store
+ git clone $REPO_URL $SUBDIR
+ cd $SUBDIR
+ git checkout $PULL_PULL_SHA
+ echo "checked out revision: $PULL_PULL_SHA to dir: $SUBDIR"
+ workingDir: /workspace
+status: {}

pkg/triggerconfig/inrepo/test_data/load_pipelinerun/task-append-steps/source.yaml

@@ -0,0 +1,11 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+ name: cheese
+ annotations:
+ "lighthouse.jenkins-x.io/appendStepsURL": "https://gist.githubusercontent.com/jstrachan/1937a809fd4223f3588db32cbb3a034f/raw/3bb56dd20cae2795e0ec4788d79e18d9486da9d9/sample-git-clone-task.yaml"
+spec:
+ steps:
+ - image: some/linter:1.2.3
+ name: lint
+ workingDir: /workspace/source