Generate the runner registration token in the github-runner-scaler an…

…d provide it to the VM instance via custom attr.
Privatehive · Aug 1, 2024 · 25bba59 · 25bba59
1 parent c364df5
commit 25bba59
Show file tree

Hide file tree

Showing 9 changed files with 220 additions and 94 deletions.
diff --git a/cloudRun.tf b/cloudRun.tf
@@ -19,6 +19,14 @@ resource "google_cloud_run_v2_service" "autoscaler" {
  }
  containers {
  image = "${local.region}-docker.pkg.dev/${local.projectId}/${google_artifact_registry_repository.ghcr.name}/privatehive/github-runner-autoscaler:latest"
+ env {
+ name = "ROUTE_WEBHOOK"
+ value = local.webhookUrl
+ }
+ env {
+ name = "WEBHOOK_SECRET"
+ value = random_password.webhook_secret.result
+ }
  env {
  name = "PROJECT_ID"
  value = local.projectId
@@ -35,6 +43,10 @@ resource "google_cloud_run_v2_service" "autoscaler" {
  name = "INSTANCE_TEMPLATE"
  value = google_compute_instance_template.runner_instance.id
  }
+ env {
+ name = "SECRET_VERSION"
+ value = google_secret_manager_secret.github_pat_token.id
+ }
  env {
  name = "RUNNER_PREFIX"
  value = var.github_runner_prefix
@@ -48,12 +60,8 @@ resource "google_cloud_run_v2_service" "autoscaler" {
  value = local.runnerLabel
  }
  env {
- name = "WEBHOOK_SECRET"
- value = random_password.webhook_secret.result
- }
- env {
- name = "ROUTE_WEBHOOK"
- value = local.webhookUrl
+ name = "GITHUB_ORG"
+ value = var.github_organization
  }
  dynamic "env" {
  for_each = var.enable_debug ? [0] : []

diff --git a/compute.tf b/compute.tf
@@ -44,7 +44,7 @@ resource "google_compute_instance_template" "runner_instance" {
  metadata_startup_script = <<EOT
 #!/bin/bash
 echo "Setup of agent '$(hostname)' started"
-apt-get update && apt-get -y install docker.io docker-buildx jq curl
+apt-get update && apt-get -y install docker.io docker-buildx curl
 useradd -d /home/agent -u 10000 agent
 usermod -aG docker agent
 newgrp docker
@@ -53,9 +53,8 @@ mkdir -p /home/agent
 chown -R agent:agent /home/agent
 pushd /home/agent
 sudo -u agent tar zxf /tmp/agent.tar.gz
-pat=$(gcloud secrets versions access latest --secret="${google_secret_manager_secret.github_pat_token.secret_id}")
-register_token=$(curl -s -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $pat" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/orgs/${var.github_organization}/actions/runners/registration-token | jq -r .token)
-sudo -u agent ./config.sh --unattended --disableupdate --ephemeral --name $(hostname) ${local.runnerLabelInstanceTemplate} --url 'https:/${var.github_organization}' --token $${register_token} --runnergroup '${var.github_runner_group}' || shutdown now
+registration_token=$(curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/registration_token" -H "Metadata-Flavor: Google")
+sudo -u agent ./config.sh --unattended --disableupdate --ephemeral --name $(hostname) ${local.runnerLabelInstanceTemplate} --url 'https:/${var.github_organization}' --token $${registration_token} --runnergroup '${var.github_runner_group}' || shutdown now
 ./bin/installdependencies.sh || shutdown now
 ./svc.sh install agent || shutdown now
 ./svc.sh start || shutdown now

diff --git a/iam.tf b/iam.tf
@@ -10,30 +10,13 @@ resource "google_project_iam_member" "cloud_run_member" {
  role = each.key
 }
 
-// ---- allow default compute-sa to access pat secret
+// ---- github-runner-sa ----
 
 resource "google_service_account" "github_runner_sa" {
  account_id = "github-runner-sa"
  display_name = "GitHub runner sa"
 }
 
-resource "google_project_iam_custom_role" "read_secret_version" {
- role_id = "AccessSecretVersion"
- title = "Access a secret payload"
- permissions = ["secretmanager.versions.access"]
-}
-
-resource "google_project_iam_member" "read_secret_version_member" {
- project = local.projectId
- member = "serviceAccount:${google_service_account.github_runner_sa.email}"
- role = google_project_iam_custom_role.read_secret_version.id
- condition {
- title = "Read secret: ${google_secret_manager_secret.github_pat_token.secret_id}"
- // The project number is needed - project id doesn't work
- expression = "resource.name == 'projects/${local.projectNumber}/secrets/${google_secret_manager_secret.github_pat_token.secret_id}/versions/latest'"
- }
-}
-
 // ---- autoscaler-sa ----
 resource "google_service_account" "autoscaler_sa" {
  account_id = "autoscaler-sa"
@@ -71,13 +54,19 @@ resource "google_project_iam_custom_role" "subnetwork_use" {
  permissions = ["compute.subnetworks.use", "compute.subnetworks.useExternalIp"]
 }
 
+resource "google_project_iam_custom_role" "read_secret_version" {
+ role_id = "AccessSecretVersion"
+ title = "Access a secret payload"
+ permissions = ["secretmanager.versions.access"]
+}
+
 // ---- autoscaler-sa roles member ----
 resource "google_project_iam_member" "manage_vm_instances_member" {
  project = local.projectId
  member = "serviceAccount:${google_service_account.autoscaler_sa.email}"
  role = google_project_iam_custom_role.manage_vm_instances.id
  condition {
- title = "VM instance administration with a fix prefix: ${var.github_runner_prefix}"
+ title = "VM instance administration with a fix prefix ${var.github_runner_prefix}"
  expression = "resource.name.startsWith('projects/${local.projectId}/zones/${local.zone}/instances/${var.github_runner_prefix}-')"
  }
 }
@@ -108,7 +97,7 @@ resource "google_project_iam_member" "create_vm_from_instance_template_member" {
  member = "serviceAccount:${google_service_account.autoscaler_sa.email}"
  role = google_project_iam_custom_role.create_vm_from_instance_template.id
  condition {
- title = "Create VM instance from instance template: ${google_compute_instance_template.runner_instance.name}"
+ title = "Create VM instance from instance template ${google_compute_instance_template.runner_instance.name}"
  expression = "resource.name == '${google_compute_instance_template.runner_instance.id}'"
  }
 }
@@ -118,7 +107,7 @@ resource "google_project_iam_member" "create_disk_member" {
  member = "serviceAccount:${google_service_account.autoscaler_sa.email}"
  role = google_project_iam_custom_role.create_disk.id
  condition {
- title = "Create disk with a fix prefix: ${var.github_runner_prefix}"
+ title = "Create disk with a fix prefix ${var.github_runner_prefix}"
  expression = "resource.name.startsWith('projects/${local.projectId}/zones/${local.zone}/disks/${var.github_runner_prefix}-')"
  }
 }
@@ -132,6 +121,17 @@ resource "google_project_iam_member" "subnetwork_use_member" {
  expression = "resource.name == '${google_compute_subnetwork.subnetwork.id}'"
  }
 }
+
+resource "google_project_iam_member" "read_secret_version_member" {
+ project = local.projectId
+ member = "serviceAccount:${google_service_account.autoscaler_sa.email}"
+ role = google_project_iam_custom_role.read_secret_version.id
+ condition {
+ title = "Read secret ${google_secret_manager_secret.github_pat_token.secret_id}"
+ // The project number is needed - project id doesn't work
+ expression = "resource.name == 'projects/${local.projectNumber}/secrets/${google_secret_manager_secret.github_pat_token.secret_id}/versions/latest'"
+ }
+}
 // -----------------------------
 
 // If "allUsers" within member, allows public access. This will not work if organization policy "Domain Restricted Sharing" is active in project

diff --git a/runner-autoscaler/README.md b/runner-autoscaler/README.md
@@ -36,7 +36,9 @@ The scaler is configured via the following environment variables:
 | ZONE | | The Google Cloud zone where the VM instance will be created. |
 | TASK_QUEUE | | The relative resource name of the Cloud Task queue. |
 | INSTANCE_TEMPLATE | | The relative resource name of the instance template from which the VM instance will be created. |
+| SECRET_VERSION | | The relative resource name of the secret version which contains the PAT |
 | RUNNER_PREFIX | runner | Prefix for the the name of a new VM instance. A random string (10 random lower case characters) will be added to make the name unique: "<prefix>-<random_string>". |
 | RUNNER_GROUP | Default | The GitHub runner group where the VM instance is expected to join as a self hosted runner. |
 | RUNNER_LABELS | self-hosted *(comma separated list)* | Only workflow jobs whose labels match **all** the configured labels will be taken into account. If only one configured label is **not** found in the workflow job it will be ignored. |
+| GITHUB_ORG | | The name of the GitHub Organization |
 | PORT | 8080 | To which port the webserver is bound. |
diff --git a/runner-autoscaler/go.mod b/runner-autoscaler/go.mod
@@ -5,6 +5,7 @@ go 1.21
 require (
  cloud.google.com/go/cloudtasks v1.12.11
  cloud.google.com/go/compute v1.27.3
+ cloud.google.com/go/secretmanager v1.13.5
  github.com/gin-gonic/gin v1.10.0
  github.com/sirupsen/logrus v1.9.3
  github.com/stretchr/testify v1.9.0
@@ -13,7 +14,7 @@ require (
 )
 
 require (
- cloud.google.com/go/auth v0.7.1 // indirect
+ cloud.google.com/go/auth v0.7.2 // indirect
  cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
  cloud.google.com/go/compute/metadata v0.5.0 // indirect
  cloud.google.com/go/iam v1.1.11 // indirect
@@ -35,7 +36,7 @@ require (
  github.com/golang/protobuf v1.5.4 // indirect
  github.com/google/s2a-go v0.1.7 // indirect
  github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
- github.com/googleapis/gax-go/v2 v2.12.5 // indirect
+ github.com/googleapis/gax-go/v2 v2.13.0 // indirect
  github.com/json-iterator/go v1.1.12 // indirect
  github.com/klauspost/cpuid/v2 v2.2.8 // indirect
  github.com/leodido/go-urn v1.4.0 // indirect
@@ -60,10 +61,10 @@ require (
  golang.org/x/sys v0.22.0 // indirect
  golang.org/x/text v0.16.0 // indirect
  golang.org/x/time v0.5.0 // indirect
- google.golang.org/api v0.188.0 // indirect
- google.golang.org/genproto v0.0.0-20240711142825-46eb208f015d // indirect
- google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect
+ google.golang.org/api v0.189.0 // indirect
+ google.golang.org/genproto v0.0.0-20240722135656-d784300faade // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade // indirect
  google.golang.org/grpc v1.65.0 // indirect
  gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/runner-autoscaler/go.sum b/runner-autoscaler/go.sum
@@ -1,8 +1,8 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14=
 cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU=
-cloud.google.com/go/auth v0.7.1 h1:Iv1bbpzJ2OIg16m94XI9/tlzZZl3cdeR3nGVGj78N7s=
-cloud.google.com/go/auth v0.7.1/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs=
+cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE=
+cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs=
 cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI=
 cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I=
 cloud.google.com/go/cloudtasks v1.12.11 h1:3Wg0x8pPaHy1f3Z6wOYQiu0KjseVZKHWIS2iNVDLsF8=
@@ -13,6 +13,8 @@ cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJ
 cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
 cloud.google.com/go/iam v1.1.11 h1:0mQ8UKSfdHLut6pH9FM3bI55KWR46ketn0PuXleDyxw=
 cloud.google.com/go/iam v1.1.11/go.mod h1:biXoiLWYIKntto2joP+62sd9uW5EpkZmKIvfNcTWlnQ=
+cloud.google.com/go/secretmanager v1.13.5 h1:tXlHvpm97mFD0Lv50N4U4zlXfkoTNay3BmpNA/W7/oI=
+cloud.google.com/go/secretmanager v1.13.5/go.mod h1:/OeZ88l5Z6nBVilV0SXgv6XJ243KP2aIhSWRMrbvDCQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg=
 github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
@@ -85,8 +87,8 @@ github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
 github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
-github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA=
-github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E=
+github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s=
+github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -189,19 +191,19 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.188.0 h1:51y8fJ/b1AaaBRJr4yWm96fPcuxSo0JcegXE3DaHQHw=
-google.golang.org/api v0.188.0/go.mod h1:VR0d+2SIiWOYG3r/jdm7adPW9hI2aRv9ETOSCQ9Beag=
+google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI=
+google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20240711142825-46eb208f015d h1:/hmn0Ku5kWij/kjGsrcJeC1T/MrJi2iNWwgAqrihFwc=
-google.golang.org/genproto v0.0.0-20240711142825-46eb208f015d/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY=
-google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY=
-google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d h1:JU0iKnSg02Gmb5ZdV8nYsKEKsP6o/FGVWTrw4i1DA9A=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
+google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg=
+google.golang.org/genproto v0.0.0-20240722135656-d784300faade/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY=
+google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade h1:WxZOF2yayUHpHSbUE6NMzumUzBxYc3YGwo0YHnbzsJY=
+google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade h1:oCRSWfwGXQsqlVdErcyTt4A93Y8fo0/9D4b1gnI++qo=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=

diff --git a/runner-autoscaler/main.go b/runner-autoscaler/main.go
@@ -6,6 +6,7 @@ import (
  "strings"
 
  "github.com/Tereius/gcp-hosted-github-runner/pkg"
+ "github.com/sirupsen/logrus"
  log "github.com/sirupsen/logrus"
 )
 
@@ -28,20 +29,30 @@ func mustGetEnv(name string) string {
 
 func main() {
 
+ logrus.SetFormatter(&logrus.JSONFormatter{
+ DisableTimestamp: true,
+ FieldMap: logrus.FieldMap{
+ logrus.FieldKeyLevel: "severity",
+ },
+ })
+ logrus.SetLevel(logrus.InfoLevel)
+
  labels := strings.Split(getEnvDefault("RUNNER_LABELS", "self-hosted"), ",")
  runnerGroup := getEnvDefault("RUNNER_GROUP", "Default")
  scaler := pkg.NewAutoscaler(pkg.AutoscalerConfig{
  RouteWebhook: getEnvDefault("ROUTE_WEBHOOK", "/webhook"),
- RouteCreateVm: getEnvDefault("ROUTE_CREATE_VM", "/create_vm"),
  RouteDeleteVm: getEnvDefault("ROUTE_DELETE_VM", "/delete_vm"),
+ RouteCreateVm: getEnvDefault("ROUTE_CREATE_VM", "/create_vm"),
  WebhookSecret: getEnvDefault("WEBHOOK_SECRET", ""),
  ProjectId: mustGetEnv("PROJECT_ID"),
  Zone: mustGetEnv("ZONE"),
  TaskQueue: mustGetEnv("TASK_QUEUE"),
  InstanceTemplate: mustGetEnv("INSTANCE_TEMPLATE"),
+ SecretVersion: mustGetEnv("SECRET_VERSION"),
  RunnerPrefix: getEnvDefault("RUNNER_PREFIX", "runner"),
  RunnerGroup: runnerGroup,
  RunnerLabels: labels,
+ GitHubOrg: mustGetEnv("GITHUB_ORG"),
  })
 
  if len(labels) == 0 {