Merge pull request #55509 from tallclair/psp-addons

Automatic merge from submit-queue (batch tested with PRs 54602, 54877, 55243, 55509, 55128). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https:/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

PodSecurityPolicies for addons

**What this PR does / why we need it**:

1. Colocate addon PodSecurityPolicy config with the addons (in a `podsecuritypolicies` subdirectory). 
2. Add policies for addons that are currently missing policies (not in the default GCE suite)
3. Remove HostPath SSL certs from several heapster deployments, so that heapster doesn't require a special PSP

**Which issue(s) this PR fixes**:
#43538

**Release note**:
```release-note
- Add PodSecurityPolicies for cluster addons
- Remove SSL cert HostPath volumes from heapster addons
```

cluster/addons/calico-policy-controller/podsecuritypolicies/calico-node-psp-binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: gce:podsecuritypolicy:calico
+ namespace: kube-system
+ labels:
+ addonmanager.kubernetes.io/mode: Reconcile
+ kubernetes.io/cluster-service: "true"
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: gce:podsecuritypolicy:privileged
+subjects:
+- kind: ServiceAccount
+ name: calico
+ namespace: kube-system

cluster/addons/cluster-monitoring/google/heapster-controller.yaml

@@ -58,26 +58,12 @@ spec:
  - /heapster
  - --source=kubernetes.summary_api:''
  - --sink=gcm
- volumeMounts:
- - name: ssl-certs
- mountPath: /etc/ssl/certs
- readOnly: true
- - name: usr-ca-certs
- mountPath: /usr/share/ca-certificates
- readOnly: true
  - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.0
  name: eventer
  command:
  - /eventer
  - --source=kubernetes:''
  - --sink=gcl
- volumeMounts:
- - name: ssl-certs
- mountPath: /etc/ssl/certs
- readOnly: true
- - name: usr-ca-certs
- mountPath: /usr/share/ca-certificates
- readOnly: true
  - image: gcr.io/google_containers/addon-resizer:1.7
  name: heapster-nanny
  resources:
@@ -136,13 +122,6 @@ spec:
  - --container=eventer
  - --poll-period=300000
  - --estimator=exponential
- volumes:
- - name: ssl-certs
- hostPath:
- path: "/etc/ssl/certs"
- - name: usr-ca-certs
- hostPath:
- path: "/usr/share/ca-certificates"
  serviceAccountName: heapster
  tolerations:
  - key: "CriticalAddonsOnly"

cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml

@@ -59,26 +59,12 @@ spec:
  - --source=kubernetes.summary_api:''
  - --sink=influxdb:http://monitoring-influxdb:8086
  - --sink=gcm:?metrics=autoscaling
- volumeMounts:
- - name: ssl-certs
- mountPath: /etc/ssl/certs
- readOnly: true
- - name: usr-ca-certs
- mountPath: /usr/share/ca-certificates
- readOnly: true
  - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.0
  name: eventer
  command:
  - /eventer
  - --source=kubernetes:''
  - --sink=gcl
- volumeMounts:
- - name: ssl-certs
- mountPath: /etc/ssl/certs
- readOnly: true
- - name: usr-ca-certs
- mountPath: /usr/share/ca-certificates
- readOnly: true
  - image: gcr.io/google_containers/addon-resizer:1.7
  name: heapster-nanny
  resources:
@@ -137,13 +123,6 @@ spec:
  - --container=eventer
  - --poll-period=300000
  - --estimator=exponential
- volumes:
- - name: ssl-certs
- hostPath:
- path: "/etc/ssl/certs"
- - name: usr-ca-certs
- hostPath:
- path: "/usr/share/ca-certificates"
  serviceAccountName: heapster
  tolerations:
  - key: "CriticalAddonsOnly"

cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml

@@ -56,13 +56,6 @@ spec:
  - /heapster
  - --source=kubernetes.summary_api:''
  - --sink=stackdriver:?cluster_name={{ cluster_name }}&min_interval_sec=100&batch_export_timeout_sec=110
- volumeMounts:
- - name: ssl-certs
- mountPath: /etc/ssl/certs
- readOnly: true
- - name: usr-ca-certs
- mountPath: /usr/share/ca-certificates
- readOnly: true
  # BEGIN_PROMETHEUS_TO_SD
  - name: prom-to-sd
  image: gcr.io/google-containers/prometheus-to-sd:v0.2.2
@@ -112,13 +105,6 @@ spec:
  - --container=heapster
  - --poll-period=300000
  - --estimator=exponential
- volumes:
- - name: ssl-certs
- hostPath:
- path: "/etc/ssl/certs"
- - name: usr-ca-certs
- hostPath:
- path: "/usr/share/ca-certificates"
  serviceAccountName: heapster
  tolerations:
  - key: "CriticalAddonsOnly"

cluster/addons/etcd-empty-dir-cleanup/etcd-empty-dir-cleanup.yaml

@@ -1,4 +1,14 @@
 apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: etcd-empty-dir-cleanup
+ namespace: kube-system
+ labels:
+ k8s-app: etcd-empty-dir-cleanup
+ kubernetes.io/cluster-service: "true"
+ addonmanager.kubernetes.io/mode: Reconcile
+---
+apiVersion: v1
 kind: Pod
 metadata:
  name: etcd-empty-dir-cleanup
@@ -8,6 +18,7 @@ metadata:
  labels:
  k8s-app: etcd-empty-dir-cleanup
 spec:
+ serviceAccountName: etcd-empty-dir-cleanup
  hostNetwork: true
  dnsPolicy: Default
  containers:

cluster/addons/etcd-empty-dir-cleanup/podsecuritypolicies/etcd-empty-dir-cleanup-psp-binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: gce:podsecuritypolicy:etcd-empty-dir-cleanup
+ namespace: kube-system
+ labels:
+ addonmanager.kubernetes.io/mode: Reconcile
+ kubernetes.io/cluster-service: "true"
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: gce:podsecuritypolicy:etcd-empty-dir-cleanup
+subjects:
+- kind: ServiceAccount
+ name: etcd-empty-dir-cleanup
+ namespace: kube-system

cluster/addons/etcd-empty-dir-cleanup/podsecuritypolicies/etcd-empty-dir-cleanup-psp-role.yaml

@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: gce:podsecuritypolicy:etcd-empty-dir-cleanup
+ namespace: kube-system
+ labels:
+ kubernetes.io/cluster-service: "true"
+ addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+ - extensions
+ resourceNames:
+ - gce.etcd-empty-dir-cleanup
+ resources:
+ - podsecuritypolicies
+ verbs:
+ - use

cluster/addons/etcd-empty-dir-cleanup/podsecuritypolicies/etcd-empty-dir-cleanup-psp.yaml

@@ -0,0 +1,31 @@
+apiVersion: extensions/v1beta1
+kind: PodSecurityPolicy
+metadata:
+ name: gce.etcd-empty-dir-cleanup
+ annotations:
+ kubernetes.io/description: 'Policy used by the etcd-empty-dir-cleanup addon.'
+ # TODO: etcd-empty-dir-cleanup should run with the default seccomp profile
+ seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*'
+ # 'runtime/default' is already the default, but must be filled in on the
+ # pod to pass admission.
+ apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default'
+ apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+ labels:
+ kubernetes.io/cluster-service: 'true'
+ addonmanager.kubernetes.io/mode: Reconcile
+spec:
+ privileged: false
+ volumes:
+ - 'secret'
+ hostNetwork: true
+ hostIPC: false
+ hostPID: false
+ runAsUser:
+ rule: 'RunAsAny'
+ seLinux:
+ rule: 'RunAsAny'
+ supplementalGroups:
+ rule: 'RunAsAny'
+ fsGroup:
+ rule: 'RunAsAny'
+ readOnlyRootFilesystem: false

cluster/addons/fluentd-elasticsearch/podsecuritypolicies/es-psp-binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: gce:podsecuritypolicy:elasticsearch-logging
+ namespace: kube-system
+ labels:
+ addonmanager.kubernetes.io/mode: Reconcile
+ kubernetes.io/cluster-service: "true"
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: gce:podsecuritypolicy:privileged
+subjects:
+- kind: ServiceAccount
+ name: elasticsearch-logging
+ namespace: kube-system

cluster/addons/ip-masq-agent/ip-masq-agent.yaml

@@ -1,3 +1,13 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: ip-masq-agent
+ namespace: kube-system
+ labels:
+ k8s-app: ip-masq-agent
+ kubernetes.io/cluster-service: "true"
+ addonmanager.kubernetes.io/mode: Reconcile
+---
 # https:/kubernetes-incubator/ip-masq-agent/blob/v2.0.0/README.md
 apiVersion: extensions/v1beta1
 kind: DaemonSet
@@ -14,6 +24,7 @@ spec:
  annotations:
  scheduler.alpha.kubernetes.io/critical-pod: ''
  spec:
+ serviceAccountName: ip-masq-agent
  hostNetwork: true
  containers:
  - name: ip-masq-agent

cluster/addons/ip-masq-agent/podsecuritypolicies/ip-masq-agent-psp-binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: gce:podsecuritypolicy:ip-masq-agent
+ namespace: kube-system
+ labels:
+ addonmanager.kubernetes.io/mode: Reconcile
+ kubernetes.io/cluster-service: "true"
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: gce:podsecuritypolicy:privileged
+subjects:
+- kind: ServiceAccount
+ name: ip-masq-agent
+ namespace: kube-system

cluster/gce/gci/configure-helper.sh

@@ -1702,14 +1702,35 @@ function start-cluster-autoscaler {
  fi
 }
 
-# A helper function for copying addon manifests and set dir/files
-# permissions.
+# A helper function for setting up addon manifests.
 #
 # $1: addon category under /etc/kubernetes
 # $2: manifest source dir
+# $3: (optional) auxilary manifest source dir
 function setup-addon-manifests {
- local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/$2"
+ local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty"
  local -r dst_dir="/etc/kubernetes/$1/$2"
+
+ copy-manifests "${src_dir}/$2" "${dst_dir}"
+
+ # If the PodSecurityPolicy admission controller is enabled,
+ # set up the corresponding addon policies.
+ if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then
+ local -r psp_dir="${src_dir}/${3:-$2}/podsecuritypolicies"
+ if [[ -d "${psp_dir}" ]]; then
+ copy-manifests "${psp_dir}" "${dst_dir}"
+ fi
+ fi
+}
+
+# A helper function for copying manifests and setting dir/files
+# permissions.
+#
+# $1: absolute source dir
+# $2: absolute destination dir
+function copy-manifests {
+ local -r src_dir="$1"
+ local -r dst_dir="$2"
  if [[ ! -d "${dst_dir}" ]]; then
  mkdir -p "${dst_dir}"
  fi
@@ -1780,7 +1801,7 @@ function start-kube-addons {
  fi
 
  if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then
-  setup-addon-manifests "addons" "podsecuritypolicies"
+ setup-addon-manifests "addons" "podsecuritypolicies"
  fi
 
  # Set up manifests of other addons.
@@ -1892,7 +1913,7 @@ EOF
  fi
  if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
  # Setup role binding for standalone node problem detector.
- setup-addon-manifests "addons" "node-problem-detector/standalone"
+ setup-addon-manifests "addons" "node-problem-detector/standalone" "node-problem-detector"
  fi
  if echo "${ADMISSION_CONTROL:-}" | grep -q "LimitRanger"; then
  setup-addon-manifests "admission-controls" "limit-range"