From 18f8db05d94608b208fa5665de0988dbcd877de0 Mon Sep 17 00:00:00 2001 From: Ellis Tarn Date: Fri, 17 Sep 2021 16:30:49 -0700 Subject: [PATCH 1/8] Implemented support for topology spread constraints --- templates/_helpers.tpl | 31 ++++++++++++++++++++++++++++++ templates/injector-deployment.yaml | 1 + templates/server-statefulset.yaml | 1 + values.yaml | 8 ++++++++ 4 files changed, 41 insertions(+) diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index 3e936f7..64ec8c6 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -248,6 +248,37 @@ Sets the injector affinity for pod placement {{ end }} {{- end -}} +{{/* +Sets the topologySpreadConstraints when running in standalone and HA modes. +*/}} +{{- define "vault.topologySpreadConstraints" -}} + {{- if and (ne .mode "dev") .Values.server.topologySpreadConstraints }} + topologySpreadConstraints: + {{ $tp := typeOf .Values.server.topologySpreadConstraints }} + {{- if eq $tp "string" }} + {{- tpl .Values.server.topologySpreadConstraints . | nindent 8 | trim }} + {{- else }} + {{- toYaml .Values.server.topologySpreadConstraints | nindent 8 }} + {{- end }} + {{ end }} +{{- end -}} + + +{{/* +Sets the injector topologySpreadConstraints for pod placement +*/}} +{{- define "injector.topologySpreadConstraints" -}} + {{- if .Values.injector.topologySpreadConstraints }} + topologySpreadConstraints: + {{ $tp := typeOf .Values.injector.topologySpreadConstraints }} + {{- if eq $tp "string" }} + {{- tpl .Values.injector.topologySpreadConstraints . | nindent 8 | trim }} + {{- else }} + {{- toYaml .Values.injector.topologySpreadConstraints | nindent 8 }} + {{- end }} + {{ end }} +{{- end -}} + {{/* Sets the toleration for pod placement when running in standalone and HA modes. */}} diff --git a/templates/injector-deployment.yaml b/templates/injector-deployment.yaml index f4a796b..d91b18c 100644 --- a/templates/injector-deployment.yaml +++ b/templates/injector-deployment.yaml @@ -29,6 +29,7 @@ spec: {{ template "injector.annotations" . }} spec: {{ template "injector.affinity" . }} + {{ template "injector.topologySpreadConstraints" . }} {{ template "injector.tolerations" . }} {{ template "injector.nodeselector" . }} {{- if .Values.injector.priorityClassName }} diff --git a/templates/server-statefulset.yaml b/templates/server-statefulset.yaml index 031b179..e3fe346 100644 --- a/templates/server-statefulset.yaml +++ b/templates/server-statefulset.yaml @@ -36,6 +36,7 @@ spec: {{ template "vault.annotations" . }} spec: {{ template "vault.affinity" . }} + {{ template "vault.topologySpreadConstraints" . }} {{ template "vault.tolerations" . }} {{ template "vault.nodeselector" . }} {{- if .Values.server.priorityClassName }} diff --git a/values.yaml b/values.yaml index 7d5d046..94de910 100644 --- a/values.yaml +++ b/values.yaml @@ -175,6 +175,10 @@ injector: component: webhook topologyKey: kubernetes.io/hostname + # Topology settings for injector pods + # ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + topologySpreadConstraints: [] + # Toleration Settings for injector pods # This should be either a multi-line string or YAML matching the Toleration array # in a PodSpec. @@ -430,6 +434,10 @@ server: component: server topologyKey: kubernetes.io/hostname + # Topology settings for server pods + # ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + topologySpreadConstraints: [] + # Toleration Settings for server pods # This should be either a multi-line string or YAML matching the Toleration array # in a PodSpec. From c77cb1f6a4f7222253f76f8602161525059fcd3f Mon Sep 17 00:00:00 2001 From: Ellis Tarn Date: Mon, 27 Sep 2021 15:19:43 -0700 Subject: [PATCH 2/8] Update values.yaml Co-authored-by: Ben Ash <32777270+benashz@users.noreply.github.com> --- values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/values.yaml b/values.yaml index 94de910..543c12b 100644 --- a/values.yaml +++ b/values.yaml @@ -436,6 +436,8 @@ server: # Topology settings for server pods # ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + # This should be either a multi-line string or YAML matching the topologySpreadConstraints array + # in a PodSpec. topologySpreadConstraints: [] # Toleration Settings for server pods From 87c3e620300f8028da302d93fb91bd2619f0d8f9 Mon Sep 17 00:00:00 2001 From: Ellis Tarn Date: Mon, 27 Sep 2021 15:19:47 -0700 Subject: [PATCH 3/8] Update values.yaml Co-authored-by: Ben Ash <32777270+benashz@users.noreply.github.com> --- values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/values.yaml b/values.yaml index 543c12b..e944c53 100644 --- a/values.yaml +++ b/values.yaml @@ -177,6 +177,8 @@ injector: # Topology settings for injector pods # ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + # This should be either a multi-line string or YAML matching the topologySpreadConstraints array + # in a PodSpec. topologySpreadConstraints: [] # Toleration Settings for injector pods From 8840187301e25293db4623c8975c06cfbbde9e48 Mon Sep 17 00:00:00 2001 From: Eric Miller Date: Mon, 22 Nov 2021 14:36:11 -0600 Subject: [PATCH 4/8] Add topologySpreadConstraints to values schema --- values.schema.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/values.schema.json b/values.schema.json index 4c0a004..ff2c41d 100644 --- a/values.schema.json +++ b/values.schema.json @@ -367,6 +367,13 @@ "string" ] }, + "topologySpreadConstraints": { + "type": [ + "null", + "array", + "string" + ] + }, "webhookAnnotations": { "type": [ "object", From f49ecb6802bea0d6ae13bc2870205534b7afbbdb Mon Sep 17 00:00:00 2001 From: Eric Miller Date: Mon, 22 Nov 2021 15:12:17 -0600 Subject: [PATCH 5/8] Implement injector deployment topology spread UTs * also remove string from the relevant schema types --- test/unit/injector-deployment.bats | 25 +++++++++++++++++++++++-- values.schema.json | 3 +-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/test/unit/injector-deployment.bats b/test/unit/injector-deployment.bats index 0f475df..52df03d 100755 --- a/test/unit/injector-deployment.bats +++ b/test/unit/injector-deployment.bats @@ -168,7 +168,7 @@ load _helpers [ "${value}" = "RELEASE-NAME-vault-agent-injector-svc,RELEASE-NAME-vault-agent-injector-svc.${namespace:-default},RELEASE-NAME-vault-agent-injector-svc.${namespace:-default}.svc" ] } -@test "injector/deployment: manual TLS adds volume mount" { +@test "injector/deployment: manual TLS adds volume mount" { cd `chart_dir` local object=$(helm template \ --show-only templates/injector-deployment.yaml \ @@ -452,6 +452,27 @@ load _helpers [ "${actual}" = "true" ] } +#-------------------------------------------------------------------- +# topologySpreadConstraints + +@test "injector/deployment: topologySpreadConstraints is null by default" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + . | tee /dev/stderr | + yq '.spec.template.spec | .topologySpreadConstraints? == null' | tee /dev/stderr) +} + +@test "injector/deployment: topologySpreadConstraints can be set as YAML" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.topologySpreadConstraints[0].foo=bar,injector.topologySpreadConstraints[1].baz=qux" \ + . | tee /dev/stderr | + yq '.spec.template.spec.topologySpreadConstraints == [{"foo": "bar"}, {"baz": "qux"}]' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + #-------------------------------------------------------------------- # tolerations @@ -695,4 +716,4 @@ load _helpers local value=$(echo $object | yq -r 'map(select(.name=="AGENT_INJECT_TEMPLATE_CONFIG_EXIT_ON_RETRY_FAILURE")) | .[] .value' | tee /dev/stderr) [ "${value}" = "false" ] -} \ No newline at end of file +} diff --git a/values.schema.json b/values.schema.json index ff2c41d..75f4d06 100644 --- a/values.schema.json +++ b/values.schema.json @@ -370,8 +370,7 @@ "topologySpreadConstraints": { "type": [ "null", - "array", - "string" + "array" ] }, "webhookAnnotations": { From 567a4386a500697262a313f6ec1a692bd9fd0a51 Mon Sep 17 00:00:00 2001 From: Eric Miller Date: Mon, 22 Nov 2021 15:21:15 -0600 Subject: [PATCH 6/8] Implement injector statefulset topology spread UTs --- test/unit/server-statefulset.bats | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats index b939051..47a342f 100755 --- a/test/unit/server-statefulset.bats +++ b/test/unit/server-statefulset.bats @@ -784,6 +784,29 @@ load _helpers [ "${actual}" = "true" ] } +#-------------------------------------------------------------------- +# topologySpreadConstraints + +@test "server/standalone-StatefulSet: topologySpreadConstraints is null by default" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-statefulset.yaml \ + . | tee /dev/stderr | + yq '.spec.template.spec | .topologySpreadConstraints? == null' | tee /dev/stderr) +} + +@test "server/standalone-StatefulSet: topologySpreadConstraints can be set as YAML" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-statefulset.yaml \ + --set "server.topologySpreadConstraints[0].foo=bar,server.topologySpreadConstraints[1].baz=qux" \ + . | tee /dev/stderr | + yq '.spec.template.spec.topologySpreadConstraints == [{"foo": "bar"}, {"baz": "qux"}]' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# tolerations @test "server/standalone-StatefulSet: tolerations not set by default" { cd `chart_dir` From 102c29955b240aa95a14e43f730e2074a998644b Mon Sep 17 00:00:00 2001 From: Eric Miller Date: Mon, 22 Nov 2021 15:24:38 -0600 Subject: [PATCH 7/8] Implement injector HA statefulset topology UTs --- test/unit/injector-deployment.bats | 2 +- test/unit/server-ha-statefulset.bats | 26 ++++++++++++++++++++++++++ test/unit/server-statefulset.bats | 2 +- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/test/unit/injector-deployment.bats b/test/unit/injector-deployment.bats index 52df03d..6796ad6 100755 --- a/test/unit/injector-deployment.bats +++ b/test/unit/injector-deployment.bats @@ -458,7 +458,7 @@ load _helpers @test "injector/deployment: topologySpreadConstraints is null by default" { cd `chart_dir` local actual=$(helm template \ - --show-only templates/injector-deployment.yaml \ + --show-only templates/injector-deployment.yaml \ . | tee /dev/stderr | yq '.spec.template.spec | .topologySpreadConstraints? == null' | tee /dev/stderr) } diff --git a/test/unit/server-ha-statefulset.bats b/test/unit/server-ha-statefulset.bats index cc77e7e..fbe5ed2 100755 --- a/test/unit/server-ha-statefulset.bats +++ b/test/unit/server-ha-statefulset.bats @@ -540,6 +540,32 @@ load _helpers [ "${actual}" = "1" ] } +#-------------------------------------------------------------------- +# topologySpreadConstraints + +@test "server/ha-StatefulSet: topologySpreadConstraints is null by default" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-statefulset.yaml \ + --set 'server.ha.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec | .topologySpreadConstraints? == null' | tee /dev/stderr) +} + +@test "server/ha-StatefulSet: topologySpreadConstraints can be set as YAML" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-statefulset.yaml \ + --set 'server.ha.enabled=true' \ + --set "server.topologySpreadConstraints[0].foo=bar,server.topologySpreadConstraints[1].baz=qux" \ + . | tee /dev/stderr | + yq '.spec.template.spec.topologySpreadConstraints == [{"foo": "bar"}, {"baz": "qux"}]' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# tolerations + @test "server/ha-StatefulSet: tolerations not set by default" { cd `chart_dir` local actual=$(helm template \ diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats index 47a342f..67be58b 100755 --- a/test/unit/server-statefulset.bats +++ b/test/unit/server-statefulset.bats @@ -790,7 +790,7 @@ load _helpers @test "server/standalone-StatefulSet: topologySpreadConstraints is null by default" { cd `chart_dir` local actual=$(helm template \ - --show-only templates/server-statefulset.yaml \ + --show-only templates/server-statefulset.yaml \ . | tee /dev/stderr | yq '.spec.template.spec | .topologySpreadConstraints? == null' | tee /dev/stderr) } From fb3126b864ff03c71a2fc9231067f958684fb093 Mon Sep 17 00:00:00 2001 From: Eric Miller Date: Tue, 30 Nov 2021 15:49:12 -0600 Subject: [PATCH 8/8] Allow topologySpreadConstraints to be a string --- values.schema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/values.schema.json b/values.schema.json index 373f91d..b48d4af 100644 --- a/values.schema.json +++ b/values.schema.json @@ -356,7 +356,8 @@ "topologySpreadConstraints": { "type": [ "null", - "array" + "array", + "string" ] }, "webhookAnnotations": {