Skip to content

Commit

Permalink
Merge pull request #135 from mresvanis/add-proxy-support-for-spec-has…
Browse files Browse the repository at this point in the history
…h-computations

MGMT-17871: Include proxy when computing openshift-apiserver spec-hash
  • Loading branch information
openshift-merge-bot[bot] authored Jun 17, 2024
2 parents dea4fc6 + 09b2bae commit 5ebadac
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 2 deletions.
9 changes: 9 additions & 0 deletions hack/assets/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,15 @@ func apiServerDeploymentJSON() {
required.Spec.Template.ObjectMeta.Labels["revision"] = "${REVISION}"
required.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchLabels["openshift-apiserver-anti-affinity"] = "true"

proxyEnvVars := []v1.EnvVar{
{Name: "HTTPS_PROXY", Value: "${HTTPS_PROXY}"},
{Name: "HTTP_PROXY", Value:"${HTTP_PROXY}"},
{Name: "NO_PROXY", Value:"${NO_PROXY}"},
}
for i, container := range required.Spec.Template.Spec.Containers {
required.Spec.Template.Spec.Containers[i].Env = append(container.Env, proxyEnvVars...)
}

jsonBytes, err := json.Marshal(required.Spec)
check(err)

Expand Down
2 changes: 1 addition & 1 deletion src/bindata/openshift-apiserver-deployment.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"replicas":1,"selector":{"matchLabels":{"apiserver":"true","app":"openshift-apiserver-a"}},"template":{"metadata":{"name":"openshift-apiserver","creationTimestamp":null,"labels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true","revision":"${REVISION}"},"annotations":{"openshift.io/required-scc":"privileged","operator.openshift.io/dep-desired.generation":"${DESIRED_GENERATION}","operator.openshift.io/dep-openshift-apiserver.config.configmap":"${CONFIG_HASH}","operator.openshift.io/dep-openshift-apiserver.etcd-client.secret":"${ETCD_CLIENT_HASH}","operator.openshift.io/dep-openshift-apiserver.etcd-serving-ca.configmap":"${ETCD_SERVING_CA_HASH}","operator.openshift.io/dep-openshift-apiserver.image-import-ca.configmap":"${IMAGE_IMPORT_CA_HASH}","operator.openshift.io/dep-openshift-apiserver.trusted-ca-bundle.configmap":"${TRUSTED_CA_BUNDLE_HASH}","target.workload.openshift.io/management":"{\"effect\": \"PreferredDuringScheduling\"}"}},"spec":{"volumes":[{"name":"node-pullsecrets","hostPath":{"path":"/var/lib/kubelet/","type":"Directory"}},{"name":"config","configMap":{"name":"config"}},{"name":"audit","configMap":{"name":"audit-${REVISION}"}},{"name":"etcd-client","secret":{"secretName":"etcd-client","defaultMode":384}},{"name":"etcd-serving-ca","configMap":{"name":"etcd-serving-ca"}},{"name":"image-import-ca","configMap":{"name":"image-import-ca","optional":true}},{"name":"serving-cert","secret":{"secretName":"serving-cert","defaultMode":384}},{"name":"trusted-ca-bundle","configMap":{"name":"trusted-ca-bundle","items":[{"key":"ca-bundle.crt","path":"tls-ca-bundle.pem"}],"optional":true}},{"name":"encryption-config","secret":{"secretName":"encryption-config-${REVISION}","defaultMode":384,"optional":true}},{"name":"audit-dir","hostPath":{"path":"/var/log/openshift-apiserver"}}],"initContainers":[{"name":"fix-audit-permissions","image":"${IMAGE}","command":["sh","-c","chmod 0700 /var/log/openshift-apiserver \u0026\u0026 touch /var/log/openshift-apiserver/audit.log \u0026\u0026 chmod 0600 /var/log/openshift-apiserver/*"],"resources":{"requests":{"cpu":"15m","memory":"50Mi"}},"volumeMounts":[{"name":"audit-dir","mountPath":"/var/log/openshift-apiserver"}],"terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":0}}],"containers":[{"name":"openshift-apiserver","image":"${IMAGE}","command":["/bin/bash","-ec"],"args":["if [ -s /var/run/configmaps/trusted-ca-bundle/tls-ca-bundle.pem ]; then\n echo \"Copying system trust bundle\"\n cp -f /var/run/configmaps/trusted-ca-bundle/tls-ca-bundle.pem /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem\nfi\nexec openshift-apiserver start --config=/var/run/configmaps/config/config.yaml -v=${VERBOSITY}\n"],"ports":[{"containerPort":8443}],"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"node-pullsecrets","readOnly":true,"mountPath":"/var/lib/kubelet/"},{"name":"config","mountPath":"/var/run/configmaps/config"},{"name":"audit","mountPath":"/var/run/configmaps/audit"},{"name":"etcd-client","mountPath":"/var/run/secrets/etcd-client"},{"name":"etcd-serving-ca","mountPath":"/var/run/configmaps/etcd-serving-ca"},{"name":"image-import-ca","mountPath":"/var/run/configmaps/image-import-ca"},{"name":"trusted-ca-bundle","mountPath":"/var/run/configmaps/trusted-ca-bundle"},{"name":"serving-cert","mountPath":"/var/run/secrets/serving-cert"},{"name":"encryption-config","mountPath":"/var/run/secrets/encryption-config"},{"name":"audit-dir","mountPath":"/var/log/openshift-apiserver"}],"livenessProbe":{"httpGet":{"path":"healthz","port":8443,"scheme":"HTTPS"},"timeoutSeconds":10,"periodSeconds":10,"successThreshold":1,"failureThreshold":3},"readinessProbe":{"httpGet":{"path":"readyz","port":8443,"scheme":"HTTPS"},"timeoutSeconds":10,"periodSeconds":5,"successThreshold":1,"failureThreshold":1},"startupProbe":{"httpGet":{"path":"healthz","port":8443,"scheme":"HTTPS"},"timeoutSeconds":10,"periodSeconds":5,"successThreshold":1,"failureThreshold":30},"terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":0,"readOnlyRootFilesystem":false}},{"name":"openshift-apiserver-check-endpoints","image":"${KUBE_APISERVER_OPERATOR_IMAGE}","command":["cluster-kube-apiserver-operator","check-endpoints"],"args":["--listen","0.0.0.0:17698","--namespace","$(POD_NAMESPACE)","--v","2"],"ports":[{"name":"check-endpoints","containerPort":17698,"protocol":"TCP"}],"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}],"resources":{"requests":{"cpu":"10m","memory":"50Mi"}},"terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"}],"terminationGracePeriodSeconds":90,"nodeSelector":{"node-role.kubernetes.io/master":""},"serviceAccountName":"openshift-apiserver-sa","affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true"}},"topologyKey":"kubernetes.io/hostname"}]}},"tolerations":[{"key":"node-role.kubernetes.io/master","operator":"Exists","effect":"NoSchedule"},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":120},{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":120}],"priorityClassName":"system-node-critical"}},"strategy":{"type":"RollingUpdate","rollingUpdate":{"maxUnavailable":1,"maxSurge":0}}}
{"replicas":1,"selector":{"matchLabels":{"apiserver":"true","app":"openshift-apiserver-a"}},"template":{"metadata":{"name":"openshift-apiserver","creationTimestamp":null,"labels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true","revision":"${REVISION}"},"annotations":{"openshift.io/required-scc":"privileged","operator.openshift.io/dep-desired.generation":"${DESIRED_GENERATION}","operator.openshift.io/dep-openshift-apiserver.config.configmap":"${CONFIG_HASH}","operator.openshift.io/dep-openshift-apiserver.etcd-client.secret":"${ETCD_CLIENT_HASH}","operator.openshift.io/dep-openshift-apiserver.etcd-serving-ca.configmap":"${ETCD_SERVING_CA_HASH}","operator.openshift.io/dep-openshift-apiserver.image-import-ca.configmap":"${IMAGE_IMPORT_CA_HASH}","operator.openshift.io/dep-openshift-apiserver.trusted-ca-bundle.configmap":"${TRUSTED_CA_BUNDLE_HASH}","target.workload.openshift.io/management":"{\"effect\": \"PreferredDuringScheduling\"}"}},"spec":{"volumes":[{"name":"node-pullsecrets","hostPath":{"path":"/var/lib/kubelet/","type":"Directory"}},{"name":"config","configMap":{"name":"config"}},{"name":"audit","configMap":{"name":"audit-${REVISION}"}},{"name":"etcd-client","secret":{"secretName":"etcd-client","defaultMode":384}},{"name":"etcd-serving-ca","configMap":{"name":"etcd-serving-ca"}},{"name":"image-import-ca","configMap":{"name":"image-import-ca","optional":true}},{"name":"serving-cert","secret":{"secretName":"serving-cert","defaultMode":384}},{"name":"trusted-ca-bundle","configMap":{"name":"trusted-ca-bundle","items":[{"key":"ca-bundle.crt","path":"tls-ca-bundle.pem"}],"optional":true}},{"name":"encryption-config","secret":{"secretName":"encryption-config-${REVISION}","defaultMode":384,"optional":true}},{"name":"audit-dir","hostPath":{"path":"/var/log/openshift-apiserver"}}],"initContainers":[{"name":"fix-audit-permissions","image":"${IMAGE}","command":["sh","-c","chmod 0700 /var/log/openshift-apiserver \u0026\u0026 touch /var/log/openshift-apiserver/audit.log \u0026\u0026 chmod 0600 /var/log/openshift-apiserver/*"],"resources":{"requests":{"cpu":"15m","memory":"50Mi"}},"volumeMounts":[{"name":"audit-dir","mountPath":"/var/log/openshift-apiserver"}],"terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":0}}],"containers":[{"name":"openshift-apiserver","image":"${IMAGE}","command":["/bin/bash","-ec"],"args":["if [ -s /var/run/configmaps/trusted-ca-bundle/tls-ca-bundle.pem ]; then\n echo \"Copying system trust bundle\"\n cp -f /var/run/configmaps/trusted-ca-bundle/tls-ca-bundle.pem /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem\nfi\nexec openshift-apiserver start --config=/var/run/configmaps/config/config.yaml -v=${VERBOSITY}\n"],"ports":[{"containerPort":8443}],"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"HTTPS_PROXY","value":"${HTTPS_PROXY}"},{"name":"HTTP_PROXY","value":"${HTTP_PROXY}"},{"name":"NO_PROXY","value":"${NO_PROXY}"}],"resources":{"requests":{"cpu":"100m","memory":"200Mi"}},"volumeMounts":[{"name":"node-pullsecrets","readOnly":true,"mountPath":"/var/lib/kubelet/"},{"name":"config","mountPath":"/var/run/configmaps/config"},{"name":"audit","mountPath":"/var/run/configmaps/audit"},{"name":"etcd-client","mountPath":"/var/run/secrets/etcd-client"},{"name":"etcd-serving-ca","mountPath":"/var/run/configmaps/etcd-serving-ca"},{"name":"image-import-ca","mountPath":"/var/run/configmaps/image-import-ca"},{"name":"trusted-ca-bundle","mountPath":"/var/run/configmaps/trusted-ca-bundle"},{"name":"serving-cert","mountPath":"/var/run/secrets/serving-cert"},{"name":"encryption-config","mountPath":"/var/run/secrets/encryption-config"},{"name":"audit-dir","mountPath":"/var/log/openshift-apiserver"}],"livenessProbe":{"httpGet":{"path":"healthz","port":8443,"scheme":"HTTPS"},"timeoutSeconds":10,"periodSeconds":10,"successThreshold":1,"failureThreshold":3},"readinessProbe":{"httpGet":{"path":"readyz","port":8443,"scheme":"HTTPS"},"timeoutSeconds":10,"periodSeconds":5,"successThreshold":1,"failureThreshold":1},"startupProbe":{"httpGet":{"path":"healthz","port":8443,"scheme":"HTTPS"},"timeoutSeconds":10,"periodSeconds":5,"successThreshold":1,"failureThreshold":30},"terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true,"runAsUser":0,"readOnlyRootFilesystem":false}},{"name":"openshift-apiserver-check-endpoints","image":"${KUBE_APISERVER_OPERATOR_IMAGE}","command":["cluster-kube-apiserver-operator","check-endpoints"],"args":["--listen","0.0.0.0:17698","--namespace","$(POD_NAMESPACE)","--v","2"],"ports":[{"name":"check-endpoints","containerPort":17698,"protocol":"TCP"}],"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"HTTPS_PROXY","value":"${HTTPS_PROXY}"},{"name":"HTTP_PROXY","value":"${HTTP_PROXY}"},{"name":"NO_PROXY","value":"${NO_PROXY}"}],"resources":{"requests":{"cpu":"10m","memory":"50Mi"}},"terminationMessagePolicy":"FallbackToLogsOnError","imagePullPolicy":"IfNotPresent"}],"terminationGracePeriodSeconds":90,"nodeSelector":{"node-role.kubernetes.io/master":""},"serviceAccountName":"openshift-apiserver-sa","affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"apiserver":"true","app":"openshift-apiserver-a","openshift-apiserver-anti-affinity":"true"}},"topologyKey":"kubernetes.io/hostname"}]}},"tolerations":[{"key":"node-role.kubernetes.io/master","operator":"Exists","effect":"NoSchedule"},{"key":"node-role.kubernetes.io/control-plane","operator":"Exists","effect":"NoExecute"},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":120},{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":120}],"priorityClassName":"system-node-critical"}},"strategy":{"type":"RollingUpdate","rollingUpdate":{"maxUnavailable":1,"maxSurge":0}}}
52 changes: 51 additions & 1 deletion src/ocp_postprocess.rs
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,31 @@ async fn get_openshift_apiserver_log_level(in_memory_etcd_client: &Arc<InMemoryK
Ok(log_level.to_string())
}

async fn get_proxy_env_vars(in_memory_etcd_client: &Arc<InMemoryK8sEtcd>) -> Result<Option<BTreeMap<String, String>>> {
let etcd_client = in_memory_etcd_client;

let cluster = get_etcd_json(
etcd_client,
&K8sResourceLocation::new(None, "OpenShiftAPIServer", "cluster", "operator.openshift.io/v1"),
)
.await?
.context("couldn't find openshiftapiserver.operator/cluster resource")?;

if let Some(proxy_config) = cluster.pointer("/spec/observedConfig/workloadcontroller/proxy") {
let vars: BTreeMap<_, _> = proxy_config
.as_object()
.context("spec.observedConfig.workloadcontroller.proxy not an object")?
.iter()
.map(|(k, v)| Ok((k.clone(), String::from(v.as_str().context("value not a string")?))))
.collect::<Result<Vec<_>>>()?
.into_iter()
.collect();
Ok(Some(vars))
} else {
Ok(None)
}
}

pub(crate) async fn fix_deployment_spec_hash_annotation(
in_memory_etcd_client: &Arc<InMemoryK8sEtcd>,
k8s_resource_location: K8sResourceLocation,
Expand Down Expand Up @@ -362,7 +387,18 @@ pub(crate) async fn fix_deployment_spec_hash_annotation(
.await
.context("could not get KUBE_APISERVER_OPERATOR_IMAGE")?;

fix_openshift_apiserver_spec_hash_annotation(metadata_annotations, revision, &log_level, &kube_apiserver_operator_image).await?
let proxy_env_vars = get_proxy_env_vars(in_memory_etcd_client)
.await
.context("could not get proxy env vars")?;

fix_openshift_apiserver_spec_hash_annotation(
metadata_annotations,
revision,
&log_level,
&kube_apiserver_operator_image,
proxy_env_vars,
)
.await?
}
Some("openshift-oauth-apiserver") => {
let container_image = dep
Expand Down Expand Up @@ -539,6 +575,7 @@ async fn fix_openshift_apiserver_spec_hash_annotation(
revision: &str,
log_level: &str,
kube_apiserver_operator_image: &str,
proxy_env_vars: Option<BTreeMap<String, String>>,
) -> Result<(), anyhow::Error> {
let bytes = include_bytes!("bindata/openshift-apiserver-deployment.json");
let mut spec_json = String::from_utf8(bytes.to_vec()).context("invalid UTF-8 string")?;
Expand Down Expand Up @@ -581,6 +618,19 @@ async fn fix_openshift_apiserver_spec_hash_annotation(
spec_json = spec_json.replace("${VERBOSITY}", log_level);
spec_json = spec_json.replace("${KUBE_APISERVER_OPERATOR_IMAGE}", kube_apiserver_operator_image);

match proxy_env_vars {
Some(vars) => {
for (key, value) in vars {
spec_json = spec_json.replace(format!("${{{}}}", key).as_str(), &value);
}
}
None => {
for var in ["HTTPS_PROXY", "HTTP_PROXY", "NO_PROXY"] {
spec_json = spec_json.replace(format!(",{{\"name\":\"{0}\",\"value\":\"${{{0}}}\"}}", var).as_str(), "");
}
}
}

let mut sha256 = Sha256::new();
sha256.update(spec_json);
let spec_hash: String = format!("{:x}", sha256.finalize());
Expand Down

0 comments on commit 5ebadac

Please sign in to comment.