From 76c0f18cdf9d79552001bc8c5a0a505b8edaae94 Mon Sep 17 00:00:00 2001 From: pratikmahajan Date: Thu, 5 Dec 2024 05:48:35 +0000 Subject: [PATCH 1/2] separate the cincinnati graph-builder and policy-engine deployments --- dist/openshift/cincinnati-microservices.yaml | 454 +++++++++++++++++++ 1 file changed, 454 insertions(+) create mode 100644 dist/openshift/cincinnati-microservices.yaml diff --git a/dist/openshift/cincinnati-microservices.yaml b/dist/openshift/cincinnati-microservices.yaml new file mode 100644 index 000000000..9feaa670f --- /dev/null +++ b/dist/openshift/cincinnati-microservices.yaml @@ -0,0 +1,454 @@ +--- +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cincinnati +objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: cincinnati-graph-builder + name: cincinnati-graph-builder + spec: + replicas: ${{MAX_REPLICAS}} + selector: + matchLabels: + app: cincinnati-graph-builder + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 0 + template: + metadata: + labels: + app: cincinnati-graph-builder + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - cincinnati-graph-builder + topologyKey: kubernetes.io/hostname + containers: + - image: ${IMAGE}:${IMAGE_TAG} + imagePullPolicy: Always + name: cincinnati-graph-builder + env: + - name: "RUST_BACKTRACE" + valueFrom: + configMapKeyRef: + key: gb.rust_backtrace + name: cincinnati + envFrom: + - configMapRef: + name: environment-secrets + command: + - ${GB_BINARY} + args: [ + "-c", "${GB_CONFIG_PATH}" + ] + ports: + - name: graph-builder + containerPort: ${{GB_PORT}} + - name: gb-public + containerPort: ${{GB_PUBLIC_PORT}} + - name: status-gb + containerPort: ${{GB_STATUS_PORT}} + livenessProbe: + httpGet: + path: /liveness + port: ${{GB_STATUS_PORT}} + initialDelaySeconds: 150 + periodSeconds: 30 + timeoutSeconds: 3 + readinessProbe: + httpGet: + path: /readiness + port: ${{GB_STATUS_PORT}} + initialDelaySeconds: 150 + periodSeconds: 30 + timeoutSeconds: 3 + resources: + limits: + cpu: ${GB_CPU_LIMIT} + memory: ${GB_MEMORY_LIMIT} + requests: + cpu: ${GB_CPU_REQUEST} + memory: ${GB_MEMORY_REQUEST} + volumeMounts: + - name: secrets + mountPath: /etc/secrets + readOnly: true + - name: configs + mountPath: /etc/configs + readOnly: true + volumes: + - name: secrets + secret: + secretName: cincinnati-credentials + - name: configs + configMap: + name: cincinnati-configs + triggers: + - type: ConfigChange + + - apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: cincinnati-policy-engine + name: cincinnati-policy-engine + spec: + replicas: ${{MAX_REPLICAS}} + selector: + matchLabels: + app: cincinnati-policy-engine + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 0 + template: + metadata: + labels: + app: cincinnati-policy-engine + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - cincinnati-policy-engine + topologyKey: kubernetes.io/hostname + containers: + - image: ${IMAGE}:${IMAGE_TAG} + name: cincinnati-policy-engine + imagePullPolicy: Always + env: + - name: ADDRESS + valueFrom: + configMapKeyRef: + key: pe.address + name: cincinnati + - name: PE_STATUS_ADDRESS + valueFrom: + configMapKeyRef: + key: pe.status.address + name: cincinnati + - name: UPSTREAM + valueFrom: + configMapKeyRef: + key: pe.upstream + name: cincinnati + - name: PE_LOG_VERBOSITY + valueFrom: + configMapKeyRef: + key: pe.log.verbosity + name: cincinnati + - name: "PE_MANDATORY_CLIENT_PARAMETERS" + valueFrom: + configMapKeyRef: + key: pe.mandatory_client_parameters + name: cincinnati + - name: "RUST_BACKTRACE" + valueFrom: + configMapKeyRef: + key: pe.rust_backtrace + name: cincinnati + command: + - ${PE_BINARY} + args: [ + "-$(PE_LOG_VERBOSITY)", + "--service.address", "$(ADDRESS)", + "--service.mandatory_client_parameters", "$(PE_MANDATORY_CLIENT_PARAMETERS)", + "--service.path_prefix", "${PE_PATH_PREFIX}", + "--service.port", "${PE_PORT}", + "--status.address", "$(PE_STATUS_ADDRESS)", + "--status.port", "${PE_STATUS_PORT}", + "--upstream.cincinnati.url", "$(UPSTREAM)", + ] + ports: + - name: policy-engine + containerPort: ${{PE_PORT}} + - name: status-pe + containerPort: ${{PE_STATUS_PORT}} + livenessProbe: + httpGet: + path: /livez + port: ${{PE_STATUS_PORT}} + initialDelaySeconds: 300 + periodSeconds: 30 + timeoutSeconds: 3 + readinessProbe: + httpGet: + path: /readyz + port: ${{PE_STATUS_PORT}} + initialDelaySeconds: 300 + periodSeconds: 30 + timeoutSeconds: 3 + resources: + limits: + cpu: ${PE_CPU_LIMIT} + memory: ${PE_MEMORY_LIMIT} + requests: + cpu: ${PE_CPU_REQUEST} + memory: ${PE_MEMORY_REQUEST} + volumes: + - name: secrets + secret: + secretName: cincinnati-credentials + - name: configs + configMap: + name: cincinnati-configs + triggers: + - type: ConfigChange + + - apiVersion: v1 + kind: Service + metadata: + name: cincinnati-graph-builder + labels: + app: cincinnati-graph-builder + spec: + ports: + - name: graph-builder + protocol: TCP + port: ${{GB_PORT}} + targetPort: ${{GB_PORT}} + - name: status-gb + protocol: TCP + port: ${{GB_STATUS_PORT}} + targetPort: ${{GB_STATUS_PORT}} + selector: + app: cincinnati-graph-builder + - apiVersion: v1 + kind: Service + metadata: + name: cincinnati-gb-public + labels: + app: cincinnati-graph-builder + spec: + ports: + - name: gb-public + protocol: TCP + port: ${{GB_PUBLIC_PORT}} + targetPort: ${{GB_PUBLIC_PORT}} + selector: + app: cincinnati-graph-builder + - apiVersion: v1 + kind: Service + metadata: + name: cincinnati-policy-engine + labels: + app: cincinnati-policy-engine + spec: + ports: + - name: policy-engine + protocol: TCP + port: 80 + targetPort: ${{PE_PORT}} + - name: status-pe + protocol: TCP + port: ${{PE_STATUS_PORT}} + targetPort: ${{PE_STATUS_PORT}} + selector: + app: cincinnati-policy-engine + + - apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + name: cincinnati-gb-pdb + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: cincinnati-graph-builder + - apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + name: cincinnati-pe-pdb + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: cincinnati-policy-engine + + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cincinnati + data: + gb.rust_backtrace: "${RUST_BACKTRACE}" + pe.address: "0.0.0.0" + pe.status.address: "0.0.0.0" + pe.upstream: "http://localhost:8080${GB_PATH_PREFIX}/graph" + pe.log.verbosity: ${{PE_LOG_VERBOSITY}} + pe.mandatory_client_parameters: "channel" + pe.rust_backtrace: "${RUST_BACKTRACE}" + - apiVersion: v1 + kind: ConfigMap + metadata: + name: environment-secrets + data: ${{ENVIRONMENT_SECRETS}} + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cincinnati-configs + annotations: + qontract.recycle: "true" + data: + gb.toml: | + verbosity = "${GB_LOG_VERBOSITY}" + + [service] + scrape_timeout_secs = ${GB_SCRAPE_TIMEOUT_SECS} + pause_secs = ${GB_PAUSE_SECS} + path_prefix = "${GB_PATH_PREFIX}" + address = "${GB_ADDRESS}" + port = ${GB_PORT} + public_port = ${GB_PUBLIC_PORT} + + [status] + address = "${GB_STATUS_ADDRESS}" + port = ${GB_STATUS_PORT} + + ${GB_PLUGIN_SETTINGS} + +parameters: + - name: IMAGE + value: "quay.io/app-sre/cincinnati" + displayName: cincinnati image + description: cincinnati docker image. Defaults to quay.io/app-sre/cincinnati + - name: IMAGE_TAG + value: "latest" + displayName: cincinnati version + description: cincinnati version which defaults to latest + - name: GB_MEMORY_LIMIT + value: "768Mi" + displayName: "Graph-builder memory limit" + description: "Maximum amount of memory (bytes) allowed for graph-builder (default: 523Mi)" + - name: GB_CPU_LIMIT + value: "750m" + displayName: "Graph-builder CPU limit" + description: "Maximum amount of CPU (millicores) allowed for graph-builder (default: 750m)" + - name: PE_MEMORY_LIMIT + value: "1Gi" + displayName: "Policy-engine memory limit" + description: "Maximum amount of memory (bytes) allowed for policy-engine (default: 512Mi)" + - name: PE_CPU_LIMIT + value: "750m" + displayName: "Policy-engine CPU limit" + description: "Maximum amount of CPU (millicores) allowed for policy-engine (default: 750m)" + - name: GB_MEMORY_REQUEST + value: "128Mi" + displayName: "Graph-builder memory request" + description: "Requested amount of memory (bytes) allowed for graph-builder (default: 128Mi)" + - name: GB_CPU_REQUEST + value: "350m" + displayName: "Graph-builder CPU request" + description: "Requested amount of CPU (millicores) allowed for graph-builder (default: 350m)" + - name: PE_MEMORY_REQUEST + value: "128Mi" + displayName: "Policy-engine memory request" + description: "Requested amount of memory (bytes) allowed for policy-engine (default: 128Mi)" + - name: PE_CPU_REQUEST + value: "350m" + displayName: "Policy-engine CPU request" + description: "Requested amount of CPU (millicores) allowed for policy-engine (default: 350m)" + - name: GB_SCRAPE_TIMEOUT_SECS + value: "300" + displayName: Graph-builder scrape timeout in seconds + - name: GB_PAUSE_SECS + value: "300" + displayName: Seconds to pause between scrapes + - name: GB_PORT + value: "8080" + displayName: Graph builder port + - name: GB_PUBLIC_PORT + value: "8090" + displayName: Graph builder public port + - name: GB_ADDRESS + value: "0.0.0.0" + displayName: Graph builder address + - name: GB_STATUS_ADDRESS + value: "0.0.0.0" + displayName: Graph builder status address + - name: GB_STATUS_PORT + value: "9080" + displayName: Graph builder status port + - name: GB_PATH_PREFIX + value: "/api/upgrades_info" + displayName: Graph builder path prefix + - name: PE_PORT + value: "8081" + displayName: Policy engine port + - name: PE_STATUS_PORT + value: "9081" + displayName: Policy engine status port + - name: PE_PATH_PREFIX + value: "/api/upgrades_info" + displayName: Policy engine path prefix + - name: GB_LOG_VERBOSITY + value: "vvv" + displayName: Graph builder log verbosity + - name: PE_LOG_VERBOSITY + value: "vv" + displayName: Policy engine log verbosity + - name: GB_CINCINNATI_REPO + value: "openshift-release-dev/ocp-release" + displayName: Graph builder quay repo + - name: GB_BINARY + value: /usr/bin/graph-builder + displayName: Path to graph-builder binary + - name: PE_BINARY + value: /usr/bin/policy-engine + displayName: Path to policy-engine binary + - name: GB_PLUGIN_SETTINGS + displayName: Graph builder plugin settings, passed through verbatim. + value: | + [[plugin_settings]] + name = "release-scrape-dockerv2" + registry = "quay.io" + repository = "openshift-release-dev/ocp-release" + fetch_concurrency = 16 + credentials_path = "/etc/secrets/registry_credentials_docker.json" + + [[plugin_settings]] + name = "github-secondary-metadata-scrape" + github_org = "openshift" + github_repo = "cincinnati-graph-data" + reference_branch = "master" + output_directory = "/tmp/cincinnati/graph-data" + oauth_token_path = "/etc/secrets/github_token.key" + + [[plugin_settings]] + name = "openshift-secondary-metadata-parse" + + [[plugin_settings]] + name = "edge-add-remove" + - name: RUST_BACKTRACE + value: "0" + displayName: Set RUST_BACKTRACE env var + - name: GB_CONFIG_PATH + value: "/etc/configs/gb.toml" + - name: ENVIRONMENT_SECRETS + value: '{ "CINCINNATI_GITHUB_SCRAPER_OAUTH_TOKEN_PATH": "/etc/secrets/github_token.key" }' + - name: MIN_REPLICAS + value: "1" + - name: MAX_REPLICAS + value: "3" + - name: PE_REQ_AVG + value: "50" From 395e6ff3ff26471291091440d7810e58daa1cbed Mon Sep 17 00:00:00 2001 From: pratikmahajan Date: Thu, 5 Dec 2024 06:20:04 +0000 Subject: [PATCH 2/2] add keda scaler and configurable pe upstream --- dist/openshift/cincinnati-microservices.yaml | 77 ++++++++++++++------ 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/dist/openshift/cincinnati-microservices.yaml b/dist/openshift/cincinnati-microservices.yaml index 9feaa670f..6a095bb91 100644 --- a/dist/openshift/cincinnati-microservices.yaml +++ b/dist/openshift/cincinnati-microservices.yaml @@ -11,7 +11,7 @@ objects: app: cincinnati-graph-builder name: cincinnati-graph-builder spec: - replicas: ${{MAX_REPLICAS}} + replicas: ${{GB_MAX_REPLICAS}} selector: matchLabels: app: cincinnati-graph-builder @@ -52,9 +52,7 @@ objects: name: environment-secrets command: - ${GB_BINARY} - args: [ - "-c", "${GB_CONFIG_PATH}" - ] + args: ["-c", "${GB_CONFIG_PATH}"] ports: - name: graph-builder containerPort: ${{GB_PORT}} @@ -99,7 +97,7 @@ objects: name: cincinnati-configs triggers: - type: ConfigChange - + - apiVersion: apps/v1 kind: Deployment metadata: @@ -107,7 +105,7 @@ objects: app: cincinnati-policy-engine name: cincinnati-policy-engine spec: - replicas: ${{MAX_REPLICAS}} + replicas: ${{PE_MAX_REPLICAS}} selector: matchLabels: app: cincinnati-policy-engine @@ -170,16 +168,24 @@ objects: name: cincinnati command: - ${PE_BINARY} - args: [ - "-$(PE_LOG_VERBOSITY)", - "--service.address", "$(ADDRESS)", - "--service.mandatory_client_parameters", "$(PE_MANDATORY_CLIENT_PARAMETERS)", - "--service.path_prefix", "${PE_PATH_PREFIX}", - "--service.port", "${PE_PORT}", - "--status.address", "$(PE_STATUS_ADDRESS)", - "--status.port", "${PE_STATUS_PORT}", - "--upstream.cincinnati.url", "$(UPSTREAM)", - ] + args: + [ + "-$(PE_LOG_VERBOSITY)", + "--service.address", + "$(ADDRESS)", + "--service.mandatory_client_parameters", + "$(PE_MANDATORY_CLIENT_PARAMETERS)", + "--service.path_prefix", + "${PE_PATH_PREFIX}", + "--service.port", + "${PE_PORT}", + "--status.address", + "$(PE_STATUS_ADDRESS)", + "--status.port", + "${PE_STATUS_PORT}", + "--upstream.cincinnati.url", + "$(UPSTREAM)", + ] ports: - name: policy-engine containerPort: ${{PE_PORT}} @@ -215,7 +221,7 @@ objects: name: cincinnati-configs triggers: - type: ConfigChange - + - apiVersion: v1 kind: Service metadata: @@ -266,7 +272,7 @@ objects: targetPort: ${{PE_STATUS_PORT}} selector: app: cincinnati-policy-engine - + - apiVersion: policy/v1 kind: PodDisruptionBudget metadata: @@ -285,7 +291,7 @@ objects: selector: matchLabels: app: cincinnati-policy-engine - + - apiVersion: v1 kind: ConfigMap metadata: @@ -294,7 +300,7 @@ objects: gb.rust_backtrace: "${RUST_BACKTRACE}" pe.address: "0.0.0.0" pe.status.address: "0.0.0.0" - pe.upstream: "http://localhost:8080${GB_PATH_PREFIX}/graph" + pe.upstream: "${PE_UPSTREAM_URL}" pe.log.verbosity: ${{PE_LOG_VERBOSITY}} pe.mandatory_client_parameters: "channel" pe.rust_backtrace: "${RUST_BACKTRACE}" @@ -327,6 +333,25 @@ objects: ${GB_PLUGIN_SETTINGS} + - apiVersion: keda.sh/v1alpha1 + kind: ScaledObject + metadata: + name: cincinnati-pe-scaler + labels: + app: cincinnati-policy-engine + spec: + scaleTargetRef: + name: cincinnati-policy-engine + maxReplicaCount: ${{PE_MAX_REPLICAS}} + minReplicaCount: ${{PE_MIN_REPLICAS}} + triggers: + - type: prometheus + metadata: + serverAddress: http://prometheus-app-sre.openshift-customer-monitoring.svc.cluster.local:9090 + metricName: cincinnati_policy_engine_graph_incoming_requests_rate + threshold: "${PE_REQ_AVG}" + query: sum by (pod) (rate(cincinnati_pe_graph_incoming_requests_total[2m])) + parameters: - name: IMAGE value: "quay.io/app-sre/cincinnati" @@ -439,6 +464,10 @@ parameters: [[plugin_settings]] name = "edge-add-remove" + - name: PE_UPSTREAM_URL + value: "http://cincinnati-graph-builder.cincinnati.svc.cluster.local:8080${GB_PATH_PREFIX}/graph" + displayName: "Policy engine upstream URL" + description: "URL for the policy engine to connect to graph builder" - name: RUST_BACKTRACE value: "0" displayName: Set RUST_BACKTRACE env var @@ -446,9 +475,13 @@ parameters: value: "/etc/configs/gb.toml" - name: ENVIRONMENT_SECRETS value: '{ "CINCINNATI_GITHUB_SCRAPER_OAUTH_TOKEN_PATH": "/etc/secrets/github_token.key" }' - - name: MIN_REPLICAS + - name: GB_MIN_REPLICAS + value: "1" + - name: PE_MIN_REPLICAS value: "1" - - name: MAX_REPLICAS + - name: GB_MAX_REPLICAS + value: "3" + - name: PE_MAX_REPLICAS value: "3" - name: PE_REQ_AVG value: "50"