From ed92494fc13b2da110d228f2e7f770c9d0eab163 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Wed, 7 Oct 2026 14:41:05 +0200 Subject: [PATCH 1/2] [deploy] Make CockroachDB and core-service resources configurable --- NEXT_RELEASE_NOTES.md | 2 +- .../terraform-commons-dss/helm.tf | 13 ++- .../terraform-commons-dss/main.tf | 9 ++ .../terraform-commons-dss/tanka.tf | 8 ++ .../templates/main.jsonnet.tmp | 10 +- .../terraform-commons-dss/variables.gen.tf | 104 ++++++++++++++++++ .../modules/terraform-aws-dss/TFVARS.gen.md | 48 ++++++++ .../modules/terraform-aws-dss/main.tf | 8 ++ .../terraform-aws-dss/variables.gen.tf | 104 ++++++++++++++++++ .../terraform-google-dss/TFVARS.gen.md | 48 ++++++++ .../modules/terraform-google-dss/main.tf | 8 ++ .../terraform-google-dss/variables.gen.tf | 104 ++++++++++++++++++ .../definitions/core_service_cpu_limit.tf | 11 ++ .../definitions/core_service_cpu_request.tf | 11 ++ .../definitions/core_service_memory_limit.tf | 11 ++ .../core_service_memory_request.tf | 11 ++ .../utils/definitions/crdb_cpu_limit.tf | 11 ++ .../utils/definitions/crdb_cpu_request.tf | 11 ++ .../utils/definitions/crdb_memory_limit.tf | 11 ++ .../utils/definitions/crdb_memory_request.tf | 11 ++ deploy/infrastructure/utils/variables.py | 8 ++ deploy/operations/ci/aws-1/variables.gen.tf | 104 ++++++++++++++++++ .../dss/templates/dss-core-service.yaml | 21 ++++ .../helm-charts/dss/values.schema.json | 16 +++ deploy/services/tanka/cockroachdb.libsonnet | 1 + deploy/services/tanka/core-service.libsonnet | 2 + deploy/services/tanka/metadata_base.libsonnet | 10 ++ deploy/services/tanka/util.libsonnet | 10 +- docs/operations/performance.md | 28 +++++ 29 files changed, 749 insertions(+), 5 deletions(-) create mode 100644 deploy/infrastructure/utils/definitions/core_service_cpu_limit.tf create mode 100644 deploy/infrastructure/utils/definitions/core_service_cpu_request.tf create mode 100644 deploy/infrastructure/utils/definitions/core_service_memory_limit.tf create mode 100644 deploy/infrastructure/utils/definitions/core_service_memory_request.tf create mode 100644 deploy/infrastructure/utils/definitions/crdb_cpu_limit.tf create mode 100644 deploy/infrastructure/utils/definitions/crdb_cpu_request.tf create mode 100644 deploy/infrastructure/utils/definitions/crdb_memory_limit.tf create mode 100644 deploy/infrastructure/utils/definitions/crdb_memory_request.tf diff --git a/NEXT_RELEASE_NOTES.md b/NEXT_RELEASE_NOTES.md index a2770019b..0377894e8 100644 --- a/NEXT_RELEASE_NOTES.md +++ b/NEXT_RELEASE_NOTES.md @@ -53,7 +53,7 @@ The release notes should contain at least the following sections: * [terraform/aws] The Kubernetes Metrics Server is now installed as an EKS add-on (`metrics-server`), which makes `kubectl top` and the Metrics API available, as on GKE clusters. If metrics-server was already installed manually in the cluster (e.g. with Helm or `kubectl apply`), uninstall it before applying to avoid conflicts during the creation of the add-on. * [deploy] AWS NLB TLS listeners (DSS gateway and Prometheus, for both Helm and Tanka) now use the `ELBSecurityPolicy-TLS13-1-2-Res-2021-06` SSL negotiation policy, which enforces a minimum of TLS 1.2 (aligning with the Google deployment) and restricts TLS 1.2 to AEAD (GCM/ChaCha20) cipher suites. Previously the AWS Load Balancer Controller default (`ELBSecurityPolicy-2016-08`) applied, which allowed TLS 1.0 and 1.1. Clients limited to TLS 1.0 or 1.1, or to CBC cipher suites, will no longer be able to connect. Unlike the Google deployment, no resources need to be provisioned by Terraform or manually to set up the policy. * [evict] RID cleanup (`db-manager evict`, including the RID cleanup default `CronJob`) now evicts the expired ISAs and subscriptions of all the DSS instances of the pool, instead of only the ones written by the DSS instance identified by `--locality`. The `--locality` flag of `db-manager evict` is deprecated and has no effect. It will be removed in a future release. -* [deploy] The CockroachDB `--cache` and `--max-sql-memory` flags can now be configured with Terraform, Tanka and Helm. See [CPU and memory allocation](docs/operations/performance.md#cpu-and-memory-allocation). +* [deploy] CPU and memory requests and limits of the CockroachDB and core-service containers and the CockroachDB `--cache` and `--max-sql-memory` flags can now be configured with Terraform, Tanka and Helm. See [CPU and memory allocation](docs/operations/performance.md#cpu-and-memory-allocation). ## Minimal database schema version diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf index 8674c9b70..ffc00d4a5 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf @@ -31,6 +31,7 @@ resource "local_file" "helm_chart_values" { "--locality-advertise-addr=zone=${var.locality}@$(hostname -f)", "--advertise-addr=$${HOSTNAME##*-}.${var.db_hostname_suffix}" ] + resources = local.helm_crdb_resources } storage = { @@ -58,7 +59,11 @@ resource "local_file" "helm_chart_values" { } dss = { - image = var.image + image = var.image + cpuRequest = var.core_service_cpu_request + memoryRequest = var.core_service_memory_request + cpuLimit = var.core_service_cpu_limit + memoryLimit = var.core_service_memory_limit conf = { pubKeys = [ @@ -273,7 +278,11 @@ resource "local_file" "helm_chart_values" { } dss = { - image = var.image + image = var.image + cpuRequest = var.core_service_cpu_request + memoryRequest = var.core_service_memory_request + cpuLimit = var.core_service_cpu_limit + memoryLimit = var.core_service_memory_limit conf = { pubKeys = [ diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf index 642fc57ce..bbf56e6df 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf @@ -3,3 +3,12 @@ locals { # Replace ':' and '/' characters from folder name by underscores. Those characters are used by AWS for contexts. workspace_location = abspath("${path.module}/../../../../build/workspace/${local.workspace_folder}") } + +locals { + # Kubernetes container resources of CockroachDB, without the requests and limits left unset. + # Only used by Helm, since the official CockroachDB chart expects a resources map. + helm_crdb_resources = { for k, v in { + requests = { for k, v in { cpu = var.crdb_cpu_request, memory = var.crdb_memory_request } : k => v if v != "" } + limits = { for k, v in { cpu = var.crdb_cpu_limit, memory = var.crdb_memory_limit } : k => v if v != "" } + } : k => v if length(v) > 0 } +} diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf index 9c6d67a69..b88745361 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf @@ -20,10 +20,18 @@ resource "local_file" "tanka_config_main" { VAR_CRDB_EXTERNAL_NODES = join(",", [for a in var.crdb_external_nodes : "'${a}'"]) VAR_STORAGE_CLASS = var.kubernetes_storage_class VAR_DOCKER_IMAGE_NAME = var.image + VAR_CORE_SERVICE_CPU_REQUEST = var.core_service_cpu_request + VAR_CORE_SERVICE_MEMORY_REQUEST = var.core_service_memory_request + VAR_CORE_SERVICE_CPU_LIMIT = var.core_service_cpu_limit + VAR_CORE_SERVICE_MEMORY_LIMIT = var.core_service_memory_limit VAR_CRDB_DOCKER_IMAGE_NAME = "cockroachdb/cockroach:${var.crdb_image_tag}" VAR_CRDB_CLUSTER_NAME = var.crdb_cluster_name VAR_CRDB_CACHE = var.crdb_cache VAR_CRDB_MAX_SQL_MEMORY = var.crdb_max_sql_memory + VAR_CRDB_CPU_REQUEST = var.crdb_cpu_request + VAR_CRDB_MEMORY_REQUEST = var.crdb_memory_request + VAR_CRDB_CPU_LIMIT = var.crdb_cpu_limit + VAR_CRDB_MEMORY_LIMIT = var.crdb_memory_limit VAR_YUGABYTE_CLOUD = var.yugabyte_cloud VAR_YUGABYTE_REGION = var.yugabyte_region VAR_YUGABYTE_ZONE = var.yugabyte_zone diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp index 03bc78b47..73fab65d2 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp @@ -27,6 +27,10 @@ local metadata = metadataBase { clusterName: '${VAR_CRDB_CLUSTER_NAME}', cache: '${VAR_CRDB_CACHE}', maxSqlMemory: '${VAR_CRDB_MAX_SQL_MEMORY}', + cpuRequest: '${VAR_CRDB_CPU_REQUEST}', + memoryRequest: '${VAR_CRDB_MEMORY_REQUEST}', + cpuLimit: '${VAR_CRDB_CPU_LIMIT}', + memoryLimit: '${VAR_CRDB_MEMORY_LIMIT}', }, yugabyte+: { image: '${VAR_YUGABYTE_DOCKER_IMAGE_NAME}', @@ -64,7 +68,11 @@ local metadata = metadataBase { hostname: '${VAR_APP_HOSTNAME}', publicEndpoint: 'https://${VAR_APP_HOSTNAME}', dumpRequests: false, - sslPolicy: '${VAR_SSL_POLICY}' + sslPolicy: '${VAR_SSL_POLICY}', + cpuRequest: '${VAR_CORE_SERVICE_CPU_REQUEST}', + memoryRequest: '${VAR_CORE_SERVICE_MEMORY_REQUEST}', + cpuLimit: '${VAR_CORE_SERVICE_CPU_LIMIT}', + memoryLimit: '${VAR_CORE_SERVICE_MEMORY_LIMIT}', }, schema_manager+: { enable: true, diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf index d7d0af311..1ea6508b8 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf @@ -310,6 +310,110 @@ variable "crdb_max_sql_memory" { } +variable "crdb_cpu_request" { + type = string + description = <<-EOT + Kubernetes CPU request of the CockroachDB containers. + Not set by default. + + Example: `3.5` + EOT + + default = "" +} + + +variable "crdb_memory_request" { + type = string + description = <<-EOT + Kubernetes memory request of the CockroachDB containers. + Not set by default. + + Example: `14Gi` + EOT + + default = "" +} + + +variable "crdb_cpu_limit" { + type = string + description = <<-EOT + Kubernetes CPU limit of the CockroachDB containers. + Not set by default. + + Example: `3.5` + EOT + + default = "" +} + + +variable "crdb_memory_limit" { + type = string + description = <<-EOT + Kubernetes memory limit of the CockroachDB containers. + Not set by default. + + Example: `14Gi` + EOT + + default = "" +} + + +variable "core_service_cpu_request" { + type = string + description = <<-EOT + Kubernetes CPU request of the core-service containers. + Not set by default. + + Example: `0.5` + EOT + + default = "" +} + + +variable "core_service_memory_request" { + type = string + description = <<-EOT + Kubernetes memory request of the core-service containers. + Not set by default. + + Example: `2Gi` + EOT + + default = "" +} + + +variable "core_service_cpu_limit" { + type = string + description = <<-EOT + Kubernetes CPU limit of the core-service containers. + Not set by default. + + Example: `1` + EOT + + default = "" +} + + +variable "core_service_memory_limit" { + type = string + description = <<-EOT + Kubernetes memory limit of the core-service containers. + Not set by default. + + Example: `2Gi` + EOT + + default = "" +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md index df4520442..08c39dcae 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md @@ -131,6 +131,30 @@ Leave empty to disable record creation.

Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)

Example: dss-che-1

+ + core_service_cpu_limit (string) +

Kubernetes CPU limit of the core-service containers. +Not set by default.

+

Example: 1

+
Default value: "" + + core_service_cpu_request (string) +

Kubernetes CPU request of the core-service containers. +Not set by default.

+

Example: 0.5

+
Default value: "" + + core_service_memory_limit (string) +

Kubernetes memory limit of the core-service containers. +Not set by default.

+

Example: 2Gi

+
Default value: "" + + core_service_memory_request (string) +

Kubernetes memory request of the core-service containers. +Not set by default.

+

Example: 2Gi

+
Default value: "" crdb_cache (string)

Size of the CockroachDB storage engine cache, passed to the --cache flag of cockroach start. @@ -146,6 +170,18 @@ The CRDB cluster name must be 6-20 characters in length, and can include lowerca and dashes (but no leading or trailing dashes). A cluster's name cannot be edited after it is created.

Example: interuss-us-production

+ + crdb_cpu_limit (string) +

Kubernetes CPU limit of the CockroachDB containers. +Not set by default.

+

Example: 3.5

+
Default value: "" + + crdb_cpu_request (string) +

Kubernetes CPU request of the CockroachDB containers. +Not set by default.

+

Example: 3.5

+
Default value: "" crdb_external_nodes (list(string))

Fully-qualified domain name of existing CRDB nodes outside of the cluster if you are joining an existing pool. @@ -172,6 +208,18 @@ From v.17, the recommended CockroachDB version is v24.1.3.

Either a percentage of the memory available to the container or an absolute size.

Example: 25% or 2GiB


Default value: "25%" + + crdb_memory_limit (string) +

Kubernetes memory limit of the CockroachDB containers. +Not set by default.

+

Example: 14Gi

+
Default value: "" + + crdb_memory_request (string) +

Kubernetes memory request of the CockroachDB containers. +Not set by default.

+

Example: 14Gi

+
Default value: "" datastore_max_open_conns (number)

Maximum number of open connections to the datastore.

diff --git a/deploy/infrastructure/modules/terraform-aws-dss/main.tf b/deploy/infrastructure/modules/terraform-aws-dss/main.tf index 4eec91adf..e4acaebe1 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/main.tf @@ -28,6 +28,14 @@ module "terraform-commons-dss" { crdb_cluster_name = var.crdb_cluster_name crdb_cache = var.crdb_cache crdb_max_sql_memory = var.crdb_max_sql_memory + crdb_cpu_request = var.crdb_cpu_request + crdb_memory_request = var.crdb_memory_request + crdb_cpu_limit = var.crdb_cpu_limit + crdb_memory_limit = var.crdb_memory_limit + core_service_cpu_request = var.core_service_cpu_request + core_service_memory_request = var.core_service_memory_request + core_service_cpu_limit = var.core_service_cpu_limit + core_service_memory_limit = var.core_service_memory_limit db_hostname_suffix = var.db_hostname_suffix datastore_type = var.datastore_type datastore_max_open_conns = var.datastore_max_open_conns diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf index d0787d597..ec2d0daa8 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf @@ -459,6 +459,110 @@ variable "crdb_max_sql_memory" { } +variable "crdb_cpu_request" { + type = string + description = <<-EOT + Kubernetes CPU request of the CockroachDB containers. + Not set by default. + + Example: `3.5` + EOT + + default = "" +} + + +variable "crdb_memory_request" { + type = string + description = <<-EOT + Kubernetes memory request of the CockroachDB containers. + Not set by default. + + Example: `14Gi` + EOT + + default = "" +} + + +variable "crdb_cpu_limit" { + type = string + description = <<-EOT + Kubernetes CPU limit of the CockroachDB containers. + Not set by default. + + Example: `3.5` + EOT + + default = "" +} + + +variable "crdb_memory_limit" { + type = string + description = <<-EOT + Kubernetes memory limit of the CockroachDB containers. + Not set by default. + + Example: `14Gi` + EOT + + default = "" +} + + +variable "core_service_cpu_request" { + type = string + description = <<-EOT + Kubernetes CPU request of the core-service containers. + Not set by default. + + Example: `0.5` + EOT + + default = "" +} + + +variable "core_service_memory_request" { + type = string + description = <<-EOT + Kubernetes memory request of the core-service containers. + Not set by default. + + Example: `2Gi` + EOT + + default = "" +} + + +variable "core_service_cpu_limit" { + type = string + description = <<-EOT + Kubernetes CPU limit of the core-service containers. + Not set by default. + + Example: `1` + EOT + + default = "" +} + + +variable "core_service_memory_limit" { + type = string + description = <<-EOT + Kubernetes memory limit of the core-service containers. + Not set by default. + + Example: `2Gi` + EOT + + default = "" +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md index 0958eef17..c638819ad 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md @@ -76,6 +76,30 @@ Example:

Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)

Example: dss-che-1

+ + core_service_cpu_limit (string) +

Kubernetes CPU limit of the core-service containers. +Not set by default.

+

Example: 1

+
Default value: "" + + core_service_cpu_request (string) +

Kubernetes CPU request of the core-service containers. +Not set by default.

+

Example: 0.5

+
Default value: "" + + core_service_memory_limit (string) +

Kubernetes memory limit of the core-service containers. +Not set by default.

+

Example: 2Gi

+
Default value: "" + + core_service_memory_request (string) +

Kubernetes memory request of the core-service containers. +Not set by default.

+

Example: 2Gi

+
Default value: "" crdb_cache (string)

Size of the CockroachDB storage engine cache, passed to the --cache flag of cockroach start. @@ -91,6 +115,18 @@ The CRDB cluster name must be 6-20 characters in length, and can include lowerca and dashes (but no leading or trailing dashes). A cluster's name cannot be edited after it is created.

Example: interuss-us-production

+ + crdb_cpu_limit (string) +

Kubernetes CPU limit of the CockroachDB containers. +Not set by default.

+

Example: 3.5

+
Default value: "" + + crdb_cpu_request (string) +

Kubernetes CPU request of the CockroachDB containers. +Not set by default.

+

Example: 3.5

+
Default value: "" crdb_external_nodes (list(string))

Fully-qualified domain name of existing CRDB nodes outside of the cluster if you are joining an existing pool. @@ -117,6 +153,18 @@ From v.17, the recommended CockroachDB version is v24.1.3.

Either a percentage of the memory available to the container or an absolute size.

Example: 25% or 2GiB


Default value: "25%" + + crdb_memory_limit (string) +

Kubernetes memory limit of the CockroachDB containers. +Not set by default.

+

Example: 14Gi

+
Default value: "" + + crdb_memory_request (string) +

Kubernetes memory request of the CockroachDB containers. +Not set by default.

+

Example: 14Gi

+
Default value: "" datastore_max_open_conns (number)

Maximum number of open connections to the datastore.

diff --git a/deploy/infrastructure/modules/terraform-google-dss/main.tf b/deploy/infrastructure/modules/terraform-google-dss/main.tf index 8ffd534f4..9882ff858 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/main.tf @@ -27,6 +27,14 @@ module "terraform-commons-dss" { crdb_cluster_name = var.crdb_cluster_name crdb_cache = var.crdb_cache crdb_max_sql_memory = var.crdb_max_sql_memory + crdb_cpu_request = var.crdb_cpu_request + crdb_memory_request = var.crdb_memory_request + crdb_cpu_limit = var.crdb_cpu_limit + crdb_memory_limit = var.crdb_memory_limit + core_service_cpu_request = var.core_service_cpu_request + core_service_memory_request = var.core_service_memory_request + core_service_cpu_limit = var.core_service_cpu_limit + core_service_memory_limit = var.core_service_memory_limit db_hostname_suffix = var.db_hostname_suffix datastore_type = var.datastore_type datastore_max_open_conns = var.datastore_max_open_conns diff --git a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf index 543ec91c6..e2776c298 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf @@ -426,6 +426,110 @@ variable "crdb_max_sql_memory" { } +variable "crdb_cpu_request" { + type = string + description = <<-EOT + Kubernetes CPU request of the CockroachDB containers. + Not set by default. + + Example: `3.5` + EOT + + default = "" +} + + +variable "crdb_memory_request" { + type = string + description = <<-EOT + Kubernetes memory request of the CockroachDB containers. + Not set by default. + + Example: `14Gi` + EOT + + default = "" +} + + +variable "crdb_cpu_limit" { + type = string + description = <<-EOT + Kubernetes CPU limit of the CockroachDB containers. + Not set by default. + + Example: `3.5` + EOT + + default = "" +} + + +variable "crdb_memory_limit" { + type = string + description = <<-EOT + Kubernetes memory limit of the CockroachDB containers. + Not set by default. + + Example: `14Gi` + EOT + + default = "" +} + + +variable "core_service_cpu_request" { + type = string + description = <<-EOT + Kubernetes CPU request of the core-service containers. + Not set by default. + + Example: `0.5` + EOT + + default = "" +} + + +variable "core_service_memory_request" { + type = string + description = <<-EOT + Kubernetes memory request of the core-service containers. + Not set by default. + + Example: `2Gi` + EOT + + default = "" +} + + +variable "core_service_cpu_limit" { + type = string + description = <<-EOT + Kubernetes CPU limit of the core-service containers. + Not set by default. + + Example: `1` + EOT + + default = "" +} + + +variable "core_service_memory_limit" { + type = string + description = <<-EOT + Kubernetes memory limit of the core-service containers. + Not set by default. + + Example: `2Gi` + EOT + + default = "" +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/infrastructure/utils/definitions/core_service_cpu_limit.tf b/deploy/infrastructure/utils/definitions/core_service_cpu_limit.tf new file mode 100644 index 000000000..902f04e89 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/core_service_cpu_limit.tf @@ -0,0 +1,11 @@ +variable "core_service_cpu_limit" { + type = string + description = <<-EOT + Kubernetes CPU limit of the core-service containers. + Not set by default. + + Example: `1` + EOT + + default = "" +} diff --git a/deploy/infrastructure/utils/definitions/core_service_cpu_request.tf b/deploy/infrastructure/utils/definitions/core_service_cpu_request.tf new file mode 100644 index 000000000..5d40f7da2 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/core_service_cpu_request.tf @@ -0,0 +1,11 @@ +variable "core_service_cpu_request" { + type = string + description = <<-EOT + Kubernetes CPU request of the core-service containers. + Not set by default. + + Example: `0.5` + EOT + + default = "" +} diff --git a/deploy/infrastructure/utils/definitions/core_service_memory_limit.tf b/deploy/infrastructure/utils/definitions/core_service_memory_limit.tf new file mode 100644 index 000000000..5f163655e --- /dev/null +++ b/deploy/infrastructure/utils/definitions/core_service_memory_limit.tf @@ -0,0 +1,11 @@ +variable "core_service_memory_limit" { + type = string + description = <<-EOT + Kubernetes memory limit of the core-service containers. + Not set by default. + + Example: `2Gi` + EOT + + default = "" +} diff --git a/deploy/infrastructure/utils/definitions/core_service_memory_request.tf b/deploy/infrastructure/utils/definitions/core_service_memory_request.tf new file mode 100644 index 000000000..2238f0f96 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/core_service_memory_request.tf @@ -0,0 +1,11 @@ +variable "core_service_memory_request" { + type = string + description = <<-EOT + Kubernetes memory request of the core-service containers. + Not set by default. + + Example: `2Gi` + EOT + + default = "" +} diff --git a/deploy/infrastructure/utils/definitions/crdb_cpu_limit.tf b/deploy/infrastructure/utils/definitions/crdb_cpu_limit.tf new file mode 100644 index 000000000..255f336c7 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/crdb_cpu_limit.tf @@ -0,0 +1,11 @@ +variable "crdb_cpu_limit" { + type = string + description = <<-EOT + Kubernetes CPU limit of the CockroachDB containers. + Not set by default. + + Example: `3.5` + EOT + + default = "" +} diff --git a/deploy/infrastructure/utils/definitions/crdb_cpu_request.tf b/deploy/infrastructure/utils/definitions/crdb_cpu_request.tf new file mode 100644 index 000000000..9ea9f2f5b --- /dev/null +++ b/deploy/infrastructure/utils/definitions/crdb_cpu_request.tf @@ -0,0 +1,11 @@ +variable "crdb_cpu_request" { + type = string + description = <<-EOT + Kubernetes CPU request of the CockroachDB containers. + Not set by default. + + Example: `3.5` + EOT + + default = "" +} diff --git a/deploy/infrastructure/utils/definitions/crdb_memory_limit.tf b/deploy/infrastructure/utils/definitions/crdb_memory_limit.tf new file mode 100644 index 000000000..1fcfe8af6 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/crdb_memory_limit.tf @@ -0,0 +1,11 @@ +variable "crdb_memory_limit" { + type = string + description = <<-EOT + Kubernetes memory limit of the CockroachDB containers. + Not set by default. + + Example: `14Gi` + EOT + + default = "" +} diff --git a/deploy/infrastructure/utils/definitions/crdb_memory_request.tf b/deploy/infrastructure/utils/definitions/crdb_memory_request.tf new file mode 100644 index 000000000..106f6b17a --- /dev/null +++ b/deploy/infrastructure/utils/definitions/crdb_memory_request.tf @@ -0,0 +1,11 @@ +variable "crdb_memory_request" { + type = string + description = <<-EOT + Kubernetes memory request of the CockroachDB containers. + Not set by default. + + Example: `14Gi` + EOT + + default = "" +} diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 1fd84edb9..f3b903b57 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -50,6 +50,14 @@ "crdb_cluster_name", "crdb_cache", "crdb_max_sql_memory", + "crdb_cpu_request", + "crdb_memory_request", + "crdb_cpu_limit", + "crdb_memory_limit", + "core_service_cpu_request", + "core_service_memory_request", + "core_service_cpu_limit", + "core_service_memory_limit", "locality", "datastore_max_open_conns", "crdb_external_nodes", diff --git a/deploy/operations/ci/aws-1/variables.gen.tf b/deploy/operations/ci/aws-1/variables.gen.tf index d0787d597..ec2d0daa8 100644 --- a/deploy/operations/ci/aws-1/variables.gen.tf +++ b/deploy/operations/ci/aws-1/variables.gen.tf @@ -459,6 +459,110 @@ variable "crdb_max_sql_memory" { } +variable "crdb_cpu_request" { + type = string + description = <<-EOT + Kubernetes CPU request of the CockroachDB containers. + Not set by default. + + Example: `3.5` + EOT + + default = "" +} + + +variable "crdb_memory_request" { + type = string + description = <<-EOT + Kubernetes memory request of the CockroachDB containers. + Not set by default. + + Example: `14Gi` + EOT + + default = "" +} + + +variable "crdb_cpu_limit" { + type = string + description = <<-EOT + Kubernetes CPU limit of the CockroachDB containers. + Not set by default. + + Example: `3.5` + EOT + + default = "" +} + + +variable "crdb_memory_limit" { + type = string + description = <<-EOT + Kubernetes memory limit of the CockroachDB containers. + Not set by default. + + Example: `14Gi` + EOT + + default = "" +} + + +variable "core_service_cpu_request" { + type = string + description = <<-EOT + Kubernetes CPU request of the core-service containers. + Not set by default. + + Example: `0.5` + EOT + + default = "" +} + + +variable "core_service_memory_request" { + type = string + description = <<-EOT + Kubernetes memory request of the core-service containers. + Not set by default. + + Example: `2Gi` + EOT + + default = "" +} + + +variable "core_service_cpu_limit" { + type = string + description = <<-EOT + Kubernetes CPU limit of the core-service containers. + Not set by default. + + Example: `1` + EOT + + default = "" +} + + +variable "core_service_memory_limit" { + type = string + description = <<-EOT + Kubernetes memory limit of the core-service containers. + Not set by default. + + Example: `2Gi` + EOT + + default = "" +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 9902a46ff..f4bc2e59e 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -92,6 +92,27 @@ spec: {{- end }} command: - core-service + {{- if or $dss.cpuRequest $dss.memoryRequest $dss.cpuLimit $dss.memoryLimit }} + resources: + {{- if or $dss.cpuRequest $dss.memoryRequest }} + requests: + {{- with $dss.cpuRequest }} + cpu: {{ . | quote }} + {{- end }} + {{- with $dss.memoryRequest }} + memory: {{ . | quote }} + {{- end }} + {{- end }} + {{- if or $dss.cpuLimit $dss.memoryLimit }} + limits: + {{- with $dss.cpuLimit }} + cpu: {{ . | quote }} + {{- end }} + {{- with $dss.memoryLimit }} + memory: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} image: {{ $dss.image }} imagePullPolicy: {{ $dss.imagePullPolicy | default "Always" }} name: core-service diff --git a/deploy/services/helm-charts/dss/values.schema.json b/deploy/services/helm-charts/dss/values.schema.json index 5e9fa6401..fd0afa4d9 100644 --- a/deploy/services/helm-charts/dss/values.schema.json +++ b/deploy/services/helm-charts/dss/values.schema.json @@ -242,6 +242,22 @@ "type": "string", "description": "Image of the DSS. Please note that the usage of the `latest` tag is discouraged to prevent accidental upgrades in case of restart. Example: `docker.io/interuss/dss:v0.15.0`. Official image releases: https://hub.docker.com/r/interuss/dss/tags" }, + "cpuRequest": { + "type": "string", + "description": "Kubernetes CPU request of the core-service containers. Not set if empty. Example: `0.5`" + }, + "memoryRequest": { + "type": "string", + "description": "Kubernetes memory request of the core-service containers. Not set if empty. Example: `2Gi`" + }, + "cpuLimit": { + "type": "string", + "description": "Kubernetes CPU limit of the core-service containers. Not set if empty. Example: `1`" + }, + "memoryLimit": { + "type": "string", + "description": "Kubernetes memory limit of the core-service containers. Not set if empty. Example: `2Gi`" + }, "conf": { "type": "object", "properties": { diff --git a/deploy/services/tanka/cockroachdb.libsonnet b/deploy/services/tanka/cockroachdb.libsonnet index 6788e3d47..2bcd6f544 100644 --- a/deploy/services/tanka/cockroachdb.libsonnet +++ b/deploy/services/tanka/cockroachdb.libsonnet @@ -55,6 +55,7 @@ local volumes = import 'volumes.libsonnet'; value: 'kubernetes-multiregion', }, ], + resources: util.resources(metadata.cockroach), livenessProbe: { httpGet: { path: '/health', diff --git a/deploy/services/tanka/core-service.libsonnet b/deploy/services/tanka/core-service.libsonnet index fd3bf56a5..8fd0fe0b2 100644 --- a/deploy/services/tanka/core-service.libsonnet +++ b/deploy/services/tanka/core-service.libsonnet @@ -1,4 +1,5 @@ local base = import 'base.libsonnet'; +local util = import 'util.libsonnet'; local volumes = import 'volumes.libsonnet'; local datastoreparameters = import 'datastoreparameters.libsonnet'; @@ -118,6 +119,7 @@ local awsLoadBalancer(metadata) = base.AWSLoadBalancerWithManagedCert(metadata, }, ] else [], volumeMounts: volumes.all(metadata).backendMounts, + resources: util.resources(metadata.backend), command: ['core-service'], args_:: { addr: ':' + metadata.backend.port, diff --git a/deploy/services/tanka/metadata_base.libsonnet b/deploy/services/tanka/metadata_base.libsonnet index 1f9d1ca76..b13c1e2b7 100644 --- a/deploy/services/tanka/metadata_base.libsonnet +++ b/deploy/services/tanka/metadata_base.libsonnet @@ -27,6 +27,11 @@ clusterName: '', cache: '25%', // Value of the --cache flag of `cockroach start` maxSqlMemory: '25%', // Value of the --max-sql-memory flag of `cockroach start` + // Kubernetes container resources, e.g. '3.5' CPU and '14Gi' memory. Not set if empty. + cpuRequest: '', + memoryRequest: '', + cpuLimit: '', + memoryLimit: '', }, yugabyte: { image: error 'must specify yugabyte db image', @@ -69,6 +74,11 @@ dumpRequests: false, certName: if $.cloud_provider == "aws" then error 'must specify certName for AWS cloud provider', # Only used by AWS sslPolicy: '', # SSL Policy Name. Only used by Google Cloud. + // Kubernetes container resources, e.g. '0.5' CPU and '2Gi' memory. Not set if empty. + cpuRequest: '', + memoryRequest: '', + cpuLimit: '', + memoryLimit: '', }, alert: { enable: false, diff --git a/deploy/services/tanka/util.libsonnet b/deploy/services/tanka/util.libsonnet index 6ce375fb6..838551529 100644 --- a/deploy/services/tanka/util.libsonnet +++ b/deploy/services/tanka/util.libsonnet @@ -4,5 +4,13 @@ objectItems(o): [[k, o[k]] for k in std.objectFields(o)], filter(o, fields): { [field]: o[field] for field in std.setInter(std.objectFields(o), fields) }, exclude(o, fields): { [field]: o[field] for field in std.setDiff(std.objectFields(o), fields) }, - makeArgs(o): ['--%s=%s' % kv for kv in $.objectItems(o)] + makeArgs(o): ['--%s=%s' % kv for kv in $.objectItems(o)], + // Kubernetes container resources from the cpuRequest, memoryRequest, cpuLimit and memoryLimit fields of o, without the empty ones. + resources(o): + local requests = { [r]: o[r + 'Request'] for r in ['cpu', 'memory'] if o[r + 'Request'] != '' }; + local limits = { [r]: o[r + 'Limit'] for r in ['cpu', 'memory'] if o[r + 'Limit'] != '' }; + { + [if requests != {} then 'requests']: requests, + [if limits != {} then 'limits']: limits, + }, } \ No newline at end of file diff --git a/docs/operations/performance.md b/docs/operations/performance.md index bb06ccda4..43cf5f924 100644 --- a/docs/operations/performance.md +++ b/docs/operations/performance.md @@ -8,8 +8,12 @@ See the detailed [section about cleanup](cleanup.md). ## CPU and memory allocation +By default, the deployment tooling does not set any Kubernetes CPU or memory requests or limits on the CockroachDB and core-service containers. +Kubernetes may then place them on nodes without enough free memory and kill them when the node runs out of memory (see issue [#1731](https://github.com/interuss/dss/issues/1731)). + The following settings are available: +* CPU and memory requests and limits of the CockroachDB and core-service containers. * CockroachDB `--cache` and `--max-sql-memory` flags (both `25%` by default). See [CockroachDB recommendations](https://www.cockroachlabs.com/docs/stable/recommended-production-settings#cache-and-sql-memory-size) before changing them. To set them: @@ -19,7 +23,11 @@ To set them: === "Terraform" Set the following variables (see `TFVARS.gen.md` for details): + * `crdb_cpu_request`, `crdb_memory_request`, `crdb_cpu_limit` and `crdb_memory_limit`, e.g. `"3.5"` and `"14Gi"` * `crdb_cache` and `crdb_max_sql_memory`, e.g. `"25%"` + * `core_service_cpu_request`, `core_service_memory_request`, `core_service_cpu_limit` and `core_service_memory_limit`, e.g. `"0.5"` and `"2Gi"` + + CPU and memory requests and limits left empty are not set on the containers. Then run `terraform apply` to regenerate the Tanka and Helm configuration. @@ -28,11 +36,22 @@ To set them: ```jsonnet cockroach+: { + cpuRequest: '3.5', + memoryRequest: '14Gi', + cpuLimit: '3.5', + memoryLimit: '14Gi', cache: '25%', maxSqlMemory: '25%', }, + backend+: { + cpuRequest: '0.5', + memoryRequest: '2Gi', + memoryLimit: '2Gi', + }, ``` + CPU and memory requests and limits left empty are not set on the containers. + === "Helm" Set the following values: @@ -41,8 +60,17 @@ To set them: conf: cache: 25% max-sql-memory: 25% + statefulset: + resources: {requests: {cpu: "3.5", memory: 14Gi}, limits: {cpu: "3.5", memory: 14Gi}} + dss: + cpuRequest: "0.5" + memoryRequest: 2Gi + memoryLimit: 2Gi ``` + CockroachDB resources are set with `statefulset.resources` of the [official CockroachDB chart](https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/values.yaml). + Core-service CPU and memory requests and limits left empty are not set on the containers. + ## The SCD global lock option !!! danger From 2abb0686e111145438b083c8af70260eec24f52f Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Fri, 23 Oct 2026 13:37:47 +0200 Subject: [PATCH 2/2] address comments --- .../terraform-commons-dss/variables.gen.tf | 2 +- .../modules/terraform-aws-dss/TFVARS.gen.md | 2 +- .../modules/terraform-aws-dss/variables.gen.tf | 2 +- .../modules/terraform-google-dss/TFVARS.gen.md | 2 +- .../modules/terraform-google-dss/variables.gen.tf | 2 +- .../utils/definitions/core_service_cpu_limit.tf | 2 +- deploy/operations/ci/aws-1/variables.gen.tf | 2 +- deploy/services/helm-charts/dss/values.example.yaml | 2 +- deploy/services/helm-charts/dss/values.schema.json | 2 +- deploy/services/helm-charts/dss/values.yaml | 2 +- docs/operations/performance.md | 12 ++++++++++-- 11 files changed, 20 insertions(+), 12 deletions(-) diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf index 1ea6508b8..7fad3d7d1 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf @@ -394,7 +394,7 @@ variable "core_service_cpu_limit" { Kubernetes CPU limit of the core-service containers. Not set by default. - Example: `1` + Example: `0.5` EOT default = "" diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md index 08c39dcae..b570d05a7 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md @@ -135,7 +135,7 @@ Leave empty to disable record creation.

core_service_cpu_limit (string)

Kubernetes CPU limit of the core-service containers. Not set by default.

-

Example: 1

+

Example: 0.5


Default value: "" core_service_cpu_request (string) diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf index ec2d0daa8..a550344f3 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf @@ -543,7 +543,7 @@ variable "core_service_cpu_limit" { Kubernetes CPU limit of the core-service containers. Not set by default. - Example: `1` + Example: `0.5` EOT default = "" diff --git a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md index c638819ad..6bc238bd2 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md @@ -80,7 +80,7 @@ Example: core_service_cpu_limit (string)

Kubernetes CPU limit of the core-service containers. Not set by default.

-

Example: 1

+

Example: 0.5


Default value: "" core_service_cpu_request (string) diff --git a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf index e2776c298..bff30203f 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf @@ -510,7 +510,7 @@ variable "core_service_cpu_limit" { Kubernetes CPU limit of the core-service containers. Not set by default. - Example: `1` + Example: `0.5` EOT default = "" diff --git a/deploy/infrastructure/utils/definitions/core_service_cpu_limit.tf b/deploy/infrastructure/utils/definitions/core_service_cpu_limit.tf index 902f04e89..807e57710 100644 --- a/deploy/infrastructure/utils/definitions/core_service_cpu_limit.tf +++ b/deploy/infrastructure/utils/definitions/core_service_cpu_limit.tf @@ -4,7 +4,7 @@ variable "core_service_cpu_limit" { Kubernetes CPU limit of the core-service containers. Not set by default. - Example: `1` + Example: `0.5` EOT default = "" diff --git a/deploy/operations/ci/aws-1/variables.gen.tf b/deploy/operations/ci/aws-1/variables.gen.tf index ec2d0daa8..a550344f3 100644 --- a/deploy/operations/ci/aws-1/variables.gen.tf +++ b/deploy/operations/ci/aws-1/variables.gen.tf @@ -543,7 +543,7 @@ variable "core_service_cpu_limit" { Kubernetes CPU limit of the core-service containers. Not set by default. - Example: `1` + Example: `0.5` EOT default = "" diff --git a/deploy/services/helm-charts/dss/values.example.yaml b/deploy/services/helm-charts/dss/values.example.yaml index 981286a16..313b744ad 100644 --- a/deploy/services/helm-charts/dss/values.example.yaml +++ b/deploy/services/helm-charts/dss/values.example.yaml @@ -22,7 +22,7 @@ dss: cockroachdb: enabled: true - # See https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/values.yaml + # See https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb-legacy/values.yaml image: tag: v24.1.3 fullnameOverride: dss-cockroachdb diff --git a/deploy/services/helm-charts/dss/values.schema.json b/deploy/services/helm-charts/dss/values.schema.json index fd0afa4d9..1e115bde9 100644 --- a/deploy/services/helm-charts/dss/values.schema.json +++ b/deploy/services/helm-charts/dss/values.schema.json @@ -252,7 +252,7 @@ }, "cpuLimit": { "type": "string", - "description": "Kubernetes CPU limit of the core-service containers. Not set if empty. Example: `1`" + "description": "Kubernetes CPU limit of the core-service containers. Not set if empty. Example: `0.5`" }, "memoryLimit": { "type": "string", diff --git a/deploy/services/helm-charts/dss/values.yaml b/deploy/services/helm-charts/dss/values.yaml index e9f72b9ad..ec338dcdd 100644 --- a/deploy/services/helm-charts/dss/values.yaml +++ b/deploy/services/helm-charts/dss/values.yaml @@ -24,7 +24,7 @@ dss: cockroachdb: enabled: true - # See https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/values.yaml + # See https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb-legacy/values.yaml image: repository: cockroachdb/cockroach tls: diff --git a/docs/operations/performance.md b/docs/operations/performance.md index 43cf5f924..d0ef79316 100644 --- a/docs/operations/performance.md +++ b/docs/operations/performance.md @@ -46,6 +46,7 @@ To set them: backend+: { cpuRequest: '0.5', memoryRequest: '2Gi', + cpuLimit: '0.5', memoryLimit: '2Gi', }, ``` @@ -61,14 +62,21 @@ To set them: cache: 25% max-sql-memory: 25% statefulset: - resources: {requests: {cpu: "3.5", memory: 14Gi}, limits: {cpu: "3.5", memory: 14Gi}} + resources: + requests: + cpu: "3.5" + memory: 14Gi + limits: + cpu: "3.5" + memory: 14Gi dss: cpuRequest: "0.5" memoryRequest: 2Gi + cpuLimit: "0.5" memoryLimit: 2Gi ``` - CockroachDB resources are set with `statefulset.resources` of the [official CockroachDB chart](https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/values.yaml). + CockroachDB resources are set with `statefulset.resources` of the [official CockroachDB chart](https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb-legacy/values.yaml). Core-service CPU and memory requests and limits left empty are not set on the containers. ## The SCD global lock option