From dc7a6263620189d0c10a1f5c08368178d08ce33b Mon Sep 17 00:00:00 2001 From: Miretpl Date: Fri, 13 Feb 2026 23:45:39 +0100 Subject: [PATCH 1/2] Add workers.celery.resources & workers.kubernetes.resources --- .../pod-template-file.kubernetes-helm-yaml | 2 +- chart/templates/NOTES.txt | 8 ++++ chart/values.schema.json | 38 ++++++++++++++- chart/values.yaml | 19 ++++++++ .../airflow_aux/test_pod_template_file.py | 26 +++++----- .../helm_tests/airflow_core/test_worker.py | 48 ++++++++----------- .../airflow_core/test_worker_sets.py | 16 +++++++ 7 files changed, 113 insertions(+), 44 deletions(-) diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index 4fa1413037ed9..96822d5ce8cf2 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -131,7 +131,7 @@ spec: {{- if or .Values.workers.kubernetes.command .Values.workers.command }} command: {{ tpl (toYaml (.Values.workers.kubernetes.command | default .Values.workers.command)) . | nindent 8 }} {{- end }} - resources: {{- toYaml .Values.workers.resources | nindent 8 }} + resources: {{- toYaml (.Values.workers.kubernetes.resources | default .Values.workers.resources) | nindent 8 }} volumeMounts: - mountPath: {{ template "airflow_logs" . }} name: logs diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index 79d967919dd9b..6f80453e8a12d 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -493,6 +493,14 @@ DEPRECATION WARNING: {{- end }} +{{- if not (empty .Values.workers.resources) }} + + DEPRECATION WARNING: + `workers.resources` has been renamed to `workers.celery.resources`/`workers.kubernetes.resources`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + {{- if not (empty .Values.webserver.defaultUser) }} DEPRECATION WARNING: diff --git a/chart/values.schema.json b/chart/values.schema.json index 74d931666da40..7cef96287b875 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -2264,7 +2264,7 @@ } }, "resources": { - "description": "Resource configuration for Airflow Celery workers and pods created with pod-template-file.", + "description": "Resource configuration for Airflow Celery workers and pods created with pod-template-file (deprecated, use `workers.celery.resources` or/and `workers.kubernetes.resources` instead).", "type": "object", "default": {}, "examples": [ @@ -3257,6 +3257,24 @@ } } } + }, + "resources": { + "description": "Resource configuration for Airflow Celery workers.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" } } }, @@ -3512,6 +3530,24 @@ } } } + }, + "resources": { + "description": "Resource configuration for pods created with pod-template-file.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" } } } diff --git a/chart/values.yaml b/chart/values.yaml index 255db1700bc83..2f04bd6888038 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -898,6 +898,7 @@ workers: containerLifecycleHooks: {} # Resource configuration for Airflow Celery workers and pods created with pod-template-file + # (deprecated, use `workers.celery.resources` or/and `workers.kubernetes.resources` instead) resources: {} # limits: # cpu: 100m @@ -1246,6 +1247,15 @@ workers: # Container level lifecycle hooks containerLifecycleHooks: {} + # Resource configuration for Airflow Celery workers + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + kubernetes: # Command to use in pod-template-file (templated) command: ~ @@ -1301,6 +1311,15 @@ workers: # Container level lifecycle hooks containerLifecycleHooks: {} + # Resource configuration for pods created with pod-template-file + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + # Airflow scheduler settings scheduler: enabled: true diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py index a1f0f03aa1dbe..1ed5a2ad09e67 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py @@ -1015,25 +1015,25 @@ def test_should_add_component_specific_labels(self): assert "test_label" in jmespath.search("metadata.labels", docs[0]) assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" - def test_should_add_resources(self): - docs = render_chart( - values={ - "workers": { - "resources": { - "requests": {"memory": "2Gi", "cpu": "1"}, - "limits": {"memory": "3Gi", "cpu": "2"}, - } - } + @pytest.mark.parametrize( + "workers_values", + [ + {"resources": {"requests": {"memory": "2Gi", "cpu": "1"}}}, + {"kubernetes": {"resources": {"requests": {"memory": "2Gi", "cpu": "1"}}}}, + { + "resources": {"limits": {"memory": "1Gi", "cpu": "2"}}, + "kubernetes": {"resources": {"requests": {"memory": "2Gi", "cpu": "1"}}}, }, + ], + ) + def test_should_add_resources(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) assert jmespath.search("spec.containers[0].resources", docs[0]) == { - "limits": { - "cpu": "2", - "memory": "3Gi", - }, "requests": { "cpu": "1", "memory": "2Gi", diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py b/helm-tests/tests/helm_tests/airflow_core/test_worker.py index beb0cdf4435b0..4a9e611b92ce0 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py @@ -821,41 +821,31 @@ def test_logs_persistence_changes_volume(self, log_values, expected_volume): assert {"name": "logs", **expected_volume} in jmespath.search("spec.template.spec.volumes", docs[0]) - def test_worker_resources_are_configurable(self): - docs = render_chart( - values={ - "workers": { - "resources": { - "limits": {"cpu": "200m", "memory": "128Mi"}, - "requests": {"cpu": "300m", "memory": "169Mi"}, - } - }, + @pytest.mark.parametrize( + "workers_values", + [ + {"resources": {"requests": {"cpu": "300m", "memory": "169Mi"}}}, + {"celery": {"resources": {"requests": {"cpu": "300m", "memory": "169Mi"}}}}, + { + "resources": {"limits": {"cpu": "100m", "memory": "128Mi"}}, + "celery": {"resources": {"requests": {"cpu": "300m", "memory": "169Mi"}}}, }, + ], + ) + def test_worker_resources_are_configurable(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) # main container - assert jmespath.search("spec.template.spec.containers[0].resources.limits.memory", docs[0]) == "128Mi" - assert jmespath.search("spec.template.spec.containers[0].resources.limits.cpu", docs[0]) == "200m" - - assert ( - jmespath.search("spec.template.spec.containers[0].resources.requests.memory", docs[0]) == "169Mi" - ) - assert jmespath.search("spec.template.spec.containers[0].resources.requests.cpu", docs[0]) == "300m" + assert jmespath.search("spec.template.spec.containers[0].resources", docs[0]) == { + "requests": {"cpu": "300m", "memory": "169Mi"} + } # initContainer wait-for-airflow-configurations - assert ( - jmespath.search("spec.template.spec.initContainers[0].resources.limits.memory", docs[0]) - == "128Mi" - ) - assert jmespath.search("spec.template.spec.initContainers[0].resources.limits.cpu", docs[0]) == "200m" - - assert ( - jmespath.search("spec.template.spec.initContainers[0].resources.requests.memory", docs[0]) - == "169Mi" - ) - assert ( - jmespath.search("spec.template.spec.initContainers[0].resources.requests.cpu", docs[0]) == "300m" - ) + assert jmespath.search("spec.template.spec.initContainers[0].resources", docs[0]) == { + "requests": {"cpu": "300m", "memory": "169Mi"} + } def test_worker_resources_are_not_added_by_default(self): docs = render_chart( diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py index 530a9bd01cee7..d9a6aeb5f89f3 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py @@ -2149,6 +2149,22 @@ def test_overwrite_kerberos_sidecar_container_lifecycle_hooks(self, values): ], }, }, + { + "celery": { + "resources": { + "requests": {"cpu": "10m", "memory": "20Mi"}, + }, + "enableDefault": False, + "sets": [ + { + "name": "test", + "resources": { + "limits": {"cpu": "3m", "memory": "4Mi"}, + }, + } + ], + }, + }, ], ) def test_overwrite_resources(self, values): From 3c3aae67f09c197fc8d17e1958557976280d8415 Mon Sep 17 00:00:00 2001 From: Miretpl Date: Fri, 13 Feb 2026 23:48:54 +0100 Subject: [PATCH 2/2] Add newsfragment --- chart/newsfragments/61890.significant.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 chart/newsfragments/61890.significant.rst diff --git a/chart/newsfragments/61890.significant.rst b/chart/newsfragments/61890.significant.rst new file mode 100644 index 0000000000000..e8208570c2af8 --- /dev/null +++ b/chart/newsfragments/61890.significant.rst @@ -0,0 +1 @@ +``workers.resources`` section is now deprecated in favor of ``workers.celery.resources`` and ``workers.kubernetes.resources``. Please update your configuration accordingly.