diff --git a/Dockerfile b/Dockerfile index 0ac9d21c318db..7b46082a7b480 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1641,22 +1641,50 @@ set -euo pipefail readonly DIRECTORY="${AIRFLOW_HOME:-/usr/local/airflow}" readonly RETENTION="${AIRFLOW__LOG_RETENTION_DAYS:-15}" readonly FREQUENCY="${AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES:-15}" +readonly MAX_PERCENT="${AIRFLOW__LOG_MAX_SIZE_PERCENT:-0}" trap "exit" INT TERM +MAX_SIZE_BYTES="${AIRFLOW__LOG_MAX_SIZE_BYTES:-0}" +if [[ "$MAX_SIZE_BYTES" -eq 0 && "$MAX_PERCENT" -gt 0 ]]; then + total_space=$(df -k "${DIRECTORY}"/logs 2>/dev/null | tail -1 | awk '{print $2}' || echo "0") + MAX_SIZE_BYTES=$(( total_space * 1024 * MAX_PERCENT / 100 )) + echo "Computed MAX_SIZE_BYTES from ${MAX_PERCENT}% of disk: ${MAX_SIZE_BYTES} bytes" +fi + +readonly MAX_SIZE_BYTES + readonly EVERY=$((FREQUENCY*60)) echo "Cleaning logs every $EVERY seconds" +if [[ "$MAX_SIZE_BYTES" -gt 0 ]]; then + echo "Max log size limit: $MAX_SIZE_BYTES bytes" +fi + +retention_days="${RETENTION}" while true; do - echo "Trimming airflow logs to ${RETENTION} days." + echo "Trimming airflow logs to ${retention_days} days." find "${DIRECTORY}"/logs \ -type d -name 'lost+found' -prune -o \ - -type f -mtime +"${RETENTION}" -name '*.log' -print0 | \ + -type f -mtime +"${retention_days}" -name '*.log' -print0 | \ xargs -0 rm -f || true + if [[ "$MAX_SIZE_BYTES" -gt 0 && "$retention_days" -ge 0 ]]; then + current_size=$(df -k "${DIRECTORY}"/logs 2>/dev/null | tail -1 | awk '{print $3}' || echo "0") + current_size=$(( current_size * 1024 )) + + if [[ "$current_size" -gt "$MAX_SIZE_BYTES" ]]; then + retention_days=$((retention_days - 1)) + echo "Size ($current_size bytes) exceeds limit ($MAX_SIZE_BYTES bytes). Reducing retention to ${retention_days} days." + continue + fi + fi + find "${DIRECTORY}"/logs -type d -empty -delete || true + retention_days="${RETENTION}" + seconds=$(( $(date -u +%s) % EVERY)) (( seconds < 1 )) || sleep $((EVERY - seconds - 1)) sleep 1 diff --git a/airflow-core/tests/unit/charts/log_groomer.py b/airflow-core/tests/unit/charts/log_groomer.py index fa66aed10e935..b516acd6210ad 100644 --- a/airflow-core/tests/unit/charts/log_groomer.py +++ b/airflow-core/tests/unit/charts/log_groomer.py @@ -218,6 +218,76 @@ def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes, freque else: assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2 + @pytest.mark.parametrize( + ("max_size_bytes", "max_size_result"), [(None, None), (1234567890, "1234567890")] + ) + def test_log_groomer_max_size_bytes_overrides(self, max_size_bytes, max_size_result): + if self.obj_name == "dag-processor": + values = { + "dagProcessor": { + "enabled": True, + "logGroomerSidecar": {"maxSizeBytes": max_size_bytes}, + } + } + else: + values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizeBytes": max_size_bytes}}} + + docs = render_chart( + values=values, + show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], + ) + + if max_size_result: + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_BYTES'].value | [0]", + docs[0], + ) + == max_size_result + ) + else: + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_BYTES'].value | [0]", + docs[0], + ) + is None + ) + + @pytest.mark.parametrize(("max_size_percent", "max_size_result"), [(None, None), (80, "80")]) + def test_log_groomer_max_size_percent_overrides(self, max_size_percent, max_size_result): + if self.obj_name == "dag-processor": + values = { + "dagProcessor": { + "enabled": True, + "logGroomerSidecar": {"maxSizePercent": max_size_percent}, + } + } + else: + values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizePercent": max_size_percent}}} + + docs = render_chart( + values=values, + show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], + ) + + if max_size_result: + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_PERCENT'].value | [0]", + docs[0], + ) + == max_size_result + ) + else: + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_PERCENT'].value | [0]", + docs[0], + ) + is None + ) + def test_log_groomer_resources(self): if self.obj_name == "dag-processor": values = { diff --git a/chart/templates/dag-processor/dag-processor-deployment.yaml b/chart/templates/dag-processor/dag-processor-deployment.yaml index d2a8254397fa3..2e4974f381752 100644 --- a/chart/templates/dag-processor/dag-processor-deployment.yaml +++ b/chart/templates/dag-processor/dag-processor-deployment.yaml @@ -218,6 +218,14 @@ spec: {{- if .Values.dagProcessor.logGroomerSidecar.frequencyMinutes }} - name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES value: "{{ .Values.dagProcessor.logGroomerSidecar.frequencyMinutes }}" + {{- end }} + {{- if .Values.dagProcessor.logGroomerSidecar.maxSizeBytes }} + - name: AIRFLOW__LOG_MAX_SIZE_BYTES + value: "{{ .Values.dagProcessor.logGroomerSidecar.maxSizeBytes | int64 }}" + {{- end }} + {{- if .Values.dagProcessor.logGroomerSidecar.maxSizePercent }} + - name: AIRFLOW__LOG_MAX_SIZE_PERCENT + value: "{{ .Values.dagProcessor.logGroomerSidecar.maxSizePercent }}" {{- end }} - name: AIRFLOW_HOME value: "{{ .Values.airflowHome }}" diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 8fc73b93c428a..26efde1061ecd 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -293,6 +293,14 @@ spec: {{- if .Values.scheduler.logGroomerSidecar.frequencyMinutes }} - name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES value: "{{ .Values.scheduler.logGroomerSidecar.frequencyMinutes }}" + {{- end }} + {{- if .Values.scheduler.logGroomerSidecar.maxSizeBytes }} + - name: AIRFLOW__LOG_MAX_SIZE_BYTES + value: "{{ .Values.scheduler.logGroomerSidecar.maxSizeBytes | int64 }}" + {{- end }} + {{- if .Values.scheduler.logGroomerSidecar.maxSizePercent }} + - name: AIRFLOW__LOG_MAX_SIZE_PERCENT + value: "{{ .Values.scheduler.logGroomerSidecar.maxSizePercent }}" {{- end }} - name: AIRFLOW_HOME value: "{{ .Values.airflowHome }}" diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index b3287945058d9..4f3b55e3e1a0b 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -253,6 +253,14 @@ spec: {{- if .Values.triggerer.logGroomerSidecar.frequencyMinutes }} - name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES value: "{{ .Values.triggerer.logGroomerSidecar.frequencyMinutes }}" + {{- end }} + {{- if .Values.triggerer.logGroomerSidecar.maxSizeBytes }} + - name: AIRFLOW__LOG_MAX_SIZE_BYTES + value: "{{ .Values.triggerer.logGroomerSidecar.maxSizeBytes | int64 }}" + {{- end }} + {{- if .Values.triggerer.logGroomerSidecar.maxSizePercent }} + - name: AIRFLOW__LOG_MAX_SIZE_PERCENT + value: "{{ .Values.triggerer.logGroomerSidecar.maxSizePercent }}" {{- end }} - name: AIRFLOW_HOME value: "{{ .Values.airflowHome }}" diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 0b628f24cc5d0..1ae802555c5b6 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -364,6 +364,14 @@ spec: {{- if .Values.workers.logGroomerSidecar.frequencyMinutes }} - name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES value: "{{ .Values.workers.logGroomerSidecar.frequencyMinutes }}" + {{- end }} + {{- if .Values.workers.logGroomerSidecar.maxSizeBytes }} + - name: AIRFLOW__LOG_MAX_SIZE_BYTES + value: "{{ .Values.workers.logGroomerSidecar.maxSizeBytes | int64 }}" + {{- end }} + {{- if .Values.workers.logGroomerSidecar.maxSizePercent }} + - name: AIRFLOW__LOG_MAX_SIZE_PERCENT + value: "{{ .Values.workers.logGroomerSidecar.maxSizePercent }}" {{- end }} - name: AIRFLOW_HOME value: "{{ .Values.airflowHome }}" diff --git a/chart/values.schema.json b/chart/values.schema.json index 25fe563f24b2d..710b2445cf525 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -13732,6 +13732,19 @@ "type": "integer", "default": 15 }, + "maxSizeBytes": { + "description": "Max size of logs directory in bytes. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled.", + "type": "integer", + "default": 0, + "minimum": 0 + }, + "maxSizePercent": { + "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if maxSizeBytes is set.", + "type": "integer", + "default": 0, + "minimum": 0, + "maximum": 100 + }, "env": { "description": "Add additional env vars to log groomer sidecar container (templated).", "items": { diff --git a/chart/values.yaml b/chart/values.yaml index 350d352f9b44f..eef0b108599ed 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -984,6 +984,12 @@ workers: # Frequency to attempt to groom logs (in minutes) frequencyMinutes: 15 + # Max size of logs in bytes. 0 = disabled + maxSizeBytes: 0 + + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set. + maxSizePercent: 0 + resources: {} # limits: # cpu: 100m @@ -1363,6 +1369,10 @@ scheduler: retentionDays: 15 # frequency to attempt to groom logs, in minutes frequencyMinutes: 15 + # Max size of logs in bytes. 0 = disabled + maxSizeBytes: 0 + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set. + maxSizePercent: 0 resources: {} # limits: # cpu: 100m @@ -2203,6 +2213,10 @@ triggerer: retentionDays: 15 # frequency to attempt to groom logs, in minutes frequencyMinutes: 15 + # Max size of logs in bytes. 0 = disabled + maxSizeBytes: 0 + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set. + maxSizePercent: 0 resources: {} # limits: # cpu: 100m @@ -2429,6 +2443,10 @@ dagProcessor: retentionDays: 15 # frequency to attempt to groom logs, in minutes frequencyMinutes: 15 + # Max size of logs in bytes. 0 = disabled + maxSizeBytes: 0 + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set. + maxSizePercent: 0 resources: {} # limits: # cpu: 100m diff --git a/helm-tests/tests/chart_utils/log_groomer.py b/helm-tests/tests/chart_utils/log_groomer.py index a5aede282af6b..5709c93796aa8 100644 --- a/helm-tests/tests/chart_utils/log_groomer.py +++ b/helm-tests/tests/chart_utils/log_groomer.py @@ -219,6 +219,76 @@ def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes, freque else: assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2 + @pytest.mark.parametrize( + ("max_size_bytes", "max_size_result"), [(None, None), (1234567890, "1234567890")] + ) + def test_log_groomer_max_size_bytes_overrides(self, max_size_bytes, max_size_result): + if self.obj_name == "dag-processor": + values = { + "dagProcessor": { + "enabled": True, + "logGroomerSidecar": {"maxSizeBytes": max_size_bytes}, + } + } + else: + values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizeBytes": max_size_bytes}}} + + docs = render_chart( + values=values, + show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], + ) + + if max_size_result: + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_BYTES'].value | [0]", + docs[0], + ) + == max_size_result + ) + else: + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_BYTES'].value | [0]", + docs[0], + ) + is None + ) + + @pytest.mark.parametrize(("max_size_percent", "max_size_result"), [(None, None), (80, "80")]) + def test_log_groomer_max_size_percent_overrides(self, max_size_percent, max_size_result): + if self.obj_name == "dag-processor": + values = { + "dagProcessor": { + "enabled": True, + "logGroomerSidecar": {"maxSizePercent": max_size_percent}, + } + } + else: + values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizePercent": max_size_percent}}} + + docs = render_chart( + values=values, + show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], + ) + + if max_size_result: + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_PERCENT'].value | [0]", + docs[0], + ) + == max_size_result + ) + else: + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_PERCENT'].value | [0]", + docs[0], + ) + is None + ) + def test_log_groomer_resources(self): if self.obj_name == "dag-processor": values = { diff --git a/scripts/docker/clean-logs.sh b/scripts/docker/clean-logs.sh index 063b0a985b685..1537036269538 100755 --- a/scripts/docker/clean-logs.sh +++ b/scripts/docker/clean-logs.sh @@ -22,22 +22,50 @@ set -euo pipefail readonly DIRECTORY="${AIRFLOW_HOME:-/usr/local/airflow}" readonly RETENTION="${AIRFLOW__LOG_RETENTION_DAYS:-15}" readonly FREQUENCY="${AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES:-15}" +readonly MAX_PERCENT="${AIRFLOW__LOG_MAX_SIZE_PERCENT:-0}" trap "exit" INT TERM +MAX_SIZE_BYTES="${AIRFLOW__LOG_MAX_SIZE_BYTES:-0}" +if [[ "$MAX_SIZE_BYTES" -eq 0 && "$MAX_PERCENT" -gt 0 ]]; then + total_space=$(df -k "${DIRECTORY}"/logs 2>/dev/null | tail -1 | awk '{print $2}' || echo "0") + MAX_SIZE_BYTES=$(( total_space * 1024 * MAX_PERCENT / 100 )) + echo "Computed MAX_SIZE_BYTES from ${MAX_PERCENT}% of disk: ${MAX_SIZE_BYTES} bytes" +fi + +readonly MAX_SIZE_BYTES + readonly EVERY=$((FREQUENCY*60)) echo "Cleaning logs every $EVERY seconds" +if [[ "$MAX_SIZE_BYTES" -gt 0 ]]; then + echo "Max log size limit: $MAX_SIZE_BYTES bytes" +fi + +retention_days="${RETENTION}" while true; do - echo "Trimming airflow logs to ${RETENTION} days." + echo "Trimming airflow logs to ${retention_days} days." find "${DIRECTORY}"/logs \ -type d -name 'lost+found' -prune -o \ - -type f -mtime +"${RETENTION}" -name '*.log' -print0 | \ + -type f -mtime +"${retention_days}" -name '*.log' -print0 | \ xargs -0 rm -f || true + if [[ "$MAX_SIZE_BYTES" -gt 0 && "$retention_days" -ge 0 ]]; then + current_size=$(df -k "${DIRECTORY}"/logs 2>/dev/null | tail -1 | awk '{print $3}' || echo "0") + current_size=$(( current_size * 1024 )) + + if [[ "$current_size" -gt "$MAX_SIZE_BYTES" ]]; then + retention_days=$((retention_days - 1)) + echo "Size ($current_size bytes) exceeds limit ($MAX_SIZE_BYTES bytes). Reducing retention to ${retention_days} days." + continue + fi + fi + find "${DIRECTORY}"/logs -type d -empty -delete || true + retention_days="${RETENTION}" + seconds=$(( $(date -u +%s) % EVERY)) (( seconds < 1 )) || sleep $((EVERY - seconds - 1)) sleep 1