Skip to content

[v3-1-test] Add LOG_MAX_SIZE environment variables to log groomer (#61559) #61950

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 1 commit into from
Feb 15, 2026
Merged

[v3-1-test] Add LOG_MAX_SIZE environment variables to log groomer (#61559) #61950

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 30 additions & 2 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1641,22 +1641,50 @@ set -euo pipefail
readonly DIRECTORY="${AIRFLOW_HOME:-/usr/local/airflow}"
readonly RETENTION="${AIRFLOW__LOG_RETENTION_DAYS:-15}"
readonly FREQUENCY="${AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES:-15}"
readonly MAX_PERCENT="${AIRFLOW__LOG_MAX_SIZE_PERCENT:-0}"

trap "exit" INT TERM

MAX_SIZE_BYTES="${AIRFLOW__LOG_MAX_SIZE_BYTES:-0}"
if [[ "$MAX_SIZE_BYTES" -eq 0 && "$MAX_PERCENT" -gt 0 ]]; then
total_space=$(df -k "${DIRECTORY}"/logs 2>/dev/null | tail -1 | awk '{print $2}' || echo "0")
MAX_SIZE_BYTES=$(( total_space * 1024 * MAX_PERCENT / 100 ))
echo "Computed MAX_SIZE_BYTES from ${MAX_PERCENT}% of disk: ${MAX_SIZE_BYTES} bytes"
fi

readonly MAX_SIZE_BYTES

readonly EVERY=$((FREQUENCY*60))

echo "Cleaning logs every $EVERY seconds"
if [[ "$MAX_SIZE_BYTES" -gt 0 ]]; then
echo "Max log size limit: $MAX_SIZE_BYTES bytes"
fi

retention_days="${RETENTION}"

while true; do
echo "Trimming airflow logs to ${RETENTION} days."
echo "Trimming airflow logs to ${retention_days} days."
find "${DIRECTORY}"/logs \
-type d -name 'lost+found' -prune -o \
-type f -mtime +"${RETENTION}" -name '*.log' -print0 | \
-type f -mtime +"${retention_days}" -name '*.log' -print0 | \
xargs -0 rm -f || true

if [[ "$MAX_SIZE_BYTES" -gt 0 && "$retention_days" -ge 0 ]]; then
current_size=$(df -k "${DIRECTORY}"/logs 2>/dev/null | tail -1 | awk '{print $3}' || echo "0")
current_size=$(( current_size * 1024 ))

if [[ "$current_size" -gt "$MAX_SIZE_BYTES" ]]; then
retention_days=$((retention_days - 1))
echo "Size ($current_size bytes) exceeds limit ($MAX_SIZE_BYTES bytes). Reducing retention to ${retention_days} days."
continue
fi
fi

find "${DIRECTORY}"/logs -type d -empty -delete || true

retention_days="${RETENTION}"

seconds=$(( $(date -u +%s) % EVERY))
(( seconds < 1 )) || sleep $((EVERY - seconds - 1))
sleep 1
Expand Down
70 changes: 70 additions & 0 deletions airflow-core/tests/unit/charts/log_groomer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,76 @@ def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes, freque
else:
assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2

@pytest.mark.parametrize(
("max_size_bytes", "max_size_result"), [(None, None), (1234567890, "1234567890")]
)
def test_log_groomer_max_size_bytes_overrides(self, max_size_bytes, max_size_result):
if self.obj_name == "dag-processor":
values = {
"dagProcessor": {
"enabled": True,
"logGroomerSidecar": {"maxSizeBytes": max_size_bytes},
}
}
else:
values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizeBytes": max_size_bytes}}}

docs = render_chart(
values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
)

if max_size_result:
assert (
jmespath.search(
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_BYTES'].value | [0]",
docs[0],
)
== max_size_result
)
else:
assert (
jmespath.search(
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_BYTES'].value | [0]",
docs[0],
)
is None
)

@pytest.mark.parametrize(("max_size_percent", "max_size_result"), [(None, None), (80, "80")])
def test_log_groomer_max_size_percent_overrides(self, max_size_percent, max_size_result):
if self.obj_name == "dag-processor":
values = {
"dagProcessor": {
"enabled": True,
"logGroomerSidecar": {"maxSizePercent": max_size_percent},
}
}
else:
values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizePercent": max_size_percent}}}

docs = render_chart(
values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
)

if max_size_result:
assert (
jmespath.search(
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_PERCENT'].value | [0]",
docs[0],
)
== max_size_result
)
else:
assert (
jmespath.search(
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_PERCENT'].value | [0]",
docs[0],
)
is None
)

def test_log_groomer_resources(self):
if self.obj_name == "dag-processor":
values = {
Expand Down
8 changes: 8 additions & 0 deletions chart/templates/dag-processor/dag-processor-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,14 @@ spec:
{{- if .Values.dagProcessor.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{ .Values.dagProcessor.logGroomerSidecar.frequencyMinutes }}"
{{- end }}
{{- if .Values.dagProcessor.logGroomerSidecar.maxSizeBytes }}
- name: AIRFLOW__LOG_MAX_SIZE_BYTES
value: "{{ .Values.dagProcessor.logGroomerSidecar.maxSizeBytes | int64 }}"
{{- end }}
{{- if .Values.dagProcessor.logGroomerSidecar.maxSizePercent }}
- name: AIRFLOW__LOG_MAX_SIZE_PERCENT
value: "{{ .Values.dagProcessor.logGroomerSidecar.maxSizePercent }}"
{{- end }}
- name: AIRFLOW_HOME
value: "{{ .Values.airflowHome }}"
Expand Down
8 changes: 8 additions & 0 deletions chart/templates/scheduler/scheduler-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,14 @@ spec:
{{- if .Values.scheduler.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{ .Values.scheduler.logGroomerSidecar.frequencyMinutes }}"
{{- end }}
{{- if .Values.scheduler.logGroomerSidecar.maxSizeBytes }}
- name: AIRFLOW__LOG_MAX_SIZE_BYTES
value: "{{ .Values.scheduler.logGroomerSidecar.maxSizeBytes | int64 }}"
{{- end }}
{{- if .Values.scheduler.logGroomerSidecar.maxSizePercent }}
- name: AIRFLOW__LOG_MAX_SIZE_PERCENT
value: "{{ .Values.scheduler.logGroomerSidecar.maxSizePercent }}"
{{- end }}
- name: AIRFLOW_HOME
value: "{{ .Values.airflowHome }}"
Expand Down
8 changes: 8 additions & 0 deletions chart/templates/triggerer/triggerer-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,14 @@ spec:
{{- if .Values.triggerer.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{ .Values.triggerer.logGroomerSidecar.frequencyMinutes }}"
{{- end }}
{{- if .Values.triggerer.logGroomerSidecar.maxSizeBytes }}
- name: AIRFLOW__LOG_MAX_SIZE_BYTES
value: "{{ .Values.triggerer.logGroomerSidecar.maxSizeBytes | int64 }}"
{{- end }}
{{- if .Values.triggerer.logGroomerSidecar.maxSizePercent }}
- name: AIRFLOW__LOG_MAX_SIZE_PERCENT
value: "{{ .Values.triggerer.logGroomerSidecar.maxSizePercent }}"
{{- end }}
- name: AIRFLOW_HOME
value: "{{ .Values.airflowHome }}"
Expand Down
8 changes: 8 additions & 0 deletions chart/templates/workers/worker-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,14 @@ spec:
{{- if .Values.workers.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{ .Values.workers.logGroomerSidecar.frequencyMinutes }}"
{{- end }}
{{- if .Values.workers.logGroomerSidecar.maxSizeBytes }}
- name: AIRFLOW__LOG_MAX_SIZE_BYTES
value: "{{ .Values.workers.logGroomerSidecar.maxSizeBytes | int64 }}"
{{- end }}
{{- if .Values.workers.logGroomerSidecar.maxSizePercent }}
- name: AIRFLOW__LOG_MAX_SIZE_PERCENT
value: "{{ .Values.workers.logGroomerSidecar.maxSizePercent }}"
{{- end }}
- name: AIRFLOW_HOME
value: "{{ .Values.airflowHome }}"
Expand Down
13 changes: 13 additions & 0 deletions chart/values.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12572,6 +12572,19 @@
"type": "integer",
"default": 15
},
"maxSizeBytes": {
"description": "Max size of logs directory in bytes. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled.",
"type": "integer",
"default": 0,
"minimum": 0
},
"maxSizePercent": {
"description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if maxSizeBytes is set.",
"type": "integer",
"default": 0,
"minimum": 0,
"maximum": 100
},
"env": {
"description": "Add additional env vars to log groomer sidecar container (templated).",
"items": {
Expand Down
18 changes: 18 additions & 0 deletions chart/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -935,6 +935,12 @@ workers:
# Frequency to attempt to groom logs (in minutes)
frequencyMinutes: 15

# Max size of logs in bytes. 0 = disabled
maxSizeBytes: 0

# Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set.
maxSizePercent: 0

resources: {}
# limits:
# cpu: 100m
Expand Down Expand Up @@ -1177,6 +1183,10 @@ scheduler:
retentionDays: 15
# frequency to attempt to groom logs, in minutes
frequencyMinutes: 15
# Max size of logs in bytes. 0 = disabled
maxSizeBytes: 0
# Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set.
maxSizePercent: 0
resources: {}
# limits:
# cpu: 100m
Expand Down Expand Up @@ -1961,6 +1971,10 @@ triggerer:
retentionDays: 15
# frequency to attempt to groom logs, in minutes
frequencyMinutes: 15
# Max size of logs in bytes. 0 = disabled
maxSizeBytes: 0
# Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set.
maxSizePercent: 0
resources: {}
# limits:
# cpu: 100m
Expand Down Expand Up @@ -2149,6 +2163,10 @@ dagProcessor:
retentionDays: 15
# frequency to attempt to groom logs, in minutes
frequencyMinutes: 15
# Max size of logs in bytes. 0 = disabled
maxSizeBytes: 0
# Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set.
maxSizePercent: 0
resources: {}
# limits:
# cpu: 100m
Expand Down
70 changes: 70 additions & 0 deletions helm-tests/tests/chart_utils/log_groomer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,76 @@ def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes, freque
else:
assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2

@pytest.mark.parametrize(
("max_size_bytes", "max_size_result"), [(None, None), (1234567890, "1234567890")]
)
def test_log_groomer_max_size_bytes_overrides(self, max_size_bytes, max_size_result):
if self.obj_name == "dag-processor":
values = {
"dagProcessor": {
"enabled": True,
"logGroomerSidecar": {"maxSizeBytes": max_size_bytes},
}
}
else:
values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizeBytes": max_size_bytes}}}

docs = render_chart(
values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
)

if max_size_result:
assert (
jmespath.search(
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_BYTES'].value | [0]",
docs[0],
)
== max_size_result
)
else:
assert (
jmespath.search(
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_BYTES'].value | [0]",
docs[0],
)
is None
)

@pytest.mark.parametrize(("max_size_percent", "max_size_result"), [(None, None), (80, "80")])
def test_log_groomer_max_size_percent_overrides(self, max_size_percent, max_size_result):
if self.obj_name == "dag-processor":
values = {
"dagProcessor": {
"enabled": True,
"logGroomerSidecar": {"maxSizePercent": max_size_percent},
}
}
else:
values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizePercent": max_size_percent}}}

docs = render_chart(
values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
)

if max_size_result:
assert (
jmespath.search(
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_PERCENT'].value | [0]",
docs[0],
)
== max_size_result
)
else:
assert (
jmespath.search(
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_PERCENT'].value | [0]",
docs[0],
)
is None
)

def test_log_groomer_resources(self):
if self.obj_name == "dag-processor":
values = {
Expand Down
32 changes: 30 additions & 2 deletions scripts/docker/clean-logs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,22 +22,50 @@ set -euo pipefail
readonly DIRECTORY="${AIRFLOW_HOME:-/usr/local/airflow}"
readonly RETENTION="${AIRFLOW__LOG_RETENTION_DAYS:-15}"
readonly FREQUENCY="${AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES:-15}"
readonly MAX_PERCENT="${AIRFLOW__LOG_MAX_SIZE_PERCENT:-0}"

trap "exit" INT TERM

MAX_SIZE_BYTES="${AIRFLOW__LOG_MAX_SIZE_BYTES:-0}"
if [[ "$MAX_SIZE_BYTES" -eq 0 && "$MAX_PERCENT" -gt 0 ]]; then
total_space=$(df -k "${DIRECTORY}"/logs 2>/dev/null | tail -1 | awk '{print $2}' || echo "0")
MAX_SIZE_BYTES=$(( total_space * 1024 * MAX_PERCENT / 100 ))
echo "Computed MAX_SIZE_BYTES from ${MAX_PERCENT}% of disk: ${MAX_SIZE_BYTES} bytes"
fi

readonly MAX_SIZE_BYTES

readonly EVERY=$((FREQUENCY*60))

echo "Cleaning logs every $EVERY seconds"
if [[ "$MAX_SIZE_BYTES" -gt 0 ]]; then
echo "Max log size limit: $MAX_SIZE_BYTES bytes"
fi

retention_days="${RETENTION}"

while true; do
echo "Trimming airflow logs to ${RETENTION} days."
echo "Trimming airflow logs to ${retention_days} days."
find "${DIRECTORY}"/logs \
-type d -name 'lost+found' -prune -o \
-type f -mtime +"${RETENTION}" -name '*.log' -print0 | \
-type f -mtime +"${retention_days}" -name '*.log' -print0 | \
xargs -0 rm -f || true

if [[ "$MAX_SIZE_BYTES" -gt 0 && "$retention_days" -ge 0 ]]; then
current_size=$(df -k "${DIRECTORY}"/logs 2>/dev/null | tail -1 | awk '{print $3}' || echo "0")
current_size=$(( current_size * 1024 ))

if [[ "$current_size" -gt "$MAX_SIZE_BYTES" ]]; then
retention_days=$((retention_days - 1))
echo "Size ($current_size bytes) exceeds limit ($MAX_SIZE_BYTES bytes). Reducing retention to ${retention_days} days."
continue
fi
fi

find "${DIRECTORY}"/logs -type d -empty -delete || true

retention_days="${RETENTION}"

seconds=$(( $(date -u +%s) % EVERY))
(( seconds < 1 )) || sleep $((EVERY - seconds - 1))
sleep 1
Expand Down
Loading