Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,9 @@
FilterParam as FilterParam,
filter_param_factory as filter_param_factory,
)
from airflow.api_fastapi.common.parameters.job import (
QueryJobTeamsFilter as QueryJobTeamsFilter,
)
from airflow.api_fastapi.common.parameters.misc import (
QueryConnectionIdPatternSearch as QueryConnectionIdPatternSearch,
QueryConnectionIdPrefixPatternSearch as QueryConnectionIdPrefixPatternSearch,
Expand Down
50 changes: 50 additions & 0 deletions airflow-core/src/airflow/api_fastapi/common/parameters/job.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

from __future__ import annotations

from typing import TYPE_CHECKING, Annotated

from fastapi import Depends, Query
from sqlalchemy import select as sql_select

from airflow.api_fastapi.common.parameters.base import BaseParam
from airflow.jobs.job import Job
from airflow.models.team import JobTeam

if TYPE_CHECKING:
from sqlalchemy.sql import Select


class _JobTeamsFilter(BaseParam[list[str]]):
"""Filter jobs by the teams they serve (via the ``JobTeam`` association)."""

def to_orm(self, select: Select) -> Select:
if self.skip_none is False:
raise ValueError(f"Cannot set 'skip_none' to False on a {type(self)}")

if not self.value:
return select

return select.where(Job.id.in_(sql_select(JobTeam.job_id).where(JobTeam.team_name.in_(self.value))))

@classmethod
def depends(cls, teams: list[str] = Query(default_factory=list)) -> _JobTeamsFilter:
return cls().set_value(teams)


QueryJobTeamsFilter = Annotated[_JobTeamsFilter, Depends(_JobTeamsFilter.depends)]
Original file line number Diff line number Diff line change
Expand Up @@ -5462,6 +5462,14 @@ paths:
- type: string
- type: 'null'
title: Executor Class
- name: teams
in: query
required: false
schema:
type: array
items:
type: string
title: Teams
responses:
'200':
description: Successful Response
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
)
from airflow.api_fastapi.common.parameters import (
FilterParam,
QueryJobTeamsFilter,
QueryLimit,
QueryOffset,
RangeFilter,
Expand Down Expand Up @@ -101,6 +102,7 @@ def get_jobs(
FilterParam[str | None],
Depends(filter_param_factory(Job.executor_class, str | None, filter_name="executor_class")),
],
teams: QueryJobTeamsFilter,
is_alive: bool | None = None,
) -> JobCollectionResponse:
"""Get all jobs."""
Expand All @@ -116,6 +118,7 @@ def get_jobs(
job_type,
hostname,
executor_class,
teams,
],
order_by=order_by,
limit=limit,
Expand Down
26 changes: 25 additions & 1 deletion airflow-core/src/airflow/cli/commands/dag_processor_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
from typing import Any

from airflow.cli.commands.daemon_utils import run_command_with_daemon_option
from airflow.configuration import conf
from airflow.dag_processing.bundles.manager import _get_configured_bundle_team_names
from airflow.dag_processing.manager import DagFileProcessorManager
from airflow.jobs.dag_processor_job_runner import DagProcessorJobRunner
from airflow.jobs.job import Job, run_job
Expand All @@ -33,12 +35,34 @@
log = logging.getLogger(__name__)


def _get_team_names(bundle_names: list[str] | None) -> list[str]:
"""
Return the teams this Dag processor serves, sorted and de-duplicated.

Teams are resolved from the bundle configuration rather than the metadata DB: the job row is
written before ``sync_bundles()`` runs, so a DB lookup would see no rows on a fresh deployment
and stale rows right after a bundle is reassigned in config. Config is the source of truth, and
is what ``airflow_health.py`` resolves teams from too.

A processor started without ``--bundle-name`` parses every configured bundle, so it serves
every configured team. A bundle mapped to no team contributes no team, so a processor parsing
only team-less (or unknown) bundles is not team-scoped and serves the empty list. Outside
multi-team mode team scoping is disabled entirely, matching ``DagFileProcessorManager``.
"""
if not conf.getboolean("core", "multi_team"):
return []

configured = _get_configured_bundle_team_names()
names = bundle_names or list(configured)
return sorted({team for name in names if (team := configured.get(name)) is not None})


def _create_dag_processor_job_runner(args: Any) -> DagProcessorJobRunner:
"""Create DagFileProcessorProcess instance."""
if args.bundle_name:
cli_utils.validate_dag_bundle_arg(args.bundle_name)
return DagProcessorJobRunner(
job=Job(bundle_names=args.bundle_name),
job=Job(bundle_names=args.bundle_name, team_names=_get_team_names(args.bundle_name)),
processor=DagFileProcessorManager(
max_runs=args.num_runs,
bundle_names_to_parse=args.bundle_name,
Expand Down
5 changes: 3 additions & 2 deletions airflow-core/src/airflow/ui/openapi-gen/queries/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -786,7 +786,7 @@ export const UseImportErrorServiceGetImportErrorsKeyFn = ({ bundleName, filename
export type JobServiceGetJobsDefaultResponse = Awaited<ReturnType<typeof JobService.getJobs>>;
export type JobServiceGetJobsQueryResult<TData = JobServiceGetJobsDefaultResponse, TError = unknown> = UseQueryResult<TData, TError>;
export const useJobServiceGetJobsKey = "JobServiceGetJobs";
export const UseJobServiceGetJobsKeyFn = ({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }: {
export const UseJobServiceGetJobsKeyFn = ({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }: {
dagId?: string;
endDateGt?: string;
endDateGte?: string;
Expand All @@ -804,7 +804,8 @@ export const UseJobServiceGetJobsKeyFn = ({ dagId, endDateGt, endDateGte, endDat
startDateGte?: string;
startDateLt?: string;
startDateLte?: string;
} = {}, queryKey?: Array<unknown>) => [useJobServiceGetJobsKey, ...(queryKey ?? [{ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }])];
teams?: string[];
} = {}, queryKey?: Array<unknown>) => [useJobServiceGetJobsKey, ...(queryKey ?? [{ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }])];
export type PluginServiceGetPluginsDefaultResponse = Awaited<ReturnType<typeof PluginService.getPlugins>>;
export type PluginServiceGetPluginsQueryResult<TData = PluginServiceGetPluginsDefaultResponse, TError = unknown> = UseQueryResult<TData, TError>;
export const usePluginServiceGetPluginsKey = "PluginServiceGetPlugins";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1561,10 +1561,11 @@ export const ensureUseImportErrorServiceGetImportErrorsData = (queryClient: Quer
* @param data.jobType
* @param data.hostname
* @param data.executorClass
* @param data.teams
* @returns JobCollectionResponse Successful Response
* @throws ApiError
*/
export const ensureUseJobServiceGetJobsData = (queryClient: QueryClient, { dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }: {
export const ensureUseJobServiceGetJobsData = (queryClient: QueryClient, { dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }: {
dagId?: string;
endDateGt?: string;
endDateGte?: string;
Expand All @@ -1582,7 +1583,8 @@ export const ensureUseJobServiceGetJobsData = (queryClient: QueryClient, { dagId
startDateGte?: string;
startDateLt?: string;
startDateLte?: string;
} = {}) => queryClient.ensureQueryData({ queryKey: Common.UseJobServiceGetJobsKeyFn({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }), queryFn: () => JobService.getJobs({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }) });
teams?: string[];
} = {}) => queryClient.ensureQueryData({ queryKey: Common.UseJobServiceGetJobsKeyFn({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }), queryFn: () => JobService.getJobs({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }) });
/**
* Get Plugins
* @param data The data for the request.
Expand Down
6 changes: 4 additions & 2 deletions airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1561,10 +1561,11 @@ export const prefetchUseImportErrorServiceGetImportErrors = (queryClient: QueryC
* @param data.jobType
* @param data.hostname
* @param data.executorClass
* @param data.teams
* @returns JobCollectionResponse Successful Response
* @throws ApiError
*/
export const prefetchUseJobServiceGetJobs = (queryClient: QueryClient, { dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }: {
export const prefetchUseJobServiceGetJobs = (queryClient: QueryClient, { dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }: {
dagId?: string;
endDateGt?: string;
endDateGte?: string;
Expand All @@ -1582,7 +1583,8 @@ export const prefetchUseJobServiceGetJobs = (queryClient: QueryClient, { dagId,
startDateGte?: string;
startDateLt?: string;
startDateLte?: string;
} = {}) => queryClient.prefetchQuery({ queryKey: Common.UseJobServiceGetJobsKeyFn({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }), queryFn: () => JobService.getJobs({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }) });
teams?: string[];
} = {}) => queryClient.prefetchQuery({ queryKey: Common.UseJobServiceGetJobsKeyFn({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }), queryFn: () => JobService.getJobs({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }) });
/**
* Get Plugins
* @param data The data for the request.
Expand Down
6 changes: 4 additions & 2 deletions airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1561,10 +1561,11 @@ export const useImportErrorServiceGetImportErrors = <TData = Common.ImportErrorS
* @param data.jobType
* @param data.hostname
* @param data.executorClass
* @param data.teams
* @returns JobCollectionResponse Successful Response
* @throws ApiError
*/
export const useJobServiceGetJobs = <TData = Common.JobServiceGetJobsDefaultResponse, TError = unknown, TQueryKey extends Array<unknown> = unknown[]>({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }: {
export const useJobServiceGetJobs = <TData = Common.JobServiceGetJobsDefaultResponse, TError = unknown, TQueryKey extends Array<unknown> = unknown[]>({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }: {
dagId?: string;
endDateGt?: string;
endDateGte?: string;
Expand All @@ -1582,7 +1583,8 @@ export const useJobServiceGetJobs = <TData = Common.JobServiceGetJobsDefaultResp
startDateGte?: string;
startDateLt?: string;
startDateLte?: string;
} = {}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useQuery<TData, TError>({ queryKey: Common.UseJobServiceGetJobsKeyFn({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }, queryKey), queryFn: () => JobService.getJobs({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }) as TData, ...options });
teams?: string[];
} = {}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useQuery<TData, TError>({ queryKey: Common.UseJobServiceGetJobsKeyFn({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }, queryKey), queryFn: () => JobService.getJobs({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }) as TData, ...options });
/**
* Get Plugins
* @param data The data for the request.
Expand Down
6 changes: 4 additions & 2 deletions airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1561,10 +1561,11 @@ export const useImportErrorServiceGetImportErrorsSuspense = <TData = Common.Impo
* @param data.jobType
* @param data.hostname
* @param data.executorClass
* @param data.teams
* @returns JobCollectionResponse Successful Response
* @throws ApiError
*/
export const useJobServiceGetJobsSuspense = <TData = Common.JobServiceGetJobsDefaultResponse, TError = unknown, TQueryKey extends Array<unknown> = unknown[]>({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }: {
export const useJobServiceGetJobsSuspense = <TData = Common.JobServiceGetJobsDefaultResponse, TError = unknown, TQueryKey extends Array<unknown> = unknown[]>({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }: {
dagId?: string;
endDateGt?: string;
endDateGte?: string;
Expand All @@ -1582,7 +1583,8 @@ export const useJobServiceGetJobsSuspense = <TData = Common.JobServiceGetJobsDef
startDateGte?: string;
startDateLt?: string;
startDateLte?: string;
} = {}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useSuspenseQuery<TData, TError>({ queryKey: Common.UseJobServiceGetJobsKeyFn({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }, queryKey), queryFn: () => JobService.getJobs({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte }) as TData, ...options });
teams?: string[];
} = {}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useSuspenseQuery<TData, TError>({ queryKey: Common.UseJobServiceGetJobsKeyFn({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }, queryKey), queryFn: () => JobService.getJobs({ dagId, endDateGt, endDateGte, endDateLt, endDateLte, executorClass, hostname, isAlive, jobState, jobType, limit, offset, orderBy, startDateGt, startDateGte, startDateLt, startDateLte, teams }) as TData, ...options });
/**
* Get Plugins
* @param data The data for the request.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3619,6 +3619,7 @@ export class JobService {
* @param data.jobType
* @param data.hostname
* @param data.executorClass
* @param data.teams
* @returns JobCollectionResponse Successful Response
* @throws ApiError
*/
Expand All @@ -3643,7 +3644,8 @@ export class JobService {
dag_id: data.dagId,
job_type: data.jobType,
hostname: data.hostname,
executor_class: data.executorClass
executor_class: data.executorClass,
teams: data.teams
},
errors: {
400: 'Bad Request',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4489,6 +4489,7 @@ export type GetJobsData = {
startDateGte?: string | null;
startDateLt?: string | null;
startDateLte?: string | null;
teams?: Array<(string)>;
};

export type GetJobsResponse = JobCollectionResponse;
Expand Down
Loading
Loading