diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index acd51783cf80b..f21907ac10385 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -116,6 +116,7 @@ body: - alibaba - amazon - anthropic + - apache-arrow - apache-beam - apache-cassandra - apache-drill diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 1801299ad8876..78469e97a8f8e 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -33,6 +33,9 @@ labelPRBasedOnFilePath: provider:anthropic: - providers/anthropic/** + provider:apache-arrow: + - providers/apache/arrow/** + provider:apache-beam: - providers/apache/beam/** diff --git a/airflow-core/docs/extra-packages-ref.rst b/airflow-core/docs/extra-packages-ref.rst index 155403b045a38..531693952ec92 100644 --- a/airflow-core/docs/extra-packages-ref.rst +++ b/airflow-core/docs/extra-packages-ref.rst @@ -188,6 +188,8 @@ custom bash/python providers). +---------------------+-----------------------------------------------------+------------------------------------------------+ | extra | install command | enables | +=====================+=====================================================+================================================+ +| apache-arrow | ``pip install 'apache-airflow[apache-arrow]'`` | Apache Arrow ADBC hooks | ++---------------------+-----------------------------------------------------+------------------------------------------------+ | apache-atlas | ``pip install 'apache-airflow[apache-atlas]'`` | Apache Atlas | +---------------------+-----------------------------------------------------+------------------------------------------------+ | apache-beam | ``pip install 'apache-airflow[apache-beam]'`` | Apache Beam operators & hooks | diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index 70d8a365de0d3..6918fd8b63769 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -241,17 +241,17 @@ Usage:                                                                                                                 breeze build-docs                                                                                                      [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-mypy | apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink |       -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging |  -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch |   -exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.db2 | ibm.mq | imap -| influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | -microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle |    -pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment |      -sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram |   -teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...                                    +apache-airflow-mypy | apache-airflow-providers | apache.arrow | apache.cassandra | apache.drill | apache.druid |       +apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy +| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 +| elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http |       +ibm.db2 | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure |            +microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage |  +opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis |    +salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard |    +tableau | task-sdk | telegram | teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...    Build documents. diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index db136fdef4daa..f2015f69a55d7 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -042f155e3eaf964145e84fd6df749ce1 +f941b35af1d912596b27d89d4863dd0a diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index ebea980e51823..ec15befcabd38 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -150,17 +150,17 @@ Usage:                                                                                                                 breeze release-management add-back-references                                                                          [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-mypy | apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink |       -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging |  -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch |   -exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.db2 | ibm.mq | imap -| influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | -microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle |    -pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment |      -sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram |   -teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...                                    +apache-airflow-mypy | apache-airflow-providers | apache.arrow | apache.cassandra | apache.drill | apache.druid |       +apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy +| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 +| elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http |       +ibm.db2 | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure |            +microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage |  +opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis |    +salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard |    +tableau | task-sdk | telegram | teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...    Command to add back references for documentation to make it backward compatible. diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index 401c2412c544b..ad57d752e8b7d 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -1f529ce2ef4cd6b5086ff168594fecba +bc1e2c3d12063d93fa3f3605b4f42de1 diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg index cef6e4d3b1d14..de7a137ca904d 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg @@ -162,17 +162,17 @@ Usage:                                                                                                                 breeze release-management classify-provider-changes                                                                    -[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        -apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy -| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.arrow | apache.cassandra | apache.drill |        +apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin +| apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |             +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Classify each provider's unreleased changes with hard-coded, high-confidence rules, flagging ambiguous commits as  'needs_llm' for an agent/skill to assess. Outputs JSON - a deterministic alternative to the random '--non-interactive' diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt index 6e02c700afdf5..44b287551c6e1 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt @@ -1 +1 @@ -4cac13b21eee8b732a46c5a15aec7a4b +0f0f937f1a0d86a58147723b1400b3ac diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index 2f9692be54fa0..628d1a972bbf1 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -151,17 +151,17 @@ Usage:                                                                                                                 breeze release-management generate-issue-content-providers                                                             -[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        -apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy -| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.arrow | apache.cassandra | apache.drill |        +apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin +| apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |             +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Generates content for issue to test the release. diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index 6afb289b5854f..91f9d9e152d75 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -bd80bf2dd63a27111b8c5745df5dfb86 +465c7719885f58cdf5fc8bd0592ba7de diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg index 64efb0af6b65b..3a4f3e5295534 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg @@ -169,19 +169,19 @@ │--provider-id     Provider_id to generate metadata for. If not specified, metadata for all packages will be       │ │generated. This is debug-only option. When this option is specified, the metadata produced for  │ │the distribution will not be written to the file, but printed via console. (airbyte | akeyless |│ -│alibaba | amazon | anthropic | apache.beam | apache.cassandra | apache.drill | apache.druid | │ -│apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | │ -│apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | │ -│apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes│ -│| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | │ -│datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | │ -│facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica│ -│| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | │ -│microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | │ -│opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | │ -│redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake │ -│| sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |│ -│yandex | ydb | zendesk)│ +│alibaba | amazon | anthropic | apache.arrow | apache.beam | apache.cassandra | apache.drill | │ +│apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | │ +│apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | │ +│apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | │ +│cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |│ +│common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | │ +│elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http │ +│| imap | influxdb | informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql │ +│| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | │ +│openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | │ +│postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | │ +│singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata│ +│| trino | vertica | vespa | weaviate | yandex | ydb | zendesk)│ │--provider-versionProvider version to generate metadata for. Only used when --provider-id is specified. Limits    │ │running metadata generation to only this version of the provider. (TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt index 7c14e04e63151..8b1eec0eef352 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt @@ -1 +1 @@ -5865810f5a4b12c9199267b7dd6459b3 +ed9d0b9e6709f6f00a87d5510d556590 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index eda0dd2e31526..ff70a78a9243c 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -201,17 +201,17 @@ Usage:                                                                                                                 breeze release-management prepare-provider-distributions                                                               -[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        -apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy -| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.arrow | apache.cassandra | apache.drill |        +apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin +| apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |             +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare sdist/whl distributions of Airflow Providers. Each provider directory is wiped with `git clean -fdx`  (preserving .venv, .idea, .vscode) before build to keep in-tree generated files out of the artifact. See dev/breeze  diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index f462d0ecff8f9..4d4703a71ae30 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -4b8e22ad991a4fed63956c9563f2fb30 +b937b6a253ce2ab2cbabb9cd8b7369d4 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index b9c71f9c81aec..4a2a8e8c474b9 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -210,17 +210,17 @@ Usage:                                                                                                                 breeze release-management prepare-provider-documentation                                                               -[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        -apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy -| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.arrow | apache.cassandra | apache.drill |        +apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin +| apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |             +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index a54080b6bfbb5..5b7184cf9c60b 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -c33a2f6d00a3a8dbec8b56c1c2d88d54 +57343f19b12400ac43ea8fc4b892ac99 diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index fc8f039759b52..ca00eadc98aeb 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -189,17 +189,17 @@ Usage:                                                                                                                 breeze release-management publish-docs                                                                                 [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-mypy | apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink |       -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging |  -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch |   -exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.db2 | ibm.mq | imap -| influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | -microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle |    -pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment |      -sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram |   -teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...                                    +apache-airflow-mypy | apache-airflow-providers | apache.arrow | apache.cassandra | apache.drill | apache.druid |       +apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy +| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 +| elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http |       +ibm.db2 | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure |            +microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage |  +opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis |    +salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard |    +tableau | task-sdk | telegram | teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...    Command to publish generated documentation to airflow-site diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index 1a7a63c1f91f4..2e522c29cf776 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -652ffdb3608cb0404403b85550fa5d27 +1a68af03bc2f8fae0d46e4a2a2266eb0 diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index 7fd604c1b66e2..36931ee17ccff 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -183,11 +183,11 @@ │--python-versions Comma separate list of Python versions to update sbom from (defaults to all historical python   │ │versions) (3.6 | 3.7 | 3.8 | 3.9 | 3.10 | 3.11 | 3.12 | 3.13 | 3.14)│ │--provider-id     Provider id to generate the requirements for (airbyte | akeyless | alibaba | amazon | anthropic │ -│| apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | │ -│apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | │ -│apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | │ -│atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | │ -│common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks | │ +│| apache.arrow | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink | │ +│apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | │ +│apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb |│ +│asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai│ +│| common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks | │ │datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | │ │facebook | ftp | git | github | google | grpc | hashicorp | http | ibm.db2 | ibm.mq | imap | │ │influxdb | informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | │ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index 815858fd19c08..17bf31294a904 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -fbadce49313f856f69861005e009e602 +d8c8fb42d8bde40b7e8252056d144184 diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index 1b0bb895c030a..fd739ee8590e2 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -204,17 +204,17 @@ Usage:                                                                                                                 breeze workflow-run publish-docs                                                                                       [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-mypy | apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink |       -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging |  -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch |   -exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.db2 | ibm.mq | imap -| influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | -microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle |    -pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment |      -sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram |   -teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...                                    +apache-airflow-mypy | apache-airflow-providers | apache.arrow | apache.cassandra | apache.drill | apache.druid |       +apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy +| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 +| elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http |       +ibm.db2 | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure |            +microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage |  +opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis |    +salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard |    +tableau | task-sdk | telegram | teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...    Trigger publish docs to S3 workflow diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index d676e74277cb2..bd1e1fde13747 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -822d1cea8d470fc7af38a81d9246a88c +c4e8633fee01c127eb062f8ccfac1c5c diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index eb1c32eac6419..61175b67f6a77 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -3065,7 +3065,7 @@ def test_upgrade_to_newer_dependencies( pytest.param( ("providers/common/sql/src/airflow/providers/common/sql/common_sql_python.py",), { - "docs-list-as-string": "amazon apache.drill apache.druid apache.hive apache.iceberg " + "docs-list-as-string": "amazon apache.arrow apache.drill apache.druid apache.hive apache.iceberg " "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks elasticsearch " "exasol google ibm.db2 informatica jdbc microsoft.mssql mysql odbc openlineage " "oracle pgvector postgres presto slack snowflake sqlite teradata trino vertica ydb", diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 27dd08f58151a..ac832abd633da 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -15,6 +15,7 @@ actionCard Acyclic acyclic adb +adbc additionalProperties adf adhoc diff --git a/providers/apache/arrow/.gitignore b/providers/apache/arrow/.gitignore new file mode 100644 index 0000000000000..bff2d7629604d --- /dev/null +++ b/providers/apache/arrow/.gitignore @@ -0,0 +1 @@ +*.iml diff --git a/providers/apache/arrow/LICENSE b/providers/apache/arrow/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/apache/arrow/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/apache/arrow/NOTICE b/providers/apache/arrow/NOTICE new file mode 100644 index 0000000000000..a51bd9390d030 --- /dev/null +++ b/providers/apache/arrow/NOTICE @@ -0,0 +1,5 @@ +Apache Airflow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/providers/apache/arrow/README.rst b/providers/apache/arrow/README.rst new file mode 100644 index 0000000000000..6c74c6d44845c --- /dev/null +++ b/providers/apache/arrow/README.rst @@ -0,0 +1,113 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + ``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the ``dev/breeze/src/airflow_breeze/templates`` DIRECTORY + +Package ``apache-airflow-providers-apache-arrow`` + +Release: ``0.1.0`` + + +`Apache Arrow `__ + + +Provider package +---------------- + +This is a provider package for ``apache.arrow`` provider. All classes for this provider package +are in ``airflow.providers.apache.arrow`` python package. + +You can find package information and changelog for the provider +in the `documentation `_. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-apache-arrow`` + +The package supports the following python versions: 3.9,3.10,3.11,3.12 + +Requirements +------------ + +======================================= ====================================== +PIP package Version required +======================================= ====================================== +``apache-airflow`` ``>=2.11.0`` +``apache-airflow-providers-common-sql`` ``>=1.28.2`` +``adbc-driver-manager`` ``>=1.7.0`` +``more-itertools`` ``>=9.0.0`` +``pyarrow`` ``>=16.1.0; python_version < "3.13"`` +``pyarrow`` ``>=18.0.0; python_version >= "3.13"`` +======================================= ====================================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified providers in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-arrow[common.sql] + + +============================================================================================================ ============== +Dependent package Extra +============================================================================================================ ============== +`apache-airflow-providers-common-sql `_ ``common.sql`` +============================================================================================================ ============== + +Optional dependencies +---------------------- + +This provider supports multiple ADBC drivers for different databases. You can install the required driver(s) as extras: + +============== ===================================================================================================================================================================== +Extra Dependency +============== ===================================================================================================================================================================== +``sqlite`` ``adbc-driver-sqlite>=1.7.0`` +``postgresql`` ``adbc-driver-postgresql>=1.7.0`` +``snowflake`` ``adbc-driver-snowflake>=1.7.0`` +``bigquery`` ``adbc-driver-bigquery>=1.7.0`` +``flightsql`` ``adbc-driver-flightsql>=1.7.0`` +``all`` ``adbc-driver-sqlite>=1.7.0``, ``adbc-driver-postgresql>=1.7.0``, ``adbc-driver-snowflake>=1.7.0``, ``adbc-driver-bigquery>=1.7.0``, ``adbc-driver-flightsql>=1.7.0`` +============== ===================================================================================================================================================================== + +To use the provider with a specific database, install the provider with the corresponding extra. For example, to use SQLite: + +.. code-block:: bash + + pip install apache-airflow-providers-apache-arrow[sqlite] + +To install all available drivers: + +.. code-block:: bash + + pip install apache-airflow-providers-apache-arrow[all] + +These extras install the Python ADBC drivers required to connect to each database. Refer to the documentation for each driver for any additional system requirements. + +The changelog for the provider package can be found in the +`changelog `_. diff --git a/providers/apache/arrow/docs/.latest-doc-only-change.txt b/providers/apache/arrow/docs/.latest-doc-only-change.txt new file mode 100644 index 0000000000000..f41e3226a6f43 --- /dev/null +++ b/providers/apache/arrow/docs/.latest-doc-only-change.txt @@ -0,0 +1 @@ +7b2ec33c7ad4998d9c9735b79593fcdcd3b9dd1f diff --git a/providers/apache/arrow/docs/changelog.rst b/providers/apache/arrow/docs/changelog.rst new file mode 100644 index 0000000000000..5e5931cb0e98e --- /dev/null +++ b/providers/apache/arrow/docs/changelog.rst @@ -0,0 +1,46 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes + and you want to add an explanation to the users on how they are supposed to deal with them. + The changelog is updated and maintained semi-automatically by release manager. + +``apache-airflow-providers-apache-arrow`` + +Changelog +--------- + +0.1.0 +..... + +.. note:: + This release of provider is only available for Airflow 2.10+ as explained in the + Apache Airflow providers support policy _. + +Initial version of the provider. + +Misc +~~~~ + +* ``Bump min Airflow version in providers to 2.10 (#49843)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Update description of provider.yaml dependencies (#50231)`` + * ``Avoid committing history for providers (#49907)`` diff --git a/providers/apache/arrow/docs/commits.rst b/providers/apache/arrow/docs/commits.rst new file mode 100644 index 0000000000000..78554447218d2 --- /dev/null +++ b/providers/apache/arrow/docs/commits.rst @@ -0,0 +1,35 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! + +Package apache-airflow-providers-apache-arrow +------------------------------------------------------ + +`ADBC: Arrow Database Connectivity `__ + + +This is detailed commit list of changes for versions provider package: ``arrow``. +For high-level changelog, see :doc:`package information including changelog `. + +.. airflow-providers-commits:: diff --git a/providers/apache/arrow/docs/conf.py b/providers/apache/arrow/docs/conf.py new file mode 100644 index 0000000000000..29c8f3976f1d7 --- /dev/null +++ b/providers/apache/arrow/docs/conf.py @@ -0,0 +1,27 @@ +# Disable Flake8 because of all the sphinx imports +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Configuration of Providers docs building.""" + +from __future__ import annotations + +import os + +os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-apache-arrow" + +from docs.provider_conf import * # noqa: F403 diff --git a/providers/apache/arrow/docs/connections/adbc.rst b/providers/apache/arrow/docs/connections/adbc.rst new file mode 100644 index 0000000000000..a747ff611a371 --- /dev/null +++ b/providers/apache/arrow/docs/connections/adbc.rst @@ -0,0 +1,196 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/connection:adbc: + +ADBC connection +=============== + +The ADBC connection type enables connection to any database that has an +`Arrow Database Connectivity (ADBC) `__ driver, +such as PostgreSQL, SQLite, DuckDB, Snowflake, BigQuery, and Flight SQL servers. + +Connections of this type are used by :class:`~airflow.providers.apache.arrow.hooks.adbc.AdbcHook`, +which builds on top of :class:`~airflow.providers.common.sql.hooks.sql.DbApiHook` and transfers +data as Apache Arrow :class:`~pyarrow.RecordBatch` objects for efficient, zero-copy bulk loads. + +Default Connection ID +--------------------- + +``adbc_default`` + +Configuring the Connection +-------------------------- + +Connection URL (Host field) + The database URI passed to the driver. For drivers that use a standard + connection string (e.g. ``postgresql://user:pass@host:5432/db`` or + ``file::memory:``), put it here. If the value contains ``::`` the hook + passes it through unchanged; otherwise the ``adbc://`` scheme prefix is + replaced with the dialect name. + +Login / Password + Convenience fields. When set, Airflow builds a URI of the form + ``://login:password@host/schema`` that is merged into + ``db_kwargs["uri"]``. Driver-specific URIs in the Host field take + precedence. + +Extra (JSON) + A JSON object with the following recognized keys: + + ``driver`` *(string)* + Python package name of the ADBC driver to load, e.g. + ``"adbc_driver_postgresql"`` or ``"adbc_driver_sqlite"``. + When omitted the hook derives the name from the dialect as + ``adbc_driver_``. + + ``entrypoint`` *(string, optional)* + Fully-qualified Python symbol used as the driver entrypoint, e.g. + ``"adbc_driver_sqlite.dbapi.connect"``. Most drivers do not need + this; it is only required when the automatic entrypoint discovery + built into ``adbc_driver_manager`` cannot locate the function. + + ``dialect`` *(string, optional)* + SQL dialect name used to build the URI scheme and derive the + default driver name. Defaults to ``"default"``. + + ``db_kwargs`` *(object, optional)* + Keyword arguments forwarded verbatim to the ADBC + ``AdbcDatabase`` constructor (database-level connection settings). + ``uri`` is always injected automatically from the Host field and + must not be repeated here. Common keys: + + .. list-table:: + :header-rows: 1 + :widths: 30 15 55 + + * - Key + - Type + - Description + * - ``username`` + - string + - Database user name (alternative to encoding it in the URI). + * - ``password`` + - string + - Database password (alternative to encoding it in the URI). + * - ``adbc.sqlite.query.batch_rows`` + - int + - SQLite: number of rows fetched per Arrow batch. + + All other keys are driver-specific. Consult the documentation for + your ADBC driver for the full list. See also the + `ADBC driver documentation `__. + + ``conn_kwargs`` *(object, optional)* + Keyword arguments forwarded verbatim to the ADBC + ``AdbcConnection`` constructor (connection-level settings within the + already-open database). Common keys defined by the ADBC + specification: + + .. list-table:: + :header-rows: 1 + :widths: 30 15 55 + + * - Key + - Type + - Description + * - ``adbc.connection.autocommit`` + - string + - Enable autocommit for this connection. Use ``"true"`` or ``"false"``. + * - ``adbc.connection.read_only`` + - string + - Open the connection in read-only mode. Use ``"true"`` or ``"false"``. + * - ``adbc.connection.current_catalog`` + - string + - Default catalog for unqualified table names. + * - ``adbc.connection.current_db_schema`` + - string + - Default schema for unqualified table names. + + All other keys are driver-specific. + +.. warning:: **Security — blast radius of db_kwargs / conn_kwargs** + + Both ``db_kwargs`` and ``conn_kwargs`` are passed through to the ADBC + driver **without filtering**. Any user or role that can edit this + connection in the Airflow UI or via the Connections API can therefore + supply arbitrary driver-level options. Depending on the driver, this + may include options that: + + * load extensions from arbitrary file-system paths (e.g. SQLite's + ``load_extension`` or DuckDB's extension loading); + * configure network endpoints or proxy addresses; + * change authentication parameters. + + Restrict access to the Connections UI and API (Airflow RBAC ``Connections`` + resource) to trusted operators only. Do not expose connection-editing + permissions to untrusted Dag authors. + + Similarly, the ``driver`` and ``entrypoint`` extras load shared libraries + or Python symbols by name. Treat them with the same level of trust as + executable code. + +Examples +-------- + +SQLite in-memory database: + +.. code-block:: json + + { + "driver": "adbc_driver_sqlite" + } + +PostgreSQL with extra driver options: + +.. code-block:: json + + { + "driver": "adbc_driver_postgresql", + "dialect": "postgresql", + "db_kwargs": { + "adbc.postgresql.query.batch_rows": 1000 + }, + "conn_kwargs": { + "adbc.connection.autocommit": "true" + } + } + +Usage +----- + +Use :class:`~airflow.providers.apache.arrow.hooks.adbc.AdbcHook` in a task to query or load data: + +.. code-block:: python + + from airflow.providers.apache.arrow.hooks.adbc import AdbcHook + + + def load_data(): + hook = AdbcHook(adbc_conn_id="my_adbc_conn") + + # Run a query and get results as a list of tuples + records = hook.get_records("SELECT id, name FROM users WHERE active = 1") + + # Bulk-insert rows in chunks of 5 000 using Arrow-native transfer + rows = [(1, "Alice"), (2, "Bob")] + hook.insert_rows( + table="users", + rows=rows, + target_fields=["id", "name"], + commit_every=5000, + ) diff --git a/providers/apache/arrow/docs/index.rst b/providers/apache/arrow/docs/index.rst new file mode 100644 index 0000000000000..4e24347bf5ee9 --- /dev/null +++ b/providers/apache/arrow/docs/index.rst @@ -0,0 +1,143 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-apache-arrow`` +========================================= + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: References + + Python API <_api/airflow/providers/apache/arrow/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/apache/arrow/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + PyPI Repository + Example Dags + Installing from sources + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Connection types + Using AdbcHook + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-apache-arrow package +------------------------------------------------------ + +`Apache Arrow `__ provider for Airflow. Offers ``AdbcHook``, +a general-purpose hook that connects to any database supported by an +`Arrow Database Connectivity (ADBC) `__ driver +(PostgreSQL, SQLite, DuckDB, Snowflake, BigQuery, Flight SQL, and more) and transfers data +as columnar Arrow ``RecordBatch`` objects for efficient, zero-copy bulk loads. + + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``apache.arrow`` provider. +All classes for this package are included in the ``airflow.providers.apache.arrow`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-apache-arrow``. +For the minimum Airflow version supported, see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``2.11.0``. + +======================================= ====================================== +PIP package Version required +======================================= ====================================== +``apache-airflow`` ``>=2.11.0`` +``apache-airflow-providers-common-sql`` ``>=1.28.2`` +``adbc-driver-manager`` ``>=1.7.0`` +``more-itertools`` ``>=9.0.0`` +``pyarrow`` ``>=16.1.0; python_version < "3.13"`` +``pyarrow`` ``>=18.0.0; python_version >= "3.13"`` +======================================= ====================================== + +Optional dependencies +--------------------- + +These extras install optional third-party libraries that enable additional features of the provider. +Install them when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-apache-arrow[sqlite] + + +============== ===================================================================================================================================================================== +Extra Dependencies +============== ===================================================================================================================================================================== +``sqlite`` ``adbc-driver-sqlite>=1.7.0`` +``snowflake`` ``adbc-driver-snowflake>=1.7.0`` +``bigquery`` ``adbc-driver-bigquery>=1.7.0`` +``flightsql`` ``adbc-driver-flightsql>=1.7.0`` +``postgresql`` ``adbc-driver-postgresql>=1.7.0`` +``all`` ``adbc-driver-sqlite>=1.7.0``, ``adbc-driver-snowflake>=1.7.0``, ``adbc-driver-bigquery>=1.7.0``, ``adbc-driver-flightsql>=1.7.0``, ``adbc-driver-postgresql>=1.7.0`` +============== ===================================================================================================================================================================== + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-apache-arrow 0.1.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-arrow 0.1.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/arrow/docs/installing-providers-from-sources.rst b/providers/apache/arrow/docs/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..fdbb17d017579 --- /dev/null +++ b/providers/apache/arrow/docs/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/installing-providers-from-sources.rst diff --git a/providers/apache/arrow/docs/integration-logos/apache-arrow.png b/providers/apache/arrow/docs/integration-logos/apache-arrow.png new file mode 100644 index 0000000000000..164401d1eeff8 Binary files /dev/null and b/providers/apache/arrow/docs/integration-logos/apache-arrow.png differ diff --git a/providers/apache/arrow/docs/operators.rst b/providers/apache/arrow/docs/operators.rst new file mode 100644 index 0000000000000..e3548650d1641 --- /dev/null +++ b/providers/apache/arrow/docs/operators.rst @@ -0,0 +1,39 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/hook:AdbcHook: + +Using AdbcHook +============== + +Use :class:`~airflow.providers.apache.arrow.hooks.adbc.AdbcHook` to interact with any database +that has an `ADBC `__ driver. The hook extends +:class:`~airflow.providers.common.sql.hooks.sql.DbApiHook`, so it works transparently with +:class:`~airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator` and the other +operators from the ``common.sql`` provider — just pass ``conn_id`` pointing at an ``adbc`` +connection. + +For direct hook usage (e.g. inside a ``@dag.task`` function), the example below demonstrates +creating a table, bulk-inserting rows via Arrow-native transfer, querying the results, and +cleaning up — all against a SQLite ADBC connection. + +.. exampleinclude:: /../../arrow/tests/system/apache/arrow/example_adbc.py + :language: python + :start-after: [START howto_adbc_hook] + :end-before: [END howto_adbc_hook] + +See :ref:`Connection types ` for how to configure the ``adbc`` connection. diff --git a/providers/apache/arrow/docs/redirects.txt b/providers/apache/arrow/docs/redirects.txt new file mode 100644 index 0000000000000..3a3c336a8474d --- /dev/null +++ b/providers/apache/arrow/docs/redirects.txt @@ -0,0 +1 @@ +connections/index.rst connections/adbc.rst diff --git a/providers/apache/arrow/docs/security.rst b/providers/apache/arrow/docs/security.rst new file mode 100644 index 0000000000000..351ff007ebf2f --- /dev/null +++ b/providers/apache/arrow/docs/security.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/security.rst diff --git a/providers/apache/arrow/provider.yaml b/providers/apache/arrow/provider.yaml new file mode 100644 index 0000000000000..d9c9f16fed0de --- /dev/null +++ b/providers/apache/arrow/provider.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-apache-arrow +name: Apache Arrow + +state: ready +lifecycle: incubation +source-date-epoch: 1744788833 +description: | + `Apache Arrow `__ provider for Airflow. Offers ``AdbcHook``, + a general-purpose hook that connects to any database supported by an + `Arrow Database Connectivity (ADBC) `__ driver + (PostgreSQL, SQLite, DuckDB, Snowflake, BigQuery, Flight SQL, and more) and transfers data + as columnar Arrow ``RecordBatch`` objects for efficient, zero-copy bulk loads. +# Note that those versions are maintained by release manager - do not update them manually +# with the exception of case where other provider in sources has >= new provider version. +# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have +# to be done in the same PR +versions: + - 0.1.0 + +integrations: + - integration-name: Apache Arrow + external-doc-url: https://arrow.apache.org/ + how-to-guide: + - /docs/apache-airflow-providers-apache-arrow/operators.rst + logo: /docs/integration-logos/apache-arrow.png + tags: [apache] + +hooks: + - integration-name: Apache Arrow + python-modules: + - airflow.providers.apache.arrow.hooks.adbc + +connection-types: + - hook-class-name: airflow.providers.apache.arrow.hooks.adbc.AdbcHook + connection-type: adbc diff --git a/providers/apache/arrow/pyproject.toml b/providers/apache/arrow/pyproject.toml new file mode 100644 index 0000000000000..fd03c2a961653 --- /dev/null +++ b/providers/apache/arrow/pyproject.toml @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==4.0.2"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-apache-arrow" +version = "0.1.0" +description = "Provider package apache-airflow-providers-apache-arrow for Apache Airflow" +readme = "README.rst" +license = "Apache-2.0" +license-files = ['LICENSE', 'NOTICE'] +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "apache.arrow", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: System :: Monitoring", +] +requires-python = ">=3.10" + +# The dependencies should be modified in place in the generated file. +# Any change in the dependencies is preserved when the file is regenerated +# Make sure to run ``prek update-providers-dependencies --all-files`` +# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` +dependencies = [ + "apache-airflow>=2.11.0", + "apache-airflow-providers-common-sql>=1.28.2", + "adbc-driver-manager>=1.7.0", + "more-itertools>=9.0.0", + "pyarrow>=16.1.0; python_version < '3.13'", + "pyarrow>=18.0.0; python_version >= '3.13'", +] + +# The optional dependencies should be modified in place in the generated file +# Any change in the dependencies is preserved when the file is regenerated +[project.optional-dependencies] +"sqlite" = [ + "adbc-driver-sqlite>=1.7.0" +] +"snowflake" = [ + "adbc-driver-snowflake>=1.7.0" +] +"bigquery" = [ + "adbc-driver-bigquery>=1.7.0" +] +"flightsql" = [ + "adbc-driver-flightsql>=1.7.0" +] +"postgresql" = [ + "adbc-driver-postgresql>=1.7.0" +] +"all" = [ + "adbc-driver-sqlite>=1.7.0", + "adbc-driver-snowflake>=1.7.0", + "adbc-driver-bigquery>=1.7.0", + "adbc-driver-flightsql>=1.7.0", + "adbc-driver-postgresql>=1.7.0" +] + +[dependency-groups] +dev = [ + "apache-airflow", + "apache-airflow-task-sdk", + "apache-airflow-devel-common", + "apache-airflow-providers-common-sql", + # Additional devel dependencies (do not remove this line and add extra development dependencies) +] + +# To build docs: +# +# uv run --group docs build-docs +# +# To enable auto-refreshing build with server: +# +# uv run --group docs build-docs --autobuild +# +# To see more options: +# +# uv run --group docs build-docs --help +# +docs = [ + "apache-airflow-devel-common[docs]" +] + +[tool.uv.sources] +# These names must match the names as defined in the pyproject.toml of the workspace items, +# *not* the workspace folder paths +apache-airflow = {workspace = true} +apache-airflow-devel-common = {workspace = true} +apache-airflow-task-sdk = {workspace = true} +apache-airflow-providers-common-sql = {workspace = true} +apache-airflow-providers-standard = {workspace = true} + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-arrow/0.1.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-arrow/0.1.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Mastodon" = "https://fosstodon.org/@airflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.apache.arrow.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.apache.arrow" + +# Explicit sdist contents so the build does not rely on VCS information +# (flit 4.0 makes --no-use-vcs the default — see https://github.com/pypa/flit/pull/782). +[tool.flit.sdist] +include = [ + "docs/", + "provider.yaml", + "src/airflow/__init__.py", + "src/airflow/providers/__init__.py", + "src/airflow/providers/apache/__init__.py", + "tests/", +] diff --git a/providers/apache/arrow/src/airflow/__init__.py b/providers/apache/arrow/src/airflow/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/src/airflow/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/src/airflow/providers/__init__.py b/providers/apache/arrow/src/airflow/providers/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/src/airflow/providers/apache/__init__.py b/providers/apache/arrow/src/airflow/providers/apache/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/src/airflow/providers/apache/arrow/LICENSE b/providers/apache/arrow/src/airflow/providers/apache/arrow/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/arrow/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/apache/arrow/src/airflow/providers/apache/arrow/__init__.py b/providers/apache/arrow/src/airflow/providers/apache/arrow/__init__.py new file mode 100644 index 0000000000000..e76bded9f7ffd --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/arrow/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +# +from __future__ import annotations + +import packaging.version + +from airflow import __version__ as airflow_version + +__all__ = ["__version__"] + +__version__ = "0.1.0" + +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "2.11.0" +): + raise RuntimeError( + f"The package `apache-airflow-providers-apache-arrow:{__version__}` needs Apache Airflow 2.11.0+" + ) diff --git a/providers/apache/arrow/src/airflow/providers/apache/arrow/get_provider_info.py b/providers/apache/arrow/src/airflow/providers/apache/arrow/get_provider_info.py new file mode 100644 index 0000000000000..7182b9372414d --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/arrow/get_provider_info.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-apache-arrow", + "name": "Apache Arrow", + "description": "`Apache Arrow `__ provider for Airflow. Offers ``AdbcHook``,\na general-purpose hook that connects to any database supported by an\n`Arrow Database Connectivity (ADBC) `__ driver\n(PostgreSQL, SQLite, DuckDB, Snowflake, BigQuery, Flight SQL, and more) and transfers data\nas columnar Arrow ``RecordBatch`` objects for efficient, zero-copy bulk loads.\n", + "integrations": [ + { + "integration-name": "Apache Arrow", + "external-doc-url": "https://arrow.apache.org/", + "how-to-guide": ["/docs/apache-airflow-providers-apache-arrow/operators.rst"], + "logo": "/docs/integration-logos/apache-arrow.png", + "tags": ["apache"], + } + ], + "hooks": [ + { + "integration-name": "Apache Arrow", + "python-modules": ["airflow.providers.apache.arrow.hooks.adbc"], + } + ], + "connection-types": [ + { + "hook-class-name": "airflow.providers.apache.arrow.hooks.adbc.AdbcHook", + "connection-type": "adbc", + } + ], + } diff --git a/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/__init__.py b/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/adbc.py b/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/adbc.py new file mode 100644 index 0000000000000..9bde7d6dc64cf --- /dev/null +++ b/providers/apache/arrow/src/airflow/providers/apache/arrow/hooks/adbc.py @@ -0,0 +1,345 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import contextlib +import pathlib +import re +import sys +from collections.abc import Callable, Iterable, Mapping +from contextlib import closing +from functools import cached_property +from importlib import resources as importlib_resources +from typing import TYPE_CHECKING, Any + +from adbc_driver_manager.dbapi import Connection, connect +from more_itertools import chunked +from pyarrow import RecordBatch, Schema, array, schema + +from airflow.providers.common.sql.dialects.dialect import Dialect +from airflow.providers.common.sql.hooks.sql import DbApiHook + +if TYPE_CHECKING: + from adbc_driver_manager.dbapi import Cursor + + +def fetch_all_handler(cursor) -> list[tuple] | None: + """Return results for DbApiHook.run().""" + if not hasattr(cursor, "description"): + raise RuntimeError( + "The database we interact with does not support DBAPI 2.0. Use operator and " + "handlers that are specifically designed for your database." + ) + if cursor.description is not None: + table = cursor.fetch_arrow_table() + return list(zip(*(column.to_pylist() for column in table.columns))) + return None + + +def replace_placeholders(sql: str, placeholder: str) -> str: + # Replace each placeholder with $1, $2, $3 ... in order + counter = [1] + + def replacer(match): + replacement = f"${counter[0]}" + counter[0] += 1 + return replacement + + return re.sub(placeholder, replacer, sql) + + +# https://arrow.apache.org/adbc/current/python/api/adbc_driver_manager.html +# https://arrow.apache.org/docs/python/ +class AdbcHook(DbApiHook): + """ + General-purpose Airflow hook for interacting with databases via the Arrow Database Connectivity (ADBC) standard. + + This hook enables connections to any database supported by an ADBC driver, using the Python ADBC driver manager. + It provides methods for executing SQL queries, inserting rows in bulk, and handling Arrow-native data transfers. + + Key Features: + - Supports chunked and batched inserts using Apache Arrow RecordBatches for efficient data transfer. + - Discovers and loads ADBC drivers dynamically based on connection extras or naming conventions. + - Handles dialect-specific connection URIs and driver entrypoints. + - Integrates with Airflow's connection system (conn_id, extras, etc.). + - Provides custom placeholder replacement for parameterized SQL queries. + - Supports both native Arrow binding and DBAPI ``executemany`` for inserts. + - Exposes configuration via connection extras: driver, entrypoint, db_kwargs, conn_kwargs, dialect. + + Connection Extras: + - driver: Name of the ADBC driver to use (e.g., "adbc_driver_postgresql"). + - entrypoint: Optional Python entrypoint for the driver. + - db_kwargs: Driver-specific database initialization options passed to + ``AdbcDatabase``. Keys and value types are defined by the driver + (e.g., ``"username"``, ``"password"`` for the PostgreSQL driver). + Do **not** put ADBC connection options here — they belong in + ``conn_kwargs``. + - conn_kwargs: ADBC connection options as string key-value pairs. + Keys must use the canonical dotted ADBC option names, for example: + ``"adbc.connection.autocommit": "true"``, + ``"adbc.connection.read_only": "true"``, + ``"adbc.connection.current_catalog": "my_catalog"``, + ``"adbc.connection.current_db_schema": "my_schema"``. + Short names such as ``autocommit`` or ``read_only`` are **not** + recognized by ADBC drivers and will raise ``NotSupportedError``. + - dialect: SQL dialect name (default: "default"). + + Example usage: + hook = AdbcHook(adbc_conn_id="my_adbc_conn") + records = hook.get_records("SELECT * FROM my_table") + + For more details, see: + - Apache Arrow ADBC Python API: https://arrow.apache.org/adbc/current/python/api/adbc_driver_manager.html + - Airflow SQL hooks: https://airflow.apache.org/docs/apache-airflow/stable/howto/custom-operator.html#hooks + """ + + conn_name_attr = "adbc_conn_id" + default_conn_name = "adbc_default" + conn_type = "adbc" + hook_name = "ADBC Connection" + supports_autocommit = True + + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Get custom field behaviour.""" + return { + "hidden_fields": ["port", "schema"], + "relabeling": {"host": "Connection URL"}, + } + + @cached_property + def _driver_path(self) -> str: + # Wheels bundle the shared library + root = importlib_resources.files(self.driver) + # The filename is always the same regardless of platform + entrypoint = root.joinpath(f"lib{self.driver}.so") + if entrypoint.is_file(): + return str(entrypoint) + + # Search sys.prefix + '/lib' (Unix, Conda on Unix) + root = pathlib.Path(sys.prefix) + for filename in (f"lib{self.driver}.so", f"lib{self.driver}.dylib"): + entrypoint = root.joinpath("lib", filename) + if entrypoint.is_file(): + return str(entrypoint) + + # Conda on Windows + entrypoint = root.joinpath("bin", f"{self.driver}.dll") + if entrypoint.is_file(): + return str(entrypoint) + + # Let the driver manager fall back to (DY)LD_LIBRARY_PATH/PATH + # (It will insert 'lib', 'so', etc. as needed) + return self.driver + + @cached_property + def uri(self) -> str: + host = self.connection.host + if host and "::" in str(host): + return str(host) + uri = self.get_uri() + return uri.replace( + f"{self.conn_type.lower().replace('_', '-')}://", + f"{self.dialect_name.lower().replace('_', '-')}://", + ) + + @cached_property + def driver(self) -> str: + return self.connection_extra_lower.get("driver") or f"adbc_driver_{self.dialect_name}" + + @cached_property + def entrypoint(self) -> str | None: + return self.connection_extra_lower.get("entrypoint") + + @cached_property + def db_kwargs(self) -> dict: + return {**{"uri": self.uri}, **self.connection_extra_lower.get("db_kwargs", {})} + + @cached_property + def conn_kwargs(self) -> dict: + return self.connection_extra_lower.get("conn_kwargs", {}) + + @cached_property + def dialect_name(self) -> str: + return self.connection_extra_lower.get("dialect", "default") + + def get_conn(self) -> Connection: + return connect( + driver=self._driver_path, + entrypoint=self.entrypoint, + db_kwargs=self.db_kwargs, + conn_kwargs=self.conn_kwargs, + autocommit=False, + ) + + def set_autocommit(self, conn: Connection, autocommit: bool) -> None: + """ + Set autocommit on the ADBC connection. + + The DBAPI attribute ``conn.autocommit`` has no effect on the underlying + ADBC driver; the real lever is the ``adbc.connection.autocommit`` option. + This override applies the option at the driver level, then calls super() + to keep the Python-level bookkeeping attribute in sync so that the base + class ``get_autocommit()`` and ``run()`` commit-guard behave correctly. + """ + conn.adbc_connection.set_autocommit(autocommit) + super().set_autocommit(conn, autocommit) + + def get_records( + self, + sql: str | list[str], + parameters: Iterable | Mapping[str, Any] | None = None, + ) -> Any: + """ + Execute the sql and return a set of records. + + :param sql: the sql statement to be executed (str) or a list of sql statements to execute + :param parameters: The parameters to render the SQL query with. + """ + return self.run(sql=sql, parameters=parameters, handler=fetch_all_handler) + + def _run_command(self, cur, sql_statement, parameters): + """Run a statement using an already open cursor.""" + if parameters: + sql_statement = replace_placeholders(sql_statement, re.escape(self.dialect.placeholder)) + + super()._run_command(cur, sql_statement, parameters) + + def _generate_insert_sql(self, table, values, target_fields=None, replace: bool = False, **kwargs) -> str: + sql_statement = super()._generate_insert_sql( + table, values, target_fields=target_fields, replace=replace, **kwargs + ) + sql_statement = replace_placeholders(sql_statement, re.escape(self.dialect.placeholder)) + + if self.log_sql: + self.log.info("Running statement: %s", sql_statement) + + return sql_statement + + @classmethod + def _to_record_batch(cls, rows, schema: Schema) -> RecordBatch: + return RecordBatch.from_arrays( + [array([row[index] for row in rows], type=field.type) for index, field in enumerate(schema)], + schema=schema, + ) + + @classmethod + def _execute_executemany(cls, cursor, statement: str, record_batch: RecordBatch) -> None: + """ + Execute a statement using cursor.executemany with a RecordBatch. + + ADBC's executemany() accepts Arrow data directly and routes it through + bind_stream internally, making this the Arrow-native fast path. + """ + cursor.executemany(statement, record_batch) + + def _resolve_execute_batch( + self, cursor, executemany: bool, fast_executemany: bool + ) -> Callable[[Cursor, str, RecordBatch], None]: + """ + Return the batch-execute callable for the given cursor. + + ADBC's executemany() natively accepts a RecordBatch, so it is always + the fast path. fast_executemany is passed through to drivers that + expose that knob (silently ignored otherwise). + """ + if fast_executemany: + with contextlib.suppress(AttributeError): + cursor.fast_executemany = True + self.log.info( + "Fast_executemany is enabled for conn_id '%s'!", + self.get_conn_id(), + ) + return self._execute_executemany + + def insert_rows( + self, + table, + rows, + target_fields=None, + commit_every=1000, + replace=False, + *, + executemany=False, + fast_executemany=False, + autocommit=False, + **kwargs, + ): + """ + Insert a collection of tuples into a table. + + Rows are inserted in chunks, each chunk (of size ``commit_every``) is + done in a new transaction. + + :param table: Name of the target table + :param rows: The rows to insert into the table + :param target_fields: The names of the columns to fill in the table + :param commit_every: The maximum number of rows to insert in one + transaction. Set to 0 to insert all rows in one transaction. + :param replace: Whether to replace instead of insert + :param executemany: If True, all rows are inserted at once in + chunks defined by the commit_every parameter. This only works if all rows + have same number of column names, but leads to better performance. + :param fast_executemany: If True, the `fast_executemany` parameter will be set on the + cursor used by `executemany` which leads to better performance, if supported by driver. + :param autocommit: What to set the connection's autocommit setting to + before executing the query. + """ + nb_rows = 0 + + with self._create_autocommit_connection(autocommit) as conn: + table_name, schema_name = Dialect.extract_schema_from_table(table) + + table_schema = conn.adbc_get_table_schema( + table_name=table_name, + db_schema_filter=schema_name, + ) + + if not target_fields: + target_fields = table_schema.names + else: + fields = {field.name: field for field in table_schema} + table_schema = schema([fields[name] for name in target_fields]) + + self.log.info("target fields: %s", target_fields) + self.log.info("table_schema: %s", table_schema) + + sql = self._generate_insert_sql( + table, + target_fields, # values not needed — parameters will come from RecordBatch + target_fields, + replace, + **kwargs, + ) + + with closing(conn.cursor()) as cur: + execute_batch = self._resolve_execute_batch( + cur, executemany=executemany, fast_executemany=fast_executemany + ) + + for chunked_rows in chunked(rows, commit_every or None): + batch = self._to_record_batch(rows=chunked_rows, schema=table_schema) + execute_batch(cur, sql, batch) + + if not autocommit: + conn.commit() + + nb_rows += batch.num_rows + self.log.info("Loaded %s rows into %s so far", nb_rows, table) + + self.log.info("Done loading. Loaded a total of %s rows into %s", nb_rows, table) diff --git a/providers/apache/arrow/tests/conftest.py b/providers/apache/arrow/tests/conftest.py new file mode 100644 index 0000000000000..f56ccce0a3f69 --- /dev/null +++ b/providers/apache/arrow/tests/conftest.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +pytest_plugins = "tests_common.pytest_plugin" diff --git a/providers/apache/arrow/tests/system/__init__.py b/providers/apache/arrow/tests/system/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/tests/system/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/tests/system/apache/__init__.py b/providers/apache/arrow/tests/system/apache/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/tests/system/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/tests/system/apache/arrow/__init__.py b/providers/apache/arrow/tests/system/apache/arrow/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/arrow/tests/system/apache/arrow/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/arrow/tests/system/apache/arrow/example_adbc.py b/providers/apache/arrow/tests/system/apache/arrow/example_adbc.py new file mode 100644 index 0000000000000..d5c052a11ff58 --- /dev/null +++ b/providers/apache/arrow/tests/system/apache/arrow/example_adbc.py @@ -0,0 +1,96 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example Dag demonstrating :class:`~airflow.providers.apache.arrow.hooks.adbc.AdbcHook`. + +Prerequisites +------------- +An Airflow connection of type ``adbc`` must exist with conn_id ``adbc_sqlite_default``: + +.. code-block:: json + + { + "conn_id": "adbc_sqlite_default", + "conn_type": "adbc", + "host": "file::memory:?cache=shared", + "extra": { + "driver": "adbc_driver_sqlite" + } + } +""" + +from __future__ import annotations + +import os +from datetime import datetime + +from airflow import DAG + +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_adbc" +CONN_ID = "adbc_sqlite_default" + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example"], +) as dag: + # [START howto_adbc_hook] + @dag.task + def create_table(): + from airflow.providers.apache.arrow.hooks.adbc import AdbcHook + + hook = AdbcHook(adbc_conn_id=CONN_ID) + hook.run("CREATE TABLE IF NOT EXISTS users ( id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + + @dag.task + def insert_rows(): + from airflow.providers.apache.arrow.hooks.adbc import AdbcHook + + hook = AdbcHook(adbc_conn_id=CONN_ID) + rows = [(1, "Alice"), (2, "Bob"), (3, "Carol")] + hook.insert_rows(table="users", rows=rows, target_fields=["id", "name"]) + + @dag.task + def query_rows(): + from airflow.providers.apache.arrow.hooks.adbc import AdbcHook + + hook = AdbcHook(adbc_conn_id=CONN_ID) + records = hook.get_records("SELECT id, name FROM users ORDER BY id") + assert len(records) == 3, f"Expected 3 rows, got {len(records)}" + + @dag.task + def drop_table(): + from airflow.providers.apache.arrow.hooks.adbc import AdbcHook + + hook = AdbcHook(adbc_conn_id=CONN_ID) + hook.run("DROP TABLE IF EXISTS users") + + create_table() >> insert_rows() >> query_rows() >> drop_table() + # [END howto_adbc_hook] + + from tests_common.test_utils.watcher import watcher + + list(dag.tasks) >> watcher() + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) +test_run = get_test_run(dag) diff --git a/providers/apache/arrow/tests/unit/__init__.py b/providers/apache/arrow/tests/unit/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/tests/unit/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/tests/unit/apache/__init__.py b/providers/apache/arrow/tests/unit/apache/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/arrow/tests/unit/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/arrow/tests/unit/apache/arrow/__init__.py b/providers/apache/arrow/tests/unit/apache/arrow/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/arrow/tests/unit/apache/arrow/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/arrow/tests/unit/apache/arrow/hooks/__init__.py b/providers/apache/arrow/tests/unit/apache/arrow/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/arrow/tests/unit/apache/arrow/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/arrow/tests/unit/apache/arrow/hooks/test_adbc.py b/providers/apache/arrow/tests/unit/apache/arrow/hooks/test_adbc.py new file mode 100644 index 0000000000000..e14fae08425ec --- /dev/null +++ b/providers/apache/arrow/tests/unit/apache/arrow/hooks/test_adbc.py @@ -0,0 +1,388 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import importlib +import json +import logging +from unittest import mock + +import pytest +from adbc_driver_manager import dbapi +from adbc_driver_manager.dbapi import Cursor +from pyarrow import field, schema, string + +from airflow.models import Connection +from airflow.providers.apache.arrow.hooks.adbc import AdbcHook +from airflow.providers.common.sql.dialects.dialect import Dialect + + +class TestAdbcHook: + def setup_method(self): + self.cur = mock.MagicMock(spec=Cursor, rowcount=0) + self.cur.fast_executemany = False + self.conn = mock.MagicMock(spec=dbapi.Connection) + self.conn.cursor.return_value = self.cur + # Provide a real pyarrow Schema so _to_record_batch can build RecordBatch + self.conn.adbc_get_table_schema.return_value = schema([field("col", string())]) + self.conn.extra_dejson = {} + # get_messages is not on the dbapi.Connection spec but is called by AdbcHookMock.get_db_log_messages + self.conn.get_messages = mock.MagicMock(return_value=[]) + conn = self.conn + + logging.root.disabled = True + + self.hook = self.make_hook_for_conn(conn) + + # Return a real pyarrow Table so fetch_arrow_table().columns works correctly. + import pyarrow as pa + + self.cur.fetch_arrow_table.return_value = pa.table({"col": [1, 2]}) + + def make_hook_for_conn(self, conn): + """ + Return an AdbcHook subclass instance bound to the provided conn. + + Tests previously redefined this subclass locally in multiple places. + This helper centralizes that logic so tests can simply call + `self.make_hook_for_conn(conn)`. + """ + + class AdbcHookMock(AdbcHook): + conn_name_attr = "adbc_default" + + @classmethod + def get_connection(cls, conn_id: str) -> Connection: + return conn + + def get_conn(self): + return conn + + @property + def dialect(self): + return Dialect(self) + + def get_db_log_messages(self, _conn) -> None: + return _conn.get_messages() + + return AdbcHookMock() + + def test_get_records_fetch_all_handler(self): + result = self.hook.get_records("SELECT 1") + assert result == [(1,), (2,)] + + def test_get_records_duplicate_column_names(self): + """Duplicate column names (e.g. a JOIN) must not collapse to fewer columns.""" + import pyarrow as pa + + # Two columns both named "id" — as produced by SELECT a.id, b.id FROM a JOIN b. + table = pa.table({"a_id": [10, 20], "b_id": [30, 40]}).rename_columns(["id", "id"]) + self.cur.fetch_arrow_table.return_value = table + + result = self.hook.get_records("SELECT a.id, b.id FROM a JOIN b ON a.k = b.k") + + assert result == [(10, 30), (20, 40)] + + def test_insert_rows_uses_executemany_with_record_batch(self): + """insert_rows must call executemany with a RecordBatch — the Arrow-native fast path.""" + rows = [("a",), ("b",)] + self.hook.insert_rows("table", rows) + + assert self.cur.executemany.called + batch = self.cur.executemany.call_args[0][1] + assert batch.num_rows == 2 + assert self.conn.commit.call_count >= 1 + + def test_insert_rows_autocommit_skips_explicit_commit(self): + """With autocommit=True the chunk loop must not call conn.commit() — drivers raise on it.""" + rows = [("a",), ("b",)] + self.hook.insert_rows("table", rows, autocommit=True) + + assert self.cur.executemany.called + assert self.conn.commit.call_count == 0 + + def test_set_autocommit_applies_adbc_option(self): + """set_autocommit must call set_autocommit() on the underlying AdbcConnection.""" + conn = mock.MagicMock() + self.hook.set_autocommit(conn, True) + conn.adbc_connection.set_autocommit.assert_called_once_with(True) + assert conn.autocommit is True + + conn.reset_mock() + self.hook.set_autocommit(conn, False) + conn.adbc_connection.set_autocommit.assert_called_once_with(False) + assert conn.autocommit is False + + def test_insert_rows_commit_every_zero_inserts_all_rows(self): + """commit_every=0 must insert all rows in a single transaction, not zero rows.""" + rows = [("a",), ("b",), ("c",)] + self.hook.insert_rows("table", rows, commit_every=0) + + assert self.cur.executemany.called + batch = self.cur.executemany.call_args[0][1] + assert batch.num_rows == 3 + assert self.conn.commit.call_count == 1 + + def test_insert_rows_fast_executemany_not_supported(self): + # Cursor that doesn't support setting fast_executemany + class NoFastExecCursor(mock.MagicMock): + def __setattr__(self, name, value): + if name == "fast_executemany": + raise AttributeError("fast_executemany not supported") + super().__setattr__(name, value) + + cur = NoFastExecCursor(spec=Cursor) + conn = mock.MagicMock() + conn.cursor.return_value = cur + conn.adbc_get_table_schema.return_value = schema([field("col", string())]) + conn.extra_dejson = {} + hook = self.make_hook_for_conn(conn) + + table = "table" + rows = [("x",), ("y",)] + + hook.insert_rows(table, rows, executemany=True, fast_executemany=True) + + assert cur.executemany.called + assert conn.commit.call_count >= 1 + + def test_insert_rows_fast_executemany_supported(self): + # Cursor that supports setting fast_executemany + cur = mock.MagicMock(spec=Cursor) + conn = mock.MagicMock() + conn.cursor.return_value = cur + conn.adbc_get_table_schema.return_value = schema([field("col", string())]) + conn.extra_dejson = {} + hook = self.make_hook_for_conn(conn) + + table = "table" + rows = [("x",), ("y",)] + + hook.insert_rows(table, rows, executemany=True, fast_executemany=True) + + assert cur.fast_executemany + assert cur.executemany.called + assert conn.commit.call_count >= 1 + + def test_insert_rows_target_fields_order_preserved(self): + """Schema must follow target_fields order so SQL and RecordBatch agree.""" + from pyarrow import int64 + + cur = mock.MagicMock(spec=Cursor) + conn = mock.MagicMock() + conn.cursor.return_value = cur + # Table declares (first TEXT, last TEXT, age INT64) — note: first before last. + conn.adbc_get_table_schema.return_value = schema( + [field("first", string()), field("last", string()), field("age", int64())] + ) + conn.extra_dejson = {} + hook = self.make_hook_for_conn(conn) + + # Insert with reversed column order and omit age. + rows = [("Smith", "Alice"), ("Jones", "Bob")] + hook.insert_rows("t", rows, target_fields=["last", "first"]) + + assert cur.executemany.called + batch = cur.executemany.call_args[0][1] # second positional arg is the RecordBatch + # Schema order must match target_fields, not the table declaration. + assert batch.schema.names == ["last", "first"] + # Values must land in the declared column order. + assert batch.column("last").to_pylist() == ["Smith", "Jones"] + assert batch.column("first").to_pylist() == ["Alice", "Bob"] + + @pytest.mark.skipif( + importlib.util.find_spec("adbc_driver_sqlite") is None, + reason="adbc_driver_sqlite not installed", + ) + def test_dbapi_connection(self, create_connection_without_db): + create_connection_without_db( + Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:?cache=shared", + extra=json.dumps( + { + "driver": "adbc_driver_sqlite", + } + ), + ) + ) + + with AdbcHook()._create_autocommit_connection() as adbc_conn: + assert isinstance(adbc_conn, dbapi.Connection) + + @pytest.mark.skipif( + importlib.util.find_spec("adbc_driver_sqlite") is None, + reason="adbc_driver_sqlite not installed", + ) + @pytest.mark.parametrize( + ("conn_kwargs", "should_raise"), + [ + pytest.param( + {"adbc.connection.autocommit": "true"}, + False, + id="canonical-dotted-name-accepted", + ), + pytest.param( + {"autocommit": "true"}, + True, + id="short-name-rejected", + ), + ], + ) + def test_conn_kwargs_canonical_names_required(self, conn_kwargs, should_raise): + """Canonical dotted ADBC option names are required; short names raise NotSupportedError. + + This test documents and enforces actual driver behaviour so that the hook + docstring and the connection docs stay accurate. + """ + import adbc_driver_sqlite.dbapi as sqlite_dbapi + from adbc_driver_manager import NotSupportedError + + if should_raise: + with sqlite_dbapi.connect("file::memory:") as conn: + with pytest.raises(NotSupportedError): + conn.adbc_connection.set_options(**conn_kwargs) + else: + with sqlite_dbapi.connect("file::memory:") as conn: + conn.adbc_connection.set_options(**conn_kwargs) + + def _make_real_conn_hook(self, connection: Connection) -> AdbcHook: + """Return an AdbcHook whose get_connection() returns *connection* but whose + get_conn() is the real implementation (needed to verify connect() kwargs).""" + + class AdbcHookWithRealGetConn(AdbcHook): + conn_name_attr = "adbc_default" + + @classmethod + def get_connection(cls, conn_id: str) -> Connection: + return connection + + return AdbcHookWithRealGetConn() + + @mock.patch("airflow.providers.apache.arrow.hooks.adbc.connect") + def test_get_conn_forwards_db_kwargs(self, mock_connect): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps( + { + "driver": "adbc_driver_sqlite", + # These are driver-specific database init options (PostgreSQL examples). + "db_kwargs": {"username": "admin", "password": "secret"}, + } + ), + ) + hook = self._make_real_conn_hook(conn) + hook.__dict__["_driver_path"] = "adbc_driver_sqlite" + + hook.get_conn() + + mock_connect.assert_called_once() + kw = mock_connect.call_args.kwargs + assert kw["db_kwargs"]["username"] == "admin" + assert kw["db_kwargs"]["password"] == "secret" + assert kw["db_kwargs"]["uri"] == "file::memory:" + + @mock.patch("airflow.providers.apache.arrow.hooks.adbc.connect") + def test_get_conn_forwards_conn_kwargs(self, mock_connect): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps( + { + "driver": "adbc_driver_sqlite", + # conn_kwargs must use the canonical dotted ADBC option names. + "conn_kwargs": { + "adbc.connection.autocommit": "true", + "adbc.connection.read_only": "true", + }, + } + ), + ) + hook = self._make_real_conn_hook(conn) + hook.__dict__["_driver_path"] = "adbc_driver_sqlite" + + hook.get_conn() + + mock_connect.assert_called_once() + kw = mock_connect.call_args.kwargs + assert kw["conn_kwargs"] == { + "adbc.connection.autocommit": "true", + "adbc.connection.read_only": "true", + } + + @mock.patch("airflow.providers.apache.arrow.hooks.adbc.connect") + def test_get_conn_forwards_entrypoint(self, mock_connect): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps( + { + "driver": "adbc_driver_sqlite", + "entrypoint": "adbc_driver_sqlite.dbapi.connect", + } + ), + ) + hook = self._make_real_conn_hook(conn) + hook.__dict__["_driver_path"] = "adbc_driver_sqlite" + + hook.get_conn() + + mock_connect.assert_called_once() + kw = mock_connect.call_args.kwargs + assert kw["entrypoint"] == "adbc_driver_sqlite.dbapi.connect" + + def test_dialect_name_from_extra(self): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps({"driver": "adbc_driver_postgresql", "dialect": "postgresql"}), + ) + hook = self.make_hook_for_conn(conn) + assert hook.dialect_name == "postgresql" + + def test_dialect_name_defaults_to_default(self): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps({"driver": "adbc_driver_sqlite"}), + ) + hook = self.make_hook_for_conn(conn) + assert hook.dialect_name == "default" + + @mock.patch("airflow.providers.apache.arrow.hooks.adbc.connect") + def test_get_conn_empty_extras_defaults(self, mock_connect): + conn = Connection( + conn_id="adbc_default", + conn_type="adbc", + host="file::memory:", + extra=json.dumps({"driver": "adbc_driver_sqlite"}), + ) + hook = self._make_real_conn_hook(conn) + hook.__dict__["_driver_path"] = "adbc_driver_sqlite" + + hook.get_conn() + + kw = mock_connect.call_args.kwargs + assert kw["conn_kwargs"] == {} + assert kw["entrypoint"] is None diff --git a/pyproject.toml b/pyproject.toml index 5179300f6f464..d932e11520971 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,9 @@ apache-airflow = "airflow.__main__:main" "anthropic" = [ "apache-airflow-providers-anthropic>=0.1.0" ] +"apache.arrow" = [ + "apache-airflow-providers-apache-arrow>=0.1.0" +] "apache.cassandra" = [ "apache-airflow-providers-apache-cassandra>=3.7.0; python_version !=\"3.14\"" ] @@ -413,6 +416,7 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-alibaba>=3.0.0", "apache-airflow-providers-amazon>=9.0.0", "apache-airflow-providers-anthropic>=0.1.0", + "apache-airflow-providers-apache-arrow>=0.1.0", "apache-airflow-providers-apache-cassandra>=3.7.0; python_version !=\"3.14\"", "apache-airflow-providers-apache-drill>=2.8.1", "apache-airflow-providers-apache-druid>=3.12.0", @@ -1106,6 +1110,8 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/amazon/tests", "$MYPY_CONFIG_FILE_DIR/providers/anthropic/src", "$MYPY_CONFIG_FILE_DIR/providers/anthropic/tests", + "$MYPY_CONFIG_FILE_DIR/providers/apache/arrow/src", + "$MYPY_CONFIG_FILE_DIR/providers/apache/arrow/tests", "$MYPY_CONFIG_FILE_DIR/providers/apache/cassandra/src", "$MYPY_CONFIG_FILE_DIR/providers/apache/cassandra/tests", "$MYPY_CONFIG_FILE_DIR/providers/apache/drill/src", @@ -1448,6 +1454,7 @@ apache-airflow-providers-akeyless = false apache-airflow-providers-alibaba = false apache-airflow-providers-amazon = false apache-airflow-providers-anthropic = false +apache-airflow-providers-apache-arrow = false apache-airflow-providers-apache-cassandra = false apache-airflow-providers-apache-drill = false apache-airflow-providers-apache-druid = false @@ -1607,6 +1614,7 @@ apache-airflow-providers-akeyless = false apache-airflow-providers-alibaba = false apache-airflow-providers-amazon = false apache-airflow-providers-anthropic = false +apache-airflow-providers-apache-arrow = false apache-airflow-providers-apache-cassandra = false apache-airflow-providers-apache-drill = false apache-airflow-providers-apache-druid = false @@ -1773,6 +1781,7 @@ apache-airflow-providers-akeyless = { workspace = true } apache-airflow-providers-alibaba = { workspace = true } apache-airflow-providers-amazon = { workspace = true } apache-airflow-providers-anthropic = { workspace = true } +apache-airflow-providers-apache-arrow = { workspace = true } apache-airflow-providers-apache-cassandra = { workspace = true } apache-airflow-providers-apache-drill = { workspace = true } apache-airflow-providers-apache-druid = { workspace = true } @@ -1915,6 +1924,7 @@ members = [ "providers/alibaba", "providers/amazon", "providers/anthropic", + "providers/apache/arrow", "providers/apache/cassandra", "providers/apache/drill", "providers/apache/druid", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index 39602e0fb2cf7..8ba4a3909ecf2 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -31,6 +31,7 @@ services: - ../../../empty:/opt/airflow/providers/alibaba/src - ../../../empty:/opt/airflow/providers/amazon/src - ../../../empty:/opt/airflow/providers/anthropic/src + - ../../../empty:/opt/airflow/providers/apache/arrow/src - ../../../empty:/opt/airflow/providers/apache/beam/src - ../../../empty:/opt/airflow/providers/apache/cassandra/src - ../../../empty:/opt/airflow/providers/apache/drill/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index e7a5a5a927d28..9e2192a203d0d 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -44,6 +44,7 @@ services: - ../../../providers/alibaba/tests:/opt/airflow/providers/alibaba/tests - ../../../providers/amazon/tests:/opt/airflow/providers/amazon/tests - ../../../providers/anthropic/tests:/opt/airflow/providers/anthropic/tests + - ../../../providers/apache/arrow/tests:/opt/airflow/providers/apache/arrow/tests - ../../../providers/apache/beam/tests:/opt/airflow/providers/apache/beam/tests - ../../../providers/apache/cassandra/tests:/opt/airflow/providers/apache/cassandra/tests - ../../../providers/apache/drill/tests:/opt/airflow/providers/apache/drill/tests diff --git a/uv.lock b/uv.lock index d4f97bd3b9923..7817a2c76141e 100644 --- a/uv.lock +++ b/uv.lock @@ -72,9 +72,9 @@ apache-airflow-providers-apache-cassandra = false apache-airflow-providers-asana = false apache-airflow-providers-oracle = false apache-airflow-providers-mysql = false -apache-airflow-providers-teradata = false apache-airflow-providers-alibaba = false apache-airflow-providers-microsoft-mssql = false +apache-airflow-providers-teradata = false apache-airflow-providers-jdbc = false apache-airflow-helm-chart = false apache-airflow-providers-anthropic = false @@ -112,6 +112,7 @@ apache-airflow-shared-serialization = false apache-airflow-scripts = false apache-airflow-providers-exasol = false apache-airflow-providers-mongo = false +apache-airflow-providers-apache-arrow = false apache-airflow-providers-apprise = false apache-airflow-providers-apache-impala = false apache-airflow-ctl = false @@ -189,6 +190,7 @@ members = [ "apache-airflow-providers-alibaba", "apache-airflow-providers-amazon", "apache-airflow-providers-anthropic", + "apache-airflow-providers-apache-arrow", "apache-airflow-providers-apache-cassandra", "apache-airflow-providers-apache-drill", "apache-airflow-providers-apache-druid", @@ -336,12 +338,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/8d/58008a9a86075827f99aa8bb75d8db515bb9c34654f95e647cda31987db7/adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d", size = 55539, upload-time = "2021-04-05T16:33:39.544Z" }, ] +[[package]] +name = "adbc-driver-bigquery" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/0d/3e7195500badfe3318d6409e3d7461354f7e3e57b3b8ee2cb5a4c0c78bbd/adbc_driver_bigquery-1.11.0.tar.gz", hash = "sha256:37fc2a90dfec1f741bc7475bdf50c744c22d05ec135d0858934117912c06077e", size = 19255, upload-time = "2026-04-07T00:17:26.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/0f/76baf47962bfa49608fcd5d38c3a46213155aeb1f58e5c8abf467ed7e4bc/adbc_driver_bigquery-1.11.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:4153bb759dcfa6c45420c4ee9a1505f3c5daa7685d10cf068a91b8876e183732", size = 9736982, upload-time = "2026-04-07T00:14:31.386Z" }, + { url = "https://files.pythonhosted.org/packages/15/0b/b88fd632ea5b26ded6f11cbfc62d154249b54d4ae95ad8bdc2e681f3cbb0/adbc_driver_bigquery-1.11.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:023744303c4900fc8e3d12cd044835d406ee8a84880458766aff0800601d4999", size = 9049987, upload-time = "2026-04-07T00:14:37.48Z" }, + { url = "https://files.pythonhosted.org/packages/9e/0a/ce0b0772f479eb451993ed9a0d139fcdaddcab651d1ce09520003163aa54/adbc_driver_bigquery-1.11.0-py3-none-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3067b90ccbb6801936567bddcc56e79540efb7e09023f0300e9f6090cb7245e1", size = 18043100, upload-time = "2026-04-07T00:14:44.159Z" }, + { url = "https://files.pythonhosted.org/packages/74/2c/f71aaea78f4859935e334b1c5791e959cb24315201789bb9f7a8fd5a94c2/adbc_driver_bigquery-1.11.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dea28d663b622e1b4c803fd063b0b959dc867218461ecb6e34b557423e956527", size = 16537326, upload-time = "2026-04-07T00:14:49.284Z" }, + { url = "https://files.pythonhosted.org/packages/f2/41/595bdc39a6e8eea2be3ccd043a10e68243b8a682d45f52b5d982674269f3/adbc_driver_bigquery-1.11.0-py3-none-win_amd64.whl", hash = "sha256:75ff98aa9071707419c300f9da74e19da375cd6524ecf09493c8131960e31c3c", size = 17698043, upload-time = "2026-04-07T00:14:53.622Z" }, +] + +[[package]] +name = "adbc-driver-flightsql" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/65/91/605b35b97aa5972a8026c0687e9996658d1d10ff7a0624173a0d0c7c5b93/adbc_driver_flightsql-1.12.0.tar.gz", hash = "sha256:300f67801ea016578e0c4bb798fc2b9024e405cfdd3f79fd77e37481a5767e45", size = 22097, upload-time = "2026-07-28T00:43:02.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/a4/5ad52abf0c4830585b86d68cff8ce07779cdee9b8cf390dff0ee21a29fa7/adbc_driver_flightsql-1.12.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:c2c7209407c180d9d0dd94846203f184ab91e116782585bf29cb968c2686ba75", size = 8146345, upload-time = "2026-07-28T00:41:22.824Z" }, + { url = "https://files.pythonhosted.org/packages/61/36/49267a4fc8c07c066dd8c36b9c3d27265a4dc0fc478cb5acb7bd6a551d80/adbc_driver_flightsql-1.12.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fb37efefceb7ceca64840c76c55e6280f3811b54dca305ec1781f9cc1a6da21e", size = 7552045, upload-time = "2026-07-28T00:41:27.147Z" }, + { url = "https://files.pythonhosted.org/packages/2b/5b/f8566aa6d05eb40225f5874f8f5f5e8b8b4a4ce866d817a5ff352f326302/adbc_driver_flightsql-1.12.0-py3-none-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:69742e4f8fcc254490aa7f8e254a3b704c24bde9d5d827a0926ea90a6b4c3a6d", size = 14981635, upload-time = "2026-07-28T00:41:33.2Z" }, + { url = "https://files.pythonhosted.org/packages/57/45/6468fb425f3c0ae868c1ca9ab8e15cba76b59e941edd7e80e892e5476728/adbc_driver_flightsql-1.12.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9adfb5e46abc77347476d09ee76dfc835fef2b4dcb0c76d08d6d35aa3a4cce17", size = 13699634, upload-time = "2026-07-28T00:41:38.232Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/6773dedc54fad73277ee2d767dd817e203e949972b0cfb197df99e00b784/adbc_driver_flightsql-1.12.0-py3-none-win_amd64.whl", hash = "sha256:098f2498778fd3efd671fbeacba80987de98ebfeb0291da77fe0a26c6fa78532", size = 15033803, upload-time = "2026-07-28T00:41:42.483Z" }, +] + [[package]] name = "adbc-driver-manager" version = "1.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9c/f8/ed6475b49a7cf35ea888d5c95e7d4bc9dc6568f9d741f14c0573d622cc1e/adbc_driver_manager-1.12.0.tar.gz", hash = "sha256:45991f0c2de369d330c6a211ca2edbcce6389c5dc81cde70461bdeb6f8f7b268", size = 217579, upload-time = "2026-07-28T00:43:03.512Z" } wheels = [ @@ -382,8 +418,8 @@ name = "adbc-driver-postgresql" version = "1.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "adbc-driver-manager", marker = "python_full_version < '3.13'" }, - { name = "importlib-resources", marker = "python_full_version < '3.13'" }, + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/9e/cc757dfc1bb5472e35bf066ee4044f6b101bef61036512e8dfb4e97e7e08/adbc_driver_postgresql-1.12.0.tar.gz", hash = "sha256:766a002531bb99b691d2b92e7d928dea21c24ea567c03a6ee1edb61fe95b9187", size = 17793, upload-time = "2026-07-28T00:43:04.481Z" } wheels = [ @@ -394,13 +430,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/02/7aa782cbb0134b09d1e67757c81332e0cd5e5697beecc8852468482193b0/adbc_driver_postgresql-1.12.0-py3-none-win_amd64.whl", hash = "sha256:5a3b5262eed6f28fb4c782b532e6a65caed1f2268fab7be736335ead49eed9dc", size = 3207946, upload-time = "2026-07-28T00:42:51.543Z" }, ] +[[package]] +name = "adbc-driver-snowflake" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/c1/561549e36abe8644bedc6b4848abb79ab41a6b0fc4b106ceef944b0c3493/adbc_driver_snowflake-1.11.0.tar.gz", hash = "sha256:7fce54ae7540c6c63aad6cc9ec2795a05647cc147c8af41ca2bd4e95455e8eae", size = 30570, upload-time = "2026-04-07T00:17:30.403Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/f3/1125c7929faf750e7a5bdcec33ca22259ce040338dae12f75f3eb8080def/adbc_driver_snowflake-1.11.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:5ebddfa5fc0b5cd7c86b5cce17bb578889c8db5ff2b756903b64e684b56829e8", size = 15094744, upload-time = "2026-04-07T00:16:55.86Z" }, + { url = "https://files.pythonhosted.org/packages/d5/1d/88d564a8941607e6e555a3d3290ecb92852d4aa5a926a1cdbe86b17a8424/adbc_driver_snowflake-1.11.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:be862f8d0b6a507dae5c0644e13be79290180532c097d97c5a95a294c0bb5302", size = 13517204, upload-time = "2026-04-07T00:17:00.137Z" }, + { url = "https://files.pythonhosted.org/packages/6f/26/baa44b8b9faaf26846dd8c0e513c7ce6862d7f35de46ee61175ac976699f/adbc_driver_snowflake-1.11.0-py3-none-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7c7d7ecef7246303648a08cee3d78c383133bea8f2e1568cf6e8750ab94b5a57", size = 27574063, upload-time = "2026-04-07T00:17:04.862Z" }, + { url = "https://files.pythonhosted.org/packages/5d/41/fe557f0d8414037dfe86fcba6338eb094a51da894050ec9a4c9f4504513d/adbc_driver_snowflake-1.11.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:07b5edbeb1b85be0882d7095641df26a24b3b4c2ba4d8edda35e3bbe89e70a06", size = 24767291, upload-time = "2026-04-07T00:17:09.763Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ca/542793fc9a164f013e3248061e922ac5d5246ce93718a5c0cb44bbd2219e/adbc_driver_snowflake-1.11.0-py3-none-win_amd64.whl", hash = "sha256:83b1a363a44ae4cb1e4956f9cc6c4e44dddd7c768a0f0a8b42d9cb8eb8906305", size = 27264722, upload-time = "2026-04-07T00:17:14.083Z" }, +] + [[package]] name = "adbc-driver-sqlite" version = "1.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "adbc-driver-manager", marker = "python_full_version < '3.13'" }, - { name = "importlib-resources", marker = "python_full_version < '3.13'" }, + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f5/02/2dc143bdd2a62c52d103d4b0ae491a347944aaed25b3d40fb11797750c70/adbc_driver_sqlite-1.12.0.tar.gz", hash = "sha256:18466a2f0c14f94cb0b17818157cc14ed6b93aef0a48ef648de945e9bac1540d", size = 12849, upload-time = "2026-07-28T00:43:05.408Z" } wheels = [ @@ -662,7 +715,7 @@ name = "aiohttp-cors" version = "0.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, + { name = "aiohttp", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/d89e846a5444b3d5eb8985a6ddb0daef3774928e1bfbce8e84ec97b0ffa7/aiohttp_cors-0.8.1.tar.gz", hash = "sha256:ccacf9cb84b64939ea15f859a146af1f662a6b1d68175754a07315e305fb1403", size = 38626, upload-time = "2025-03-31T14:16:20.048Z" } wheels = [ @@ -994,6 +1047,7 @@ all = [ { name = "apache-airflow-providers-alibaba" }, { name = "apache-airflow-providers-amazon", extra = ["aiobotocore", "python3-saml", "s3fs"] }, { name = "apache-airflow-providers-anthropic" }, + { name = "apache-airflow-providers-apache-arrow" }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*'" }, { name = "apache-airflow-providers-apache-drill" }, { name = "apache-airflow-providers-apache-druid" }, @@ -1113,6 +1167,9 @@ amazon-aws-auth = [ anthropic = [ { name = "apache-airflow-providers-anthropic" }, ] +apache-arrow = [ + { name = "apache-airflow-providers-apache-arrow" }, +] apache-atlas = [ { name = "atlasclient" }, ] @@ -1559,6 +1616,8 @@ requires-dist = [ { name = "apache-airflow-providers-amazon", extras = ["s3fs"], marker = "extra == 's3fs'", editable = "providers/amazon" }, { name = "apache-airflow-providers-anthropic", marker = "extra == 'all'", editable = "providers/anthropic" }, { name = "apache-airflow-providers-anthropic", marker = "extra == 'anthropic'", editable = "providers/anthropic" }, + { name = "apache-airflow-providers-apache-arrow", marker = "extra == 'all'", editable = "providers/apache/arrow" }, + { name = "apache-airflow-providers-apache-arrow", marker = "extra == 'apache-arrow'", editable = "providers/apache/arrow" }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*' and extra == 'all'", editable = "providers/apache/cassandra" }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*' and extra == 'apache-cassandra'", editable = "providers/apache/cassandra" }, { name = "apache-airflow-providers-apache-drill", marker = "extra == 'all'", editable = "providers/apache/drill" }, @@ -1768,7 +1827,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.12.5" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-arrow", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] [package.metadata.requires-dev] ci-image = [ @@ -3276,6 +3335,83 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +[[package]] +name = "apache-airflow-providers-apache-arrow" +version = "0.1.0" +source = { editable = "providers/apache/arrow" } +dependencies = [ + { name = "adbc-driver-manager" }, + { name = "apache-airflow" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "more-itertools" }, + { name = "pyarrow" }, +] + +[package.optional-dependencies] +all = [ + { name = "adbc-driver-bigquery" }, + { name = "adbc-driver-flightsql" }, + { name = "adbc-driver-postgresql" }, + { name = "adbc-driver-snowflake" }, + { name = "adbc-driver-sqlite" }, +] +bigquery = [ + { name = "adbc-driver-bigquery" }, +] +flightsql = [ + { name = "adbc-driver-flightsql" }, +] +postgresql = [ + { name = "adbc-driver-postgresql" }, +] +snowflake = [ + { name = "adbc-driver-snowflake" }, +] +sqlite = [ + { name = "adbc-driver-sqlite" }, +] + +[package.dev-dependencies] +dev = [ + { name = "apache-airflow" }, + { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "apache-airflow-task-sdk" }, +] +docs = [ + { name = "apache-airflow-devel-common", extra = ["docs"] }, +] + +[package.metadata] +requires-dist = [ + { name = "adbc-driver-bigquery", marker = "extra == 'all'", specifier = ">=1.7.0" }, + { name = "adbc-driver-bigquery", marker = "extra == 'bigquery'", specifier = ">=1.7.0" }, + { name = "adbc-driver-flightsql", marker = "extra == 'all'", specifier = ">=1.7.0" }, + { name = "adbc-driver-flightsql", marker = "extra == 'flightsql'", specifier = ">=1.7.0" }, + { name = "adbc-driver-manager", specifier = ">=1.7.0" }, + { name = "adbc-driver-postgresql", marker = "extra == 'all'", specifier = ">=1.7.0" }, + { name = "adbc-driver-postgresql", marker = "extra == 'postgresql'", specifier = ">=1.7.0" }, + { name = "adbc-driver-snowflake", marker = "extra == 'all'", specifier = ">=1.7.0" }, + { name = "adbc-driver-snowflake", marker = "extra == 'snowflake'", specifier = ">=1.7.0" }, + { name = "adbc-driver-sqlite", marker = "extra == 'all'", specifier = ">=1.7.0" }, + { name = "adbc-driver-sqlite", marker = "extra == 'sqlite'", specifier = ">=1.7.0" }, + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "more-itertools", specifier = ">=9.0.0" }, + { name = "pyarrow", marker = "python_full_version < '3.13'", specifier = ">=16.1.0" }, + { name = "pyarrow", marker = "python_full_version >= '3.13'", specifier = ">=18.0.0" }, +] +provides-extras = ["sqlite", "snowflake", "bigquery", "flightsql", "postgresql", "all"] + +[package.metadata.requires-dev] +dev = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "apache-airflow-task-sdk", editable = "task-sdk" }, +] +docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] + [[package]] name = "apache-airflow-providers-apache-cassandra" version = "3.9.5" @@ -11355,7 +11491,7 @@ name = "colorful" version = "0.5.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "python_full_version < '3.15' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/82/31/109ef4bedeb32b4202e02ddb133162457adc4eb890a9ed9c05c9dd126ed0/colorful-0.5.8.tar.gz", hash = "sha256:bb16502b198be2f1c42ba3c52c703d5f651d826076817185f0294c1a549a7445", size = 209361, upload-time = "2025-10-29T11:53:21.663Z" } wheels = [ @@ -18221,9 +18357,9 @@ name = "opencensus" version = "0.11.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "google-api-core" }, - { name = "opencensus-context" }, - { name = "six" }, + { name = "google-api-core", marker = "python_full_version < '3.15'" }, + { name = "opencensus-context", marker = "python_full_version < '3.15'" }, + { name = "six", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/15/a7/a46dcffa1b63084f9f17fe3c8cb20724c4c8f91009fd0b2cfdb27d5d2b35/opencensus-0.11.4.tar.gz", hash = "sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2", size = 64966, upload-time = "2024-01-03T18:04:07.085Z" } wheels = [ @@ -21747,14 +21883,14 @@ name = "ray" version = "2.57.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "filelock" }, - { name = "jsonschema" }, - { name = "msgpack" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "requests" }, + { name = "click", marker = "python_full_version < '3.15'" }, + { name = "filelock", marker = "python_full_version < '3.15'" }, + { name = "jsonschema", marker = "python_full_version < '3.15'" }, + { name = "msgpack", marker = "python_full_version < '3.15'" }, + { name = "packaging", marker = "python_full_version < '3.15'" }, + { name = "protobuf", marker = "python_full_version < '3.15'" }, + { name = "pyyaml", marker = "python_full_version < '3.15'" }, + { name = "requests", marker = "python_full_version < '3.15'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/0a/18/2e608519b656ff1aa61492e9dcca965a4703965e1ccb0409eecabfd76a51/ray-2.57.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e14f60ac542f3e9136b10e30cd6602f4e8ba48e482983d3b9221193dd7a91ae5", size = 66760376, upload-time = "2026-08-11T00:20:34.684Z" }, @@ -21779,20 +21915,20 @@ wheels = [ [package.optional-dependencies] default = [ - { name = "aiohttp" }, - { name = "aiohttp-cors" }, - { name = "colorful" }, - { name = "grpcio" }, - { name = "opencensus" }, - { name = "opentelemetry-exporter-prometheus" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "prometheus-client" }, - { name = "py-spy" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "smart-open" }, - { name = "virtualenv" }, + { name = "aiohttp", marker = "python_full_version < '3.15'" }, + { name = "aiohttp-cors", marker = "python_full_version < '3.15'" }, + { name = "colorful", marker = "python_full_version < '3.15'" }, + { name = "grpcio", marker = "python_full_version < '3.15'" }, + { name = "opencensus", marker = "python_full_version < '3.15'" }, + { name = "opentelemetry-exporter-prometheus", marker = "python_full_version < '3.15'" }, + { name = "opentelemetry-proto", marker = "python_full_version < '3.15'" }, + { name = "opentelemetry-sdk", marker = "python_full_version < '3.15'" }, + { name = "prometheus-client", marker = "python_full_version < '3.15'" }, + { name = "py-spy", marker = "python_full_version < '3.15'" }, + { name = "pydantic", marker = "python_full_version < '3.15'" }, + { name = "requests", marker = "python_full_version < '3.15'" }, + { name = "smart-open", marker = "python_full_version < '3.15'" }, + { name = "virtualenv", marker = "python_full_version < '3.15'" }, ] [[package]] @@ -22914,8 +23050,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "python_full_version >= '3.14' or platform_machine != 'arm64' or sys_platform != 'darwin'" }, - { name = "jeepney", marker = "python_full_version >= '3.14' or platform_machine != 'arm64' or sys_platform != 'darwin'" }, + { name = "cryptography", marker = "(python_full_version >= '3.14' and sys_platform == 'darwin') or (python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (platform_machine != 'arm64' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "jeepney", marker = "(python_full_version >= '3.14' and sys_platform == 'darwin') or (python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (platform_machine != 'arm64' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -23114,7 +23250,7 @@ name = "smart-open" version = "8.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wrapt" }, + { name = "wrapt", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/53/9c513747547fd595d5c143259129ea8b9c3ea2f6b7bb9dcea2b1966ded3c/smart_open-8.0.1.tar.gz", hash = "sha256:18b1c4496003c6902be17c15f032b5c319f307c89c6ae9e6b028b508bed8b2cf", size = 61882, upload-time = "2026-07-15T13:56:10.492Z" } wheels = [