diff --git a/providers/oracle/README.rst b/providers/oracle/README.rst index 1d41b1299b794..44620641ca500 100644 --- a/providers/oracle/README.rst +++ b/providers/oracle/README.rst @@ -23,20 +23,23 @@ Package ``apache-airflow-providers-oracle`` -Release: ``4.6.2`` +Release: ``4.6.3`` `Oracle `__ +**Deprecated.** All functionality has moved, unchanged, to +``apache-airflow-providers-oracle-oracledb``. See the changelog below for the migration guide. Provider package ---------------- This is a provider package for ``oracle`` provider. All classes for this provider package -are in ``airflow.providers.oracle`` python package. +are in ``airflow.providers.oracle`` python package, and now re-export the equivalent +classes from ``airflow.providers.oracle.oracledb``. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -50,14 +53,14 @@ The package supports the following python versions: 3.10,3.11,3.12,3.13,3.14 Requirements ------------ -========================================== ================== -PIP package Version required -========================================== ================== -``apache-airflow`` ``>=2.11.0`` -``apache-airflow-providers-common-compat`` ``>=1.8.0`` -``apache-airflow-providers-common-sql`` ``>=1.32.0`` -``oracledb`` ``>=2.3.0`` -========================================== ================== +=========================================== ================== +PIP package Version required +=========================================== ================== +``apache-airflow`` ``>=2.11.0`` +``apache-airflow-providers-common-compat`` ``>=1.8.0`` +``apache-airflow-providers-common-sql`` ``>=1.32.0`` +``apache-airflow-providers-oracle-oracledb`` ``>=4.6.3`` +=========================================== ================== Optional cross provider package dependencies -------------------------------------------- @@ -89,4 +92,4 @@ Extra Dependencies =============== ============================================================================================================================================================================================================================================ The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/oracle/docs/changelog.rst b/providers/oracle/docs/changelog.rst index 8abc97ccf577d..386f3cd11c187 100644 --- a/providers/oracle/docs/changelog.rst +++ b/providers/oracle/docs/changelog.rst @@ -27,6 +27,39 @@ Changelog --------- +4.6.3 +..... + +``apache-airflow-providers-oracle`` is now **deprecated**. All functionality has moved, +unchanged, to the new ``apache-airflow-providers-oracle-oracledb`` provider. Installing +``apache-airflow-providers-oracle`` now pulls in ``apache-airflow-providers-oracle-oracledb`` +as a dependency, and the old import paths keep re-exporting the same classes so existing +Dags keep working, but every import now raises a ``DeprecatedImportWarning``. + +To migrate: + +* Add ``apache-airflow-providers-oracle-oracledb`` to your dependencies (you can drop + ``apache-airflow-providers-oracle`` once you finish migrating). +* Update your imports: + + ============================================================================== ============================================================================ + Old import (deprecated) New import + ============================================================================== ============================================================================ + ``airflow.providers.oracle.hooks.oracle.OracleHook`` ``airflow.providers.oracle.oracledb.hooks.oracle.OracleHook`` + ``airflow.providers.oracle.hooks.handlers`` ``airflow.providers.oracle.oracledb.hooks.handlers`` + ``airflow.providers.oracle.operators.oracle.OracleStoredProcedureOperator`` ``airflow.providers.oracle.oracledb.operators.oracle.OracleStoredProcedureOperator`` + ``airflow.providers.oracle.transfers.oracle_to_oracle.OracleToOracleOperator`` ``airflow.providers.oracle.oracledb.transfers.oracle_to_oracle.OracleToOracleOperator`` + ``airflow.providers.oracle.assets.oracle`` ``airflow.providers.oracle.oracledb.assets.oracle`` + ============================================================================== ============================================================================ + +Connection type (``oracle``), connection ids, and behavior are unchanged — only the +python import path moves. + +Misc +~~~~ + +* ``Deprecate apache-airflow-providers-oracle in favor of apache-airflow-providers-oracle-oracledb`` + 4.6.2 ..... diff --git a/providers/oracle/docs/connections/oracle.rst b/providers/oracle/docs/connections/oracle.rst index a941758d35fda..68835a9f4d3d9 100644 --- a/providers/oracle/docs/connections/oracle.rst +++ b/providers/oracle/docs/connections/oracle.rst @@ -19,6 +19,12 @@ .. _howto/connection:oracle: +.. warning:: + ``apache-airflow-providers-oracle`` is deprecated. Install + ``apache-airflow-providers-oracle-oracledb`` instead and see its + `connection documentation `__. + The ``oracle`` connection type and its configuration are unchanged. + Oracle Connection ================= The Oracle connection type provides connection to a Oracle database. diff --git a/providers/oracle/docs/index.rst b/providers/oracle/docs/index.rst index def6b48c2c2d9..c5b8043e44de6 100644 --- a/providers/oracle/docs/index.rst +++ b/providers/oracle/docs/index.rst @@ -19,6 +19,10 @@ ``apache-airflow-providers-oracle`` =================================== +.. warning:: + This provider is **deprecated**. All functionality has moved, unchanged, to + ``apache-airflow-providers-oracle-oracledb``. See the :doc:`changelog ` + for the migration guide. .. toctree:: :hidden: @@ -76,14 +80,17 @@ apache-airflow-providers-oracle package `Oracle `__ +This provider is deprecated. All functionality has moved, unchanged, to +``apache-airflow-providers-oracle-oracledb``. -Release: 4.6.2 +Release: 4.6.3 Provider package ---------------- This package is for the ``oracle`` provider. -All classes for this package are included in the ``airflow.providers.oracle`` python package. +All classes for this package are included in the ``airflow.providers.oracle`` python package, +and now re-export the equivalent classes from ``airflow.providers.oracle.oracledb``. Installation ------------ @@ -97,14 +104,14 @@ Requirements The minimum Apache Airflow version supported by this provider distribution is ``2.11.0``. -========================================== ================== -PIP package Version required -========================================== ================== -``apache-airflow`` ``>=2.11.0`` -``apache-airflow-providers-common-compat`` ``>=1.8.0`` -``apache-airflow-providers-common-sql`` ``>=1.32.0`` -``oracledb`` ``>=2.3.0`` -========================================== ================== +=========================================== ================== +PIP package Version required +=========================================== ================== +``apache-airflow`` ``>=2.11.0`` +``apache-airflow-providers-common-compat`` ``>=1.8.0`` +``apache-airflow-providers-common-sql`` ``>=1.32.0`` +``apache-airflow-providers-oracle-oracledb`` ``>=4.6.3`` +=========================================== ================== Optional cross provider package dependencies -------------------------------------------- @@ -149,5 +156,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-oracle 4.6.2 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-oracle 4.6.2 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-oracle 4.6.3 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-oracle 4.6.3 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/oracle/docs/operators.rst b/providers/oracle/docs/operators.rst index c72ef24c6c821..72b58584dfc6c 100644 --- a/providers/oracle/docs/operators.rst +++ b/providers/oracle/docs/operators.rst @@ -19,6 +19,13 @@ .. _howto/operator:OracleOperator: +.. warning:: + ``apache-airflow-providers-oracle`` is deprecated. Install + ``apache-airflow-providers-oracle-oracledb`` instead and see its + `operators documentation `__. + The classes below keep working from their old import paths for now, but emit a + deprecation warning; see the :doc:`changelog ` for the full migration guide. + SQLExecuteQueryOperator to connect to Oracle ============================================ diff --git a/providers/oracle/oracledb/.gitignore b/providers/oracle/oracledb/.gitignore new file mode 100644 index 0000000000000..bff2d7629604d --- /dev/null +++ b/providers/oracle/oracledb/.gitignore @@ -0,0 +1 @@ +*.iml diff --git a/providers/oracle/oracledb/LICENSE b/providers/oracle/oracledb/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/oracle/oracledb/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/oracle/oracledb/NOTICE b/providers/oracle/oracledb/NOTICE new file mode 100644 index 0000000000000..a51bd9390d030 --- /dev/null +++ b/providers/oracle/oracledb/NOTICE @@ -0,0 +1,5 @@ +Apache Airflow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/providers/oracle/oracledb/README.rst b/providers/oracle/oracledb/README.rst new file mode 100644 index 0000000000000..4bae8beac5437 --- /dev/null +++ b/providers/oracle/oracledb/README.rst @@ -0,0 +1,94 @@ + +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + ``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the ``dev/breeze/src/airflow_breeze/templates`` DIRECTORY + +Package ``apache-airflow-providers-oracle-oracledb`` + +Release: ``4.6.3`` + + +`Oracle `__ + +This provider was extracted from ``apache-airflow-providers-oracle``, which is now +deprecated. See the ``apache-airflow-providers-oracle`` changelog for migration notes. + +Provider package +---------------- + +This is a provider package for ``oracle`` provider. All classes for this provider package +are in ``airflow.providers.oracle.oracledb`` python package. + +You can find package information and changelog for the provider +in the `documentation `_. + +Installation +------------ + +You can install this package on top of an existing Airflow installation (see ``Requirements`` below +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-oracle-oracledb`` + +The package supports the following python versions: 3.10,3.11,3.12,3.13,3.14 + +Requirements +------------ + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=2.11.0`` +``apache-airflow-providers-common-compat`` ``>=1.8.0`` +``apache-airflow-providers-common-sql`` ``>=1.32.0`` +``oracledb`` ``>=2.3.0`` +========================================== ================== + +Optional cross provider package dependencies +-------------------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified providers in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-oracle-oracledb[openlineage] + + +============================================================================================================== =============== +Dependent package Extra +============================================================================================================== =============== +`apache-airflow-providers-openlineage `_ ``openlineage`` +============================================================================================================== =============== + +Optional dependencies +---------------------- + +=============== ============================================================================================================================================================================================================================================ +Extra Dependencies +=============== ============================================================================================================================================================================================================================================ +``numpy`` ``numpy>=1.22.4; python_version<'3.11'``, ``numpy>=1.23.2; python_version=='3.11'``, ``numpy>=1.26.0; python_version=='3.12'``, ``numpy>=2.1.0; python_version>='3.13' and python_version<'3.14'``, ``numpy>=2.4.3; python_version>='3.14'`` +``openlineage`` ``apache-airflow-providers-openlineage`` +=============== ============================================================================================================================================================================================================================================ + +The changelog for the provider package can be found in the +`changelog `_. diff --git a/providers/oracle/oracledb/docs/.latest-doc-only-change.txt b/providers/oracle/oracledb/docs/.latest-doc-only-change.txt new file mode 100644 index 0000000000000..3f35346f79b81 --- /dev/null +++ b/providers/oracle/oracledb/docs/.latest-doc-only-change.txt @@ -0,0 +1 @@ +134348e1895ad54cfa4d3a75a78bafe872328b11 diff --git a/providers/oracle/oracledb/docs/changelog.rst b/providers/oracle/oracledb/docs/changelog.rst new file mode 100644 index 0000000000000..eca5feda45058 --- /dev/null +++ b/providers/oracle/oracledb/docs/changelog.rst @@ -0,0 +1,53 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes + and you want to add an explanation to the users on how they are supposed to deal with them. + The changelog is updated and maintained semi-automatically by release manager. + +``apache-airflow-providers-oracle-oracledb`` + + +Changelog +--------- + +4.6.3 +..... + +This is the first release of ``apache-airflow-providers-oracle-oracledb``. It is extracted +from ``apache-airflow-providers-oracle``, whose ``4.6.3`` release deprecates the +``airflow.providers.oracle`` python package in favor of this one. + +All classes moved unchanged (same behavior, same connection type ``oracle``, same +``oracle_default`` connection id) to their new import paths: + +============================================================================== ============================================================================ +Old import (``airflow.providers.oracle``, deprecated) New import (``airflow.providers.oracle.oracledb``) +============================================================================== ============================================================================ +``airflow.providers.oracle.hooks.oracle.OracleHook`` ``airflow.providers.oracle.oracledb.hooks.oracle.OracleHook`` +``airflow.providers.oracle.hooks.handlers`` ``airflow.providers.oracle.oracledb.hooks.handlers`` +``airflow.providers.oracle.operators.oracle.OracleStoredProcedureOperator`` ``airflow.providers.oracle.oracledb.operators.oracle.OracleStoredProcedureOperator`` +``airflow.providers.oracle.transfers.oracle_to_oracle.OracleToOracleOperator`` ``airflow.providers.oracle.oracledb.transfers.oracle_to_oracle.OracleToOracleOperator`` +``airflow.providers.oracle.assets.oracle`` ``airflow.providers.oracle.oracledb.assets.oracle`` +============================================================================== ============================================================================ + +To migrate, replace ``apache-airflow-providers-oracle`` with +``apache-airflow-providers-oracle-oracledb`` in your dependencies and update the imports +above in your Dags and plugins. The old import paths keep working for now but emit a +deprecation warning and will be removed in a future release. diff --git a/providers/oracle/oracledb/docs/commits.rst b/providers/oracle/oracledb/docs/commits.rst new file mode 100644 index 0000000000000..a4b6ed4ee8828 --- /dev/null +++ b/providers/oracle/oracledb/docs/commits.rst @@ -0,0 +1,35 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! + +Package apache-airflow-providers-oracle-oracledb +------------------------------------------------------ + +`Oracle `__ + + +This is detailed commit list of changes for versions provider package: ``oracle.oracledb``. +For high-level changelog, see :doc:`package information including changelog `. + +.. airflow-providers-commits:: diff --git a/providers/oracle/oracledb/docs/conf.py b/providers/oracle/oracledb/docs/conf.py new file mode 100644 index 0000000000000..aa8a81a1bf94f --- /dev/null +++ b/providers/oracle/oracledb/docs/conf.py @@ -0,0 +1,27 @@ +# Disable Flake8 because of all the sphinx imports +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Configuration of Providers docs building.""" + +from __future__ import annotations + +import os + +os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-oracle-oracledb" + +from docs.provider_conf import * # noqa: F403 diff --git a/providers/oracle/oracledb/docs/connections/oracle.rst b/providers/oracle/oracledb/docs/connections/oracle.rst new file mode 100644 index 0000000000000..2c07c0db4564d --- /dev/null +++ b/providers/oracle/oracledb/docs/connections/oracle.rst @@ -0,0 +1,117 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +.. _howto/connection:oracle: + +Oracle Connection +================= +The Oracle connection type provides connection to a Oracle database. + +Configuring the Connection +-------------------------- + +Host (optional) + The host to connect to. + +Schema (optional) + Specify the schema name to be used in the database. + +Login (optional) + Specify the user name to connect. + +Password (optional) + Specify the password to connect. + +Extra (optional) + Specify the extra parameters (as json dictionary) that can be used in Oracle + connection. The following parameters are supported: + + * ``events`` - Whether or not to initialize Oracle in events mode. + * ``mode`` - one of ``sysdba``, ``sysasm``, ``sysoper``, ``sysbkp``, ``sysdgd``, ``syskmt`` or ``sysrac`` + which are defined at the module level, Default mode is connecting. + * ``purity`` - one of ``new``, ``self``, ``default``. Specify the session acquired from the pool. + configuration parameter. + * ``dsn``. Specify a Data Source Name (and ignore Host). + * ``sid`` or ``service_name``. Use to form DSN instead of Schema. + * ``module`` (str) - This write-only attribute sets the module column in the v$session table. + The maximum length for this string is 48 and if you exceed this length you will get ORA-24960. + * ``thick_mode`` (bool) - Specify whether to use python-oracledb in thick mode. Defaults to False. + If set to True, you must have the Oracle Client libraries installed. + See `oracledb docs `__ for more info. + * ``thick_mode_lib_dir`` (str) - Path to use to find the Oracle Client libraries when using thick mode. + If not specified, defaults to the standard way of locating the Oracle Client library on the OS. + See `oracledb docs `__ for more info. + * ``thick_mode_config_dir`` (str) - Path to use to find the Oracle Client library configuration files when using thick mode. + If not specified, defaults to the standard way of locating the Oracle Client library configuration files on the OS. + See `oracledb docs `__ for more info. + * ``fetch_decimals`` (bool) - Specify whether numbers should be fetched as ``decimal.Decimal`` values. + See `defaults.fetch_decimals `_ for more info. + * ``fetch_lobs`` (bool) - Specify whether to fetch strings/bytes for CLOBs or BLOBs instead of locators. + See `defaults.fetch_lobs `_ for more info. + + + Connect using ``dsn``, Host and ``sid``, Host and ``service_name``, + or only Host `(OracleHook.getconn Documentation) `_. + + For example: + + .. code-block:: python + + Host = "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=dbhost.example.com)(PORT=1521))(CONNECT_DATA=(SERVICE_NAME=orclpdb1)))" + + or + + .. code-block:: python + + Host = "dbhost.example.com" + Schema = "orclpdb1" + + or + + .. code-block:: python + + Host = "dbhost.example.com" + Schema = "orcl" + + + More details on all Oracle connect parameters supported can be found in `oracledb documentation + `_. + + Information on creating an Oracle Connection through the web user interface can be found in Airflow's :doc:`Managing Connections Documentation `. + + + Example "extras" field: + + .. code-block:: json + + { + "events": false, + "mode": "sysdba", + "purity": "new" + } + + When specifying the connection as URI (in :envvar:`AIRFLOW_CONN_{CONN_ID}` variable) you should specify it + following the standard syntax of DB connections, where extras are passed as parameters + of the URI (note that all components of the URI should be URL-encoded). + + For example: + + .. code-block:: bash + + export AIRFLOW_CONN_ORACLE_DEFAULT='oracle://oracle_user:XXXXXXXXXXXX@1.1.1.1:1521?encoding=UTF-8&nencoding=UTF-8&threaded=False&events=False&mode=sysdba&purity=new' diff --git a/providers/oracle/oracledb/docs/index.rst b/providers/oracle/oracledb/docs/index.rst new file mode 100644 index 0000000000000..6be83f542656c --- /dev/null +++ b/providers/oracle/oracledb/docs/index.rst @@ -0,0 +1,155 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-oracle-oracledb`` +============================================= + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Connection types + Operators + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: References + + Python API <_api/airflow/providers/oracle/oracledb/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/oracle/oracledb/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + Example Dags <_api/airflow/providers/oracle/oracledb/example_dags/index> + PyPI Repository + Installing from sources + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-oracle-oracledb package +------------------------------------------------------ + +`Oracle `__ + +This provider was extracted from ``apache-airflow-providers-oracle``, which is now +deprecated. See the ``apache-airflow-providers-oracle`` changelog for migration notes. + +Release: 4.6.3 + +Provider package +---------------- + +This package is for the ``oracle`` provider. +All classes for this package are included in the ``airflow.providers.oracle.oracledb`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-oracle-oracledb``. +For the minimum Airflow version supported, see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``2.11.0``. + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=2.11.0`` +``apache-airflow-providers-common-compat`` ``>=1.8.0`` +``apache-airflow-providers-common-sql`` ``>=1.32.0`` +``oracledb`` ``>=2.3.0`` +========================================== ================== + +Optional cross provider package dependencies +-------------------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified provider distributions in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-oracle-oracledb[openlineage] + + +============================================================================================================== =============== +Dependent package Extra +============================================================================================================== =============== +`apache-airflow-providers-openlineage `_ ``openlineage`` +============================================================================================================== =============== + +Optional dependencies +--------------------- + +These extras install optional third-party libraries that enable additional features of the provider. +Install them when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-oracle-oracledb[numpy] + + +=============== ============================================================================================================================================================================================================================================ +Extra Dependencies +=============== ============================================================================================================================================================================================================================================ +``numpy`` ``numpy>=1.22.4; python_version<'3.11'``, ``numpy>=1.23.2; python_version=='3.11'``, ``numpy>=1.26.0; python_version=='3.12'``, ``numpy>=2.1.0; python_version>='3.13' and python_version<'3.14'``, ``numpy>=2.4.3; python_version>='3.14'`` +``openlineage`` ``apache-airflow-providers-openlineage`` +=============== ============================================================================================================================================================================================================================================ + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-oracle-oracledb 4.6.3 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-oracle-oracledb 4.6.3 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/oracle/oracledb/docs/installing-providers-from-sources.rst b/providers/oracle/oracledb/docs/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..fdbb17d017579 --- /dev/null +++ b/providers/oracle/oracledb/docs/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/installing-providers-from-sources.rst diff --git a/providers/oracle/oracledb/docs/integration-logos/Oracle.png b/providers/oracle/oracledb/docs/integration-logos/Oracle.png new file mode 100644 index 0000000000000..f411428777198 Binary files /dev/null and b/providers/oracle/oracledb/docs/integration-logos/Oracle.png differ diff --git a/providers/oracle/oracledb/docs/operators.rst b/providers/oracle/oracledb/docs/operators.rst new file mode 100644 index 0000000000000..53c00e3c81820 --- /dev/null +++ b/providers/oracle/oracledb/docs/operators.rst @@ -0,0 +1,74 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +.. _howto/operator:OracleOperator: + +SQLExecuteQueryOperator to connect to Oracle +============================================ + +Use the :class:`SQLExecuteQueryOperator` to execute +Oracle commands in a `Oracle `__ database. + +.. note:: + Previously, ``OracleStoredProcedureOperator`` was used to perform this kind of operation. After deprecation this has been removed. Please use ``SQLExecuteQueryOperator`` instead. + +Using the Operator +^^^^^^^^^^^^^^^^^^ + +Use the ``conn_id`` argument to connect to your Oracle instance where +the connection metadata is structured as follows: + +.. list-table:: Oracle Airflow Connection Metadata + :widths: 25 25 + :header-rows: 1 + + * - Parameter + - Input + * - Host: string + - Oracle database hostname + * - Schema: string + - Schema to execute SQL operations on by default + * - Login: string + - Oracle database user + * - Password: string + - Oracle database user password + * - Port: int + - Oracle database port (default: 1521) + * - Extra: JSON + - Additional connection configuration, such as DSN string: + ``{"dsn": "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=)(PORT=1521))(CONNECT_DATA=(SERVICE_NAME=)))"}`` + +An example usage of the SQLExecuteQueryOperator to connect to Oracle is as follows: + +.. exampleinclude:: /../tests/system/oracle/oracledb/example_oracle.py + :language: python + :start-after: [START howto_operator_oracle] + :end-before: [END howto_operator_oracle] + + +Reference +^^^^^^^^^ +For further information, look at: + +* `Oracle Documentation `__ + +.. note:: + + Parameters given via SQLExecuteQueryOperator() are given first-place priority + relative to parameters set via Airflow connection metadata (such as ``schema``, ``login``, ``password`` etc). diff --git a/providers/oracle/oracledb/docs/security.rst b/providers/oracle/oracledb/docs/security.rst new file mode 100644 index 0000000000000..351ff007ebf2f --- /dev/null +++ b/providers/oracle/oracledb/docs/security.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/security.rst diff --git a/providers/oracle/oracledb/provider.yaml b/providers/oracle/oracledb/provider.yaml new file mode 100644 index 0000000000000..5997a31fa78f5 --- /dev/null +++ b/providers/oracle/oracledb/provider.yaml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-oracle-oracledb +name: Oracle +description: | + `Oracle `__ + + This provider was extracted from ``apache-airflow-providers-oracle``, which is now + deprecated. See the ``apache-airflow-providers-oracle`` changelog for migration notes. + +state: ready +lifecycle: production +source-date-epoch: 1783356304 +# Note that those versions are maintained by release manager - do not update them manually +# with the exception of case where other provider in sources has >= new provider version. +# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have +# to be done in the same PR +versions: + - 4.6.3 + +integrations: + - integration-name: Oracle + external-doc-url: https://www.oracle.com/en/database/ + how-to-guide: + - /docs/apache-airflow-providers-oracle-oracledb/operators.rst + logo: /docs/integration-logos/Oracle.png + tags: [software] + +operators: + - integration-name: Oracle + python-modules: + - airflow.providers.oracle.oracledb.operators.oracle + +asset-uris: + - schemes: [oracle] + handler: airflow.providers.oracle.oracledb.assets.oracle.sanitize_uri + factory: airflow.providers.oracle.oracledb.assets.oracle.create_asset + to_openlineage_converter: airflow.providers.oracle.oracledb.assets.oracle.convert_asset_to_openlineage + +# dataset has been renamed to asset in Airflow 3.0 +# This is kept for backward compatibility. +dataset-uris: + - schemes: [oracle] + handler: airflow.providers.oracle.oracledb.assets.oracle.sanitize_uri + factory: airflow.providers.oracle.oracledb.assets.oracle.create_asset + to_openlineage_converter: airflow.providers.oracle.oracledb.assets.oracle.convert_asset_to_openlineage + +hooks: + - integration-name: Oracle + python-modules: + - airflow.providers.oracle.oracledb.hooks.handlers + - airflow.providers.oracle.oracledb.hooks.oracle + +transfers: + - source-integration-name: Oracle + target-integration-name: Oracle + python-module: airflow.providers.oracle.oracledb.transfers.oracle_to_oracle + +connection-types: + - hook-class-name: airflow.providers.oracle.oracledb.hooks.oracle.OracleHook + hook-name: "Oracle" + connection-type: oracle diff --git a/providers/oracle/oracledb/pyproject.toml b/providers/oracle/oracledb/pyproject.toml new file mode 100644 index 0000000000000..bf426b4df76c6 --- /dev/null +++ b/providers/oracle/oracledb/pyproject.toml @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==3.12.0"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-oracle-oracledb" +version = "4.6.3" +description = "Provider package apache-airflow-providers-oracle-oracledb for Apache Airflow" +readme = "README.rst" +license = "Apache-2.0" +license-files = ['LICENSE', 'NOTICE'] +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "oracle", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: System :: Monitoring", +] +requires-python = ">=3.10" + +# The dependencies should be modified in place in the generated file. +# Any change in the dependencies is preserved when the file is regenerated +# Make sure to run ``prek update-providers-dependencies --all-files`` +# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` +dependencies = [ + "apache-airflow>=2.11.0", + "apache-airflow-providers-common-compat>=1.8.0", + "apache-airflow-providers-common-sql>=1.32.0", + "oracledb>=2.3.0", +] + +# The optional dependencies should be modified in place in the generated file +# Any change in the dependencies is preserved when the file is regenerated +[project.optional-dependencies] +"numpy" = [ + "numpy>=1.22.4; python_version<'3.11'", + "numpy>=1.23.2; python_version=='3.11'", + "numpy>=1.26.0; python_version=='3.12'", + "numpy>=2.1.0; python_version>='3.13' and python_version<'3.14'", + "numpy>=2.4.3; python_version>='3.14'", +] +"openlineage" = [ + "apache-airflow-providers-openlineage" +] + +[dependency-groups] +dev = [ + "apache-airflow", + "apache-airflow-task-sdk", + "apache-airflow-devel-common", + "apache-airflow-providers-common-compat", + "apache-airflow-providers-common-sql", + "apache-airflow-providers-openlineage", + # Additional devel dependencies (do not remove this line and add extra development dependencies) + "numpy>=1.22.4; python_version<'3.11'", + "numpy>=1.23.2; python_version=='3.11'", + "numpy>=1.26.0; python_version=='3.12'", + "numpy>=2.1.0; python_version>='3.13' and python_version<'3.14'", + "numpy>=2.4.3; python_version>='3.14'", +] + +# To build docs: +# +# uv run --group docs build-docs +# +# To enable auto-refreshing build with server: +# +# uv run --group docs build-docs --autobuild +# +# To see more options: +# +# uv run --group docs build-docs --help +# +docs = [ + "apache-airflow-devel-common[docs]" +] + +[tool.uv.sources] +# These names must match the names as defined in the pyproject.toml of the workspace items, +# *not* the workspace folder paths +apache-airflow = {workspace = true} +apache-airflow-devel-common = {workspace = true} +apache-airflow-task-sdk = {workspace = true} +apache-airflow-providers-common-sql = {workspace = true} +apache-airflow-providers-standard = {workspace = true} + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-oracle-oracledb/4.6.3" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-oracle-oracledb/4.6.3/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Mastodon" = "https://fosstodon.org/@airflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.oracle.oracledb.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.oracle.oracledb" + +# Explicit sdist contents so the build does not rely on VCS information +# (flit 4.0 makes --no-use-vcs the default — see https://github.com/pypa/flit/pull/782). +[tool.flit.sdist] +include = [ + "docs/", + "provider.yaml", + "src/airflow/__init__.py", + "src/airflow/providers/__init__.py", + "src/airflow/providers/oracle/__init__.py", + "tests/", +] diff --git a/providers/oracle/oracledb/src/airflow/__init__.py b/providers/oracle/oracledb/src/airflow/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/oracle/oracledb/src/airflow/providers/__init__.py b/providers/oracle/oracledb/src/airflow/providers/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/__init__.py b/providers/oracle/oracledb/src/airflow/providers/oracle/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/__init__.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/__init__.py new file mode 100644 index 0000000000000..042f8ba880793 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +# +from __future__ import annotations + +import packaging.version + +from airflow import __version__ as airflow_version + +__all__ = ["__version__"] + +__version__ = "4.6.3" + +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "2.11.0" +): + raise RuntimeError( + f"The package `apache-airflow-providers-oracle-oracledb:{__version__}` needs Apache Airflow 2.11.0+" + ) diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/assets/__init__.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/assets/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/assets/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/assets/oracle.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/assets/oracle.py new file mode 100644 index 0000000000000..19942df7ab10b --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/assets/oracle.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from airflow.providers.common.compat.assets import Asset + +if TYPE_CHECKING: + from urllib.parse import SplitResult + + from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset + + +def sanitize_uri(uri: SplitResult) -> SplitResult: + if not uri.netloc: + raise ValueError("URI format oracle:// must contain a host") + if uri.port is None: + host = uri.netloc.rstrip(":") + uri = uri._replace(netloc=f"{host}:1521") + if len(uri.path.split("/")) != 4: # Leading slash, service name, schema, and table names. + raise ValueError("URI format oracle:// must contain service name, schema, and table names") + return uri + + +def create_asset( + *, + host: str, + port: int = 1521, + service_name: str, + schema: str, + table: str, + extra: dict | None = None, +) -> Asset: + return Asset(uri=f"oracle://{host}:{port}/{service_name}/{schema}/{table}", extra=extra) + + +def convert_asset_to_openlineage(asset: Asset, lineage_context) -> OpenLineageDataset: + """Translate Asset with valid AIP-60 uri to OpenLineage with assistance from the hook.""" + from urllib.parse import urlsplit + + from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset + + parsed = urlsplit(asset.uri) + _, service_name, schema, table = parsed.path.split( + "/" + ) # Leading slash, service_name, schema, and table names. + return OpenLineageDataset(namespace=f"oracle://{parsed.netloc}", name=f"{service_name}.{schema}.{table}") diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/example_dags/__init__.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/example_dags/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/example_dags/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/example_dags/example_oracle.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/example_dags/example_oracle.py new file mode 100644 index 0000000000000..8d118da5d88df --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/example_dags/example_oracle.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from datetime import datetime + +from airflow import DAG +from airflow.providers.oracle.oracledb.operators.oracle import OracleStoredProcedureOperator + +with DAG( + max_active_runs=1, + max_active_tasks=3, + catchup=False, + start_date=datetime(2023, 1, 1), + dag_id="example_oracle", + tags=["example"], +) as dag: + # [START howto_oracle_stored_procedure_operator_with_list_inout] + + opr_stored_procedure_with_list_input_output = OracleStoredProcedureOperator( + task_id="opr_stored_procedure_with_list_input_output", + oracle_conn_id="oracle", + procedure="TEST_PROCEDURE", + parameters=[3, int], + ) + + # [END howto_oracle_stored_procedure_operator_with_list_inout] + + # [START howto_oracle_stored_procedure_operator_with_dict_inout] + + opr_stored_procedure_with_dict_input_output = OracleStoredProcedureOperator( + task_id="opr_stored_procedure_with_dict_input_output", + oracle_conn_id="oracle", + procedure="TEST_PROCEDURE", + parameters={"val_in": 3, "val_out": int}, + ) + + # [END howto_oracle_stored_procedure_operator_with_dict_inout] diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/example_dags/example_oracle_fetch.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/example_dags/example_oracle_fetch.py new file mode 100644 index 0000000000000..06a8694f00456 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/example_dags/example_oracle_fetch.py @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from datetime import datetime + +from airflow import DAG +from airflow.operators.empty import EmptyOperator +from airflow.operators.python import PythonOperator +from airflow.providers.oracle.oracledb.hooks.oracle import OracleHook + +DOC = """ +### Example: Simple Oracle fetch + +This DAG demonstrates using `OracleHook` to read from Oracle and push rows +into XCom for downstream tasks. Adapt this pattern for your transfers. + +**Prereqs** +- Define a connection `oracle_default` in Airflow. +- Ensure a table `demo_table(id NUMBER, name VARCHAR2(100))` exists (or edit the SQL). +""" + + +def fetch_rows(): + """Fetch sample rows from Oracle table DEMO_TABLE.""" + hook = OracleHook(oracle_conn_id="oracle_default") + sql = "SELECT id, name FROM demo_table WHERE ROWNUM <= 5" + rows = hook.get_records(sql) + return rows + + +def print_rows(ti=None): + """Print rows pulled from Oracle.""" + rows = ti.xcom_pull(task_ids="fetch_rows") + for r in rows or []: + print(r) + + +with DAG( + dag_id="example_oracle_fetch", + start_date=datetime(2024, 1, 1), + schedule="@daily", + catchup=False, + doc_md=DOC, + tags=["example", "oracle"], +): + start = EmptyOperator(task_id="start") + + fetch = PythonOperator( + task_id="fetch_rows", + python_callable=fetch_rows, + ) + + show = PythonOperator( + task_id="print_rows", + python_callable=print_rows, + ) + + start >> fetch >> show diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/get_provider_info.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/get_provider_info.py new file mode 100644 index 0000000000000..6c812188cc306 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/get_provider_info.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-oracle-oracledb", + "name": "Oracle", + "description": "`Oracle `__\n\nThis provider was extracted from ``apache-airflow-providers-oracle``, which is now\ndeprecated. See the ``apache-airflow-providers-oracle`` changelog for migration notes.\n", + "integrations": [ + { + "integration-name": "Oracle", + "external-doc-url": "https://www.oracle.com/en/database/", + "how-to-guide": ["/docs/apache-airflow-providers-oracle-oracledb/operators.rst"], + "logo": "/docs/integration-logos/Oracle.png", + "tags": ["software"], + } + ], + "operators": [ + { + "integration-name": "Oracle", + "python-modules": ["airflow.providers.oracle.oracledb.operators.oracle"], + } + ], + "asset-uris": [ + { + "schemes": ["oracle"], + "handler": "airflow.providers.oracle.oracledb.assets.oracle.sanitize_uri", + "factory": "airflow.providers.oracle.oracledb.assets.oracle.create_asset", + "to_openlineage_converter": "airflow.providers.oracle.oracledb.assets.oracle.convert_asset_to_openlineage", + } + ], + "dataset-uris": [ + { + "schemes": ["oracle"], + "handler": "airflow.providers.oracle.oracledb.assets.oracle.sanitize_uri", + "factory": "airflow.providers.oracle.oracledb.assets.oracle.create_asset", + "to_openlineage_converter": "airflow.providers.oracle.oracledb.assets.oracle.convert_asset_to_openlineage", + } + ], + "hooks": [ + { + "integration-name": "Oracle", + "python-modules": [ + "airflow.providers.oracle.oracledb.hooks.handlers", + "airflow.providers.oracle.oracledb.hooks.oracle", + ], + } + ], + "transfers": [ + { + "source-integration-name": "Oracle", + "target-integration-name": "Oracle", + "python-module": "airflow.providers.oracle.oracledb.transfers.oracle_to_oracle", + } + ], + "connection-types": [ + { + "hook-class-name": "airflow.providers.oracle.oracledb.hooks.oracle.OracleHook", + "hook-name": "Oracle", + "connection-type": "oracle", + } + ], + } diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/hooks/__init__.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/hooks/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/hooks/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/hooks/handlers.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/hooks/handlers.py new file mode 100644 index 0000000000000..d75bcfc6e7e20 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/hooks/handlers.py @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import oracledb + + +def _read_lob(val): + if isinstance(val, oracledb.LOB): + return val.read() + return val + + +def _read_lobs(row): + if row is not None: + return tuple([_read_lob(value) for value in row]) + return row + + +def fetch_all_handler(cursor) -> list[tuple] | None: + """Return results for DbApiHook.run(). If oracledb.LOB objects are present, then those will be read.""" + if not hasattr(cursor, "description"): + raise RuntimeError( + "The database we interact with does not support DBAPI 2.0. Use operator and " + "handlers that are specifically designed for your database." + ) + if cursor.description is not None: + results = [_read_lobs(row) for row in cursor.fetchall()] + return results + return None + + +def fetch_one_handler(cursor) -> tuple | None: + """Return first result for DbApiHook.run(). If oracledb.LOB objects are present, then those will be read.""" + if not hasattr(cursor, "description"): + raise RuntimeError( + "The database we interact with does not support DBAPI 2.0. Use operator and " + "handlers that are specifically designed for your database." + ) + if cursor.description is not None: + return _read_lobs(cursor.fetchone()) + return None diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/hooks/oracle.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/hooks/oracle.py new file mode 100644 index 0000000000000..bfa0714ac3e15 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/hooks/oracle.py @@ -0,0 +1,566 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import math +import warnings +from collections.abc import Iterable, Mapping +from datetime import datetime +from typing import TYPE_CHECKING, Any + +import oracledb + +if TYPE_CHECKING: + from airflow.models.connection import Connection + from airflow.providers.openlineage.sqlparser import DatabaseInfo + +from airflow.providers.common.sql.hooks.lineage import send_sql_hook_lineage +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.providers.oracle.oracledb.hooks import handlers + +DEFAULT_DB_PORT = 1521 +PARAM_TYPES = {bool, float, int, str} + + +def _map_param(value): + if value in PARAM_TYPES: + # In this branch, value is a Python type; calling it produces + # an instance of the type which is understood by the Oracle driver + # in the out parameter mapping mechanism. + value = value() + return value + + +def _get_bool(val): + if isinstance(val, bool): + return val + if isinstance(val, str): + val = val.lower().strip() + if val == "true": + return True + if val == "false": + return False + return None + + +def _get_first_bool(*vals): + for val in vals: + converted = _get_bool(val) + if isinstance(converted, bool): + return converted + return None + + +class OracleHook(DbApiHook): + """ + Interact with Oracle SQL. + + :param oracle_conn_id: The :ref:`Oracle connection id ` + used for Oracle credentials. + :param thick_mode: Specify whether to use python-oracledb in thick mode. Defaults to False. + If set to True, you must have the Oracle Client libraries installed. + See `oracledb docs` + for more info. + :param thick_mode_lib_dir: Path to use to find the Oracle Client libraries when using thick mode. + If not specified, defaults to the standard way of locating the Oracle Client library on the OS. + See `oracledb docs + ` + for more info. + :param thick_mode_config_dir: Path to use to find the Oracle Client library + configuration files when using thick mode. + If not specified, defaults to the standard way of locating the Oracle Client + library configuration files on the OS. + See `oracledb docs + ` + for more info. + :param fetch_decimals: Specify whether numbers should be fetched as ``decimal.Decimal`` values. + See `defaults.fetch_decimals + ` + for more info. + :param fetch_lobs: Specify whether to fetch strings/bytes for CLOBs or BLOBs instead of locators. + See `defaults.fetch_lobs + ` + for more info. + """ + + conn_name_attr = "oracle_conn_id" + default_conn_name = "oracle_default" + conn_type = "oracle" + hook_name = "Oracle" + + _test_connection_sql = "select 1 from dual" + supports_autocommit = True + + def __init__( + self, + *args, + thick_mode: bool | None = None, + thick_mode_lib_dir: str | None = None, + thick_mode_config_dir: str | None = None, + fetch_decimals: bool | None = None, + fetch_lobs: bool | None = None, + **kwargs, + ) -> None: + super().__init__(*args, **kwargs) + + self.thick_mode = thick_mode + self.thick_mode_lib_dir = thick_mode_lib_dir + self.thick_mode_config_dir = thick_mode_config_dir + self.fetch_decimals = fetch_decimals + self.fetch_lobs = fetch_lobs + self._service_name: str | None = None + self._sid: str | None = None + + @property + def service_name(self) -> str | None: + if self._service_name is None: + self._service_name = self.get_connection(self.get_conn_id()).extra_dejson.get("service_name") + return self._service_name + + @property + def sid(self) -> str | None: + if self._sid is None: + self._sid = self.get_connection(self.get_conn_id()).extra_dejson.get("sid") + return self._sid + + def get_conn(self) -> oracledb.Connection: + """ + Get an Oracle connection object. + + Optional parameters for using a custom DSN connection (instead of using + a server alias from tnsnames.ora) The dsn (data source name) is the TNS + entry (from the Oracle names server or tnsnames.ora file), or is a + string like the one returned from ``makedsn()``. + + :param dsn: the data source name for the Oracle server + :param service_name: the db_unique_name of the database + that you are connecting to (CONNECT_DATA part of TNS) + :param sid: Oracle System ID that identifies a particular + database on a system + :param wallet_location: Specify the directory where the wallet can be found. + :param wallet_password: the password to use to decrypt the wallet, if it is encrypted. + For Oracle Autonomous Database this is the password created when downloading the wallet. + :param ssl_server_cert_dn: Specify the distinguished name (DN) which should be matched + with the server. This value is ignored if the ``ssl_server_dn_match`` parameter is not + set to the value True. + :param ssl_server_dn_match: Specify whether the server certificate distinguished name + (DN) should be matched in addition to the regular certificate verification that is performed. + :param cclass: the connection class to use for Database Resident Connection Pooling (DRCP). + :param pool_name: the name of the DRCP pool when using multi-pool DRCP with Oracle Database 23.4, or higher. + + You can set these parameters in the extra fields of your connection + as in + + .. code-block:: python + + {"dsn": ("(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=host)(PORT=1521))(CONNECT_DATA=(SID=sid)))")} + + see more param detail in `oracledb.connect + `_ + + + """ + conn = self.get_connection(self.get_conn_id()) + conn_config: dict[str, Any] = {"user": conn.login, "password": conn.password} + sid = conn.extra_dejson.get("sid") + mod = conn.extra_dejson.get("module") + schema = conn.schema + + # Enable oracledb thick mode if thick_mode is set to True + # Parameters take precedence over connection config extra + # Defaults to use thin mode if not provided in params or connection config extra + thick_mode = _get_first_bool(self.thick_mode, conn.extra_dejson.get("thick_mode")) + if thick_mode is True: + if self.thick_mode_lib_dir is None: + self.thick_mode_lib_dir = conn.extra_dejson.get("thick_mode_lib_dir") + if not isinstance(self.thick_mode_lib_dir, (str, type(None))): + raise TypeError( + f"thick_mode_lib_dir expected str or None, " + f"got {type(self.thick_mode_lib_dir).__name__}" + ) + if self.thick_mode_config_dir is None: + self.thick_mode_config_dir = conn.extra_dejson.get("thick_mode_config_dir") + if not isinstance(self.thick_mode_config_dir, (str, type(None))): + raise TypeError( + f"thick_mode_config_dir expected str or None, " + f"got {type(self.thick_mode_config_dir).__name__}" + ) + oracledb.init_oracle_client( + lib_dir=self.thick_mode_lib_dir, config_dir=self.thick_mode_config_dir + ) + + # Set oracledb Defaults Attributes if provided + # (https://python-oracledb.readthedocs.io/en/latest/api_manual/defaults.html) + fetch_decimals = _get_first_bool(self.fetch_decimals, conn.extra_dejson.get("fetch_decimals")) + if isinstance(fetch_decimals, bool): + oracledb.defaults.fetch_decimals = fetch_decimals + + fetch_lobs = _get_first_bool(self.fetch_lobs, conn.extra_dejson.get("fetch_lobs")) + if isinstance(fetch_lobs, bool): + oracledb.defaults.fetch_lobs = fetch_lobs + + # Set up DSN + service_name = conn.extra_dejson.get("service_name") + # Fall back to conn.schema as service_name when not explicitly set in extras. + # The UI Schema field maps to conn.schema which is the Oracle service name. + if not service_name and not sid and schema: + service_name = schema + port = conn.port if conn.port else DEFAULT_DB_PORT + if conn.host and sid and not service_name: + conn_config["dsn"] = oracledb.makedsn(conn.host, port, sid) + elif conn.host and service_name and not sid: + conn_config["dsn"] = oracledb.makedsn(conn.host, port, service_name=service_name) + else: + dsn = conn.extra_dejson.get("dsn") + if dsn is None: + dsn = conn.host or "" + if conn.port is not None: + dsn += f":{conn.port}" + if service_name: + dsn += f"/{service_name}" + conn_config["dsn"] = dsn + + if "events" in conn.extra_dejson: + conn_config["events"] = conn.extra_dejson.get("events") + + # Map the connection extra "mode"/"purity" string (e.g. "sysdba") to the + # matching python-oracledb AuthMode/Purity enum member by name. Compare + # against None explicitly: Purity.DEFAULT is 0, so a truthiness check + # would silently drop it. + if mode_name := conn.extra_dejson.get("mode"): + auth_mode = getattr(oracledb.AuthMode, mode_name.upper(), None) + if auth_mode is not None: + conn_config["mode"] = auth_mode + + if purity_name := conn.extra_dejson.get("purity"): + purity = getattr(oracledb.Purity, purity_name.upper(), None) + if purity is not None: + conn_config["purity"] = purity + + expire_time = conn.extra_dejson.get("expire_time") + if expire_time: + conn_config["expire_time"] = expire_time + + for name in [ + "wallet_location", + "wallet_password", + "ssl_server_cert_dn", + "ssl_server_dn_match", + "cclass", + "pool_name", + ]: + value = conn.extra_dejson.get(name) + if value is not None: + conn_config[name] = value + + oracle_conn = oracledb.connect(**conn_config) + if mod is not None: + oracle_conn.module = mod + + # if Connection.schema is defined, set schema after connecting successfully + # cannot be part of conn_config + # https://python-oracledb.readthedocs.io/en/latest/api_manual/connection.html?highlight=schema#Connection.current_schema + # Only set schema when not using conn.schema as Service Name + if schema and service_name: + oracle_conn.current_schema = schema + + return oracle_conn + + def get_records( + self, + sql: str | list[str], + parameters: Iterable | Mapping[str, Any] | None = None, + ) -> Any: + """ + Execute the sql and return a set of records. + + :param sql: the sql statement to be executed (str) or a list of sql statements to execute + :param parameters: The parameters to render the SQL query with. + """ + return self.run(sql=sql, parameters=parameters, handler=handlers.fetch_all_handler) + + def get_first(self, sql: str | list[str], parameters: Iterable | Mapping[str, Any] | None = None) -> Any: + """ + Execute the sql and return the first resulting row. + + :param sql: the sql statement to be executed (str) or a list of sql statements to execute + :param parameters: The parameters to render the SQL query with. + """ + return self.run(sql=sql, parameters=parameters, handler=handlers.fetch_one_handler) + + def insert_rows( + self, + table: str, + rows: list[tuple], + target_fields=None, + commit_every: int = 1000, + replace: bool | None = False, + **kwargs, + ) -> None: + """ + Insert a collection of tuples into a table. + + All data to insert are treated as one transaction. Changes from standard + DbApiHook implementation: + + - Oracle SQL queries can not be terminated with a semicolon (``;``). + - Replace NaN values with NULL using ``numpy.nan_to_num`` (not using + ``is_nan()`` because of input types error for strings). + - Coerce datetime cells to Oracle DATETIME format during insert. + + :param table: target Oracle table, use dot notation to target a + specific database + :param rows: the rows to insert into the table + :param target_fields: the names of the columns to fill in the table + :param commit_every: the maximum number of rows to insert in one transaction + Default 1000, Set greater than 0. + Set 1 to insert each row in each single transaction + :param replace: Whether to replace instead of insert. Currently not implemented. + """ + if replace: + warnings.warn( + "Using 'replace=True' does not implement any replace functionality currently.", + category=UserWarning, + stacklevel=2, + ) + try: + import numpy as np + except ImportError: + np = None # type: ignore + + if target_fields: + target_fields = ", ".join(target_fields) + target_fields = f"({target_fields})" + else: + target_fields = "" + conn = self.get_conn() + if self.supports_autocommit: + self.set_autocommit(conn, False) + cur = conn.cursor() + i = 0 + sql = None # not generated unless we actually process at least one chunk + for row in rows: + i += 1 + lst = [] + for cell in row: + if isinstance(cell, str): + lst.append("'" + str(cell).replace("'", "''") + "'") + elif cell is None or isinstance(cell, float) and math.isnan(cell): # coerce numpy NaN to NULL + lst.append("NULL") + elif np and isinstance(cell, np.datetime64): + lst.append(f"'{cell}'") + elif isinstance(cell, datetime): + lst.append(f"to_date('{cell:%Y-%m-%d %H:%M:%S}','YYYY-MM-DD HH24:MI:SS')") + else: + lst.append(str(cell)) + values = tuple(lst) + sql = f"INSERT /*+ APPEND */ INTO {table} {target_fields} VALUES ({','.join(values)})" + cur.execute(sql) + if i % commit_every == 0: + conn.commit() + self.log.info("Loaded %s into %s rows so far", i, table) + conn.commit() + + if sql: + # We only send lineage once, not for each value collection, to save memory. + send_sql_hook_lineage(context=self, sql=sql, row_count=i) + + cur.close() + conn.close() + self.log.info("Done loading. Loaded a total of %s rows", i) + + def bulk_insert_rows( + self, + table: str, + rows: list[tuple], + target_fields: list[str] | None = None, + commit_every: int = 5000, + sequence_column: str | None = None, + sequence_name: str | None = None, + ): + """ + Perform bulk inserts efficiently for Oracle DB. + + This uses prepared statements via `executemany()`. For best performance, + pass in `rows` as an iterator. + + :param table: target Oracle table, use dot notation to target a + specific database + :param rows: the rows to insert into the table + :param target_fields: the names of the columns to fill in the table, default None. + If None, each rows should have some order as table columns name + :param commit_every: the maximum number of rows to insert in one transaction + Default 5000. Set greater than 0. Set 1 to insert each row in each transaction + :param sequence_column: the column name to which the sequence will be applied, default None. + :param sequence_name: the names of the sequence_name in the table, default None. + """ + if not rows: + raise ValueError("parameter rows could not be None or empty iterable") + conn = self.get_conn() + if self.supports_autocommit: + self.set_autocommit(conn, False) + cursor = conn.cursor() + values_base = target_fields or rows[0] + + if bool(sequence_column) ^ bool(sequence_name): + raise ValueError( + "Parameters 'sequence_column' and 'sequence_name' must be provided together or not at all." + ) + + if sequence_column and sequence_name: + columns = ( + f"({', '.join([sequence_column] + target_fields)})" + if target_fields + else f"({sequence_column})" + ) + value_placeholders = ", ".join( + [f"{sequence_name}.NEXTVAL"] + [f":{i}" for i in range(1, len(values_base) + 1)] + ) + else: + columns = f"({', '.join(target_fields)})" if target_fields else "" + value_placeholders = ", ".join(f":{i}" for i in range(1, len(values_base) + 1)) + prepared_stm = f"insert into {table} {columns} values ({value_placeholders})" + + row_count = 0 + # Chunk the rows + row_chunk = [] + for row in rows: + row_chunk.append(row) + row_count += 1 + if row_count % commit_every == 0: + cursor.prepare(prepared_stm) + cursor.executemany(None, row_chunk) + conn.commit() + self.log.info("[%s] inserted %s rows", table, row_count) + # Empty chunk + row_chunk = [] + # Commit the leftover chunk + if row_chunk: + cursor.prepare(prepared_stm) + cursor.executemany(None, row_chunk) + conn.commit() + self.log.info("[%s] inserted %s rows", table, row_count) + # We only send lineage once, not for each value collection, to save memory. + send_sql_hook_lineage(context=self, sql=prepared_stm, row_count=row_count) + cursor.close() + conn.close() + + def callproc( + self, + identifier: str, + autocommit: bool = False, + parameters: list | dict | None = None, + ) -> list | dict | tuple | None: + """ + Call the stored procedure identified by the provided string. + + Any OUT parameters must be provided with a value of either the + expected Python type (e.g., `int`) or an instance of that type. + + The return value is a list or mapping that includes parameters in + both directions; the actual return type depends on the type of the + provided `parameters` argument. + + See + https://python-oracledb.readthedocs.io/en/latest/api_manual/cursor.html#Cursor.var + for further reference. + """ + if parameters is None: + parameters = [] + + args = ",".join( + f":{name}" + for name in (parameters if isinstance(parameters, dict) else range(1, len(parameters) + 1)) + ) + + sql = f"BEGIN {identifier}({args}); END;" + + def handler(cursor): + if cursor.bindvars is None: + return + + if isinstance(cursor.bindvars, list): + return [v.getvalue() for v in cursor.bindvars] + + if isinstance(cursor.bindvars, dict): + return {n: v.getvalue() for (n, v) in cursor.bindvars.items()} + + raise TypeError(f"Unexpected bindvars: {cursor.bindvars!r}") + + result = self.run( + sql, + autocommit=autocommit, + parameters=( + {name: _map_param(value) for (name, value) in parameters.items()} + if isinstance(parameters, dict) + else [_map_param(value) for value in parameters] + ), + handler=handler, + ) + + return result + + def get_openlineage_database_info(self, connection: Connection) -> DatabaseInfo: + """Return Oracle specific information for OpenLineage.""" + from airflow.providers.openlineage.sqlparser import DatabaseInfo + + return DatabaseInfo( + scheme=self.get_openlineage_database_dialect(connection), + authority=DbApiHook.get_openlineage_authority_part(connection, default_port=DEFAULT_DB_PORT), + information_schema_table_name="ALL_TAB_COLUMNS", + information_schema_columns=[ + "owner", + "table_name", + "column_name", + "column_id", + "data_type", + ], + database=self.service_name or self.sid, + normalize_name_method=lambda name: name.upper(), + ) + + def get_openlineage_database_dialect(self, _) -> str: + """Return database dialect.""" + return "oracle" + + def get_openlineage_default_schema(self) -> str | None: + """Return current schema.""" + return self.get_first("SELECT SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') FROM dual")[0] + + def get_uri(self) -> str: + """Get the URI for the Oracle connection.""" + conn = self.get_connection(self.get_conn_id()) + login = conn.login + password = conn.password + host = conn.host + port = conn.port or DEFAULT_DB_PORT + service_name = conn.extra_dejson.get("service_name") + sid = conn.extra_dejson.get("sid") + + if sid and service_name: + raise ValueError("At most one allowed for 'sid', and 'service name'.") + + uri = f"oracle+oracledb://{login}:{password}@{host}:{port}" + if service_name: + uri = f"{uri}?service_name={service_name}" + elif sid: + uri = f"{uri}/{sid}" + elif conn.schema: + uri = f"{uri}/{conn.schema}" + + return uri diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/operators/__init__.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/operators/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/operators/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/operators/oracle.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/operators/oracle.py new file mode 100644 index 0000000000000..5c189124e77c0 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/operators/oracle.py @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import re +from collections.abc import Sequence +from typing import TYPE_CHECKING + +import oracledb + +from airflow.providers.common.compat.sdk import BaseOperator +from airflow.providers.oracle.oracledb.hooks.oracle import OracleHook + +if TYPE_CHECKING: + from airflow.providers.common.compat.sdk import Context + + +class OracleStoredProcedureOperator(BaseOperator): + """ + Executes stored procedure in a specific Oracle database. + + :param procedure: name of stored procedure to call (templated) + :param oracle_conn_id: The :ref:`Oracle connection id ` + reference to a specific Oracle database. + :param parameters: (optional, templated) the parameters provided in the call + + If *do_xcom_push* is *True*, the numeric exit code emitted by + the database is pushed to XCom under key ``ORA`` in case of failure. + """ + + template_fields: Sequence[str] = ( + "parameters", + "procedure", + ) + ui_color = "#ededed" + + def __init__( + self, + *, + procedure: str, + oracle_conn_id: str = "oracle_default", + parameters: dict | list | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.oracle_conn_id = oracle_conn_id + self.procedure = procedure + self.parameters = parameters + + def execute(self, context: Context): + self.log.info("Executing: %s", self.procedure) + hook = OracleHook(oracle_conn_id=self.oracle_conn_id) + try: + return hook.callproc(self.procedure, autocommit=True, parameters=self.parameters) + except oracledb.DatabaseError as e: + if not self.do_xcom_push or not context: + raise + ti = context["ti"] + code_match = re.search("^ORA-(\\d+):.+", str(e)) + if code_match: + ti.xcom_push(key="ORA", value=code_match.group(1)) + raise diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/transfers/__init__.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/transfers/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/transfers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/transfers/oracle_to_oracle.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/transfers/oracle_to_oracle.py new file mode 100644 index 0000000000000..0c9d8b9d1b331 --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/transfers/oracle_to_oracle.py @@ -0,0 +1,89 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from airflow.providers.common.compat.sdk import BaseOperator +from airflow.providers.oracle.oracledb.hooks.oracle import OracleHook + +if TYPE_CHECKING: + from airflow.providers.common.compat.sdk import Context + + +class OracleToOracleOperator(BaseOperator): + """ + Moves data from Oracle to Oracle. + + :param oracle_destination_conn_id: destination Oracle connection. + :param destination_table: destination table to insert rows. + :param oracle_source_conn_id: :ref:`Source Oracle connection `. + :param source_sql: SQL query to execute against the source Oracle + database. (templated) + :param source_sql_params: Parameters to use in sql query. (templated) + :param rows_chunk: number of rows per chunk to commit. + """ + + template_fields: Sequence[str] = ("source_sql", "source_sql_params") + template_fields_renderers = {"source_sql": "sql", "source_sql_params": "py"} + ui_color = "#e08c8c" + + def __init__( + self, + *, + oracle_destination_conn_id: str, + destination_table: str, + oracle_source_conn_id: str, + source_sql: str, + source_sql_params: dict | None = None, + rows_chunk: int = 5000, + **kwargs, + ) -> None: + super().__init__(**kwargs) + if source_sql_params is None: + source_sql_params = {} + self.oracle_destination_conn_id = oracle_destination_conn_id + self.destination_table = destination_table + self.oracle_source_conn_id = oracle_source_conn_id + self.source_sql = source_sql + self.source_sql_params = source_sql_params + self.rows_chunk = rows_chunk + + def _execute(self, src_hook, dest_hook, context) -> None: + with src_hook.get_conn() as src_conn: + cursor = src_conn.cursor() + self.log.info("Querying data from source: %s", self.oracle_source_conn_id) + cursor.execute(self.source_sql, self.source_sql_params) + target_fields = [field[0] for field in cursor.description] + + rows_total = 0 + for rows in iter(lambda: cursor.fetchmany(self.rows_chunk), []): + dest_hook.bulk_insert_rows( + self.destination_table, rows, target_fields=target_fields, commit_every=self.rows_chunk + ) + rows_total += len(rows) + self.log.info("Total inserted: %s rows", rows_total) + + self.log.info("Finished data transfer.") + cursor.close() + + def execute(self, context: Context) -> None: + src_hook = OracleHook(oracle_conn_id=self.oracle_source_conn_id) + dest_hook = OracleHook(oracle_conn_id=self.oracle_destination_conn_id) + self._execute(src_hook, dest_hook, context) diff --git a/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/version_compat.py b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/version_compat.py new file mode 100644 index 0000000000000..ff0f446dc5efc --- /dev/null +++ b/providers/oracle/oracledb/src/airflow/providers/oracle/oracledb/version_compat.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + + +def get_base_airflow_version_tuple() -> tuple[int, int, int]: + from packaging.version import Version + + from airflow import __version__ + + airflow_version = Version(__version__) + return airflow_version.major, airflow_version.minor, airflow_version.micro + + +AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0) + +__all__ = [ + "AIRFLOW_V_3_0_PLUS", +] diff --git a/providers/oracle/oracledb/tests/conftest.py b/providers/oracle/oracledb/tests/conftest.py new file mode 100644 index 0000000000000..f56ccce0a3f69 --- /dev/null +++ b/providers/oracle/oracledb/tests/conftest.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +pytest_plugins = "tests_common.pytest_plugin" diff --git a/providers/oracle/oracledb/tests/system/__init__.py b/providers/oracle/oracledb/tests/system/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/oracle/oracledb/tests/system/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/oracle/oracledb/tests/system/oracle/__init__.py b/providers/oracle/oracledb/tests/system/oracle/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/oracle/oracledb/tests/system/oracle/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/tests/system/oracle/oracledb/__init__.py b/providers/oracle/oracledb/tests/system/oracle/oracledb/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/oracle/oracledb/tests/system/oracle/oracledb/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/tests/system/oracle/oracledb/example_oracle.py b/providers/oracle/oracledb/tests/system/oracle/oracledb/example_oracle.py new file mode 100644 index 0000000000000..b3cda3a09b568 --- /dev/null +++ b/providers/oracle/oracledb/tests/system/oracle/oracledb/example_oracle.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +This is an example DAG for the use of the SQLExecuteQueryOperator with Oracle. +""" + +from __future__ import annotations + +import os +from datetime import datetime + +from airflow import DAG +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator + +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_oracle" + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2025, 1, 1), + default_args={"conn_id": "oracle_conn_id"}, + tags=["example"], + catchup=False, +) as dag: + # [START howto_operator_oracle] + + # Example of creating a task that calls a common CREATE TABLE sql command. + create_table_oracle_task = SQLExecuteQueryOperator( + task_id="create_table_oracle", + sql=r""" + BEGIN + EXECUTE IMMEDIATE ' + CREATE TABLE employees ( + id NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name VARCHAR2(50), + salary NUMBER(10, 2), + hire_date DATE DEFAULT SYSDATE + )'; + END; + """, + ) + + # [END howto_operator_oracle] + + insert_data_oracle_task = SQLExecuteQueryOperator( + task_id="insert_data_oracle", + sql=r""" + BEGIN + INSERT INTO employees (name, salary) VALUES ('Alice', 50000); + INSERT INTO employees (name, salary) VALUES ('Bob', 60000); + END; + """, + ) + + select_data_oracle_task = SQLExecuteQueryOperator( + task_id="select_data_oracle", + sql=r""" + SELECT * FROM employees + """, + ) + + drop_table_oracle_task = SQLExecuteQueryOperator( + task_id="drop_table_oracle", + sql="DROP TABLE employees", + ) + + (create_table_oracle_task >> insert_data_oracle_task >> select_data_oracle_task >> drop_table_oracle_task) + + from tests_common.test_utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) +test_run = get_test_run(dag) diff --git a/providers/oracle/oracledb/tests/unit/__init__.py b/providers/oracle/oracledb/tests/unit/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/oracle/oracledb/tests/unit/oracle/__init__.py b/providers/oracle/oracledb/tests/unit/oracle/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/tests/unit/oracle/oracledb/__init__.py b/providers/oracle/oracledb/tests/unit/oracle/oracledb/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/oracledb/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/tests/unit/oracle/oracledb/assets/__init__.py b/providers/oracle/oracledb/tests/unit/oracle/oracledb/assets/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/oracledb/assets/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/tests/unit/oracle/oracledb/assets/test_oracle.py b/providers/oracle/oracledb/tests/unit/oracle/oracledb/assets/test_oracle.py new file mode 100644 index 0000000000000..1c7a0f1364700 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/oracledb/assets/test_oracle.py @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import urllib.parse + +import pytest + +from airflow.providers.common.compat.assets import Asset +from airflow.providers.oracle.oracledb.assets.oracle import ( + convert_asset_to_openlineage, + create_asset, + sanitize_uri, +) + + +@pytest.mark.parametrize( + ("original", "normalized"), + [ + pytest.param( + "oracle://example.com:1234/orcl/HR/employees", + "oracle://example.com:1234/orcl/HR/employees", + id="normalized", + ), + pytest.param( + "oracle://example.com/orcl/HR/employees", + "oracle://example.com:1521/orcl/HR/employees", + id="default-port", + ), + ], +) +def test_sanitize_uri_pass(original: str, normalized: str) -> None: + uri_i = urllib.parse.urlsplit(original) + uri_o = sanitize_uri(uri_i) + assert urllib.parse.urlunsplit(uri_o) == normalized + + +@pytest.mark.parametrize( + "value", + [ + pytest.param("oracle://", id="blank"), + pytest.param("oracle:///orcl/HR/employees", id="no-host"), + pytest.param("oracle://example.com/orcl/employees", id="missing-component"), + pytest.param("oracle://example.com/orcl/HR/employees/column", id="extra-component"), + ], +) +def test_sanitize_uri_fail(value: str) -> None: + uri_i = urllib.parse.urlsplit(value) + with pytest.raises(ValueError, match="URI format oracle:// must contain"): + sanitize_uri(uri_i) + + +def test_sanitize_uri_fail_non_port() -> None: + uri_i = urllib.parse.urlsplit("oracle://example.com:abcd/orcl/HR/employees") + with pytest.raises(ValueError, match="Port could not be cast to integer value as 'abcd'"): + sanitize_uri(uri_i) + + +@pytest.mark.parametrize( + ("host", "service_name", "schema", "table", "port", "expected_uri"), + [ + pytest.param( + "example.com", + "orcl", + "HR", + "employees", + 1521, + "oracle://example.com:1521/orcl/HR/employees", + id="default-port", + ), + pytest.param( + "example.com", + "orcl", + "HR", + "employees", + 1522, + "oracle://example.com:1522/orcl/HR/employees", + id="custom-port", + ), + ], +) +def test_create_asset( + host: str, service_name: str, schema: str, table: str, port: int, expected_uri: str +) -> None: + result = create_asset(host=host, service_name=service_name, schema=schema, table=table, port=port) + assert result == Asset(uri=expected_uri) + + +@pytest.mark.parametrize( + ("uri", "expected_namespace", "expected_name"), + [ + pytest.param( + "oracle://example.com:1521/orcl/HR/employees", + "oracle://example.com:1521", + "orcl.HR.employees", + id="default-port", + ), + pytest.param( + "oracle://db-host:1522/prod/SCHEMA/users", + "oracle://db-host:1522", + "prod.SCHEMA.users", + id="custom-port", + ), + ], +) +def test_convert_asset_to_openlineage(uri: str, expected_namespace: str, expected_name: str) -> None: + asset = Asset(uri=uri) + ol_dataset = convert_asset_to_openlineage(asset=asset, lineage_context=None) + assert ol_dataset.namespace == expected_namespace + assert ol_dataset.name == expected_name diff --git a/providers/oracle/oracledb/tests/unit/oracle/oracledb/hooks/__init__.py b/providers/oracle/oracledb/tests/unit/oracle/oracledb/hooks/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/oracledb/hooks/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/tests/unit/oracle/oracledb/hooks/test_handlers.py b/providers/oracle/oracledb/tests/unit/oracle/oracledb/hooks/test_handlers.py new file mode 100644 index 0000000000000..f087b274d5ba5 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/oracledb/hooks/test_handlers.py @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest.mock import MagicMock + +from airflow.providers.oracle.oracledb.hooks.handlers import ( + fetch_all_handler, + fetch_one_handler, +) + +from unit.oracle.test_utils import mock_oracle_lob + + +class TestHandlers: + def test_fetch_all_handler(self): + cursor = MagicMock() + cursor.description = [("col1", "int"), ("col2", "string")] + cursor.fetchall.return_value = [(1, mock_oracle_lob("hello"))] + + assert fetch_all_handler(cursor) == [(1, "hello")] + + cursor.description = None + assert fetch_all_handler(cursor) is None + + def test_fetch_one_handler(self): + cursor = MagicMock() + cursor.description = [("col1", "int")] + cursor.fetchone.return_value = (mock_oracle_lob("hello"),) + + assert fetch_one_handler(cursor) == ("hello",) + + cursor.description = None + assert fetch_one_handler(cursor) is None diff --git a/providers/oracle/oracledb/tests/unit/oracle/oracledb/hooks/test_oracle.py b/providers/oracle/oracledb/tests/unit/oracle/oracledb/hooks/test_oracle.py new file mode 100644 index 0000000000000..11302b3a9308b --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/oracledb/hooks/test_oracle.py @@ -0,0 +1,710 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +from datetime import datetime +from unittest import mock + +import numpy as np +import pytest + +import oracledb +from airflow.models import Connection +from airflow.providers.oracle.oracledb.hooks.oracle import OracleHook + +from unit.oracle.test_utils import mock_oracle_lob + + +class TestOracleHookConn: + def setup_method(self): + self.connection = Connection( + login="login", password="password", host="host", port=1521, extra='{"service_name": "schema"}' + ) + + self.db_hook = OracleHook() + self.db_hook.get_connection = mock.Mock() + self.db_hook.get_connection.return_value = self.connection + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_host(self, mock_connect): + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert args == () + assert kwargs["user"] == "login" + assert kwargs["password"] == "password" + assert kwargs["dsn"] == oracledb.makedsn("host", 1521, service_name="schema") + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_host_alternative_port(self, mock_connect): + self.connection.port = 1522 + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert args == () + assert kwargs["user"] == "login" + assert kwargs["password"] == "password" + assert kwargs["dsn"] == oracledb.makedsn("host", self.connection.port, service_name="schema") + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_sid(self, mock_connect): + dsn_sid = {"dsn": "ignored", "sid": "sid"} + self.connection.extra = json.dumps(dsn_sid) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert args == () + assert kwargs["dsn"] == oracledb.makedsn("host", self.connection.port, dsn_sid["sid"]) + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_service_name(self, mock_connect): + dsn_service_name = {"dsn": "ignored", "service_name": "service_name"} + self.connection.extra = json.dumps(dsn_service_name) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert args == () + assert kwargs["dsn"] == oracledb.makedsn( + "host", self.connection.port, service_name=dsn_service_name["service_name"] + ) + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_mode(self, mock_connect): + mode = { + "sysdba": oracledb.AUTH_MODE_SYSDBA, + "sysasm": oracledb.AUTH_MODE_SYSASM, + "sysoper": oracledb.AUTH_MODE_SYSOPER, + "sysbkp": oracledb.AUTH_MODE_SYSBKP, + "sysdgd": oracledb.AUTH_MODE_SYSDGD, + "syskmt": oracledb.AUTH_MODE_SYSKMT, + } + first = True + for mod in mode: + self.connection.extra = json.dumps({"mode": mod}) + self.db_hook.get_conn() + if first: + assert mock_connect.call_count == 1 + first = False + args, kwargs = mock_connect.call_args + assert args == () + assert kwargs["mode"] == mode.get(mod) + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_events(self, mock_connect): + self.connection.extra = json.dumps({"events": True}) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert args == () + assert kwargs["events"] is True + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_purity(self, mock_connect): + purity = { + "new": oracledb.PURITY_NEW, + "self": oracledb.PURITY_SELF, + "default": oracledb.PURITY_DEFAULT, + } + first = True + for pur in purity: + self.connection.extra = json.dumps({"purity": pur}) + self.db_hook.get_conn() + if first: + assert mock_connect.call_count == 1 + first = False + args, kwargs = mock_connect.call_args + assert args == () + assert kwargs["purity"] == purity.get(pur) + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_expire_time(self, mock_connect): + self.connection.extra = json.dumps({"expire_time": 10}) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert args == () + assert kwargs["expire_time"] == 10 + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_schema_as_service_name(self, mock_connect): + """When service_name and sid are not in extras, conn.schema should be used as service_name.""" + self.connection.schema = "MY_SERVICE" + self.connection.extra = json.dumps({}) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert kwargs["dsn"] == oracledb.makedsn("host", 1521, service_name="MY_SERVICE") + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_schema_not_used_when_service_name_set(self, mock_connect): + """Explicit service_name in extras takes precedence over conn.schema.""" + self.connection.schema = "MY_SCHEMA" + self.connection.extra = json.dumps({"service_name": "EXPLICIT_SVC"}) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert kwargs["dsn"] == oracledb.makedsn("host", 1521, service_name="EXPLICIT_SVC") + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_schema_not_used_when_sid_set(self, mock_connect): + """Explicit sid in extras takes precedence over conn.schema.""" + self.connection.schema = "MY_SCHEMA" + self.connection.extra = json.dumps({"sid": "MY_SID"}) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + args, kwargs = mock_connect.call_args + assert kwargs["dsn"] == oracledb.makedsn("host", 1521, "MY_SID") + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_set_current_schema(self, mock_connect): + self.connection.schema = "schema_name" + self.connection.extra = json.dumps({"service_name": "service_name"}) + assert self.db_hook.get_conn().current_schema == self.connection.schema + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.init_oracle_client") + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_set_thick_mode_extra(self, mock_connect, mock_init_client): + thick_mode_test = { + "thick_mode": True, + "thick_mode_lib_dir": "/opt/oracle/instantclient", + "thick_mode_config_dir": "/opt/oracle/config", + } + self.connection.extra = json.dumps(thick_mode_test) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + assert mock_init_client.call_count == 1 + args, kwargs = mock_init_client.call_args + assert args == () + assert kwargs["lib_dir"] == thick_mode_test["thick_mode_lib_dir"] + assert kwargs["config_dir"] == thick_mode_test["thick_mode_config_dir"] + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.init_oracle_client") + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_set_thick_mode_extra_str(self, mock_connect, mock_init_client): + thick_mode_test = {"thick_mode": "True"} + self.connection.extra = json.dumps(thick_mode_test) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + assert mock_init_client.call_count == 1 + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.init_oracle_client") + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_set_thick_mode_params(self, mock_connect, mock_init_client): + # Verify params overrides connection config extra + thick_mode_test = { + "thick_mode": False, + "thick_mode_lib_dir": "/opt/oracle/instantclient", + "thick_mode_config_dir": "/opt/oracle/config", + } + self.connection.extra = json.dumps(thick_mode_test) + db_hook = OracleHook(thick_mode=True, thick_mode_lib_dir="/test", thick_mode_config_dir="/test_conf") + db_hook.get_connection = mock.Mock() + db_hook.get_connection.return_value = self.connection + db_hook.get_conn() + assert mock_connect.call_count == 1 + assert mock_init_client.call_count == 1 + args, kwargs = mock_init_client.call_args + assert args == () + assert kwargs["lib_dir"] == "/test" + assert kwargs["config_dir"] == "/test_conf" + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.init_oracle_client") + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_thick_mode_defaults_to_false(self, mock_connect, mock_init_client): + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + assert mock_init_client.call_count == 0 + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.init_oracle_client") + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_thick_mode_dirs_defaults(self, mock_connect, mock_init_client): + thick_mode_test = {"thick_mode": True} + self.connection.extra = json.dumps(thick_mode_test) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + assert mock_init_client.call_count == 1 + args, kwargs = mock_init_client.call_args + assert args == () + assert kwargs["lib_dir"] is None + assert kwargs["config_dir"] is None + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_oracledb_defaults_attributes_default_values(self, mock_connect): + default_fetch_decimals = oracledb.defaults.fetch_decimals + default_fetch_lobs = oracledb.defaults.fetch_lobs + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + # Check that OracleHook.get_conn() doesn't try to set defaults if not provided + assert oracledb.defaults.fetch_decimals == default_fetch_decimals + assert oracledb.defaults.fetch_lobs == default_fetch_lobs + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_set_oracledb_defaults_attributes_extra(self, mock_connect): + defaults_test = {"fetch_decimals": True, "fetch_lobs": False} + self.connection.extra = json.dumps(defaults_test) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + assert oracledb.defaults.fetch_decimals == defaults_test["fetch_decimals"] + assert oracledb.defaults.fetch_lobs == defaults_test["fetch_lobs"] + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_set_oracledb_defaults_attributes_extra_str(self, mock_connect): + defaults_test = {"fetch_decimals": "True", "fetch_lobs": "False"} + self.connection.extra = json.dumps(defaults_test) + self.db_hook.get_conn() + assert mock_connect.call_count == 1 + assert oracledb.defaults.fetch_decimals is True + assert oracledb.defaults.fetch_lobs is False + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_set_oracledb_defaults_attributes_params(self, mock_connect): + # Verify params overrides connection config extra + defaults_test = {"fetch_decimals": False, "fetch_lobs": True} + self.connection.extra = json.dumps(defaults_test) + db_hook = OracleHook(fetch_decimals=True, fetch_lobs=False) + db_hook.get_connection = mock.Mock() + db_hook.get_connection.return_value = self.connection + db_hook.get_conn() + assert mock_connect.call_count == 1 + assert oracledb.defaults.fetch_decimals is True + assert oracledb.defaults.fetch_lobs is False + + def test_type_checking_thick_mode_lib_dir(self): + thick_mode_lib_dir_test = {"thick_mode": True, "thick_mode_lib_dir": 1} + self.connection.extra = json.dumps(thick_mode_lib_dir_test) + with pytest.raises(TypeError, match=r"thick_mode_lib_dir expected str or None, got.*"): + self.db_hook.get_conn() + + def test_type_checking_thick_mode_config_dir(self): + thick_mode_config_dir_test = {"thick_mode": True, "thick_mode_config_dir": 1} + self.connection.extra = json.dumps(thick_mode_config_dir_test) + with pytest.raises(TypeError, match=r"thick_mode_config_dir expected str or None, got.*"): + self.db_hook.get_conn() + + @pytest.mark.parametrize( + ("connection_params", "expected_uri"), + [ + pytest.param( + {"extra": '{"service_name": "service"}', "schema": None, "port": 1521}, + "oracle+oracledb://login:password@host:1521?service_name=service", + id="service_name_in_extra", + ), + pytest.param( + {"extra": '{"sid": "sid"}', "schema": None, "port": 1521}, + "oracle+oracledb://login:password@host:1521/sid", + id="sid_in_extra", + ), + pytest.param( + {"extra": "{}", "schema": "db_schema", "port": 1521}, + "oracle+oracledb://login:password@host:1521/db_schema", + id="schema_only", + ), + pytest.param( + {"extra": "{}", "schema": None, "port": 1521}, + "oracle+oracledb://login:password@host:1521", + id="no_schema_no_extra", + ), + pytest.param( + {"extra": "{}", "schema": "db_schema", "port": None}, + "oracle+oracledb://login:password@host:1521/db_schema", + id="schema_only_default_port", + ), + pytest.param( + {"extra": '{"service_name": "service"}', "schema": "db_schema", "port": 1521}, + "oracle+oracledb://login:password@host:1521?service_name=service", + id="service_name_with_schema", + ), + ], + ) + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_uri(self, mock_connect, connection_params, expected_uri): + self.connection.extra = connection_params["extra"] + self.connection.schema = connection_params["schema"] + self.connection.port = connection_params["port"] + + uri = self.db_hook.get_uri() + assert uri == expected_uri + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.oracledb.connect") + def test_get_conn_with_various_params(self, mock_connect): + """Verify wallet/SSL, connection class, and pool parameters + are passed to oracledb.connect.""" + params = { + "wallet_location": "/tmp/wallet", + "wallet_password": "secret", + "ssl_server_cert_dn": "CN=dbserver,OU=DB,O=Oracle,L=BLR,C=IN", + "ssl_server_dn_match": True, + "cclass": "MY_APP_CLASS", + "pool_name": "POOL_1", + } + self.connection.extra = json.dumps(params) + self.db_hook.get_conn() + + assert mock_connect.call_count == 1 + _, kwargs = mock_connect.call_args + + for key, value in params.items(): + assert kwargs[key] == value + + +class TestOracleHook: + def setup_method(self): + self.cur = mock.MagicMock(rowcount=0) + self.conn = mock.MagicMock() + self.conn.cursor.return_value = self.cur + conn = self.conn + + class UnitTestOracleHook(OracleHook): + conn_name_attr = "test_conn_id" + + def get_conn(self): + return conn + + self.db_hook = UnitTestOracleHook() + + def test_run_without_parameters(self): + sql = "SQL" + self.db_hook.run(sql) + self.cur.execute.assert_called_once_with(sql) + assert self.conn.commit.called + + def test_run_with_parameters(self): + sql = "SQL" + param = ("p1", "p2") + self.db_hook.run(sql, parameters=param) + self.cur.execute.assert_called_once_with(sql, param) + assert self.conn.commit.called + + @mock.patch("airflow.providers.common.sql.hooks.sql.send_sql_hook_lineage") + def test_run_hook_lineage(self, mock_send_lineage): + statement = "SELECT 1" + self.cur.fetchall.return_value = [] + + self.db_hook.run(statement) + + mock_send_lineage.assert_called() + call_kw = mock_send_lineage.call_args.kwargs + assert call_kw["context"] is self.db_hook + assert call_kw["sql"] == statement + assert call_kw["sql_parameters"] is None + assert call_kw["cur"] is self.cur + + @mock.patch("airflow.providers.common.sql.hooks.sql.send_sql_hook_lineage") + @mock.patch("airflow.providers.common.sql.hooks.sql.DbApiHook._get_pandas_df") + def test_get_df_hook_lineage(self, mock_get_pandas_df, mock_send_lineage): + sql = "SELECT 1" + parameters = ("x",) + self.db_hook.get_df(sql, parameters=parameters) + + mock_send_lineage.assert_called_once() + call_kw = mock_send_lineage.call_args.kwargs + assert call_kw["context"] is self.db_hook + assert call_kw["sql"] == sql + assert call_kw["sql_parameters"] == parameters + + @mock.patch("airflow.providers.common.sql.hooks.sql.send_sql_hook_lineage") + @mock.patch("airflow.providers.common.sql.hooks.sql.DbApiHook._get_pandas_df_by_chunks") + def test_get_df_by_chunks_hook_lineage(self, mock_get_pandas_df_by_chunks, mock_send_lineage): + sql = "SELECT 1" + parameters = ("x",) + self.db_hook.get_df_by_chunks(sql, parameters=parameters, chunksize=1) + + mock_send_lineage.assert_called_once() + call_kw = mock_send_lineage.call_args.kwargs + assert call_kw["context"] is self.db_hook + assert call_kw["sql"] == sql + assert call_kw["sql_parameters"] == parameters + + def test_insert_rows_with_fields(self): + rows = [ + ( + "'basestr_with_quote", + None, + np.nan, + np.datetime64("2019-01-24T01:02:03"), + datetime(2019, 1, 24), + 1, + 10.24, + "str", + ) + ] + target_fields = [ + "basestring", + "none", + "numpy_nan", + "numpy_datetime64", + "datetime", + "int", + "float", + "str", + ] + self.db_hook.insert_rows("table", rows, target_fields) + self.cur.execute.assert_called_once_with( + "INSERT /*+ APPEND */ INTO table " + "(basestring, none, numpy_nan, numpy_datetime64, datetime, int, float, str) " + "VALUES ('''basestr_with_quote',NULL,NULL,'2019-01-24T01:02:03'," + "to_date('2019-01-24 00:00:00','YYYY-MM-DD HH24:MI:SS'),1,10.24,'str')" + ) + + def test_insert_rows_without_fields(self): + rows = [ + ( + "'basestr_with_quote", + None, + np.nan, + np.datetime64("2019-01-24T01:02:03"), + datetime(2019, 1, 24), + 1, + 10.24, + "str", + ) + ] + self.db_hook.insert_rows("table", rows) + self.cur.execute.assert_called_once_with( + "INSERT /*+ APPEND */ INTO table " + " VALUES ('''basestr_with_quote',NULL,NULL,'2019-01-24T01:02:03'," + "to_date('2019-01-24 00:00:00','YYYY-MM-DD HH24:MI:SS'),1,10.24,'str')" + ) + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.send_sql_hook_lineage") + def test_insert_rows_hook_lineage(self, mock_send_lineage): + rows = [("a", "b", "c")] + target_fields = ["col1", "col2", "col3"] + self.db_hook.insert_rows("table", rows, target_fields) + + mock_send_lineage.assert_called() + call_kw = mock_send_lineage.call_args.kwargs + assert call_kw["context"] is self.db_hook + assert call_kw["sql"] == "INSERT /*+ APPEND */ INTO table (col1, col2, col3) VALUES ('a','b','c')" + assert call_kw["row_count"] == 1 + + def test_bulk_insert_rows_with_fields(self): + rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] + target_fields = ["col1", "col2", "col3"] + self.db_hook.bulk_insert_rows("table", rows, target_fields) + self.cur.prepare.assert_called_once_with("insert into table (col1, col2, col3) values (:1, :2, :3)") + self.cur.executemany.assert_called_once_with(None, rows) + + def test_bulk_insert_rows_with_commit_every(self): + rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] + target_fields = ["col1", "col2", "col3"] + self.db_hook.bulk_insert_rows("table", rows, target_fields, commit_every=2) + calls = [ + mock.call("insert into table (col1, col2, col3) values (:1, :2, :3)"), + mock.call("insert into table (col1, col2, col3) values (:1, :2, :3)"), + ] + self.cur.prepare.assert_has_calls(calls) + calls = [ + mock.call(None, rows[:2]), + mock.call(None, rows[2:]), + ] + self.cur.executemany.assert_has_calls(calls, any_order=True) + + def test_bulk_insert_rows_without_fields(self): + rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] + self.db_hook.bulk_insert_rows("table", rows) + self.cur.prepare.assert_called_once_with("insert into table values (:1, :2, :3)") + self.cur.executemany.assert_called_once_with(None, rows) + + @mock.patch("airflow.providers.oracle.oracledb.hooks.oracle.send_sql_hook_lineage") + def test_bulk_insert_rows_hook_lineage(self, mock_send_lineage): + rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] + target_fields = ["col1", "col2", "col3"] + self.db_hook.bulk_insert_rows("table", rows, target_fields) + + mock_send_lineage.assert_called_once() + call_kw = mock_send_lineage.call_args.kwargs + assert call_kw["context"] is self.db_hook + assert call_kw["sql"] == "insert into table (col1, col2, col3) values (:1, :2, :3)" + assert call_kw["row_count"] == 3 + + def test_bulk_insert_rows_no_rows(self): + rows = [] + with pytest.raises(ValueError, match="parameter rows could not be None or empty iterable"): + self.db_hook.bulk_insert_rows("table", rows) + + def test_bulk_insert_sequence_field(self): + rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] + target_fields = ["col1", "col2", "col3"] + sequence_column = "id" + sequence_name = "my_sequence" + self.db_hook.bulk_insert_rows( + "table", rows, target_fields, sequence_column=sequence_column, sequence_name=sequence_name + ) + self.cur.prepare.assert_called_once_with( + "insert into table (id, col1, col2, col3) values (my_sequence.NEXTVAL, :1, :2, :3)" + ) + self.cur.executemany.assert_called_once_with(None, rows) + + def test_bulk_insert_sequence_without_parameter(self): + SEQUENCE_COLUMN_OR_NAME_PROVIDED = ( + "Parameters 'sequence_column' and 'sequence_name' must be provided together or not at all." + ) + rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] + target_fields = ["col1", "col2", "col3"] + sequence_column = "id" + sequence_name = None + with pytest.raises(ValueError, match=SEQUENCE_COLUMN_OR_NAME_PROVIDED): + self.db_hook.bulk_insert_rows( + "table", rows, target_fields, sequence_column=sequence_column, sequence_name=sequence_name + ) + + sequence_column = None + sequence_name = "my_sequence" + with pytest.raises(ValueError, match=SEQUENCE_COLUMN_OR_NAME_PROVIDED): + self.db_hook.bulk_insert_rows( + "table", rows, target_fields, sequence_column=sequence_column, sequence_name=sequence_name + ) + + def test_bulk_insert_commit_leftovers_only_if_exists(self): + rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9), (1, 2, 3), (4, 5, 6), (7, 8, 9)] + target_fields = ["col1", "col2", "col3"] + sequence_column = "id" + sequence_name = "my_sequence" + + self.db_hook.bulk_insert_rows( + "table", + rows, + target_fields, + sequence_column=sequence_column, + sequence_name=sequence_name, + commit_every=3, + ) + + # executemany should be called exactly 2 times because there is no leftovers + assert self.cur.executemany.call_count == 2 + + def test_callproc_none(self): + parameters = None + + class bindvar(int): + def getvalue(self): + return self + + self.cur.bindvars = None + result = self.db_hook.callproc("proc", True, parameters) + assert self.cur.execute.mock_calls == [mock.call("BEGIN proc(); END;")] + assert result == parameters + + def test_callproc_dict(self): + parameters = {"a": 1, "b": 2, "c": 3} + + class bindvar(int): + def getvalue(self): + return self + + self.cur.bindvars = {k: bindvar(v) for k, v in parameters.items()} + result = self.db_hook.callproc("proc", True, parameters) + assert self.cur.execute.mock_calls == [mock.call("BEGIN proc(:a,:b,:c); END;", parameters)] + assert result == parameters + + def test_callproc_list(self): + parameters = [1, 2, 3] + + class bindvar(int): + def getvalue(self): + return self + + self.cur.bindvars = list(map(bindvar, parameters)) + result = self.db_hook.callproc("proc", True, parameters) + assert self.cur.execute.mock_calls == [mock.call("BEGIN proc(:1,:2,:3); END;", parameters)] + assert result == parameters + + def test_callproc_out_param(self): + parameters = [1, int, float, bool, str] + + def bindvar(value): + m = mock.Mock() + m.getvalue.return_value = value + return m + + self.cur.bindvars = [bindvar(p() if type(p) is type else p) for p in parameters] + result = self.db_hook.callproc("proc", True, parameters) + expected = [1, 0, 0.0, False, ""] + assert self.cur.execute.mock_calls == [mock.call("BEGIN proc(:1,:2,:3,:4,:5); END;", expected)] + assert result == expected + + def test_test_connection_use_dual_table(self): + self.cur.fetchone.return_value = (1,) + status, message = self.db_hook.test_connection() + self.cur.execute.assert_called_once_with("select 1 from dual") + assert status is True + assert message == "Connection successfully tested" + + def test_get_openlineage_database_info_with_service_name(self): + conn = Connection( + conn_id="oracle_default", + conn_type="oracle", + host="localhost", + port=1521, + extra='{"service_name": "ORCLPDB1"}', + ) + hook = OracleHook(oracle_conn_id="oracle_default") + hook.get_connection = lambda _: conn + + assert hook.service_name == "ORCLPDB1" + db_info = hook.get_openlineage_database_info(conn) + assert db_info.scheme == "oracle" + assert db_info.authority == "localhost:1521" + assert db_info.database == "ORCLPDB1" + assert db_info.normalize_name_method("employees") == "EMPLOYEES" + assert db_info.information_schema_table_name == "ALL_TAB_COLUMNS" + assert "owner" in db_info.information_schema_columns + + def test_get_openlineage_database_info_with_sid(self): + conn = Connection( + conn_id="oracle_default", + conn_type="oracle", + host="dbhost", + port=1521, + extra='{"sid": "XE"}', + ) + hook = OracleHook(oracle_conn_id="oracle_default") + hook.get_connection = lambda _: conn + + assert hook.sid == "XE" + db_info = hook.get_openlineage_database_info(conn) + assert db_info.scheme == "oracle" + assert db_info.authority == "dbhost:1521" + assert db_info.database == "XE" + assert db_info.normalize_name_method("employees") == "EMPLOYEES" + assert db_info.information_schema_table_name == "ALL_TAB_COLUMNS" + assert "owner" in db_info.information_schema_columns + + def test_get_first(self): + statement = "SQL" + + self.cur.fetchone.return_value = (mock_oracle_lob("hello"),) + + assert self.db_hook.get_first(statement) == ("hello",) + + assert self.conn.close.call_count == 1 + assert self.cur.close.call_count == 1 + self.cur.execute.assert_called_once_with(statement) + + def test_get_records(self): + statement = "SQL" + + self.cur.fetchall.return_value = (mock_oracle_lob("hello"),), (mock_oracle_lob("world"),) + + assert self.db_hook.get_records(statement) == [("hello",), ("world",)] + + assert self.conn.close.call_count == 1 + assert self.cur.close.call_count == 1 + self.cur.execute.assert_called_once_with(statement) diff --git a/providers/oracle/oracledb/tests/unit/oracle/oracledb/operators/__init__.py b/providers/oracle/oracledb/tests/unit/oracle/oracledb/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/oracledb/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/tests/unit/oracle/oracledb/operators/test_oracle.py b/providers/oracle/oracledb/tests/unit/oracle/oracledb/operators/test_oracle.py new file mode 100644 index 0000000000000..d1bb3921c5f61 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/oracledb/operators/test_oracle.py @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import random +import re +from unittest import mock + +import pytest + +import oracledb +from airflow.models import TaskInstance +from airflow.providers.oracle.oracledb.hooks.oracle import OracleHook +from airflow.providers.oracle.oracledb.operators.oracle import OracleStoredProcedureOperator + +from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS + + +class TestOracleStoredProcedureOperator: + @mock.patch.object(OracleHook, "run", autospec=OracleHook.run) + def test_execute(self, mock_run): + procedure = "test" + oracle_conn_id = "oracle_default" + parameters = {"parameter": "value"} + context = "test_context" + task_id = "test_task_id" + + operator = OracleStoredProcedureOperator( + procedure=procedure, + oracle_conn_id=oracle_conn_id, + parameters=parameters, + task_id=task_id, + ) + result = operator.execute(context=context) + assert result is mock_run.return_value + mock_run.assert_called_once_with( + mock.ANY, + "BEGIN test(:parameter); END;", + autocommit=True, + parameters=parameters, + handler=mock.ANY, + ) + + @mock.patch.object(OracleHook, "callproc", autospec=OracleHook.callproc) + def test_push_oracle_exit_to_xcom(self, mock_callproc, request, dag_maker): + # Test pulls the value previously pushed to xcom and checks if it's the same + procedure = "test_push" + oracle_conn_id = "oracle_default" + parameters = {"parameter": "value"} + task_id = "test_push" + ora_exit_code = f"{random.randrange(10**5):05}" + error = f"ORA-{ora_exit_code}: This is a five-digit ORA error code" + mock_callproc.side_effect = oracledb.DatabaseError(error) + + if AIRFLOW_V_3_0_PLUS: + run_task = request.getfixturevalue("run_task") + task = OracleStoredProcedureOperator( + procedure=procedure, oracle_conn_id=oracle_conn_id, parameters=parameters, task_id=task_id + ) + run_task(task=task) + assert run_task.xcom.get(task_id=task.task_id, key="ORA") == ora_exit_code + else: + with dag_maker(dag_id=f"dag_{request.node.name}"): + task = OracleStoredProcedureOperator( + procedure=procedure, oracle_conn_id=oracle_conn_id, parameters=parameters, task_id=task_id + ) + dr = dag_maker.create_dagrun(run_id=task_id) + ti = TaskInstance(task=task, run_id=dr.run_id) + with pytest.raises(oracledb.DatabaseError, match=re.escape(error)): + ti.run() + assert ti.xcom_pull(task_ids=task.task_id, key="ORA") == ora_exit_code diff --git a/providers/oracle/oracledb/tests/unit/oracle/oracledb/test_utils.py b/providers/oracle/oracledb/tests/unit/oracle/oracledb/test_utils.py new file mode 100644 index 0000000000000..4132475cabd67 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/oracledb/test_utils.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from unittest.mock import MagicMock + +import oracledb + + +def mock_oracle_lob(value): + mock_lob = MagicMock(spec=oracledb.LOB) + mock_lob.read.return_value = value + return mock_lob diff --git a/providers/oracle/oracledb/tests/unit/oracle/oracledb/transfers/__init__.py b/providers/oracle/oracledb/tests/unit/oracle/oracledb/transfers/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/oracledb/transfers/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/oracle/oracledb/tests/unit/oracle/oracledb/transfers/test_oracle_to_oracle.py b/providers/oracle/oracledb/tests/unit/oracle/oracledb/transfers/test_oracle_to_oracle.py new file mode 100644 index 0000000000000..e286b7daae2d2 --- /dev/null +++ b/providers/oracle/oracledb/tests/unit/oracle/oracledb/transfers/test_oracle_to_oracle.py @@ -0,0 +1,70 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock +from unittest.mock import MagicMock + +from airflow.providers.oracle.oracledb.transfers.oracle_to_oracle import OracleToOracleOperator + + +class TestOracleToOracleTransfer: + def test_execute(self): + oracle_destination_conn_id = "oracle_destination_conn_id" + destination_table = "destination_table" + oracle_source_conn_id = "oracle_source_conn_id" + source_sql = "select sysdate from dual where trunc(sysdate) = :p_data" + source_sql_params = {":p_data": "2018-01-01"} + rows_chunk = 5000 + cursor_description = [ + ("id", "", 39, None, 38, 0, 0), + ("description", "", 60, 240, None, None, 1), + ] + cursor_rows = [[1, "description 1"], [2, "description 2"]] + + mock_dest_hook = MagicMock() + mock_src_hook = MagicMock() + mock_src_conn = mock_src_hook.get_conn.return_value.__enter__.return_value + mock_cursor = mock_src_conn.cursor.return_value + mock_cursor.description.__iter__.return_value = cursor_description + mock_cursor.fetchmany.side_effect = [cursor_rows, []] + + op = OracleToOracleOperator( + task_id="copy_data", + oracle_destination_conn_id=oracle_destination_conn_id, + destination_table=destination_table, + oracle_source_conn_id=oracle_source_conn_id, + source_sql=source_sql, + source_sql_params=source_sql_params, + rows_chunk=rows_chunk, + ) + + op._execute(mock_src_hook, mock_dest_hook, None) + + assert mock_src_hook.get_conn.called + assert mock_src_conn.cursor.called + mock_cursor.execute.assert_called_once_with(source_sql, source_sql_params) + + calls = [ + mock.call(rows_chunk), + mock.call(rows_chunk), + ] + mock_cursor.fetchmany.assert_has_calls(calls) + mock_dest_hook.bulk_insert_rows.assert_called_once_with( + destination_table, cursor_rows, commit_every=rows_chunk, target_fields=["id", "description"] + ) diff --git a/providers/oracle/provider.yaml b/providers/oracle/provider.yaml index 97cfe31aa01cb..4e650a9e5bf11 100644 --- a/providers/oracle/provider.yaml +++ b/providers/oracle/provider.yaml @@ -21,14 +21,22 @@ name: Oracle description: | `Oracle `__ + .. deprecated:: + This provider is deprecated. It now only re-exports the classes from + ``apache-airflow-providers-oracle-oracledb`` for backward compatibility. + Install ``apache-airflow-providers-oracle-oracledb`` and update your imports + from ``airflow.providers.oracle`` to ``airflow.providers.oracle.oracledb``. + See the changelog for a migration guide. + state: ready -lifecycle: production +lifecycle: deprecated source-date-epoch: 1783356304 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 4.6.3 - 4.6.2 - 4.6.1 - 4.6.0 diff --git a/providers/oracle/pyproject.toml b/providers/oracle/pyproject.toml index d23df241f91eb..92b42052be7cf 100644 --- a/providers/oracle/pyproject.toml +++ b/providers/oracle/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-oracle" -version = "4.6.2" +version = "4.6.3" description = "Provider package apache-airflow-providers-oracle for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -62,7 +62,7 @@ dependencies = [ "apache-airflow>=2.11.0", "apache-airflow-providers-common-compat>=1.8.0", "apache-airflow-providers-common-sql>=1.32.0", - "oracledb>=2.3.0", + "apache-airflow-providers-oracle-oracledb>=4.6.3", ] # The optional dependencies should be modified in place in the generated file @@ -118,11 +118,12 @@ apache-airflow = {workspace = true} apache-airflow-devel-common = {workspace = true} apache-airflow-task-sdk = {workspace = true} apache-airflow-providers-common-sql = {workspace = true} +apache-airflow-providers-oracle-oracledb = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-oracle/4.6.2" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-oracle/4.6.2/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-oracle/4.6.3" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-oracle/4.6.3/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/oracle/src/airflow/providers/oracle/__init__.py b/providers/oracle/src/airflow/providers/oracle/__init__.py index 2b56adad50d21..2e8fd75aabc67 100644 --- a/providers/oracle/src/airflow/providers/oracle/__init__.py +++ b/providers/oracle/src/airflow/providers/oracle/__init__.py @@ -29,7 +29,15 @@ __all__ = ["__version__"] -__version__ = "4.6.2" +# ``apache-airflow-providers-oracle`` is deprecated and now hosts only a re-export shim +# (see hooks/, operators/, transfers/, assets/ in this package). The real implementation +# lives in the ``apache-airflow-providers-oracle-oracledb`` distribution, installed as a +# dependency of this package, under ``airflow.providers.oracle.oracledb``. Extending +# ``__path__`` here lets that sibling distribution's ``oracledb`` subpackage be found +# under this namespace regardless of package install/discovery order. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) + +__version__ = "4.6.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/oracle/src/airflow/providers/oracle/assets/oracle.py b/providers/oracle/src/airflow/providers/oracle/assets/oracle.py index 19942df7ab10b..dbbe51bd3a883 100644 --- a/providers/oracle/src/airflow/providers/oracle/assets/oracle.py +++ b/providers/oracle/src/airflow/providers/oracle/assets/oracle.py @@ -14,50 +14,24 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Deprecated. Use ``airflow.providers.oracle.oracledb.assets.oracle`` instead.""" from __future__ import annotations -from typing import TYPE_CHECKING - -from airflow.providers.common.compat.assets import Asset - -if TYPE_CHECKING: - from urllib.parse import SplitResult - - from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset - - -def sanitize_uri(uri: SplitResult) -> SplitResult: - if not uri.netloc: - raise ValueError("URI format oracle:// must contain a host") - if uri.port is None: - host = uri.netloc.rstrip(":") - uri = uri._replace(netloc=f"{host}:1521") - if len(uri.path.split("/")) != 4: # Leading slash, service name, schema, and table names. - raise ValueError("URI format oracle:// must contain service name, schema, and table names") - return uri - - -def create_asset( - *, - host: str, - port: int = 1521, - service_name: str, - schema: str, - table: str, - extra: dict | None = None, -) -> Asset: - return Asset(uri=f"oracle://{host}:{port}/{service_name}/{schema}/{table}", extra=extra) - - -def convert_asset_to_openlineage(asset: Asset, lineage_context) -> OpenLineageDataset: - """Translate Asset with valid AIP-60 uri to OpenLineage with assistance from the hook.""" - from urllib.parse import urlsplit - - from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset - - parsed = urlsplit(asset.uri) - _, service_name, schema, table = parsed.path.split( - "/" - ) # Leading slash, service_name, schema, and table names. - return OpenLineageDataset(namespace=f"oracle://{parsed.netloc}", name=f"{service_name}.{schema}.{table}") +from airflow.utils.deprecation_tools import add_deprecated_classes + +__deprecated_classes = { + __name__: { + "sanitize_uri": "airflow.providers.oracle.oracledb.assets.oracle.sanitize_uri", + "create_asset": "airflow.providers.oracle.oracledb.assets.oracle.create_asset", + "convert_asset_to_openlineage": ( + "airflow.providers.oracle.oracledb.assets.oracle.convert_asset_to_openlineage" + ), + }, +} + +add_deprecated_classes( + __deprecated_classes, + __name__, + extra_message="`apache-airflow-providers-oracle` is deprecated, install `apache-airflow-providers-oracle-oracledb` instead", +) diff --git a/providers/oracle/src/airflow/providers/oracle/example_dags/example_oracle.py b/providers/oracle/src/airflow/providers/oracle/example_dags/example_oracle.py index 6eab1f4de1bae..e5fc5567043f4 100644 --- a/providers/oracle/src/airflow/providers/oracle/example_dags/example_oracle.py +++ b/providers/oracle/src/airflow/providers/oracle/example_dags/example_oracle.py @@ -19,7 +19,10 @@ from datetime import datetime from airflow import DAG -from airflow.providers.oracle.operators.oracle import OracleStoredProcedureOperator + +# `apache-airflow-providers-oracle` is deprecated; import from +# `apache-airflow-providers-oracle-oracledb` instead. +from airflow.providers.oracle.oracledb.operators.oracle import OracleStoredProcedureOperator with DAG( max_active_runs=1, diff --git a/providers/oracle/src/airflow/providers/oracle/example_dags/example_oracle_fetch.py b/providers/oracle/src/airflow/providers/oracle/example_dags/example_oracle_fetch.py index 9ecb494b26e19..e70758d9bda84 100644 --- a/providers/oracle/src/airflow/providers/oracle/example_dags/example_oracle_fetch.py +++ b/providers/oracle/src/airflow/providers/oracle/example_dags/example_oracle_fetch.py @@ -22,7 +22,10 @@ from airflow import DAG from airflow.operators.empty import EmptyOperator from airflow.operators.python import PythonOperator -from airflow.providers.oracle.hooks.oracle import OracleHook + +# `apache-airflow-providers-oracle` is deprecated; import from +# `apache-airflow-providers-oracle-oracledb` instead. +from airflow.providers.oracle.oracledb.hooks.oracle import OracleHook DOC = """ ### Example: Simple Oracle fetch diff --git a/providers/oracle/src/airflow/providers/oracle/hooks/handlers.py b/providers/oracle/src/airflow/providers/oracle/hooks/handlers.py index d75bcfc6e7e20..581494746ba65 100644 --- a/providers/oracle/src/airflow/providers/oracle/hooks/handlers.py +++ b/providers/oracle/src/airflow/providers/oracle/hooks/handlers.py @@ -15,43 +15,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - -import oracledb - - -def _read_lob(val): - if isinstance(val, oracledb.LOB): - return val.read() - return val - - -def _read_lobs(row): - if row is not None: - return tuple([_read_lob(value) for value in row]) - return row +"""Deprecated. Use ``airflow.providers.oracle.oracledb.hooks.handlers`` instead.""" +from __future__ import annotations -def fetch_all_handler(cursor) -> list[tuple] | None: - """Return results for DbApiHook.run(). If oracledb.LOB objects are present, then those will be read.""" - if not hasattr(cursor, "description"): - raise RuntimeError( - "The database we interact with does not support DBAPI 2.0. Use operator and " - "handlers that are specifically designed for your database." - ) - if cursor.description is not None: - results = [_read_lobs(row) for row in cursor.fetchall()] - return results - return None +from airflow.utils.deprecation_tools import add_deprecated_classes +__deprecated_classes = { + __name__: { + "fetch_all_handler": "airflow.providers.oracle.oracledb.hooks.handlers.fetch_all_handler", + "fetch_one_handler": "airflow.providers.oracle.oracledb.hooks.handlers.fetch_one_handler", + }, +} -def fetch_one_handler(cursor) -> tuple | None: - """Return first result for DbApiHook.run(). If oracledb.LOB objects are present, then those will be read.""" - if not hasattr(cursor, "description"): - raise RuntimeError( - "The database we interact with does not support DBAPI 2.0. Use operator and " - "handlers that are specifically designed for your database." - ) - if cursor.description is not None: - return _read_lobs(cursor.fetchone()) - return None +add_deprecated_classes( + __deprecated_classes, + __name__, + extra_message="`apache-airflow-providers-oracle` is deprecated, install `apache-airflow-providers-oracle-oracledb` instead", +) diff --git a/providers/oracle/src/airflow/providers/oracle/hooks/oracle.py b/providers/oracle/src/airflow/providers/oracle/hooks/oracle.py index 285dfe71215b9..d03ba75ec2338 100644 --- a/providers/oracle/src/airflow/providers/oracle/hooks/oracle.py +++ b/providers/oracle/src/airflow/providers/oracle/hooks/oracle.py @@ -15,552 +15,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - -import math -import warnings -from collections.abc import Iterable, Mapping -from datetime import datetime -from typing import TYPE_CHECKING, Any - -import oracledb - -if TYPE_CHECKING: - from airflow.models.connection import Connection - from airflow.providers.openlineage.sqlparser import DatabaseInfo - -from airflow.providers.common.sql.hooks.lineage import send_sql_hook_lineage -from airflow.providers.common.sql.hooks.sql import DbApiHook -from airflow.providers.oracle.hooks import handlers - -DEFAULT_DB_PORT = 1521 -PARAM_TYPES = {bool, float, int, str} - - -def _map_param(value): - if value in PARAM_TYPES: - # In this branch, value is a Python type; calling it produces - # an instance of the type which is understood by the Oracle driver - # in the out parameter mapping mechanism. - value = value() - return value - - -def _get_bool(val): - if isinstance(val, bool): - return val - if isinstance(val, str): - val = val.lower().strip() - if val == "true": - return True - if val == "false": - return False - return None - - -def _get_first_bool(*vals): - for val in vals: - converted = _get_bool(val) - if isinstance(converted, bool): - return converted - return None - - -class OracleHook(DbApiHook): - """ - Interact with Oracle SQL. - - :param oracle_conn_id: The :ref:`Oracle connection id ` - used for Oracle credentials. - :param thick_mode: Specify whether to use python-oracledb in thick mode. Defaults to False. - If set to True, you must have the Oracle Client libraries installed. - See `oracledb docs` - for more info. - :param thick_mode_lib_dir: Path to use to find the Oracle Client libraries when using thick mode. - If not specified, defaults to the standard way of locating the Oracle Client library on the OS. - See `oracledb docs - ` - for more info. - :param thick_mode_config_dir: Path to use to find the Oracle Client library - configuration files when using thick mode. - If not specified, defaults to the standard way of locating the Oracle Client - library configuration files on the OS. - See `oracledb docs - ` - for more info. - :param fetch_decimals: Specify whether numbers should be fetched as ``decimal.Decimal`` values. - See `defaults.fetch_decimals - ` - for more info. - :param fetch_lobs: Specify whether to fetch strings/bytes for CLOBs or BLOBs instead of locators. - See `defaults.fetch_lobs - ` - for more info. - """ - - conn_name_attr = "oracle_conn_id" - default_conn_name = "oracle_default" - conn_type = "oracle" - hook_name = "Oracle" - - _test_connection_sql = "select 1 from dual" - supports_autocommit = True - - def __init__( - self, - *args, - thick_mode: bool | None = None, - thick_mode_lib_dir: str | None = None, - thick_mode_config_dir: str | None = None, - fetch_decimals: bool | None = None, - fetch_lobs: bool | None = None, - **kwargs, - ) -> None: - super().__init__(*args, **kwargs) - - self.thick_mode = thick_mode - self.thick_mode_lib_dir = thick_mode_lib_dir - self.thick_mode_config_dir = thick_mode_config_dir - self.fetch_decimals = fetch_decimals - self.fetch_lobs = fetch_lobs - self._service_name: str | None = None - self._sid: str | None = None - - @property - def service_name(self) -> str | None: - if self._service_name is None: - self._service_name = self.get_connection(self.get_conn_id()).extra_dejson.get("service_name") - return self._service_name - - @property - def sid(self) -> str | None: - if self._sid is None: - self._sid = self.get_connection(self.get_conn_id()).extra_dejson.get("sid") - return self._sid - - def get_conn(self) -> oracledb.Connection: - """ - Get an Oracle connection object. - - Optional parameters for using a custom DSN connection (instead of using - a server alias from tnsnames.ora) The dsn (data source name) is the TNS - entry (from the Oracle names server or tnsnames.ora file), or is a - string like the one returned from ``makedsn()``. - - :param dsn: the data source name for the Oracle server - :param service_name: the db_unique_name of the database - that you are connecting to (CONNECT_DATA part of TNS) - :param sid: Oracle System ID that identifies a particular - database on a system - :param wallet_location: Specify the directory where the wallet can be found. - :param wallet_password: the password to use to decrypt the wallet, if it is encrypted. - For Oracle Autonomous Database this is the password created when downloading the wallet. - :param ssl_server_cert_dn: Specify the distinguished name (DN) which should be matched - with the server. This value is ignored if the ``ssl_server_dn_match`` parameter is not - set to the value True. - :param ssl_server_dn_match: Specify whether the server certificate distinguished name - (DN) should be matched in addition to the regular certificate verification that is performed. - :param cclass: the connection class to use for Database Resident Connection Pooling (DRCP). - :param pool_name: the name of the DRCP pool when using multi-pool DRCP with Oracle Database 23.4, or higher. - - You can set these parameters in the extra fields of your connection - as in - - .. code-block:: python - - {"dsn": ("(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=host)(PORT=1521))(CONNECT_DATA=(SID=sid)))")} - - see more param detail in `oracledb.connect - `_ - - - """ - conn = self.get_connection(self.get_conn_id()) - conn_config: dict[str, Any] = {"user": conn.login, "password": conn.password} - sid = conn.extra_dejson.get("sid") - mod = conn.extra_dejson.get("module") - schema = conn.schema - - # Enable oracledb thick mode if thick_mode is set to True - # Parameters take precedence over connection config extra - # Defaults to use thin mode if not provided in params or connection config extra - thick_mode = _get_first_bool(self.thick_mode, conn.extra_dejson.get("thick_mode")) - if thick_mode is True: - if self.thick_mode_lib_dir is None: - self.thick_mode_lib_dir = conn.extra_dejson.get("thick_mode_lib_dir") - if not isinstance(self.thick_mode_lib_dir, (str, type(None))): - raise TypeError( - f"thick_mode_lib_dir expected str or None, " - f"got {type(self.thick_mode_lib_dir).__name__}" - ) - if self.thick_mode_config_dir is None: - self.thick_mode_config_dir = conn.extra_dejson.get("thick_mode_config_dir") - if not isinstance(self.thick_mode_config_dir, (str, type(None))): - raise TypeError( - f"thick_mode_config_dir expected str or None, " - f"got {type(self.thick_mode_config_dir).__name__}" - ) - oracledb.init_oracle_client( - lib_dir=self.thick_mode_lib_dir, config_dir=self.thick_mode_config_dir - ) - - # Set oracledb Defaults Attributes if provided - # (https://python-oracledb.readthedocs.io/en/latest/api_manual/defaults.html) - fetch_decimals = _get_first_bool(self.fetch_decimals, conn.extra_dejson.get("fetch_decimals")) - if isinstance(fetch_decimals, bool): - oracledb.defaults.fetch_decimals = fetch_decimals - - fetch_lobs = _get_first_bool(self.fetch_lobs, conn.extra_dejson.get("fetch_lobs")) - if isinstance(fetch_lobs, bool): - oracledb.defaults.fetch_lobs = fetch_lobs - - # Set up DSN - service_name = conn.extra_dejson.get("service_name") - # Fall back to conn.schema as service_name when not explicitly set in extras. - # The UI Schema field maps to conn.schema which is the Oracle service name. - if not service_name and not sid and schema: - service_name = schema - port = conn.port if conn.port else DEFAULT_DB_PORT - if conn.host and sid and not service_name: - conn_config["dsn"] = oracledb.makedsn(conn.host, port, sid) - elif conn.host and service_name and not sid: - conn_config["dsn"] = oracledb.makedsn(conn.host, port, service_name=service_name) - else: - dsn = conn.extra_dejson.get("dsn") - if dsn is None: - dsn = conn.host or "" - if conn.port is not None: - dsn += f":{conn.port}" - if service_name: - dsn += f"/{service_name}" - conn_config["dsn"] = dsn - - if "events" in conn.extra_dejson: - conn_config["events"] = conn.extra_dejson.get("events") - - # Map the connection extra "mode"/"purity" string (e.g. "sysdba") to the - # matching python-oracledb AuthMode/Purity enum member by name. Compare - # against None explicitly: Purity.DEFAULT is 0, so a truthiness check - # would silently drop it. - if mode_name := conn.extra_dejson.get("mode"): - auth_mode = getattr(oracledb.AuthMode, mode_name.upper(), None) - if auth_mode is not None: - conn_config["mode"] = auth_mode - - if purity_name := conn.extra_dejson.get("purity"): - purity = getattr(oracledb.Purity, purity_name.upper(), None) - if purity is not None: - conn_config["purity"] = purity - - expire_time = conn.extra_dejson.get("expire_time") - if expire_time: - conn_config["expire_time"] = expire_time +"""Deprecated. Use ``airflow.providers.oracle.oracledb.hooks.oracle`` instead.""" - for name in [ - "wallet_location", - "wallet_password", - "ssl_server_cert_dn", - "ssl_server_dn_match", - "cclass", - "pool_name", - ]: - value = conn.extra_dejson.get(name) - if value is not None: - conn_config[name] = value - - oracle_conn = oracledb.connect(**conn_config) - if mod is not None: - oracle_conn.module = mod - - # if Connection.schema is defined, set schema after connecting successfully - # cannot be part of conn_config - # https://python-oracledb.readthedocs.io/en/latest/api_manual/connection.html?highlight=schema#Connection.current_schema - # Only set schema when not using conn.schema as Service Name - if schema and service_name: - oracle_conn.current_schema = schema - - return oracle_conn - - def get_records( - self, - sql: str | list[str], - parameters: Iterable | Mapping[str, Any] | None = None, - ) -> Any: - """ - Execute the sql and return a set of records. - - :param sql: the sql statement to be executed (str) or a list of sql statements to execute - :param parameters: The parameters to render the SQL query with. - """ - return self.run(sql=sql, parameters=parameters, handler=handlers.fetch_all_handler) - - def get_first(self, sql: str | list[str], parameters: Iterable | Mapping[str, Any] | None = None) -> Any: - """ - Execute the sql and return the first resulting row. - - :param sql: the sql statement to be executed (str) or a list of sql statements to execute - :param parameters: The parameters to render the SQL query with. - """ - return self.run(sql=sql, parameters=parameters, handler=handlers.fetch_one_handler) - - def insert_rows( - self, - table: str, - rows: list[tuple], - target_fields=None, - commit_every: int = 1000, - replace: bool | None = False, - **kwargs, - ) -> None: - """ - Insert a collection of tuples into a table. - - All data to insert are treated as one transaction. Changes from standard - DbApiHook implementation: - - - Oracle SQL queries can not be terminated with a semicolon (``;``). - - Replace NaN values with NULL using ``numpy.nan_to_num`` (not using - ``is_nan()`` because of input types error for strings). - - Coerce datetime cells to Oracle DATETIME format during insert. - - :param table: target Oracle table, use dot notation to target a - specific database - :param rows: the rows to insert into the table - :param target_fields: the names of the columns to fill in the table - :param commit_every: the maximum number of rows to insert in one transaction - Default 1000, Set greater than 0. - Set 1 to insert each row in each single transaction - :param replace: Whether to replace instead of insert. Currently not implemented. - """ - if replace: - warnings.warn( - "Using 'replace=True' does not implement any replace functionality currently.", - category=UserWarning, - stacklevel=2, - ) - try: - import numpy as np - except ImportError: - np = None # type: ignore - - if target_fields: - target_fields = ", ".join(target_fields) - target_fields = f"({target_fields})" - else: - target_fields = "" - conn = self.get_conn() - if self.supports_autocommit: - self.set_autocommit(conn, False) - cur = conn.cursor() - i = 0 - sql = None # not generated unless we actually process at least one chunk - for row in rows: - i += 1 - lst = [] - for cell in row: - if isinstance(cell, str): - lst.append("'" + str(cell).replace("'", "''") + "'") - elif cell is None or isinstance(cell, float) and math.isnan(cell): # coerce numpy NaN to NULL - lst.append("NULL") - elif np and isinstance(cell, np.datetime64): - lst.append(f"'{cell}'") - elif isinstance(cell, datetime): - lst.append(f"to_date('{cell:%Y-%m-%d %H:%M:%S}','YYYY-MM-DD HH24:MI:SS')") - else: - lst.append(str(cell)) - values = tuple(lst) - sql = f"INSERT /*+ APPEND */ INTO {table} {target_fields} VALUES ({','.join(values)})" - cur.execute(sql) - if i % commit_every == 0: - conn.commit() - self.log.info("Loaded %s into %s rows so far", i, table) - conn.commit() - - if sql: - # We only send lineage once, not for each value collection, to save memory. - send_sql_hook_lineage(context=self, sql=sql, row_count=i) - - cur.close() - conn.close() - self.log.info("Done loading. Loaded a total of %s rows", i) - - def bulk_insert_rows( - self, - table: str, - rows: list[tuple], - target_fields: list[str] | None = None, - commit_every: int = 5000, - sequence_column: str | None = None, - sequence_name: str | None = None, - ): - """ - Perform bulk inserts efficiently for Oracle DB. - - This uses prepared statements via `executemany()`. For best performance, - pass in `rows` as an iterator. - - :param table: target Oracle table, use dot notation to target a - specific database - :param rows: the rows to insert into the table - :param target_fields: the names of the columns to fill in the table, default None. - If None, each rows should have some order as table columns name - :param commit_every: the maximum number of rows to insert in one transaction - Default 5000. Set greater than 0. Set 1 to insert each row in each transaction - :param sequence_column: the column name to which the sequence will be applied, default None. - :param sequence_name: the names of the sequence_name in the table, default None. - """ - if not rows: - raise ValueError("parameter rows could not be None or empty iterable") - conn = self.get_conn() - if self.supports_autocommit: - self.set_autocommit(conn, False) - cursor = conn.cursor() - values_base = target_fields or rows[0] - - if bool(sequence_column) ^ bool(sequence_name): - raise ValueError( - "Parameters 'sequence_column' and 'sequence_name' must be provided together or not at all." - ) - - if sequence_column and sequence_name: - columns = ( - f"({', '.join([sequence_column] + target_fields)})" - if target_fields - else f"({sequence_column})" - ) - value_placeholders = ", ".join( - [f"{sequence_name}.NEXTVAL"] + [f":{i}" for i in range(1, len(values_base) + 1)] - ) - else: - columns = f"({', '.join(target_fields)})" if target_fields else "" - value_placeholders = ", ".join(f":{i}" for i in range(1, len(values_base) + 1)) - prepared_stm = f"insert into {table} {columns} values ({value_placeholders})" - - row_count = 0 - # Chunk the rows - row_chunk = [] - for row in rows: - row_chunk.append(row) - row_count += 1 - if row_count % commit_every == 0: - cursor.prepare(prepared_stm) - cursor.executemany(None, row_chunk) - conn.commit() - self.log.info("[%s] inserted %s rows", table, row_count) - # Empty chunk - row_chunk = [] - # Commit the leftover chunk - if row_chunk: - cursor.prepare(prepared_stm) - cursor.executemany(None, row_chunk) - conn.commit() - self.log.info("[%s] inserted %s rows", table, row_count) - # We only send lineage once, not for each value collection, to save memory. - send_sql_hook_lineage(context=self, sql=prepared_stm, row_count=row_count) - cursor.close() - conn.close() - - def callproc( - self, - identifier: str, - autocommit: bool = False, - parameters: list | dict | None = None, - ) -> list | dict | tuple | None: - """ - Call the stored procedure identified by the provided string. - - Any OUT parameters must be provided with a value of either the - expected Python type (e.g., `int`) or an instance of that type. - - The return value is a list or mapping that includes parameters in - both directions; the actual return type depends on the type of the - provided `parameters` argument. - - See - https://python-oracledb.readthedocs.io/en/latest/api_manual/cursor.html#Cursor.var - for further reference. - """ - if parameters is None: - parameters = [] - - args = ",".join( - f":{name}" - for name in (parameters if isinstance(parameters, dict) else range(1, len(parameters) + 1)) - ) - - sql = f"BEGIN {identifier}({args}); END;" - - def handler(cursor): - if cursor.bindvars is None: - return - - if isinstance(cursor.bindvars, list): - return [v.getvalue() for v in cursor.bindvars] - - if isinstance(cursor.bindvars, dict): - return {n: v.getvalue() for (n, v) in cursor.bindvars.items()} - - raise TypeError(f"Unexpected bindvars: {cursor.bindvars!r}") - - result = self.run( - sql, - autocommit=autocommit, - parameters=( - {name: _map_param(value) for (name, value) in parameters.items()} - if isinstance(parameters, dict) - else [_map_param(value) for value in parameters] - ), - handler=handler, - ) - - return result - - def get_openlineage_database_info(self, connection: Connection) -> DatabaseInfo: - """Return Oracle specific information for OpenLineage.""" - from airflow.providers.openlineage.sqlparser import DatabaseInfo - - return DatabaseInfo( - scheme=self.get_openlineage_database_dialect(connection), - authority=DbApiHook.get_openlineage_authority_part(connection, default_port=DEFAULT_DB_PORT), - information_schema_table_name="ALL_TAB_COLUMNS", - information_schema_columns=[ - "owner", - "table_name", - "column_name", - "column_id", - "data_type", - ], - database=self.service_name or self.sid, - normalize_name_method=lambda name: name.upper(), - ) - - def get_openlineage_database_dialect(self, _) -> str: - """Return database dialect.""" - return "oracle" - - def get_openlineage_default_schema(self) -> str | None: - """Return current schema.""" - return self.get_first("SELECT SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') FROM dual")[0] - - def get_uri(self) -> str: - """Get the URI for the Oracle connection.""" - conn = self.get_connection(self.get_conn_id()) - login = conn.login - password = conn.password - host = conn.host - port = conn.port or DEFAULT_DB_PORT - service_name = conn.extra_dejson.get("service_name") - sid = conn.extra_dejson.get("sid") +from __future__ import annotations - if sid and service_name: - raise ValueError("At most one allowed for 'sid', and 'service name'.") +from airflow.utils.deprecation_tools import add_deprecated_classes - uri = f"oracle+oracledb://{login}:{password}@{host}:{port}" - if service_name: - uri = f"{uri}?service_name={service_name}" - elif sid: - uri = f"{uri}/{sid}" - elif conn.schema: - uri = f"{uri}/{conn.schema}" +__deprecated_classes = { + __name__: { + "OracleHook": "airflow.providers.oracle.oracledb.hooks.oracle.OracleHook", + }, +} - return uri +add_deprecated_classes( + __deprecated_classes, + __name__, + extra_message="`apache-airflow-providers-oracle` is deprecated, install `apache-airflow-providers-oracle-oracledb` instead", +) diff --git a/providers/oracle/src/airflow/providers/oracle/operators/oracle.py b/providers/oracle/src/airflow/providers/oracle/operators/oracle.py index a81b79bc77660..fb15bf3756793 100644 --- a/providers/oracle/src/airflow/providers/oracle/operators/oracle.py +++ b/providers/oracle/src/airflow/providers/oracle/operators/oracle.py @@ -15,63 +15,22 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - -import re -from collections.abc import Sequence -from typing import TYPE_CHECKING - -import oracledb - -from airflow.providers.common.compat.sdk import BaseOperator -from airflow.providers.oracle.hooks.oracle import OracleHook - -if TYPE_CHECKING: - from airflow.providers.common.compat.sdk import Context - +"""Deprecated. Use ``airflow.providers.oracle.oracledb.operators.oracle`` instead.""" -class OracleStoredProcedureOperator(BaseOperator): - """ - Executes stored procedure in a specific Oracle database. - - :param procedure: name of stored procedure to call (templated) - :param oracle_conn_id: The :ref:`Oracle connection id ` - reference to a specific Oracle database. - :param parameters: (optional, templated) the parameters provided in the call - - If *do_xcom_push* is *True*, the numeric exit code emitted by - the database is pushed to XCom under key ``ORA`` in case of failure. - """ - - template_fields: Sequence[str] = ( - "parameters", - "procedure", - ) - ui_color = "#ededed" - - def __init__( - self, - *, - procedure: str, - oracle_conn_id: str = "oracle_default", - parameters: dict | list | None = None, - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.oracle_conn_id = oracle_conn_id - self.procedure = procedure - self.parameters = parameters +from __future__ import annotations - def execute(self, context: Context): - self.log.info("Executing: %s", self.procedure) - hook = OracleHook(oracle_conn_id=self.oracle_conn_id) - try: - return hook.callproc(self.procedure, autocommit=True, parameters=self.parameters) - except oracledb.DatabaseError as e: - if not self.do_xcom_push or not context: - raise - ti = context["ti"] - code_match = re.search("^ORA-(\\d+):.+", str(e)) - if code_match: - ti.xcom_push(key="ORA", value=code_match.group(1)) - raise +from airflow.utils.deprecation_tools import add_deprecated_classes + +__deprecated_classes = { + __name__: { + "OracleStoredProcedureOperator": ( + "airflow.providers.oracle.oracledb.operators.oracle.OracleStoredProcedureOperator" + ), + }, +} + +add_deprecated_classes( + __deprecated_classes, + __name__, + extra_message="`apache-airflow-providers-oracle` is deprecated, install `apache-airflow-providers-oracle-oracledb` instead", +) diff --git a/providers/oracle/src/airflow/providers/oracle/transfers/oracle_to_oracle.py b/providers/oracle/src/airflow/providers/oracle/transfers/oracle_to_oracle.py index da4cfb66b9d4a..6589d024ad134 100644 --- a/providers/oracle/src/airflow/providers/oracle/transfers/oracle_to_oracle.py +++ b/providers/oracle/src/airflow/providers/oracle/transfers/oracle_to_oracle.py @@ -15,75 +15,22 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - -from collections.abc import Sequence -from typing import TYPE_CHECKING - -from airflow.providers.common.compat.sdk import BaseOperator -from airflow.providers.oracle.hooks.oracle import OracleHook - -if TYPE_CHECKING: - from airflow.providers.common.compat.sdk import Context - - -class OracleToOracleOperator(BaseOperator): - """ - Moves data from Oracle to Oracle. +"""Deprecated. Use ``airflow.providers.oracle.oracledb.transfers.oracle_to_oracle`` instead.""" - :param oracle_destination_conn_id: destination Oracle connection. - :param destination_table: destination table to insert rows. - :param oracle_source_conn_id: :ref:`Source Oracle connection `. - :param source_sql: SQL query to execute against the source Oracle - database. (templated) - :param source_sql_params: Parameters to use in sql query. (templated) - :param rows_chunk: number of rows per chunk to commit. - """ - - template_fields: Sequence[str] = ("source_sql", "source_sql_params") - template_fields_renderers = {"source_sql": "sql", "source_sql_params": "py"} - ui_color = "#e08c8c" - - def __init__( - self, - *, - oracle_destination_conn_id: str, - destination_table: str, - oracle_source_conn_id: str, - source_sql: str, - source_sql_params: dict | None = None, - rows_chunk: int = 5000, - **kwargs, - ) -> None: - super().__init__(**kwargs) - if source_sql_params is None: - source_sql_params = {} - self.oracle_destination_conn_id = oracle_destination_conn_id - self.destination_table = destination_table - self.oracle_source_conn_id = oracle_source_conn_id - self.source_sql = source_sql - self.source_sql_params = source_sql_params - self.rows_chunk = rows_chunk - - def _execute(self, src_hook, dest_hook, context) -> None: - with src_hook.get_conn() as src_conn: - cursor = src_conn.cursor() - self.log.info("Querying data from source: %s", self.oracle_source_conn_id) - cursor.execute(self.source_sql, self.source_sql_params) - target_fields = [field[0] for field in cursor.description] - - rows_total = 0 - for rows in iter(lambda: cursor.fetchmany(self.rows_chunk), []): - dest_hook.bulk_insert_rows( - self.destination_table, rows, target_fields=target_fields, commit_every=self.rows_chunk - ) - rows_total += len(rows) - self.log.info("Total inserted: %s rows", rows_total) - - self.log.info("Finished data transfer.") - cursor.close() +from __future__ import annotations - def execute(self, context: Context) -> None: - src_hook = OracleHook(oracle_conn_id=self.oracle_source_conn_id) - dest_hook = OracleHook(oracle_conn_id=self.oracle_destination_conn_id) - self._execute(src_hook, dest_hook, context) +from airflow.utils.deprecation_tools import add_deprecated_classes + +__deprecated_classes = { + __name__: { + "OracleToOracleOperator": ( + "airflow.providers.oracle.oracledb.transfers.oracle_to_oracle.OracleToOracleOperator" + ), + }, +} + +add_deprecated_classes( + __deprecated_classes, + __name__, + extra_message="`apache-airflow-providers-oracle` is deprecated, install `apache-airflow-providers-oracle-oracledb` instead", +) diff --git a/providers/oracle/tests/unit/oracle/assets/test_oracle.py b/providers/oracle/tests/unit/oracle/assets/test_oracle.py index 82d44fdaf1b34..b83efcefa872f 100644 --- a/providers/oracle/tests/unit/oracle/assets/test_oracle.py +++ b/providers/oracle/tests/unit/oracle/assets/test_oracle.py @@ -14,112 +14,44 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - from __future__ import annotations -import urllib.parse +import warnings -import pytest +from airflow.providers.oracle.oracledb.assets import oracle as oracledb_asset +from airflow.utils.deprecation_tools import DeprecatedImportWarning -from airflow.providers.common.compat.assets import Asset -from airflow.providers.oracle.assets.oracle import ( - convert_asset_to_openlineage, - create_asset, - sanitize_uri, -) +class TestDeprecatedAssetsImport: + """`airflow.providers.oracle.assets.oracle` is deprecated; it must keep re-exporting + the real functions from `airflow.providers.oracle.oracledb.assets.oracle` unchanged.""" -@pytest.mark.parametrize( - ("original", "normalized"), - [ - pytest.param( - "oracle://example.com:1234/orcl/HR/employees", - "oracle://example.com:1234/orcl/HR/employees", - id="normalized", - ), - pytest.param( - "oracle://example.com/orcl/HR/employees", - "oracle://example.com:1521/orcl/HR/employees", - id="default-port", - ), - ], -) -def test_sanitize_uri_pass(original: str, normalized: str) -> None: - uri_i = urllib.parse.urlsplit(original) - uri_o = sanitize_uri(uri_i) - assert urllib.parse.urlunsplit(uri_o) == normalized + def test_sanitize_uri_redirects_and_warns(self): + import airflow.providers.oracle.assets.oracle as deprecated_module + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + sanitize_uri = deprecated_module.sanitize_uri -@pytest.mark.parametrize( - "value", - [ - pytest.param("oracle://", id="blank"), - pytest.param("oracle:///orcl/HR/employees", id="no-host"), - pytest.param("oracle://example.com/orcl/employees", id="missing-component"), - pytest.param("oracle://example.com/orcl/HR/employees/column", id="extra-component"), - ], -) -def test_sanitize_uri_fail(value: str) -> None: - uri_i = urllib.parse.urlsplit(value) - with pytest.raises(ValueError, match="URI format oracle:// must contain"): - sanitize_uri(uri_i) + assert sanitize_uri is oracledb_asset.sanitize_uri + assert any(issubclass(w.category, DeprecatedImportWarning) for w in captured_warnings) + def test_create_asset_redirects_and_warns(self): + import airflow.providers.oracle.assets.oracle as deprecated_module -def test_sanitize_uri_fail_non_port() -> None: - uri_i = urllib.parse.urlsplit("oracle://example.com:abcd/orcl/HR/employees") - with pytest.raises(ValueError, match="Port could not be cast to integer value as 'abcd'"): - sanitize_uri(uri_i) + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + create_asset = deprecated_module.create_asset + assert create_asset is oracledb_asset.create_asset + assert any(issubclass(w.category, DeprecatedImportWarning) for w in captured_warnings) -@pytest.mark.parametrize( - ("host", "service_name", "schema", "table", "port", "expected_uri"), - [ - pytest.param( - "example.com", - "orcl", - "HR", - "employees", - 1521, - "oracle://example.com:1521/orcl/HR/employees", - id="default-port", - ), - pytest.param( - "example.com", - "orcl", - "HR", - "employees", - 1522, - "oracle://example.com:1522/orcl/HR/employees", - id="custom-port", - ), - ], -) -def test_create_asset( - host: str, service_name: str, schema: str, table: str, port: int, expected_uri: str -) -> None: - result = create_asset(host=host, service_name=service_name, schema=schema, table=table, port=port) - assert result == Asset(uri=expected_uri) + def test_convert_asset_to_openlineage_redirects_and_warns(self): + import airflow.providers.oracle.assets.oracle as deprecated_module + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + convert_asset_to_openlineage = deprecated_module.convert_asset_to_openlineage -@pytest.mark.parametrize( - ("uri", "expected_namespace", "expected_name"), - [ - pytest.param( - "oracle://example.com:1521/orcl/HR/employees", - "oracle://example.com:1521", - "orcl.HR.employees", - id="default-port", - ), - pytest.param( - "oracle://db-host:1522/prod/SCHEMA/users", - "oracle://db-host:1522", - "prod.SCHEMA.users", - id="custom-port", - ), - ], -) -def test_convert_asset_to_openlineage(uri: str, expected_namespace: str, expected_name: str) -> None: - asset = Asset(uri=uri) - ol_dataset = convert_asset_to_openlineage(asset=asset, lineage_context=None) - assert ol_dataset.namespace == expected_namespace - assert ol_dataset.name == expected_name + assert convert_asset_to_openlineage is oracledb_asset.convert_asset_to_openlineage + assert any(issubclass(w.category, DeprecatedImportWarning) for w in captured_warnings) diff --git a/providers/oracle/tests/unit/oracle/hooks/test_handlers.py b/providers/oracle/tests/unit/oracle/hooks/test_handlers.py index 398e206243297..94e8423b51c57 100644 --- a/providers/oracle/tests/unit/oracle/hooks/test_handlers.py +++ b/providers/oracle/tests/unit/oracle/hooks/test_handlers.py @@ -17,33 +17,32 @@ # under the License. from __future__ import annotations -from unittest.mock import MagicMock +import warnings -from airflow.providers.oracle.hooks.handlers import ( - fetch_all_handler, - fetch_one_handler, -) +from airflow.providers.oracle.oracledb.hooks import handlers as oracledb_handlers +from airflow.utils.deprecation_tools import DeprecatedImportWarning -from unit.oracle.test_utils import mock_oracle_lob +class TestDeprecatedHandlersImport: + """`airflow.providers.oracle.hooks.handlers` is deprecated; it must keep re-exporting + the real functions from `airflow.providers.oracle.oracledb.hooks.handlers` unchanged.""" -class TestHandlers: - def test_fetch_all_handler(self): - cursor = MagicMock() - cursor.description = [("col1", "int"), ("col2", "string")] - cursor.fetchall.return_value = [(1, mock_oracle_lob("hello"))] + def test_fetch_all_handler_redirects_and_warns(self): + import airflow.providers.oracle.hooks.handlers as deprecated_module - assert fetch_all_handler(cursor) == [(1, "hello")] + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + fetch_all_handler = deprecated_module.fetch_all_handler - cursor.description = None - assert fetch_all_handler(cursor) is None + assert fetch_all_handler is oracledb_handlers.fetch_all_handler + assert any(issubclass(w.category, DeprecatedImportWarning) for w in captured_warnings) - def test_fetch_one_handler(self): - cursor = MagicMock() - cursor.description = [("col1", "int")] - cursor.fetchone.return_value = (mock_oracle_lob("hello"),) + def test_fetch_one_handler_redirects_and_warns(self): + import airflow.providers.oracle.hooks.handlers as deprecated_module - assert fetch_one_handler(cursor) == ("hello",) + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + fetch_one_handler = deprecated_module.fetch_one_handler - cursor.description = None - assert fetch_one_handler(cursor) is None + assert fetch_one_handler is oracledb_handlers.fetch_one_handler + assert any(issubclass(w.category, DeprecatedImportWarning) for w in captured_warnings) diff --git a/providers/oracle/tests/unit/oracle/hooks/test_oracle.py b/providers/oracle/tests/unit/oracle/hooks/test_oracle.py index e007361a26785..329217575e962 100644 --- a/providers/oracle/tests/unit/oracle/hooks/test_oracle.py +++ b/providers/oracle/tests/unit/oracle/hooks/test_oracle.py @@ -17,694 +17,30 @@ # under the License. from __future__ import annotations -import json -from datetime import datetime -from unittest import mock +import warnings -import numpy as np -import oracledb -import pytest +from airflow.providers.oracle.oracledb.hooks.oracle import OracleHook as OracleDbOracleHook +from airflow.utils.deprecation_tools import DeprecatedImportWarning -from airflow.models import Connection -from airflow.providers.oracle.hooks.oracle import OracleHook -from unit.oracle.test_utils import mock_oracle_lob +class TestDeprecatedOracleHookImport: + """`airflow.providers.oracle.hooks.oracle` is deprecated; it must keep re-exporting + the real class from `airflow.providers.oracle.oracledb.hooks.oracle` unchanged.""" + def test_attribute_access_redirects_to_oracledb_and_warns(self): + import airflow.providers.oracle.hooks.oracle as deprecated_module -class TestOracleHookConn: - def setup_method(self): - self.connection = Connection( - login="login", password="password", host="host", port=1521, extra='{"service_name": "schema"}' - ) + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + oracle_hook_cls = deprecated_module.OracleHook - self.db_hook = OracleHook() - self.db_hook.get_connection = mock.Mock() - self.db_hook.get_connection.return_value = self.connection + assert oracle_hook_cls is OracleDbOracleHook + assert any(issubclass(w.category, DeprecatedImportWarning) for w in captured_warnings) - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_host(self, mock_connect): - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - args, kwargs = mock_connect.call_args - assert args == () - assert kwargs["user"] == "login" - assert kwargs["password"] == "password" - assert kwargs["dsn"] == oracledb.makedsn("host", 1521, service_name="schema") + def test_from_import_redirects_to_oracledb_and_warns(self): + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + from airflow.providers.oracle.hooks.oracle import OracleHook - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_host_alternative_port(self, mock_connect): - self.connection.port = 1522 - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - args, kwargs = mock_connect.call_args - assert args == () - assert kwargs["user"] == "login" - assert kwargs["password"] == "password" - assert kwargs["dsn"] == oracledb.makedsn("host", self.connection.port, service_name="schema") - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_sid(self, mock_connect): - dsn_sid = {"dsn": "ignored", "sid": "sid"} - self.connection.extra = json.dumps(dsn_sid) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - args, kwargs = mock_connect.call_args - assert args == () - assert kwargs["dsn"] == oracledb.makedsn("host", self.connection.port, dsn_sid["sid"]) - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_service_name(self, mock_connect): - dsn_service_name = {"dsn": "ignored", "service_name": "service_name"} - self.connection.extra = json.dumps(dsn_service_name) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - args, kwargs = mock_connect.call_args - assert args == () - assert kwargs["dsn"] == oracledb.makedsn( - "host", self.connection.port, service_name=dsn_service_name["service_name"] - ) - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_mode(self, mock_connect): - mode = { - "sysdba": oracledb.AUTH_MODE_SYSDBA, - "sysasm": oracledb.AUTH_MODE_SYSASM, - "sysoper": oracledb.AUTH_MODE_SYSOPER, - "sysbkp": oracledb.AUTH_MODE_SYSBKP, - "sysdgd": oracledb.AUTH_MODE_SYSDGD, - "syskmt": oracledb.AUTH_MODE_SYSKMT, - } - first = True - for mod in mode: - self.connection.extra = json.dumps({"mode": mod}) - self.db_hook.get_conn() - if first: - assert mock_connect.call_count == 1 - first = False - args, kwargs = mock_connect.call_args - assert args == () - assert kwargs["mode"] == mode.get(mod) - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_events(self, mock_connect): - self.connection.extra = json.dumps({"events": True}) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - args, kwargs = mock_connect.call_args - assert args == () - assert kwargs["events"] is True - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_purity(self, mock_connect): - purity = { - "new": oracledb.PURITY_NEW, - "self": oracledb.PURITY_SELF, - "default": oracledb.PURITY_DEFAULT, - } - first = True - for pur in purity: - self.connection.extra = json.dumps({"purity": pur}) - self.db_hook.get_conn() - if first: - assert mock_connect.call_count == 1 - first = False - args, kwargs = mock_connect.call_args - assert args == () - assert kwargs["purity"] == purity.get(pur) - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_expire_time(self, mock_connect): - self.connection.extra = json.dumps({"expire_time": 10}) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - args, kwargs = mock_connect.call_args - assert args == () - assert kwargs["expire_time"] == 10 - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_schema_as_service_name(self, mock_connect): - """When service_name and sid are not in extras, conn.schema should be used as service_name.""" - self.connection.schema = "MY_SERVICE" - self.connection.extra = json.dumps({}) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - args, kwargs = mock_connect.call_args - assert kwargs["dsn"] == oracledb.makedsn("host", 1521, service_name="MY_SERVICE") - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_schema_not_used_when_service_name_set(self, mock_connect): - """Explicit service_name in extras takes precedence over conn.schema.""" - self.connection.schema = "MY_SCHEMA" - self.connection.extra = json.dumps({"service_name": "EXPLICIT_SVC"}) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - args, kwargs = mock_connect.call_args - assert kwargs["dsn"] == oracledb.makedsn("host", 1521, service_name="EXPLICIT_SVC") - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_schema_not_used_when_sid_set(self, mock_connect): - """Explicit sid in extras takes precedence over conn.schema.""" - self.connection.schema = "MY_SCHEMA" - self.connection.extra = json.dumps({"sid": "MY_SID"}) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - args, kwargs = mock_connect.call_args - assert kwargs["dsn"] == oracledb.makedsn("host", 1521, "MY_SID") - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_set_current_schema(self, mock_connect): - self.connection.schema = "schema_name" - self.connection.extra = json.dumps({"service_name": "service_name"}) - assert self.db_hook.get_conn().current_schema == self.connection.schema - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.init_oracle_client") - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_set_thick_mode_extra(self, mock_connect, mock_init_client): - thick_mode_test = { - "thick_mode": True, - "thick_mode_lib_dir": "/opt/oracle/instantclient", - "thick_mode_config_dir": "/opt/oracle/config", - } - self.connection.extra = json.dumps(thick_mode_test) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - assert mock_init_client.call_count == 1 - args, kwargs = mock_init_client.call_args - assert args == () - assert kwargs["lib_dir"] == thick_mode_test["thick_mode_lib_dir"] - assert kwargs["config_dir"] == thick_mode_test["thick_mode_config_dir"] - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.init_oracle_client") - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_set_thick_mode_extra_str(self, mock_connect, mock_init_client): - thick_mode_test = {"thick_mode": "True"} - self.connection.extra = json.dumps(thick_mode_test) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - assert mock_init_client.call_count == 1 - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.init_oracle_client") - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_set_thick_mode_params(self, mock_connect, mock_init_client): - # Verify params overrides connection config extra - thick_mode_test = { - "thick_mode": False, - "thick_mode_lib_dir": "/opt/oracle/instantclient", - "thick_mode_config_dir": "/opt/oracle/config", - } - self.connection.extra = json.dumps(thick_mode_test) - db_hook = OracleHook(thick_mode=True, thick_mode_lib_dir="/test", thick_mode_config_dir="/test_conf") - db_hook.get_connection = mock.Mock() - db_hook.get_connection.return_value = self.connection - db_hook.get_conn() - assert mock_connect.call_count == 1 - assert mock_init_client.call_count == 1 - args, kwargs = mock_init_client.call_args - assert args == () - assert kwargs["lib_dir"] == "/test" - assert kwargs["config_dir"] == "/test_conf" - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.init_oracle_client") - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_thick_mode_defaults_to_false(self, mock_connect, mock_init_client): - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - assert mock_init_client.call_count == 0 - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.init_oracle_client") - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_thick_mode_dirs_defaults(self, mock_connect, mock_init_client): - thick_mode_test = {"thick_mode": True} - self.connection.extra = json.dumps(thick_mode_test) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - assert mock_init_client.call_count == 1 - args, kwargs = mock_init_client.call_args - assert args == () - assert kwargs["lib_dir"] is None - assert kwargs["config_dir"] is None - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_oracledb_defaults_attributes_default_values(self, mock_connect): - default_fetch_decimals = oracledb.defaults.fetch_decimals - default_fetch_lobs = oracledb.defaults.fetch_lobs - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - # Check that OracleHook.get_conn() doesn't try to set defaults if not provided - assert oracledb.defaults.fetch_decimals == default_fetch_decimals - assert oracledb.defaults.fetch_lobs == default_fetch_lobs - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_set_oracledb_defaults_attributes_extra(self, mock_connect): - defaults_test = {"fetch_decimals": True, "fetch_lobs": False} - self.connection.extra = json.dumps(defaults_test) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - assert oracledb.defaults.fetch_decimals == defaults_test["fetch_decimals"] - assert oracledb.defaults.fetch_lobs == defaults_test["fetch_lobs"] - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_set_oracledb_defaults_attributes_extra_str(self, mock_connect): - defaults_test = {"fetch_decimals": "True", "fetch_lobs": "False"} - self.connection.extra = json.dumps(defaults_test) - self.db_hook.get_conn() - assert mock_connect.call_count == 1 - assert oracledb.defaults.fetch_decimals is True - assert oracledb.defaults.fetch_lobs is False - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_set_oracledb_defaults_attributes_params(self, mock_connect): - # Verify params overrides connection config extra - defaults_test = {"fetch_decimals": False, "fetch_lobs": True} - self.connection.extra = json.dumps(defaults_test) - db_hook = OracleHook(fetch_decimals=True, fetch_lobs=False) - db_hook.get_connection = mock.Mock() - db_hook.get_connection.return_value = self.connection - db_hook.get_conn() - assert mock_connect.call_count == 1 - assert oracledb.defaults.fetch_decimals is True - assert oracledb.defaults.fetch_lobs is False - - def test_type_checking_thick_mode_lib_dir(self): - thick_mode_lib_dir_test = {"thick_mode": True, "thick_mode_lib_dir": 1} - self.connection.extra = json.dumps(thick_mode_lib_dir_test) - with pytest.raises(TypeError, match=r"thick_mode_lib_dir expected str or None, got.*"): - self.db_hook.get_conn() - - def test_type_checking_thick_mode_config_dir(self): - thick_mode_config_dir_test = {"thick_mode": True, "thick_mode_config_dir": 1} - self.connection.extra = json.dumps(thick_mode_config_dir_test) - with pytest.raises(TypeError, match=r"thick_mode_config_dir expected str or None, got.*"): - self.db_hook.get_conn() - - @pytest.mark.parametrize( - ("connection_params", "expected_uri"), - [ - pytest.param( - {"extra": '{"service_name": "service"}', "schema": None, "port": 1521}, - "oracle+oracledb://login:password@host:1521?service_name=service", - id="service_name_in_extra", - ), - pytest.param( - {"extra": '{"sid": "sid"}', "schema": None, "port": 1521}, - "oracle+oracledb://login:password@host:1521/sid", - id="sid_in_extra", - ), - pytest.param( - {"extra": "{}", "schema": "db_schema", "port": 1521}, - "oracle+oracledb://login:password@host:1521/db_schema", - id="schema_only", - ), - pytest.param( - {"extra": "{}", "schema": None, "port": 1521}, - "oracle+oracledb://login:password@host:1521", - id="no_schema_no_extra", - ), - pytest.param( - {"extra": "{}", "schema": "db_schema", "port": None}, - "oracle+oracledb://login:password@host:1521/db_schema", - id="schema_only_default_port", - ), - pytest.param( - {"extra": '{"service_name": "service"}', "schema": "db_schema", "port": 1521}, - "oracle+oracledb://login:password@host:1521?service_name=service", - id="service_name_with_schema", - ), - ], - ) - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_uri(self, mock_connect, connection_params, expected_uri): - self.connection.extra = connection_params["extra"] - self.connection.schema = connection_params["schema"] - self.connection.port = connection_params["port"] - - uri = self.db_hook.get_uri() - assert uri == expected_uri - - @mock.patch("airflow.providers.oracle.hooks.oracle.oracledb.connect") - def test_get_conn_with_various_params(self, mock_connect): - """Verify wallet/SSL, connection class, and pool parameters - are passed to oracledb.connect.""" - params = { - "wallet_location": "/tmp/wallet", - "wallet_password": "secret", - "ssl_server_cert_dn": "CN=dbserver,OU=DB,O=Oracle,L=BLR,C=IN", - "ssl_server_dn_match": True, - "cclass": "MY_APP_CLASS", - "pool_name": "POOL_1", - } - self.connection.extra = json.dumps(params) - self.db_hook.get_conn() - - assert mock_connect.call_count == 1 - _, kwargs = mock_connect.call_args - - for key, value in params.items(): - assert kwargs[key] == value - - -class TestOracleHook: - def setup_method(self): - self.cur = mock.MagicMock(rowcount=0) - self.conn = mock.MagicMock() - self.conn.cursor.return_value = self.cur - conn = self.conn - - class UnitTestOracleHook(OracleHook): - conn_name_attr = "test_conn_id" - - def get_conn(self): - return conn - - self.db_hook = UnitTestOracleHook() - - def test_run_without_parameters(self): - sql = "SQL" - self.db_hook.run(sql) - self.cur.execute.assert_called_once_with(sql) - assert self.conn.commit.called - - def test_run_with_parameters(self): - sql = "SQL" - param = ("p1", "p2") - self.db_hook.run(sql, parameters=param) - self.cur.execute.assert_called_once_with(sql, param) - assert self.conn.commit.called - - @mock.patch("airflow.providers.common.sql.hooks.sql.send_sql_hook_lineage") - def test_run_hook_lineage(self, mock_send_lineage): - statement = "SELECT 1" - self.cur.fetchall.return_value = [] - - self.db_hook.run(statement) - - mock_send_lineage.assert_called() - call_kw = mock_send_lineage.call_args.kwargs - assert call_kw["context"] is self.db_hook - assert call_kw["sql"] == statement - assert call_kw["sql_parameters"] is None - assert call_kw["cur"] is self.cur - - @mock.patch("airflow.providers.common.sql.hooks.sql.send_sql_hook_lineage") - @mock.patch("airflow.providers.common.sql.hooks.sql.DbApiHook._get_pandas_df") - def test_get_df_hook_lineage(self, mock_get_pandas_df, mock_send_lineage): - sql = "SELECT 1" - parameters = ("x",) - self.db_hook.get_df(sql, parameters=parameters) - - mock_send_lineage.assert_called_once() - call_kw = mock_send_lineage.call_args.kwargs - assert call_kw["context"] is self.db_hook - assert call_kw["sql"] == sql - assert call_kw["sql_parameters"] == parameters - - @mock.patch("airflow.providers.common.sql.hooks.sql.send_sql_hook_lineage") - @mock.patch("airflow.providers.common.sql.hooks.sql.DbApiHook._get_pandas_df_by_chunks") - def test_get_df_by_chunks_hook_lineage(self, mock_get_pandas_df_by_chunks, mock_send_lineage): - sql = "SELECT 1" - parameters = ("x",) - self.db_hook.get_df_by_chunks(sql, parameters=parameters, chunksize=1) - - mock_send_lineage.assert_called_once() - call_kw = mock_send_lineage.call_args.kwargs - assert call_kw["context"] is self.db_hook - assert call_kw["sql"] == sql - assert call_kw["sql_parameters"] == parameters - - def test_insert_rows_with_fields(self): - rows = [ - ( - "'basestr_with_quote", - None, - np.nan, - np.datetime64("2019-01-24T01:02:03"), - datetime(2019, 1, 24), - 1, - 10.24, - "str", - ) - ] - target_fields = [ - "basestring", - "none", - "numpy_nan", - "numpy_datetime64", - "datetime", - "int", - "float", - "str", - ] - self.db_hook.insert_rows("table", rows, target_fields) - self.cur.execute.assert_called_once_with( - "INSERT /*+ APPEND */ INTO table " - "(basestring, none, numpy_nan, numpy_datetime64, datetime, int, float, str) " - "VALUES ('''basestr_with_quote',NULL,NULL,'2019-01-24T01:02:03'," - "to_date('2019-01-24 00:00:00','YYYY-MM-DD HH24:MI:SS'),1,10.24,'str')" - ) - - def test_insert_rows_without_fields(self): - rows = [ - ( - "'basestr_with_quote", - None, - np.nan, - np.datetime64("2019-01-24T01:02:03"), - datetime(2019, 1, 24), - 1, - 10.24, - "str", - ) - ] - self.db_hook.insert_rows("table", rows) - self.cur.execute.assert_called_once_with( - "INSERT /*+ APPEND */ INTO table " - " VALUES ('''basestr_with_quote',NULL,NULL,'2019-01-24T01:02:03'," - "to_date('2019-01-24 00:00:00','YYYY-MM-DD HH24:MI:SS'),1,10.24,'str')" - ) - - @mock.patch("airflow.providers.oracle.hooks.oracle.send_sql_hook_lineage") - def test_insert_rows_hook_lineage(self, mock_send_lineage): - rows = [("a", "b", "c")] - target_fields = ["col1", "col2", "col3"] - self.db_hook.insert_rows("table", rows, target_fields) - - mock_send_lineage.assert_called() - call_kw = mock_send_lineage.call_args.kwargs - assert call_kw["context"] is self.db_hook - assert call_kw["sql"] == "INSERT /*+ APPEND */ INTO table (col1, col2, col3) VALUES ('a','b','c')" - assert call_kw["row_count"] == 1 - - def test_bulk_insert_rows_with_fields(self): - rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] - target_fields = ["col1", "col2", "col3"] - self.db_hook.bulk_insert_rows("table", rows, target_fields) - self.cur.prepare.assert_called_once_with("insert into table (col1, col2, col3) values (:1, :2, :3)") - self.cur.executemany.assert_called_once_with(None, rows) - - def test_bulk_insert_rows_with_commit_every(self): - rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] - target_fields = ["col1", "col2", "col3"] - self.db_hook.bulk_insert_rows("table", rows, target_fields, commit_every=2) - calls = [ - mock.call("insert into table (col1, col2, col3) values (:1, :2, :3)"), - mock.call("insert into table (col1, col2, col3) values (:1, :2, :3)"), - ] - self.cur.prepare.assert_has_calls(calls) - calls = [ - mock.call(None, rows[:2]), - mock.call(None, rows[2:]), - ] - self.cur.executemany.assert_has_calls(calls, any_order=True) - - def test_bulk_insert_rows_without_fields(self): - rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] - self.db_hook.bulk_insert_rows("table", rows) - self.cur.prepare.assert_called_once_with("insert into table values (:1, :2, :3)") - self.cur.executemany.assert_called_once_with(None, rows) - - @mock.patch("airflow.providers.oracle.hooks.oracle.send_sql_hook_lineage") - def test_bulk_insert_rows_hook_lineage(self, mock_send_lineage): - rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] - target_fields = ["col1", "col2", "col3"] - self.db_hook.bulk_insert_rows("table", rows, target_fields) - - mock_send_lineage.assert_called_once() - call_kw = mock_send_lineage.call_args.kwargs - assert call_kw["context"] is self.db_hook - assert call_kw["sql"] == "insert into table (col1, col2, col3) values (:1, :2, :3)" - assert call_kw["row_count"] == 3 - - def test_bulk_insert_rows_no_rows(self): - rows = [] - with pytest.raises(ValueError, match="parameter rows could not be None or empty iterable"): - self.db_hook.bulk_insert_rows("table", rows) - - def test_bulk_insert_sequence_field(self): - rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] - target_fields = ["col1", "col2", "col3"] - sequence_column = "id" - sequence_name = "my_sequence" - self.db_hook.bulk_insert_rows( - "table", rows, target_fields, sequence_column=sequence_column, sequence_name=sequence_name - ) - self.cur.prepare.assert_called_once_with( - "insert into table (id, col1, col2, col3) values (my_sequence.NEXTVAL, :1, :2, :3)" - ) - self.cur.executemany.assert_called_once_with(None, rows) - - def test_bulk_insert_sequence_without_parameter(self): - SEQUENCE_COLUMN_OR_NAME_PROVIDED = ( - "Parameters 'sequence_column' and 'sequence_name' must be provided together or not at all." - ) - rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9)] - target_fields = ["col1", "col2", "col3"] - sequence_column = "id" - sequence_name = None - with pytest.raises(ValueError, match=SEQUENCE_COLUMN_OR_NAME_PROVIDED): - self.db_hook.bulk_insert_rows( - "table", rows, target_fields, sequence_column=sequence_column, sequence_name=sequence_name - ) - - sequence_column = None - sequence_name = "my_sequence" - with pytest.raises(ValueError, match=SEQUENCE_COLUMN_OR_NAME_PROVIDED): - self.db_hook.bulk_insert_rows( - "table", rows, target_fields, sequence_column=sequence_column, sequence_name=sequence_name - ) - - def test_bulk_insert_commit_leftovers_only_if_exists(self): - rows = [(1, 2, 3), (4, 5, 6), (7, 8, 9), (1, 2, 3), (4, 5, 6), (7, 8, 9)] - target_fields = ["col1", "col2", "col3"] - sequence_column = "id" - sequence_name = "my_sequence" - - self.db_hook.bulk_insert_rows( - "table", - rows, - target_fields, - sequence_column=sequence_column, - sequence_name=sequence_name, - commit_every=3, - ) - - # executemany should be called exactly 2 times because there is no leftovers - assert self.cur.executemany.call_count == 2 - - def test_callproc_none(self): - parameters = None - - class bindvar(int): - def getvalue(self): - return self - - self.cur.bindvars = None - result = self.db_hook.callproc("proc", True, parameters) - assert self.cur.execute.mock_calls == [mock.call("BEGIN proc(); END;")] - assert result == parameters - - def test_callproc_dict(self): - parameters = {"a": 1, "b": 2, "c": 3} - - class bindvar(int): - def getvalue(self): - return self - - self.cur.bindvars = {k: bindvar(v) for k, v in parameters.items()} - result = self.db_hook.callproc("proc", True, parameters) - assert self.cur.execute.mock_calls == [mock.call("BEGIN proc(:a,:b,:c); END;", parameters)] - assert result == parameters - - def test_callproc_list(self): - parameters = [1, 2, 3] - - class bindvar(int): - def getvalue(self): - return self - - self.cur.bindvars = list(map(bindvar, parameters)) - result = self.db_hook.callproc("proc", True, parameters) - assert self.cur.execute.mock_calls == [mock.call("BEGIN proc(:1,:2,:3); END;", parameters)] - assert result == parameters - - def test_callproc_out_param(self): - parameters = [1, int, float, bool, str] - - def bindvar(value): - m = mock.Mock() - m.getvalue.return_value = value - return m - - self.cur.bindvars = [bindvar(p() if type(p) is type else p) for p in parameters] - result = self.db_hook.callproc("proc", True, parameters) - expected = [1, 0, 0.0, False, ""] - assert self.cur.execute.mock_calls == [mock.call("BEGIN proc(:1,:2,:3,:4,:5); END;", expected)] - assert result == expected - - def test_test_connection_use_dual_table(self): - self.cur.fetchone.return_value = (1,) - status, message = self.db_hook.test_connection() - self.cur.execute.assert_called_once_with("select 1 from dual") - assert status is True - assert message == "Connection successfully tested" - - def test_get_openlineage_database_info_with_service_name(self): - conn = Connection( - conn_id="oracle_default", - conn_type="oracle", - host="localhost", - port=1521, - extra='{"service_name": "ORCLPDB1"}', - ) - hook = OracleHook(oracle_conn_id="oracle_default") - hook.get_connection = lambda _: conn - - assert hook.service_name == "ORCLPDB1" - db_info = hook.get_openlineage_database_info(conn) - assert db_info.scheme == "oracle" - assert db_info.authority == "localhost:1521" - assert db_info.database == "ORCLPDB1" - assert db_info.normalize_name_method("employees") == "EMPLOYEES" - assert db_info.information_schema_table_name == "ALL_TAB_COLUMNS" - assert "owner" in db_info.information_schema_columns - - def test_get_openlineage_database_info_with_sid(self): - conn = Connection( - conn_id="oracle_default", - conn_type="oracle", - host="dbhost", - port=1521, - extra='{"sid": "XE"}', - ) - hook = OracleHook(oracle_conn_id="oracle_default") - hook.get_connection = lambda _: conn - - assert hook.sid == "XE" - db_info = hook.get_openlineage_database_info(conn) - assert db_info.scheme == "oracle" - assert db_info.authority == "dbhost:1521" - assert db_info.database == "XE" - assert db_info.normalize_name_method("employees") == "EMPLOYEES" - assert db_info.information_schema_table_name == "ALL_TAB_COLUMNS" - assert "owner" in db_info.information_schema_columns - - def test_get_first(self): - statement = "SQL" - - self.cur.fetchone.return_value = (mock_oracle_lob("hello"),) - - assert self.db_hook.get_first(statement) == ("hello",) - - assert self.conn.close.call_count == 1 - assert self.cur.close.call_count == 1 - self.cur.execute.assert_called_once_with(statement) - - def test_get_records(self): - statement = "SQL" - - self.cur.fetchall.return_value = (mock_oracle_lob("hello"),), (mock_oracle_lob("world"),) - - assert self.db_hook.get_records(statement) == [("hello",), ("world",)] - - assert self.conn.close.call_count == 1 - assert self.cur.close.call_count == 1 - self.cur.execute.assert_called_once_with(statement) + assert OracleHook is OracleDbOracleHook + assert any(issubclass(w.category, DeprecatedImportWarning) for w in captured_warnings) diff --git a/providers/oracle/tests/unit/oracle/operators/test_oracle.py b/providers/oracle/tests/unit/oracle/operators/test_oracle.py index 6fde023ff6639..b84786020fb27 100644 --- a/providers/oracle/tests/unit/oracle/operators/test_oracle.py +++ b/providers/oracle/tests/unit/oracle/operators/test_oracle.py @@ -16,70 +16,32 @@ # under the License. from __future__ import annotations -import random -import re -from unittest import mock +import warnings -import oracledb -import pytest +from airflow.providers.oracle.oracledb.operators.oracle import ( + OracleStoredProcedureOperator as OracleDbOracleStoredProcedureOperator, +) +from airflow.utils.deprecation_tools import DeprecatedImportWarning -from airflow.models import TaskInstance -from airflow.providers.oracle.hooks.oracle import OracleHook -from airflow.providers.oracle.operators.oracle import OracleStoredProcedureOperator -from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS +class TestDeprecatedOracleStoredProcedureOperatorImport: + """`airflow.providers.oracle.operators.oracle` is deprecated; it must keep re-exporting + the real class from `airflow.providers.oracle.oracledb.operators.oracle` unchanged.""" + def test_attribute_access_redirects_to_oracledb_and_warns(self): + import airflow.providers.oracle.operators.oracle as deprecated_module -class TestOracleStoredProcedureOperator: - @mock.patch.object(OracleHook, "run", autospec=OracleHook.run) - def test_execute(self, mock_run): - procedure = "test" - oracle_conn_id = "oracle_default" - parameters = {"parameter": "value"} - context = "test_context" - task_id = "test_task_id" + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + operator_cls = deprecated_module.OracleStoredProcedureOperator - operator = OracleStoredProcedureOperator( - procedure=procedure, - oracle_conn_id=oracle_conn_id, - parameters=parameters, - task_id=task_id, - ) - result = operator.execute(context=context) - assert result is mock_run.return_value - mock_run.assert_called_once_with( - mock.ANY, - "BEGIN test(:parameter); END;", - autocommit=True, - parameters=parameters, - handler=mock.ANY, - ) + assert operator_cls is OracleDbOracleStoredProcedureOperator + assert any(issubclass(w.category, DeprecatedImportWarning) for w in captured_warnings) - @mock.patch.object(OracleHook, "callproc", autospec=OracleHook.callproc) - def test_push_oracle_exit_to_xcom(self, mock_callproc, request, dag_maker): - # Test pulls the value previously pushed to xcom and checks if it's the same - procedure = "test_push" - oracle_conn_id = "oracle_default" - parameters = {"parameter": "value"} - task_id = "test_push" - ora_exit_code = f"{random.randrange(10**5):05}" - error = f"ORA-{ora_exit_code}: This is a five-digit ORA error code" - mock_callproc.side_effect = oracledb.DatabaseError(error) + def test_from_import_redirects_to_oracledb_and_warns(self): + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + from airflow.providers.oracle.operators.oracle import OracleStoredProcedureOperator - if AIRFLOW_V_3_0_PLUS: - run_task = request.getfixturevalue("run_task") - task = OracleStoredProcedureOperator( - procedure=procedure, oracle_conn_id=oracle_conn_id, parameters=parameters, task_id=task_id - ) - run_task(task=task) - assert run_task.xcom.get(task_id=task.task_id, key="ORA") == ora_exit_code - else: - with dag_maker(dag_id=f"dag_{request.node.name}"): - task = OracleStoredProcedureOperator( - procedure=procedure, oracle_conn_id=oracle_conn_id, parameters=parameters, task_id=task_id - ) - dr = dag_maker.create_dagrun(run_id=task_id) - ti = TaskInstance(task=task, run_id=dr.run_id) - with pytest.raises(oracledb.DatabaseError, match=re.escape(error)): - ti.run() - assert ti.xcom_pull(task_ids=task.task_id, key="ORA") == ora_exit_code + assert OracleStoredProcedureOperator is OracleDbOracleStoredProcedureOperator + assert any(issubclass(w.category, DeprecatedImportWarning) for w in captured_warnings) diff --git a/providers/oracle/tests/unit/oracle/transfers/test_oracle_to_oracle.py b/providers/oracle/tests/unit/oracle/transfers/test_oracle_to_oracle.py index e2e66706da852..14e3123adcfc1 100644 --- a/providers/oracle/tests/unit/oracle/transfers/test_oracle_to_oracle.py +++ b/providers/oracle/tests/unit/oracle/transfers/test_oracle_to_oracle.py @@ -17,54 +17,33 @@ # under the License. from __future__ import annotations -from unittest import mock -from unittest.mock import MagicMock +import warnings -from airflow.providers.oracle.transfers.oracle_to_oracle import OracleToOracleOperator +from airflow.providers.oracle.oracledb.transfers.oracle_to_oracle import ( + OracleToOracleOperator as OracleDbOracleToOracleOperator, +) +from airflow.utils.deprecation_tools import DeprecatedImportWarning -class TestOracleToOracleTransfer: - def test_execute(self): - oracle_destination_conn_id = "oracle_destination_conn_id" - destination_table = "destination_table" - oracle_source_conn_id = "oracle_source_conn_id" - source_sql = "select sysdate from dual where trunc(sysdate) = :p_data" - source_sql_params = {":p_data": "2018-01-01"} - rows_chunk = 5000 - cursor_description = [ - ("id", "", 39, None, 38, 0, 0), - ("description", "", 60, 240, None, None, 1), - ] - cursor_rows = [[1, "description 1"], [2, "description 2"]] +class TestDeprecatedOracleToOracleOperatorImport: + """`airflow.providers.oracle.transfers.oracle_to_oracle` is deprecated; it must keep + re-exporting the real class from `airflow.providers.oracle.oracledb.transfers.oracle_to_oracle` + unchanged.""" - mock_dest_hook = MagicMock() - mock_src_hook = MagicMock() - mock_src_conn = mock_src_hook.get_conn.return_value.__enter__.return_value - mock_cursor = mock_src_conn.cursor.return_value - mock_cursor.description.__iter__.return_value = cursor_description - mock_cursor.fetchmany.side_effect = [cursor_rows, []] + def test_attribute_access_redirects_to_oracledb_and_warns(self): + import airflow.providers.oracle.transfers.oracle_to_oracle as deprecated_module - op = OracleToOracleOperator( - task_id="copy_data", - oracle_destination_conn_id=oracle_destination_conn_id, - destination_table=destination_table, - oracle_source_conn_id=oracle_source_conn_id, - source_sql=source_sql, - source_sql_params=source_sql_params, - rows_chunk=rows_chunk, - ) + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + operator_cls = deprecated_module.OracleToOracleOperator - op._execute(mock_src_hook, mock_dest_hook, None) + assert operator_cls is OracleDbOracleToOracleOperator + assert any(issubclass(w.category, DeprecatedImportWarning) for w in captured_warnings) - assert mock_src_hook.get_conn.called - assert mock_src_conn.cursor.called - mock_cursor.execute.assert_called_once_with(source_sql, source_sql_params) + def test_from_import_redirects_to_oracledb_and_warns(self): + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + from airflow.providers.oracle.transfers.oracle_to_oracle import OracleToOracleOperator - calls = [ - mock.call(rows_chunk), - mock.call(rows_chunk), - ] - mock_cursor.fetchmany.assert_has_calls(calls) - mock_dest_hook.bulk_insert_rows.assert_called_once_with( - destination_table, cursor_rows, commit_every=rows_chunk, target_fields=["id", "description"] - ) + assert OracleToOracleOperator is OracleDbOracleToOracleOperator + assert any(issubclass(w.category, DeprecatedImportWarning) for w in captured_warnings) diff --git a/pyproject.toml b/pyproject.toml index b09bdf933d02b..4593ae92da24b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1829,6 +1829,7 @@ apache-airflow-providers-openlineage = { workspace = true } apache-airflow-providers-opensearch = { workspace = true } apache-airflow-providers-opsgenie = { workspace = true } apache-airflow-providers-oracle = { workspace = true } +apache-airflow-providers-oracle-oracledb = { workspace = true } apache-airflow-providers-pagerduty = { workspace = true } apache-airflow-providers-papermill = { workspace = true } apache-airflow-providers-pgvector = { workspace = true } @@ -1970,6 +1971,7 @@ members = [ "providers/opensearch", "providers/opsgenie", "providers/oracle", + "providers/oracle/oracledb", "providers/pagerduty", "providers/papermill", "providers/pgvector",