From 8c12be37014df5b0b4354ba8d1eb1560340c6824 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 04:01:54 +0000 Subject: [PATCH 1/8] build: prepare PyAirbyte slim package boundaries --- airbyte/__init__.py | 43 +++++++++---- airbyte/_executors/python.py | 57 +++++++++++++----- airbyte/_executors/util.py | 1 + airbyte/_message_iterators.py | 10 +--- airbyte/cloud/sync_results.py | 32 +++++++--- airbyte/cloud/workspaces.py | 21 ++++--- airbyte/constants.py | 8 +++ airbyte/logs.py | 5 +- airbyte/progress.py | 22 ++++--- pyproject.toml | 45 ++++++++++++++ uv.lock | 110 ++++++++++++++++++++++++++++++++++ 11 files changed, 289 insertions(+), 65 deletions(-) diff --git a/airbyte/__init__.py b/airbyte/__init__.py index 976c5a5f6..207fa4917 100644 --- a/airbyte/__init__.py +++ b/airbyte/__init__.py @@ -123,21 +123,44 @@ from __future__ import annotations -from typing import TYPE_CHECKING +# ruff: noqa: F822 # Lazy exports are resolved by __getattr__ at runtime. +from importlib import import_module +from typing import TYPE_CHECKING, Protocol from airbyte import registry -from airbyte.caches.bigquery import BigQueryCache -from airbyte.caches.duckdb import DuckDBCache -from airbyte.caches.util import get_colab_cache, get_default_cache, new_local_cache -from airbyte.datasets import CachedDataset -from airbyte.destinations.base import Destination -from airbyte.destinations.util import get_destination from airbyte.records import StreamRecord from airbyte.registry import get_available_connectors -from airbyte.results import ReadResult, WriteResult from airbyte.secrets import SecretSourceEnum, get_secret -from airbyte.sources.base import Source -from airbyte.sources.util import get_source + + +class _LazyExport(Protocol): + def __call__(self, *args: object, **kwargs: object) -> object: ... + + +_LAZY_EXPORTS = { + "BigQueryCache": "airbyte.caches.bigquery", + "CachedDataset": "airbyte.datasets", + "Destination": "airbyte.destinations.base", + "DuckDBCache": "airbyte.caches.duckdb", + "ReadResult": "airbyte.results", + "Source": "airbyte.sources.base", + "WriteResult": "airbyte.results", + "get_colab_cache": "airbyte.caches.util", + "get_default_cache": "airbyte.caches.util", + "get_destination": "airbyte.destinations.util", + "get_source": "airbyte.sources.util", + "new_local_cache": "airbyte.caches.util", +} + + +def __getattr__(name: str) -> _LazyExport: + if name not in _LAZY_EXPORTS: + raise AttributeError(f"module 'airbyte' has no attribute {name!r}") + + module = import_module(_LAZY_EXPORTS[name]) + value = getattr(module, name) + globals()[name] = value + return value # Submodules imported here for documentation reasons: https://github.com/mitmproxy/pdoc/issues/757 diff --git a/airbyte/_executors/python.py b/airbyte/_executors/python.py index 57a5e0094..beb2a886f 100644 --- a/airbyte/_executors/python.py +++ b/airbyte/_executors/python.py @@ -17,7 +17,7 @@ from airbyte._util.meta import is_windows from airbyte._util.telemetry import EventState, log_install_state from airbyte._util.venv_util import get_bin_dir -from airbyte.constants import DEFAULT_INSTALL_DIR, NO_UV +from airbyte.constants import DEFAULT_CONNECTOR_PYTHON_VERSIONS, DEFAULT_INSTALL_DIR, NO_UV if TYPE_CHECKING: @@ -45,7 +45,7 @@ def __init__( install_root: (Optional.) The root directory where the virtual environment will be created. If not provided, the current working directory will be used. use_python: (Optional.) Python interpreter specification: - - True: Use current Python interpreter + - True: Use uv-managed default connector Python versions - False: Use Docker instead (handled by factory) - Path: Use interpreter at this path or interpreter name/command - str: Use uv-managed Python version (semver patterns like "3.12", "3.11.5") @@ -98,6 +98,41 @@ def _run_subprocess_and_raise_on_failure(self, args: list[str]) -> None: log_text=result.stderr.decode("utf-8"), ) + def _create_venv(self, python_override: str | None) -> None: + uv_cmd_prefix = ["uv"] if not NO_UV else [] + python_clause: list[str] = ["--python", python_override] if python_override else [] + venv_cmd: list[str] = [ + *(uv_cmd_prefix or [sys.executable, "-m"]), + "venv", + str(self._get_venv_path()), + *python_clause, + ] + print( + f"Creating '{self.name}' virtual environment with command '{' '.join(venv_cmd)}'", + file=sys.stderr, + ) + self._run_subprocess_and_raise_on_failure(venv_cmd) + + def _create_venv_with_fallbacks(self, python_versions: tuple[str, ...]) -> None: + failures: list[exc.AirbyteSubprocessFailedError] = [] + for python_version in python_versions: + try: + self._create_venv(python_version) + except exc.AirbyteSubprocessFailedError as ex: + failures.append(ex) + else: + self.use_python = python_version + return + + raise exc.AirbyteConnectorInstallationError( + message="Connector virtual environment could not be created.", + context={ + "connector_name": self.name, + "python_versions": python_versions, + }, + log_text=[str(failure.log_text or "") for failure in failures], + ) + def uninstall(self) -> None: if self._get_venv_path().exists(): rmtree(str(self._get_venv_path())) @@ -134,20 +169,10 @@ def install(self) -> None: elif not NO_UV and isinstance(self.use_python, str): python_override = self.use_python - uv_cmd_prefix = ["uv"] if not NO_UV else [] - python_clause: list[str] = ["--python", python_override] if python_override else [] - - venv_cmd: list[str] = [ - *(uv_cmd_prefix or [sys.executable, "-m"]), - "venv", - str(self._get_venv_path()), - *python_clause, - ] - print( - f"Creating '{self.name}' virtual environment with command '{' '.join(venv_cmd)}'", - file=sys.stderr, - ) - self._run_subprocess_and_raise_on_failure(venv_cmd) + if not NO_UV and self.use_python is True: + self._create_venv_with_fallbacks(DEFAULT_CONNECTOR_PYTHON_VERSIONS) + else: + self._create_venv(python_override) install_cmd = ( [ diff --git a/airbyte/_executors/util.py b/airbyte/_executors/util.py index 7a30a1bf1..c0b744be0 100644 --- a/airbyte/_executors/util.py +++ b/airbyte/_executors/util.py @@ -270,6 +270,7 @@ def get_connector_executor( # noqa: PLR0912, PLR0913, PLR0914, PLR0915, C901 # case InstallType.PYTHON: pip_url = metadata.pypi_package_name pip_url = f"{pip_url}=={version}" if version else pip_url + use_python = True case _: docker_image = True diff --git a/airbyte/_message_iterators.py b/airbyte/_message_iterators.py index e6efd5a11..ef5a09ebe 100644 --- a/airbyte/_message_iterators.py +++ b/airbyte/_message_iterators.py @@ -4,13 +4,12 @@ from __future__ import annotations import sys -from collections.abc import Iterator +from datetime import UTC, datetime from typing import IO, TYPE_CHECKING, cast import pydantic from typing_extensions import final -from airbyte_cdk.utils.datetime_helpers import ab_datetime_now from airbyte_protocol.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -26,7 +25,6 @@ if TYPE_CHECKING: - import datetime from collections.abc import Callable, Generator, Iterable, Iterator from pathlib import Path @@ -39,7 +37,7 @@ def _new_stream_success_message(stream_name: str) -> AirbyteMessage: type=Type.TRACE, trace=AirbyteTraceMessage( type=TraceType.STREAM_STATUS, - emitted_at=ab_datetime_now().timestamp(), + emitted_at=datetime.now(tz=UTC).timestamp(), stream_status=AirbyteStreamStatusTraceMessage( stream_descriptor=StreamDescriptor( name=stream_name, @@ -104,9 +102,7 @@ def generator() -> Generator[AirbyteMessage, None, None]: stream=stream_name, data=record, emitted_at=int( - cast( - "datetime.datetime", record.get(AB_EXTRACTED_AT_COLUMN) - ).timestamp() + cast("datetime", record.get(AB_EXTRACTED_AT_COLUMN)).timestamp() ), # `meta` and `namespace` are not handled: meta=None, diff --git a/airbyte/cloud/sync_results.py b/airbyte/cloud/sync_results.py index 90ef9e807..d611596b8 100644 --- a/airbyte/cloud/sync_results.py +++ b/airbyte/cloud/sync_results.py @@ -103,31 +103,41 @@ import time from collections.abc import Iterator, Mapping from dataclasses import asdict, dataclass +from datetime import UTC, datetime from typing import TYPE_CHECKING, Any from typing_extensions import final -from airbyte_cdk.utils.datetime_helpers import ab_datetime_parse - from airbyte._util import api_util -from airbyte.caches._utils._dest_to_cache import destination_to_cache from airbyte.cloud.constants import FAILED_STATUSES, FINAL_STATUSES -from airbyte.datasets import CachedDataset from airbyte.exceptions import AirbyteConnectionSyncError, AirbyteConnectionSyncTimeoutError +def _parse_datetime(value: str | int) -> datetime: + if isinstance(value, int) or ( + isinstance(value, str) + and (value.isdigit() or (value.startswith("-") and value[1:].isdigit())) + ): + return datetime.fromtimestamp(int(value), tz=UTC) + + if not isinstance(value, str): + raise TypeError(f"Could not parse datetime string: {value}") + + normalized = value.replace("Z", "+00:00") + return datetime.fromisoformat(normalized) + + DEFAULT_SYNC_TIMEOUT_SECONDS = 30 * 60 # 30 minutes """The default timeout for waiting for a sync job to complete, in seconds.""" if TYPE_CHECKING: - from datetime import datetime - import sqlalchemy from airbyte._util.api_imports import ConnectionResponse, JobResponse, JobStatusEnum from airbyte.caches.base import CacheBase from airbyte.cloud.connections import CloudConnection from airbyte.cloud.workspaces import CloudWorkspace + from airbyte.datasets import CachedDataset # noqa: TC004 @dataclass @@ -168,7 +178,7 @@ def records_synced(self) -> int: def created_at(self) -> datetime: """Return the creation time of the attempt.""" timestamp = self._get_attempt_data()["createdAt"] - return ab_datetime_parse(timestamp) + return _parse_datetime(timestamp) def _get_attempt_data(self) -> dict[str, Any]: """Get attempt data from the provided attempt data.""" @@ -307,7 +317,7 @@ def records_synced(self) -> int: def start_time(self) -> datetime: """Return the start time of the sync job in UTC.""" try: - return ab_datetime_parse(self._fetch_latest_job_info().start_time) + return _parse_datetime(self._fetch_latest_job_info().start_time) except (ValueError, TypeError) as e: if "Invalid isoformat string" in str(e): job_info_raw = api_util._make_config_api_request( # noqa: SLF001 @@ -321,7 +331,7 @@ def start_time(self) -> datetime: ) raw_start_time = job_info_raw.get("startTime") if raw_start_time: - return ab_datetime_parse(raw_start_time) + return _parse_datetime(raw_start_time) raise def _fetch_job_with_attempts(self) -> dict[str, Any]: @@ -421,6 +431,8 @@ def get_sql_cache(self) -> CacheBase: if self._cache: return self._cache + from airbyte.caches._utils._dest_to_cache import destination_to_cache # noqa: PLC0415 + destination_configuration = self._get_destination_configuration() self._cache = destination_to_cache(destination_configuration=destination_configuration) return self._cache @@ -449,6 +461,8 @@ def get_dataset(self, stream_name: str) -> CachedDataset: (catalog information) to the `CachedDataset` object via the "Get stream properties" API: https://reference.airbyte.com/reference/getstreamproperties """ + from airbyte.datasets import CachedDataset # noqa: PLC0415 + return CachedDataset( self.get_sql_cache(), stream_name=stream_name, diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index 05e41e20f..4ce5f6bf6 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -38,7 +38,7 @@ from dataclasses import dataclass, field from functools import cached_property from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal, overload +from typing import TYPE_CHECKING, Any, Literal, Protocol, overload import yaml @@ -54,10 +54,16 @@ CustomCloudSourceDefinition, ) from airbyte.cloud.organizations import CloudOrganization -from airbyte.destinations.base import Destination from airbyte.exceptions import AirbyteError +class _DestinationLike(Protocol): + name: str + + @property + def _hydrated_config(self) -> dict[str, Any]: ... + + if TYPE_CHECKING: from collections.abc import Callable @@ -415,7 +421,7 @@ def deploy_source( def deploy_destination( self, name: str, - destination: Destination | dict[str, Any], + destination: _DestinationLike | dict[str, Any], *, unique: bool = True, random_name_suffix: bool = False, @@ -432,12 +438,11 @@ def deploy_destination( are not allowed. Defaults to `True`. random_name_suffix: Whether to append a random suffix to the name. """ - if isinstance(destination, Destination): - destination_conf_dict = destination._hydrated_config.copy() # noqa: SLF001 (non-public API) - destination_conf_dict["destinationType"] = destination.name.replace("destination-", "") - # raise ValueError(destination_conf_dict) - else: + if isinstance(destination, dict): destination_conf_dict = destination.copy() + else: + destination_conf_dict = destination._hydrated_config.copy() # noqa: SLF001 + destination_conf_dict["destinationType"] = destination.name.replace("destination-", "") if "destinationType" not in destination_conf_dict: raise exc.PyAirbyteInputError( message="Missing `destinationType` in configuration dictionary.", diff --git a/airbyte/constants.py b/airbyte/constants.py index a2716f219..c419a7500 100644 --- a/airbyte/constants.py +++ b/airbyte/constants.py @@ -133,6 +133,14 @@ def _str_to_bool(value: str) -> bool: directories for permissions reasons. """ + +DEFAULT_CONNECTOR_PYTHON_VERSIONS: tuple[str, ...] = tuple( + version.strip() + for version in os.getenv("AIRBYTE_CONNECTOR_PYTHON_VERSIONS", "3.12,3.11").split(",") + if version.strip() +) +"""Default uv-managed Python versions to try for Python connector execution.""" + TEMP_FILE_CLEANUP = _str_to_bool( os.getenv( key="AIRBYTE_TEMP_FILE_CLEANUP", diff --git a/airbyte/logs.py b/airbyte/logs.py index b65195515..3d4e09529 100644 --- a/airbyte/logs.py +++ b/airbyte/logs.py @@ -17,14 +17,13 @@ import sys import tempfile import warnings +from datetime import UTC, datetime from functools import lru_cache from pathlib import Path import structlog import ulid -from airbyte_cdk.utils.datetime_helpers import ab_datetime_now - def _str_to_bool(value: str) -> bool: """Convert a string value of an environment values to a boolean value.""" @@ -145,7 +144,7 @@ def get_global_file_logger() -> logging.Logger | None: for handler in logger.handlers: logger.removeHandler(handler) - yyyy_mm_dd: str = ab_datetime_now().strftime("%Y-%m-%d") + yyyy_mm_dd: str = datetime.now(tz=UTC).strftime("%Y-%m-%d") folder = AIRBYTE_LOGGING_ROOT / yyyy_mm_dd try: folder.mkdir(parents=True, exist_ok=True) diff --git a/airbyte/progress.py b/airbyte/progress.py index ad984f8a3..a4ffb6b28 100644 --- a/airbyte/progress.py +++ b/airbyte/progress.py @@ -16,7 +16,6 @@ from __future__ import annotations -import datetime import importlib import json import math @@ -25,6 +24,7 @@ import time from collections import defaultdict from contextlib import suppress +from datetime import UTC, datetime from enum import Enum, auto from typing import IO, TYPE_CHECKING, Any, Literal, cast @@ -33,7 +33,6 @@ from rich.live import Live as RichLive from rich.markdown import Markdown as RichMarkdown -from airbyte_cdk.utils.datetime_helpers import ab_datetime_now from airbyte_protocol.models import ( AirbyteMessage, AirbyteStreamStatus, @@ -115,7 +114,7 @@ def _to_time_str(timestamp: float) -> str: For now, we'll just use UTC to avoid breaking tests. In the future, we should return a local time string. """ - datetime_obj = datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc) + datetime_obj = datetime.fromtimestamp(timestamp, tz=UTC) datetime_obj = datetime_obj.astimezone() return datetime_obj.strftime("%H:%M:%S") @@ -423,7 +422,7 @@ def _log_sync_start(self) -> None: """Log the start of a sync operation.""" self._print_info_message( f"Started `{self.job_description}` sync at " - f"`{ab_datetime_now().strftime('%H:%M:%S')}`..." + f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`..." ) # We access a non-public API here (noqa: SLF001) to get the runtime info for participants. self._send_telemetry( @@ -432,9 +431,8 @@ def _log_sync_start(self) -> None: ) def _log_sync_cancel(self) -> None: - print( - f"Canceled `{self.job_description}` sync at `{ab_datetime_now().strftime('%H:%M:%S')}`." - ) + time_now = datetime.now(tz=UTC).strftime("%H:%M:%S") + print(f"Canceled `{self.job_description}` sync at `{time_now}`.") self._send_telemetry( state=EventState.CANCELED, event_type=EventType.SYNC, @@ -443,7 +441,7 @@ def _log_sync_cancel(self) -> None: def _log_stream_read_start(self, stream_name: str) -> None: print( f"Read started on stream `{stream_name}` at " - f"`{ab_datetime_now().strftime('%H:%M:%S')}`..." + f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`..." ) self.stream_read_start_times[stream_name] = time.time() @@ -452,14 +450,14 @@ def log_stream_start(self, stream_name: str) -> None: if stream_name not in self.stream_read_start_times: self._print_info_message( f"Read started on stream `{stream_name}` at " - f"`{ab_datetime_now().strftime('%H:%M:%S')}`..." + f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`..." ) self.stream_read_start_times[stream_name] = time.time() def _log_stream_read_end(self, stream_name: str) -> None: self._print_info_message( f"Read completed on stream `{stream_name}` at " - f"`{ab_datetime_now().strftime('%H:%M:%S')}`..." + f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`..." ) self.stream_read_end_times[stream_name] = time.time() @@ -620,7 +618,7 @@ def log_success( print( f"Completed `{self.job_description}` sync at " - f"`{ab_datetime_now().strftime('%H:%M:%S')}`{streams_str}." + f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`{streams_str}." ) self._log_read_metrics() self._send_telemetry( @@ -638,7 +636,7 @@ def log_failure( self._stop_rich_view() self._print_info_message( f"Failed `{self.job_description}` sync at " - f"`{ab_datetime_now().strftime('%H:%M:%S')}`." + f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`." ) self._send_telemetry( state=EventState.FAILED, diff --git a/pyproject.toml b/pyproject.toml index 4f557514f..4d7407f9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,52 @@ dependencies = [ ] [dependency-groups] +slim = [ + "airbyte-api>=0.53.0,<1.0", + "airbyte-protocol-models-pdv2>=0.13.0,<1.0", + "cryptography>=44.0.0,<45.0.0", + "google-auth>=2.27.0,<3.0", + "google-cloud-secret-manager>=2.17.0,<3.0", + "jsonschema>=3.2.0,<5.0", + "orjson>=3.10,<4.0", + "overrides>=7.4.0,<8.0", + "pydantic>=2.0,<3.0", + "pydantic-core", + "python-dotenv>=1.0.1,<2.0", + "python-ulid>=3.0.0,<4.0", + "pyyaml>=6.0.2,<7.0", + "requests!=3.32.0", + "rich>=13.7.0,<14.0", + "structlog>=24.4.0,<25.0", + "typing-extensions", + "uuid7>=0.1.0,<1.0", + "uv>=0.5.0,<0.9.0", +] +full = [ + "airbyte-cdk>=7.3.9,<8.0.0", + "cyclopts>=4.0,<5.0", + # TODO: Restore broader version constraints after verifying compatibility + # "duckdb>=1.4.0,<2.0", + # "duckdb-engine>=0.17.0,<1.0", + "duckdb==1.4.3", # Pinned to match uv.lock + "duckdb-engine==0.17.0", # Pinned to match uv.lock + "fastmcp>=3.0,<4.0", + "fastmcp-extensions>=0.3.0,<1.0.0", + "google-cloud-bigquery>=3.12.0,<4.0", + "google-cloud-bigquery-storage>=2.25.0,<3.0", + "pandas>=1.5.3,<3.0", + "psycopg[binary,pool]>=3.1.19,<4.0", + "psycopg2-binary>=2.9.9,<3.0", + "pyarrow>=16.1,<22.0", + "snowflake-connector-python>=3.12.2,<4.0", + "snowflake-sqlalchemy>=1.6.1,<2.0", + # TODO: Restore broader version constraints after verifying compatibility + # "sqlalchemy>=1.4.51,!=2.0.36,<3.0", + "sqlalchemy==2.0.43", # Pinned to match uv.lock + "sqlalchemy-bigquery==1.12.0", +] dev = [ + { include-group = "full" }, "coverage>=7.5.1,<8.0", "deptry>=0.21.1,<1.0", "docker>=7.1.0,<8.0", diff --git a/uv.lock b/uv.lock index 060c4ea6e..cbec76f82 100644 --- a/uv.lock +++ b/uv.lock @@ -165,14 +165,26 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "airbyte-cdk" }, { name = "coverage" }, + { name = "cyclopts" }, { name = "deptry" }, { name = "docker" }, + { name = "duckdb" }, + { name = "duckdb-engine" }, { name = "faker" }, + { name = "fastmcp" }, + { name = "fastmcp-extensions" }, { name = "freezegun" }, + { name = "google-cloud-bigquery" }, + { name = "google-cloud-bigquery-storage" }, + { name = "pandas" }, { name = "pandas-stubs" }, { name = "pdoc" }, { name = "poethepoet" }, + { name = "psycopg", extra = ["binary", "pool"] }, + { name = "psycopg2-binary" }, + { name = "pyarrow" }, { name = "pydantic-ai" }, { name = "pyrefly" }, { name = "pytest" }, @@ -181,6 +193,10 @@ dev = [ { name = "pytest-timeout" }, { name = "responses" }, { name = "ruff" }, + { name = "snowflake-connector-python" }, + { name = "snowflake-sqlalchemy" }, + { name = "sqlalchemy" }, + { name = "sqlalchemy-bigquery" }, { name = "sqlalchemy2-stubs" }, { name = "tomli" }, { name = "types-jsonschema" }, @@ -188,6 +204,45 @@ dev = [ { name = "types-requests" }, { name = "viztracer" }, ] +full = [ + { name = "airbyte-cdk" }, + { name = "cyclopts" }, + { name = "duckdb" }, + { name = "duckdb-engine" }, + { name = "fastmcp" }, + { name = "fastmcp-extensions" }, + { name = "google-cloud-bigquery" }, + { name = "google-cloud-bigquery-storage" }, + { name = "pandas" }, + { name = "psycopg", extra = ["binary", "pool"] }, + { name = "psycopg2-binary" }, + { name = "pyarrow" }, + { name = "snowflake-connector-python" }, + { name = "snowflake-sqlalchemy" }, + { name = "sqlalchemy" }, + { name = "sqlalchemy-bigquery" }, +] +slim = [ + { name = "airbyte-api" }, + { name = "airbyte-protocol-models-pdv2" }, + { name = "cryptography" }, + { name = "google-auth" }, + { name = "google-cloud-secret-manager" }, + { name = "jsonschema" }, + { name = "orjson" }, + { name = "overrides" }, + { name = "pydantic" }, + { name = "pydantic-core" }, + { name = "python-dotenv" }, + { name = "python-ulid" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "rich" }, + { name = "structlog" }, + { name = "typing-extensions" }, + { name = "uuid7" }, + { name = "uv" }, +] [package.metadata] requires-dist = [ @@ -230,14 +285,26 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "airbyte-cdk", specifier = ">=7.3.9,<8.0.0" }, { name = "coverage", specifier = ">=7.5.1,<8.0" }, + { name = "cyclopts", specifier = ">=4.0,<5.0" }, { name = "deptry", specifier = ">=0.21.1,<1.0" }, { name = "docker", specifier = ">=7.1.0,<8.0" }, + { name = "duckdb", specifier = "==1.4.3" }, + { name = "duckdb-engine", specifier = "==0.17.0" }, { name = "faker", specifier = ">=21.0.0,<22.0" }, + { name = "fastmcp", specifier = ">=3.0,<4.0" }, + { name = "fastmcp-extensions", specifier = ">=0.3.0,<1.0.0" }, { name = "freezegun", specifier = ">=1.4.0,<2.0" }, + { name = "google-cloud-bigquery", specifier = ">=3.12.0,<4.0" }, + { name = "google-cloud-bigquery-storage", specifier = ">=2.25.0,<3.0" }, + { name = "pandas", specifier = ">=1.5.3,<3.0" }, { name = "pandas-stubs", specifier = ">=2.1.4.231218" }, { name = "pdoc", specifier = ">=16.0.0,<17.0" }, { name = "poethepoet", specifier = ">=0.26.1,<0.32.0" }, + { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.1.19,<4.0" }, + { name = "psycopg2-binary", specifier = ">=2.9.9,<3.0" }, + { name = "pyarrow", specifier = ">=16.1,<22.0" }, { name = "pydantic-ai", specifier = ">=0.2.16" }, { name = "pyrefly", specifier = ">=0.38.0,<0.39.0" }, { name = "pytest", specifier = ">=8.2.0,<9.0" }, @@ -246,6 +313,10 @@ dev = [ { name = "pytest-timeout", specifier = ">=2.3.1,<3.0" }, { name = "responses", specifier = ">=0.25.0,<1.0" }, { name = "ruff", specifier = ">=0.8.2,<0.9.0" }, + { name = "snowflake-connector-python", specifier = ">=3.12.2,<4.0" }, + { name = "snowflake-sqlalchemy", specifier = ">=1.6.1,<2.0" }, + { name = "sqlalchemy", specifier = "==2.0.43" }, + { name = "sqlalchemy-bigquery", specifier = "==1.12.0" }, { name = "sqlalchemy2-stubs", specifier = ">=0.0.2a38" }, { name = "tomli", specifier = ">=2.0,<3.0" }, { name = "types-jsonschema", specifier = ">=4.20.0.0" }, @@ -253,6 +324,45 @@ dev = [ { name = "types-requests", specifier = "==2.31.0.4" }, { name = "viztracer", specifier = ">=0.16.3,<1.1.0" }, ] +full = [ + { name = "airbyte-cdk", specifier = ">=7.3.9,<8.0.0" }, + { name = "cyclopts", specifier = ">=4.0,<5.0" }, + { name = "duckdb", specifier = "==1.4.3" }, + { name = "duckdb-engine", specifier = "==0.17.0" }, + { name = "fastmcp", specifier = ">=3.0,<4.0" }, + { name = "fastmcp-extensions", specifier = ">=0.3.0,<1.0.0" }, + { name = "google-cloud-bigquery", specifier = ">=3.12.0,<4.0" }, + { name = "google-cloud-bigquery-storage", specifier = ">=2.25.0,<3.0" }, + { name = "pandas", specifier = ">=1.5.3,<3.0" }, + { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.1.19,<4.0" }, + { name = "psycopg2-binary", specifier = ">=2.9.9,<3.0" }, + { name = "pyarrow", specifier = ">=16.1,<22.0" }, + { name = "snowflake-connector-python", specifier = ">=3.12.2,<4.0" }, + { name = "snowflake-sqlalchemy", specifier = ">=1.6.1,<2.0" }, + { name = "sqlalchemy", specifier = "==2.0.43" }, + { name = "sqlalchemy-bigquery", specifier = "==1.12.0" }, +] +slim = [ + { name = "airbyte-api", specifier = ">=0.53.0,<1.0" }, + { name = "airbyte-protocol-models-pdv2", specifier = ">=0.13.0,<1.0" }, + { name = "cryptography", specifier = ">=44.0.0,<45.0.0" }, + { name = "google-auth", specifier = ">=2.27.0,<3.0" }, + { name = "google-cloud-secret-manager", specifier = ">=2.17.0,<3.0" }, + { name = "jsonschema", specifier = ">=3.2.0,<5.0" }, + { name = "orjson", specifier = ">=3.10,<4.0" }, + { name = "overrides", specifier = ">=7.4.0,<8.0" }, + { name = "pydantic", specifier = ">=2.0,<3.0" }, + { name = "pydantic-core" }, + { name = "python-dotenv", specifier = ">=1.0.1,<2.0" }, + { name = "python-ulid", specifier = ">=3.0.0,<4.0" }, + { name = "pyyaml", specifier = ">=6.0.2,<7.0" }, + { name = "requests", specifier = "!=3.32.0" }, + { name = "rich", specifier = ">=13.7.0,<14.0" }, + { name = "structlog", specifier = ">=24.4.0,<25.0" }, + { name = "typing-extensions" }, + { name = "uuid7", specifier = ">=0.1.0,<1.0" }, + { name = "uv", specifier = ">=0.5.0,<0.9.0" }, +] [[package]] name = "airbyte-api" From 957093a48dedfa7894c913c195599b18e003bc49 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 04:06:55 +0000 Subject: [PATCH 2/8] fix: restore Python 3.10 compatibility --- airbyte/__init__.py | 17 +++++++++++------ airbyte/_message_iterators.py | 4 ++-- airbyte/cloud/sync_results.py | 8 ++++---- airbyte/cloud/workspaces.py | 6 ++++-- airbyte/logs.py | 4 ++-- airbyte/progress.py | 18 +++++++++--------- 6 files changed, 32 insertions(+), 25 deletions(-) diff --git a/airbyte/__init__.py b/airbyte/__init__.py index 207fa4917..0e44f4527 100644 --- a/airbyte/__init__.py +++ b/airbyte/__init__.py @@ -125,7 +125,7 @@ # ruff: noqa: F822 # Lazy exports are resolved by __getattr__ at runtime. from importlib import import_module -from typing import TYPE_CHECKING, Protocol +from typing import TYPE_CHECKING from airbyte import registry from airbyte.records import StreamRecord @@ -133,10 +133,6 @@ from airbyte.secrets import SecretSourceEnum, get_secret -class _LazyExport(Protocol): - def __call__(self, *args: object, **kwargs: object) -> object: ... - - _LAZY_EXPORTS = { "BigQueryCache": "airbyte.caches.bigquery", "CachedDataset": "airbyte.datasets", @@ -153,7 +149,7 @@ def __call__(self, *args: object, **kwargs: object) -> object: ... } -def __getattr__(name: str) -> _LazyExport: +def __getattr__(name: str) -> object: if name not in _LAZY_EXPORTS: raise AttributeError(f"module 'airbyte' has no attribute {name!r}") @@ -184,6 +180,15 @@ def __getattr__(name: str) -> _LazyExport: secrets, sources, ) + from airbyte.caches.bigquery import BigQueryCache + from airbyte.caches.duckdb import DuckDBCache + from airbyte.caches.util import get_colab_cache, get_default_cache, new_local_cache + from airbyte.datasets import CachedDataset + from airbyte.destinations.base import Destination + from airbyte.destinations.util import get_destination + from airbyte.results import ReadResult, WriteResult + from airbyte.sources.base import Source + from airbyte.sources.util import get_source __all__ = [ diff --git a/airbyte/_message_iterators.py b/airbyte/_message_iterators.py index ef5a09ebe..155475b4c 100644 --- a/airbyte/_message_iterators.py +++ b/airbyte/_message_iterators.py @@ -4,7 +4,7 @@ from __future__ import annotations import sys -from datetime import UTC, datetime +from datetime import datetime, timezone from typing import IO, TYPE_CHECKING, cast import pydantic @@ -37,7 +37,7 @@ def _new_stream_success_message(stream_name: str) -> AirbyteMessage: type=Type.TRACE, trace=AirbyteTraceMessage( type=TraceType.STREAM_STATUS, - emitted_at=datetime.now(tz=UTC).timestamp(), + emitted_at=datetime.now(tz=timezone.utc).timestamp(), stream_status=AirbyteStreamStatusTraceMessage( stream_descriptor=StreamDescriptor( name=stream_name, diff --git a/airbyte/cloud/sync_results.py b/airbyte/cloud/sync_results.py index d611596b8..3732cbeca 100644 --- a/airbyte/cloud/sync_results.py +++ b/airbyte/cloud/sync_results.py @@ -103,7 +103,7 @@ import time from collections.abc import Iterator, Mapping from dataclasses import asdict, dataclass -from datetime import UTC, datetime +from datetime import datetime, timezone from typing import TYPE_CHECKING, Any from typing_extensions import final @@ -118,7 +118,7 @@ def _parse_datetime(value: str | int) -> datetime: isinstance(value, str) and (value.isdigit() or (value.startswith("-") and value[1:].isdigit())) ): - return datetime.fromtimestamp(int(value), tz=UTC) + return datetime.fromtimestamp(int(value), tz=timezone.utc) if not isinstance(value, str): raise TypeError(f"Could not parse datetime string: {value}") @@ -137,7 +137,7 @@ def _parse_datetime(value: str | int) -> datetime: from airbyte.caches.base import CacheBase from airbyte.cloud.connections import CloudConnection from airbyte.cloud.workspaces import CloudWorkspace - from airbyte.datasets import CachedDataset # noqa: TC004 + from airbyte.datasets import CachedDataset @dataclass @@ -496,7 +496,7 @@ def streams( """ return self._SyncResultStreams(self) - class _SyncResultStreams(Mapping[str, CachedDataset]): + class _SyncResultStreams(Mapping[str, "CachedDataset"]): """A mapping of stream names to cached datasets.""" def __init__( diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index 4ce5f6bf6..ef07365db 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -61,13 +61,15 @@ class _DestinationLike(Protocol): name: str @property - def _hydrated_config(self) -> dict[str, Any]: ... + def _hydrated_config(self) -> dict[str, Any]: + return {} if TYPE_CHECKING: from collections.abc import Callable from airbyte._util.api_imports import WorkspaceResponse + from airbyte.destinations.base import Destination from airbyte.secrets.base import SecretString from airbyte.sources.base import Source @@ -421,7 +423,7 @@ def deploy_source( def deploy_destination( self, name: str, - destination: _DestinationLike | dict[str, Any], + destination: Destination | _DestinationLike | dict[str, Any], *, unique: bool = True, random_name_suffix: bool = False, diff --git a/airbyte/logs.py b/airbyte/logs.py index 3d4e09529..24fccdb8f 100644 --- a/airbyte/logs.py +++ b/airbyte/logs.py @@ -17,7 +17,7 @@ import sys import tempfile import warnings -from datetime import UTC, datetime +from datetime import datetime, timezone from functools import lru_cache from pathlib import Path @@ -144,7 +144,7 @@ def get_global_file_logger() -> logging.Logger | None: for handler in logger.handlers: logger.removeHandler(handler) - yyyy_mm_dd: str = datetime.now(tz=UTC).strftime("%Y-%m-%d") + yyyy_mm_dd: str = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d") folder = AIRBYTE_LOGGING_ROOT / yyyy_mm_dd try: folder.mkdir(parents=True, exist_ok=True) diff --git a/airbyte/progress.py b/airbyte/progress.py index a4ffb6b28..8e2e87b48 100644 --- a/airbyte/progress.py +++ b/airbyte/progress.py @@ -24,7 +24,7 @@ import time from collections import defaultdict from contextlib import suppress -from datetime import UTC, datetime +from datetime import datetime, timezone from enum import Enum, auto from typing import IO, TYPE_CHECKING, Any, Literal, cast @@ -114,7 +114,7 @@ def _to_time_str(timestamp: float) -> str: For now, we'll just use UTC to avoid breaking tests. In the future, we should return a local time string. """ - datetime_obj = datetime.fromtimestamp(timestamp, tz=UTC) + datetime_obj = datetime.fromtimestamp(timestamp, tz=timezone.utc) datetime_obj = datetime_obj.astimezone() return datetime_obj.strftime("%H:%M:%S") @@ -422,7 +422,7 @@ def _log_sync_start(self) -> None: """Log the start of a sync operation.""" self._print_info_message( f"Started `{self.job_description}` sync at " - f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`..." + f"`{datetime.now(tz=timezone.utc).strftime('%H:%M:%S')}`..." ) # We access a non-public API here (noqa: SLF001) to get the runtime info for participants. self._send_telemetry( @@ -431,7 +431,7 @@ def _log_sync_start(self) -> None: ) def _log_sync_cancel(self) -> None: - time_now = datetime.now(tz=UTC).strftime("%H:%M:%S") + time_now = datetime.now(tz=timezone.utc).strftime("%H:%M:%S") print(f"Canceled `{self.job_description}` sync at `{time_now}`.") self._send_telemetry( state=EventState.CANCELED, @@ -441,7 +441,7 @@ def _log_sync_cancel(self) -> None: def _log_stream_read_start(self, stream_name: str) -> None: print( f"Read started on stream `{stream_name}` at " - f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`..." + f"`{datetime.now(tz=timezone.utc).strftime('%H:%M:%S')}`..." ) self.stream_read_start_times[stream_name] = time.time() @@ -450,14 +450,14 @@ def log_stream_start(self, stream_name: str) -> None: if stream_name not in self.stream_read_start_times: self._print_info_message( f"Read started on stream `{stream_name}` at " - f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`..." + f"`{datetime.now(tz=timezone.utc).strftime('%H:%M:%S')}`..." ) self.stream_read_start_times[stream_name] = time.time() def _log_stream_read_end(self, stream_name: str) -> None: self._print_info_message( f"Read completed on stream `{stream_name}` at " - f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`..." + f"`{datetime.now(tz=timezone.utc).strftime('%H:%M:%S')}`..." ) self.stream_read_end_times[stream_name] = time.time() @@ -618,7 +618,7 @@ def log_success( print( f"Completed `{self.job_description}` sync at " - f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`{streams_str}." + f"`{datetime.now(tz=timezone.utc).strftime('%H:%M:%S')}`{streams_str}." ) self._log_read_metrics() self._send_telemetry( @@ -636,7 +636,7 @@ def log_failure( self._stop_rich_view() self._print_info_message( f"Failed `{self.job_description}` sync at " - f"`{datetime.now(tz=UTC).strftime('%H:%M:%S')}`." + f"`{datetime.now(tz=timezone.utc).strftime('%H:%M:%S')}`." ) self._send_telemetry( state=EventState.FAILED, From 40816090f203d378123b4964003c9cd589053386 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 04:09:42 +0000 Subject: [PATCH 3/8] fix: preserve top-level lazy export typing --- airbyte/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte/__init__.py b/airbyte/__init__.py index 0e44f4527..a53205579 100644 --- a/airbyte/__init__.py +++ b/airbyte/__init__.py @@ -125,7 +125,7 @@ # ruff: noqa: F822 # Lazy exports are resolved by __getattr__ at runtime. from importlib import import_module -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, cast from airbyte import registry from airbyte.records import StreamRecord @@ -149,12 +149,12 @@ } -def __getattr__(name: str) -> object: +def __getattr__(name: str) -> Any: # noqa: ANN401 if name not in _LAZY_EXPORTS: raise AttributeError(f"module 'airbyte' has no attribute {name!r}") module = import_module(_LAZY_EXPORTS[name]) - value = getattr(module, name) + value = cast("Any", getattr(module, name)) globals()[name] = value return value From ba6dca91a0c4971a81bdcdca2779bb34a05f10e3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 04:14:39 +0000 Subject: [PATCH 4/8] fix: validate destinationType for destination dictionaries --- airbyte/cloud/workspaces.py | 9 +-- pyproject.toml | 4 ++ tests/unit_tests/test_cloud_api_util.py | 38 +++++++++++++ uv.lock | 76 +++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 4 deletions(-) diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index ef07365db..2879fe35e 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -445,10 +445,11 @@ def deploy_destination( else: destination_conf_dict = destination._hydrated_config.copy() # noqa: SLF001 destination_conf_dict["destinationType"] = destination.name.replace("destination-", "") - if "destinationType" not in destination_conf_dict: - raise exc.PyAirbyteInputError( - message="Missing `destinationType` in configuration dictionary.", - ) + + if "destinationType" not in destination_conf_dict: + raise exc.PyAirbyteInputError( + message="Missing `destinationType` in configuration dictionary.", + ) if random_name_suffix: name += f" (ID: {text_util.generate_random_suffix()})" diff --git a/pyproject.toml b/pyproject.toml index 4d7407f9c..6c966cbbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,8 @@ authors = [ ] requires-python = ">=3.10,<3.13" dynamic = ["version"] +# `project.dependencies` remains the published full install requirement set. +# Keep it in sync with `dependency-groups.slim`, `dependency-groups.full`, and `uv.lock`. dependencies = [ "airbyte-api>=0.53.0,<1.0", "airbyte-cdk>=7.3.9,<8.0.0", @@ -51,6 +53,7 @@ dependencies = [ ] [dependency-groups] +# These uv groups model install boundaries for local workflows and lockfile management. slim = [ "airbyte-api>=0.53.0,<1.0", "airbyte-protocol-models-pdv2>=0.13.0,<1.0", @@ -73,6 +76,7 @@ slim = [ "uv>=0.5.0,<0.9.0", ] full = [ + { include-group = "slim" }, "airbyte-cdk>=7.3.9,<8.0.0", "cyclopts>=4.0,<5.0", # TODO: Restore broader version constraints after verifying compatibility diff --git a/tests/unit_tests/test_cloud_api_util.py b/tests/unit_tests/test_cloud_api_util.py index eb7a41ac9..8c2ff9874 100644 --- a/tests/unit_tests/test_cloud_api_util.py +++ b/tests/unit_tests/test_cloud_api_util.py @@ -4,10 +4,12 @@ from __future__ import annotations from types import SimpleNamespace +from typing import Any import pytest import requests from airbyte._util import api_util +from airbyte.cloud.workspaces import CloudWorkspace from airbyte.exceptions import AirbyteWorkspaceNotEmptyError, PyAirbyteInputError from airbyte.secrets.base import SecretString from airbyte_api import api, models @@ -107,6 +109,42 @@ def _list_workspaces_response( ) +def test_deploy_destination_rejects_dict_without_destination_type() -> None: + workspace = CloudWorkspace(workspace_id="workspace-id", bearer_token="token") + + with pytest.raises(PyAirbyteInputError, match="Missing `destinationType`"): + workspace.deploy_destination( + name="test-destination", + destination={"host": "example.com"}, + ) + + +def test_deploy_destination_forwards_destination_type( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured_config: dict[str, Any] | None = None + + def create_destination( + *, + config: dict[str, Any], + **_: Any, + ) -> SimpleNamespace: + nonlocal captured_config + captured_config = config + return SimpleNamespace(destination_id="destination-id") + + monkeypatch.setattr(api_util, "create_destination", create_destination) + + workspace = CloudWorkspace(workspace_id="workspace-id", bearer_token="token") + workspace.deploy_destination( + name="test-destination", + destination={"destinationType": "duckdb", "path": "/tmp/test.duckdb"}, + unique=False, + ) + + assert captured_config == {"destinationType": "duckdb", "path": "/tmp/test.duckdb"} + + def test_create_workspace_forwards_request( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/uv.lock b/uv.lock index cbec76f82..c14be5424 100644 --- a/uv.lock +++ b/uv.lock @@ -165,8 +165,11 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "airbyte-api" }, { name = "airbyte-cdk" }, + { name = "airbyte-protocol-models-pdv2" }, { name = "coverage" }, + { name = "cryptography" }, { name = "cyclopts" }, { name = "deptry" }, { name = "docker" }, @@ -176,8 +179,13 @@ dev = [ { name = "fastmcp" }, { name = "fastmcp-extensions" }, { name = "freezegun" }, + { name = "google-auth" }, { name = "google-cloud-bigquery" }, { name = "google-cloud-bigquery-storage" }, + { name = "google-cloud-secret-manager" }, + { name = "jsonschema" }, + { name = "orjson" }, + { name = "overrides" }, { name = "pandas" }, { name = "pandas-stubs" }, { name = "pdoc" }, @@ -185,42 +193,72 @@ dev = [ { name = "psycopg", extra = ["binary", "pool"] }, { name = "psycopg2-binary" }, { name = "pyarrow" }, + { name = "pydantic" }, { name = "pydantic-ai" }, + { name = "pydantic-core" }, { name = "pyrefly" }, { name = "pytest" }, { name = "pytest-docker" }, { name = "pytest-mock" }, { name = "pytest-timeout" }, + { name = "python-dotenv" }, + { name = "python-ulid" }, + { name = "pyyaml" }, + { name = "requests" }, { name = "responses" }, + { name = "rich" }, { name = "ruff" }, { name = "snowflake-connector-python" }, { name = "snowflake-sqlalchemy" }, { name = "sqlalchemy" }, { name = "sqlalchemy-bigquery" }, { name = "sqlalchemy2-stubs" }, + { name = "structlog" }, { name = "tomli" }, { name = "types-jsonschema" }, { name = "types-pyyaml" }, { name = "types-requests" }, + { name = "typing-extensions" }, + { name = "uuid7" }, + { name = "uv" }, { name = "viztracer" }, ] full = [ + { name = "airbyte-api" }, { name = "airbyte-cdk" }, + { name = "airbyte-protocol-models-pdv2" }, + { name = "cryptography" }, { name = "cyclopts" }, { name = "duckdb" }, { name = "duckdb-engine" }, { name = "fastmcp" }, { name = "fastmcp-extensions" }, + { name = "google-auth" }, { name = "google-cloud-bigquery" }, { name = "google-cloud-bigquery-storage" }, + { name = "google-cloud-secret-manager" }, + { name = "jsonschema" }, + { name = "orjson" }, + { name = "overrides" }, { name = "pandas" }, { name = "psycopg", extra = ["binary", "pool"] }, { name = "psycopg2-binary" }, { name = "pyarrow" }, + { name = "pydantic" }, + { name = "pydantic-core" }, + { name = "python-dotenv" }, + { name = "python-ulid" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "rich" }, { name = "snowflake-connector-python" }, { name = "snowflake-sqlalchemy" }, { name = "sqlalchemy" }, { name = "sqlalchemy-bigquery" }, + { name = "structlog" }, + { name = "typing-extensions" }, + { name = "uuid7" }, + { name = "uv" }, ] slim = [ { name = "airbyte-api" }, @@ -285,8 +323,11 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "airbyte-api", specifier = ">=0.53.0,<1.0" }, { name = "airbyte-cdk", specifier = ">=7.3.9,<8.0.0" }, + { name = "airbyte-protocol-models-pdv2", specifier = ">=0.13.0,<1.0" }, { name = "coverage", specifier = ">=7.5.1,<8.0" }, + { name = "cryptography", specifier = ">=44.0.0,<45.0.0" }, { name = "cyclopts", specifier = ">=4.0,<5.0" }, { name = "deptry", specifier = ">=0.21.1,<1.0" }, { name = "docker", specifier = ">=7.1.0,<8.0" }, @@ -296,8 +337,13 @@ dev = [ { name = "fastmcp", specifier = ">=3.0,<4.0" }, { name = "fastmcp-extensions", specifier = ">=0.3.0,<1.0.0" }, { name = "freezegun", specifier = ">=1.4.0,<2.0" }, + { name = "google-auth", specifier = ">=2.27.0,<3.0" }, { name = "google-cloud-bigquery", specifier = ">=3.12.0,<4.0" }, { name = "google-cloud-bigquery-storage", specifier = ">=2.25.0,<3.0" }, + { name = "google-cloud-secret-manager", specifier = ">=2.17.0,<3.0" }, + { name = "jsonschema", specifier = ">=3.2.0,<5.0" }, + { name = "orjson", specifier = ">=3.10,<4.0" }, + { name = "overrides", specifier = ">=7.4.0,<8.0" }, { name = "pandas", specifier = ">=1.5.3,<3.0" }, { name = "pandas-stubs", specifier = ">=2.1.4.231218" }, { name = "pdoc", specifier = ">=16.0.0,<17.0" }, @@ -305,42 +351,72 @@ dev = [ { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.1.19,<4.0" }, { name = "psycopg2-binary", specifier = ">=2.9.9,<3.0" }, { name = "pyarrow", specifier = ">=16.1,<22.0" }, + { name = "pydantic", specifier = ">=2.0,<3.0" }, { name = "pydantic-ai", specifier = ">=0.2.16" }, + { name = "pydantic-core" }, { name = "pyrefly", specifier = ">=0.38.0,<0.39.0" }, { name = "pytest", specifier = ">=8.2.0,<9.0" }, { name = "pytest-docker", specifier = ">=3.1.1,<4.0" }, { name = "pytest-mock", specifier = ">=3.14.0,<4.0" }, { name = "pytest-timeout", specifier = ">=2.3.1,<3.0" }, + { name = "python-dotenv", specifier = ">=1.0.1,<2.0" }, + { name = "python-ulid", specifier = ">=3.0.0,<4.0" }, + { name = "pyyaml", specifier = ">=6.0.2,<7.0" }, + { name = "requests", specifier = "!=3.32.0" }, { name = "responses", specifier = ">=0.25.0,<1.0" }, + { name = "rich", specifier = ">=13.7.0,<14.0" }, { name = "ruff", specifier = ">=0.8.2,<0.9.0" }, { name = "snowflake-connector-python", specifier = ">=3.12.2,<4.0" }, { name = "snowflake-sqlalchemy", specifier = ">=1.6.1,<2.0" }, { name = "sqlalchemy", specifier = "==2.0.43" }, { name = "sqlalchemy-bigquery", specifier = "==1.12.0" }, { name = "sqlalchemy2-stubs", specifier = ">=0.0.2a38" }, + { name = "structlog", specifier = ">=24.4.0,<25.0" }, { name = "tomli", specifier = ">=2.0,<3.0" }, { name = "types-jsonschema", specifier = ">=4.20.0.0" }, { name = "types-pyyaml", specifier = ">=6.0.12.12" }, { name = "types-requests", specifier = "==2.31.0.4" }, + { name = "typing-extensions" }, + { name = "uuid7", specifier = ">=0.1.0,<1.0" }, + { name = "uv", specifier = ">=0.5.0,<0.9.0" }, { name = "viztracer", specifier = ">=0.16.3,<1.1.0" }, ] full = [ + { name = "airbyte-api", specifier = ">=0.53.0,<1.0" }, { name = "airbyte-cdk", specifier = ">=7.3.9,<8.0.0" }, + { name = "airbyte-protocol-models-pdv2", specifier = ">=0.13.0,<1.0" }, + { name = "cryptography", specifier = ">=44.0.0,<45.0.0" }, { name = "cyclopts", specifier = ">=4.0,<5.0" }, { name = "duckdb", specifier = "==1.4.3" }, { name = "duckdb-engine", specifier = "==0.17.0" }, { name = "fastmcp", specifier = ">=3.0,<4.0" }, { name = "fastmcp-extensions", specifier = ">=0.3.0,<1.0.0" }, + { name = "google-auth", specifier = ">=2.27.0,<3.0" }, { name = "google-cloud-bigquery", specifier = ">=3.12.0,<4.0" }, { name = "google-cloud-bigquery-storage", specifier = ">=2.25.0,<3.0" }, + { name = "google-cloud-secret-manager", specifier = ">=2.17.0,<3.0" }, + { name = "jsonschema", specifier = ">=3.2.0,<5.0" }, + { name = "orjson", specifier = ">=3.10,<4.0" }, + { name = "overrides", specifier = ">=7.4.0,<8.0" }, { name = "pandas", specifier = ">=1.5.3,<3.0" }, { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.1.19,<4.0" }, { name = "psycopg2-binary", specifier = ">=2.9.9,<3.0" }, { name = "pyarrow", specifier = ">=16.1,<22.0" }, + { name = "pydantic", specifier = ">=2.0,<3.0" }, + { name = "pydantic-core" }, + { name = "python-dotenv", specifier = ">=1.0.1,<2.0" }, + { name = "python-ulid", specifier = ">=3.0.0,<4.0" }, + { name = "pyyaml", specifier = ">=6.0.2,<7.0" }, + { name = "requests", specifier = "!=3.32.0" }, + { name = "rich", specifier = ">=13.7.0,<14.0" }, { name = "snowflake-connector-python", specifier = ">=3.12.2,<4.0" }, { name = "snowflake-sqlalchemy", specifier = ">=1.6.1,<2.0" }, { name = "sqlalchemy", specifier = "==2.0.43" }, { name = "sqlalchemy-bigquery", specifier = "==1.12.0" }, + { name = "structlog", specifier = ">=24.4.0,<25.0" }, + { name = "typing-extensions" }, + { name = "uuid7", specifier = ">=0.1.0,<1.0" }, + { name = "uv", specifier = ">=0.5.0,<0.9.0" }, ] slim = [ { name = "airbyte-api", specifier = ">=0.53.0,<1.0" }, From 3352d81cd88f8e5f321f5e049b967865a465b2be Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 04:22:08 +0000 Subject: [PATCH 5/8] build: add airbyte-slim release artifacts --- .github/workflows/release_drafter.yml | 4 +- pyproject.toml | 1 + scripts/build_slim_package.uv | 127 ++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 1 deletion(-) create mode 100755 scripts/build_slim_package.uv diff --git a/.github/workflows/release_drafter.yml b/.github/workflows/release_drafter.yml index 25703f24d..d733f3122 100644 --- a/.github/workflows/release_drafter.yml +++ b/.github/workflows/release_drafter.yml @@ -42,7 +42,9 @@ jobs: env: VERSION: ${{ steps.dry-run.outputs.tag-name }} # Standardize by removing preceding 'v' char: - run: UV_DYNAMIC_VERSIONING_BYPASS="${VERSION#v}" uv build + run: | + UV_DYNAMIC_VERSIONING_BYPASS="${VERSION#v}" uv build + UV_DYNAMIC_VERSIONING_BYPASS="${VERSION#v}" uv run scripts/build_slim_package.uv - name: Create or update draft release uses: aaronsteers/semantic-pr-release-drafter@v1.0.0 diff --git a/pyproject.toml b/pyproject.toml index 6c966cbbb..466a1aaf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -209,6 +209,7 @@ coverage-html = { shell = "coverage html -d htmlcov && open htmlcov/index.html" coverage-reset = { shell = "coverage erase" } check = { shell = "ruff check . && pyrefly check && pytest --collect-only -qq" } +build-slim = { cmd = "scripts/build_slim_package.uv", help = "Build the airbyte-slim package into dist/" } docs-generate = {env = {PDOC_ALLOW_EXEC = "1"}, cmd = "python -m docs.generate run"} docs-preview = {shell = "poe docs-generate && open docs/generated/index.html"} diff --git a/scripts/build_slim_package.uv b/scripts/build_slim_package.uv new file mode 100755 index 000000000..8c35fd88b --- /dev/null +++ b/scripts/build_slim_package.uv @@ -0,0 +1,127 @@ +#!/usr/bin/env -S uv run --script +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# ruff: noqa: EXE003 +# /// script +# requires-python = ">=3.10,<3.15" +# dependencies = ["tomli>=2.0,<3.0", "tomli-w", "uv-dynamic-versioning>=0.8.2,<1.0"] +# /// +"""Build the `airbyte-slim` distribution from the shared PyAirbyte source tree.""" + +from __future__ import annotations + +import argparse +import shutil +import subprocess +from pathlib import Path +from typing import TypeAlias, cast + +import tomli as tomllib +import tomli_w + + +REPO_ROOT = Path(__file__).resolve().parents[1] +PYPROJECT_PATH = REPO_ROOT / "pyproject.toml" +TomlValue: TypeAlias = str | int | float | bool | list["TomlValue"] | dict[str, "TomlValue"] | None +SLIM_EXCLUDED_PACKAGES = [ + "airbyte/caches", + "airbyte/cli", + "airbyte/destinations", + "airbyte/mcp", + "airbyte/shared/sql_processor.py", + "airbyte/sources", + "airbyte/_processors/sql", +] + + +def _load_pyproject() -> dict[str, TomlValue]: + return tomllib.loads(PYPROJECT_PATH.read_text()) + + +def _build_slim_pyproject(full_pyproject: dict[str, TomlValue]) -> dict[str, TomlValue]: + slim_pyproject = dict(full_pyproject) + dependency_groups = cast(dict[str, TomlValue], full_pyproject["dependency-groups"]) + slim_dependencies = cast(list[TomlValue], dependency_groups["slim"]) + + project = dict(cast(dict[str, TomlValue], full_pyproject["project"])) + project["name"] = "airbyte-slim" + project["requires-python"] = ">=3.14,<3.15" + project["dependencies"] = slim_dependencies + project.pop("scripts", None) + slim_pyproject["project"] = project + + tool = cast(dict[str, TomlValue], full_pyproject["tool"]) + hatch = dict(cast(dict[str, TomlValue], tool["hatch"])) + build = dict(cast(dict[str, TomlValue], hatch["build"])) + targets = dict(cast(dict[str, TomlValue], build["targets"])) + wheel = dict(cast(dict[str, TomlValue], targets["wheel"])) + wheel["packages"] = ["airbyte"] + wheel["exclude"] = SLIM_EXCLUDED_PACKAGES + targets["wheel"] = wheel + sdist = dict(cast(dict[str, TomlValue], targets.get("sdist", {}))) + sdist["exclude"] = SLIM_EXCLUDED_PACKAGES + targets["sdist"] = sdist + build["targets"] = targets + hatch["build"] = build + slim_pyproject["tool"] = {**tool, "hatch": hatch} + + slim_pyproject.pop("dependency-groups", None) + return slim_pyproject + + +def _copy_project(build_dir: Path) -> None: + if build_dir.exists(): + shutil.rmtree(build_dir) + + ignore = shutil.ignore_patterns( + ".git", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".venv", + ".venv-*", + "build", + "dist", + "__pycache__", + "*.pyc", + ) + shutil.copytree(REPO_ROOT, build_dir, ignore=ignore) + + +def build_airbyte_slim(dist_dir: Path, build_dir: Path) -> None: + """Build the slim wheel and sdist artifacts.""" + slim_pyproject = _build_slim_pyproject(_load_pyproject()) + _copy_project(build_dir) + + build_pyproject = build_dir / "pyproject.toml" + build_pyproject.write_bytes(tomli_w.dumps(slim_pyproject).encode()) + + dist_dir.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["uv", "build", "--out-dir", str(dist_dir)], + cwd=build_dir, + check=True, + ) + + +def main() -> None: + """Build the `airbyte-slim` package.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--dist-dir", + type=Path, + default=REPO_ROOT / "dist", + help="Directory for generated distribution artifacts.", + ) + parser.add_argument( + "--build-dir", + type=Path, + default=REPO_ROOT / "build" / "airbyte-slim", + help="Temporary directory used to stage the slim build.", + ) + args = parser.parse_args() + + build_airbyte_slim(dist_dir=args.dist_dir, build_dir=args.build_dir) + + +if __name__ == "__main__": + main() From 52bd2a80b613e361ab4121eddc7d25f12ad3d4a3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 05:24:17 +0000 Subject: [PATCH 6/8] ci: clarify release variant build step --- .github/workflows/release_drafter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_drafter.yml b/.github/workflows/release_drafter.yml index d733f3122..6280280fc 100644 --- a/.github/workflows/release_drafter.yml +++ b/.github/workflows/release_drafter.yml @@ -41,7 +41,7 @@ jobs: - name: Build package env: VERSION: ${{ steps.dry-run.outputs.tag-name }} - # Standardize by removing preceding 'v' char: + # Build the main and slim variants with same version. run: | UV_DYNAMIC_VERSIONING_BYPASS="${VERSION#v}" uv build UV_DYNAMIC_VERSIONING_BYPASS="${VERSION#v}" uv run scripts/build_slim_package.uv From 9cbc54db4c50922295781d9d917d861d37c5ef3c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 05:30:40 +0000 Subject: [PATCH 7/8] build: harden slim package helper paths --- airbyte/_util/datetime_parse.py | 25 +++++++++++ airbyte/cloud/sync_results.py | 24 +++-------- scripts/build_slim_package.uv | 18 ++++++-- tests/unit_tests/test_datetime_parse.py | 55 +++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 21 deletions(-) create mode 100644 airbyte/_util/datetime_parse.py create mode 100644 tests/unit_tests/test_datetime_parse.py diff --git a/airbyte/_util/datetime_parse.py b/airbyte/_util/datetime_parse.py new file mode 100644 index 000000000..34ef666c3 --- /dev/null +++ b/airbyte/_util/datetime_parse.py @@ -0,0 +1,25 @@ +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +"""Datetime parsing helpers.""" + +from __future__ import annotations + +from datetime import datetime, timezone + + +def parse_datetime(value: str | int) -> datetime: + """Parse an ISO datetime string or Unix timestamp as a timezone-aware datetime.""" + if isinstance(value, int) or ( + isinstance(value, str) + and (value.isdigit() or (value.startswith("-") and value[1:].isdigit())) + ): + return datetime.fromtimestamp(int(value), tz=timezone.utc) + + if not isinstance(value, str): + raise TypeError(f"Could not parse datetime string: {value}") + + normalized = value.replace("Z", "+00:00") + parsed = datetime.fromisoformat(normalized) + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + + return parsed diff --git a/airbyte/cloud/sync_results.py b/airbyte/cloud/sync_results.py index 3732cbeca..2f8c07d9e 100644 --- a/airbyte/cloud/sync_results.py +++ b/airbyte/cloud/sync_results.py @@ -103,34 +103,22 @@ import time from collections.abc import Iterator, Mapping from dataclasses import asdict, dataclass -from datetime import datetime, timezone from typing import TYPE_CHECKING, Any from typing_extensions import final from airbyte._util import api_util +from airbyte._util.datetime_parse import parse_datetime from airbyte.cloud.constants import FAILED_STATUSES, FINAL_STATUSES from airbyte.exceptions import AirbyteConnectionSyncError, AirbyteConnectionSyncTimeoutError -def _parse_datetime(value: str | int) -> datetime: - if isinstance(value, int) or ( - isinstance(value, str) - and (value.isdigit() or (value.startswith("-") and value[1:].isdigit())) - ): - return datetime.fromtimestamp(int(value), tz=timezone.utc) - - if not isinstance(value, str): - raise TypeError(f"Could not parse datetime string: {value}") - - normalized = value.replace("Z", "+00:00") - return datetime.fromisoformat(normalized) - - DEFAULT_SYNC_TIMEOUT_SECONDS = 30 * 60 # 30 minutes """The default timeout for waiting for a sync job to complete, in seconds.""" if TYPE_CHECKING: + from datetime import datetime + import sqlalchemy from airbyte._util.api_imports import ConnectionResponse, JobResponse, JobStatusEnum @@ -178,7 +166,7 @@ def records_synced(self) -> int: def created_at(self) -> datetime: """Return the creation time of the attempt.""" timestamp = self._get_attempt_data()["createdAt"] - return _parse_datetime(timestamp) + return parse_datetime(timestamp) def _get_attempt_data(self) -> dict[str, Any]: """Get attempt data from the provided attempt data.""" @@ -317,7 +305,7 @@ def records_synced(self) -> int: def start_time(self) -> datetime: """Return the start time of the sync job in UTC.""" try: - return _parse_datetime(self._fetch_latest_job_info().start_time) + return parse_datetime(self._fetch_latest_job_info().start_time) except (ValueError, TypeError) as e: if "Invalid isoformat string" in str(e): job_info_raw = api_util._make_config_api_request( # noqa: SLF001 @@ -331,7 +319,7 @@ def start_time(self) -> datetime: ) raw_start_time = job_info_raw.get("startTime") if raw_start_time: - return _parse_datetime(raw_start_time) + return parse_datetime(raw_start_time) raise def _fetch_job_with_attempts(self) -> dict[str, Any]: diff --git a/scripts/build_slim_package.uv b/scripts/build_slim_package.uv index 8c35fd88b..9a0836806 100755 --- a/scripts/build_slim_package.uv +++ b/scripts/build_slim_package.uv @@ -31,6 +31,7 @@ SLIM_EXCLUDED_PACKAGES = [ "airbyte/sources", "airbyte/_processors/sql", ] +SAFE_BUILD_PARENT = REPO_ROOT / "build" def _load_pyproject() -> dict[str, TomlValue]: @@ -69,8 +70,17 @@ def _build_slim_pyproject(full_pyproject: dict[str, TomlValue]) -> dict[str, Tom def _copy_project(build_dir: Path) -> None: - if build_dir.exists(): - shutil.rmtree(build_dir) + resolved_build_dir = build_dir.resolve() + safe_build_parent = SAFE_BUILD_PARENT.resolve() + if ( + resolved_build_dir == Path("/") + or resolved_build_dir == REPO_ROOT + or resolved_build_dir.parent != safe_build_parent + ): + raise ValueError(f"Refusing to delete unsafe build_dir: {resolved_build_dir}") + + if resolved_build_dir.exists(): + shutil.rmtree(resolved_build_dir) ignore = shutil.ignore_patterns( ".git", @@ -84,11 +94,13 @@ def _copy_project(build_dir: Path) -> None: "__pycache__", "*.pyc", ) - shutil.copytree(REPO_ROOT, build_dir, ignore=ignore) + shutil.copytree(REPO_ROOT, resolved_build_dir, ignore=ignore) def build_airbyte_slim(dist_dir: Path, build_dir: Path) -> None: """Build the slim wheel and sdist artifacts.""" + dist_dir = dist_dir.resolve() + build_dir = build_dir.resolve() slim_pyproject = _build_slim_pyproject(_load_pyproject()) _copy_project(build_dir) diff --git a/tests/unit_tests/test_datetime_parse.py b/tests/unit_tests/test_datetime_parse.py new file mode 100644 index 000000000..11cea7616 --- /dev/null +++ b/tests/unit_tests/test_datetime_parse.py @@ -0,0 +1,55 @@ +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +"""Unit tests for datetime parsing helpers.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from airbyte._util.datetime_parse import parse_datetime + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + pytest.param( + 1_767_225_600, + datetime(2026, 1, 1, tzinfo=timezone.utc), + id="integer-unix-timestamp", + ), + pytest.param( + "1767225600", + datetime(2026, 1, 1, tzinfo=timezone.utc), + id="string-unix-timestamp", + ), + pytest.param( + "2026-01-01T00:00:00Z", + datetime(2026, 1, 1, tzinfo=timezone.utc), + id="rfc3339-zulu", + ), + pytest.param( + "2026-01-01T01:00:00+01:00", + datetime(2026, 1, 1, tzinfo=timezone.utc), + id="timezone-offset", + ), + pytest.param( + "2026-01-01", + datetime(2026, 1, 1, tzinfo=timezone.utc), + id="date-only", + ), + ], +) +def test_parse_datetime(value: str | int, expected: datetime) -> None: + assert parse_datetime(value) == expected + + +@pytest.mark.parametrize( + "value", + [ + pytest.param(object(), id="object"), + pytest.param("not-a-date", id="invalid-string"), + ], +) +def test_parse_datetime_rejects_invalid_values(value: object) -> None: + with pytest.raises((TypeError, ValueError)): + parse_datetime(value) # type: ignore[arg-type] From bb99bc286b39c019d41701830a1bd9e4f325c68f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 06:38:41 +0000 Subject: [PATCH 8/8] build: make slim package first-class --- .github/workflows/release_drafter.yml | 2 +- airbyte/version.py | 12 ++- packages/airbyte-slim/README.md | 91 +++++++++++++++++ packages/airbyte-slim/airbyte | 1 + packages/airbyte-slim/pyproject.toml | 70 +++++++++++++ pyproject.toml | 2 +- scripts/build_slim_package.uv | 139 -------------------------- 7 files changed, 175 insertions(+), 142 deletions(-) create mode 100644 packages/airbyte-slim/README.md create mode 120000 packages/airbyte-slim/airbyte create mode 100644 packages/airbyte-slim/pyproject.toml delete mode 100755 scripts/build_slim_package.uv diff --git a/.github/workflows/release_drafter.yml b/.github/workflows/release_drafter.yml index 6280280fc..8a355a9e4 100644 --- a/.github/workflows/release_drafter.yml +++ b/.github/workflows/release_drafter.yml @@ -44,7 +44,7 @@ jobs: # Build the main and slim variants with same version. run: | UV_DYNAMIC_VERSIONING_BYPASS="${VERSION#v}" uv build - UV_DYNAMIC_VERSIONING_BYPASS="${VERSION#v}" uv run scripts/build_slim_package.uv + UV_DYNAMIC_VERSIONING_BYPASS="${VERSION#v}" uv build packages/airbyte-slim --out-dir dist - name: Create or update draft release uses: aaronsteers/semantic-pr-release-drafter@v1.0.0 diff --git a/airbyte/version.py b/airbyte/version.py index 90aab62be..aa2f917de 100644 --- a/airbyte/version.py +++ b/airbyte/version.py @@ -6,7 +6,17 @@ import importlib.metadata -airbyte_version = importlib.metadata.version("airbyte") +def _get_installed_version() -> str: + for distribution_name in ("airbyte", "airbyte-slim"): + try: + return importlib.metadata.version(distribution_name) + except importlib.metadata.PackageNotFoundError: + continue + + raise importlib.metadata.PackageNotFoundError("airbyte") + + +airbyte_version = _get_installed_version() def get_version() -> str: diff --git a/packages/airbyte-slim/README.md b/packages/airbyte-slim/README.md new file mode 100644 index 000000000..1ea8d3f48 --- /dev/null +++ b/packages/airbyte-slim/README.md @@ -0,0 +1,91 @@ +# PyAirbyte + +PyAirbyte brings the power of Airbyte to every Python developer. PyAirbyte provides a set of utilities to use Airbyte connectors in Python. + +[![PyPI version](https://badge.fury.io/py/airbyte.svg)](https://badge.fury.io/py/airbyte) +[![PyPI - Downloads](https://img.shields.io/pypi/dm/airbyte)](https://pypi.org/project/airbyte/) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/airbyte)](https://pypi.org/project/airbyte/) +[![Star on GitHub](https://img.shields.io/github/stars/airbytehq/pyairbyte.svg?style=social&label=★%20on%20GitHub)](https://github.com/airbytehq/pyairbyte) + +## Getting Started + +Watch this [Getting Started Loom video](https://www.loom.com/share/3de81ca3ce914feca209bf83777efa3f?sid=8804e8d7-096c-4aaa-a8a4-9eb93a44e850) or run one of our Quickstart tutorials below to see how you can use PyAirbyte in your python code. + +* [Basic Demo](https://github.com/airbytehq/quickstarts/blob/main/pyairbyte_notebooks/PyAirbyte_Basic_Features_Demo.ipynb) +* [CoinAPI](https://github.com/airbytehq/quickstarts/blob/main/pyairbyte_notebooks/PyAirbyte_CoinAPI_Demo.ipynb) +* [GA4](https://github.com/airbytehq/quickstarts/blob/main/pyairbyte_notebooks/PyAirbyte_GA4_Demo.ipynb) +* [Shopify](https://github.com/airbytehq/quickstarts/blob/main/pyairbyte_notebooks/PyAirbyte_Shopify_Demo.ipynb) +* [GitHub](https://github.com/airbytehq/quickstarts/blob/main/pyairbyte_notebooks/PyAirbyte_Github_Incremental_Demo.ipynb) +* [Postgres (cache)](https://github.com/airbytehq/quickstarts/blob/main/pyairbyte_notebooks/PyAirbyte_Postgres_Custom_Cache_Demo.ipynb) + +## Connector Installation + +### Declarative Source Installation + +For Declarative Sources defined in YAML, the installation process will is to simply download the yaml file from the `connectors.airbyte.com` public URLs, and to run them directly as YAML. + +Declarative sources have the fastest download times, due to the simplicity and each of install. + +In some cases, you may get better stability by using `docker_image=True` in `get_source()`/`get_destination()`, due to the fact that all dependencies are locked within the docker image. + +### Python Installation + +Generally, when Python-based installation is possible, it will be performed automatically given a Python-based connector name. + +In some cases, you may get better stability by using `docker_image=True` in `get_source()`/`get_destination()`, due to the fact that all dependencies are locked within the docker image. + +#### Installing Connectors with `uv` + +By default, beginning with version `0.29.0`, PyAirbyte defaults to [`uv`](https://docs.astral.sh/uv) instead of `pip` for Python connector installation. Compared with `pip`, `uv` is much faster. It also provides the unique ability of specifying different versions of Python than PyAirbyte is using, and even Python versions which are not already pre-installed on the local workstation. + +If you prefer to fall back to the prior `pip`-based installation methods, set the env var `AIRBYTE_NO_UV=true`. + +#### Installing Connectors With a Custom Python Version + +In both `get_source()` and `get_destination()`, you can provide a `use_python` input arg that is equal to the desired version of Python that you with to use for the given connector. This can be helpful if an older connector doesn't support the version of Python that you are using for PyAirbyte itself. + +For example, assuming PyAirbyte is running on Python 3.12, you can install a connector using Python 3.10.13 with the following code snippet: + +```py +import airbyte as ab + +source = ab.get_source( + "source-faker", + use_python="3.10.17", +) +``` + +### Installing Connectors with Docker + +For any connector (`get_source()`/`get_destination()`), you can specify the `docker_image` argument to `True` to prefer Docker over other default installation methods or `docker_image=MY_IMAGE` to leverage a specific docker image tag for the execution. + +## Contributing + +To learn how you can contribute to PyAirbyte, please see our [PyAirbyte Contributors Guide](./docs/CONTRIBUTING.md). + +## Frequently asked Questions + +**1. Does PyAirbyte replace Airbyte?** +No. PyAirbyte is a Python library that allows you to use Airbyte connectors in Python, but it does not have orchestration +or scheduling capabilities, nor does is provide logging, alerting, or other features for managing pipelines in +production. Airbyte is a full-fledged data integration platform that provides connectors, orchestration, and scheduling capabilities. + +**2. What is the PyAirbyte cache? Is it a destination?** +Yes and no. You can think of it as a built-in destination implementation, but we avoid the word "destination" in our docs to prevent confusion with our certified destinations list [here](https://docs.airbyte.com/integrations/destinations/). + +**3. Does PyAirbyte work with data orchestration frameworks like Airflow, Dagster, and Snowpark,** +Yes, it should. Please give it a try and report any problems you see. Also, drop us a note if works for you! + +**4. Can I use PyAirbyte to develop or test when developing Airbyte sources?** +Yes, you can. PyAirbyte makes it easy to test connectors in Python, and you can use it to develop new local connectors +as well as existing already-published ones. + +**5. Can I develop traditional ETL pipelines with PyAirbyte?** +Yes. Just pick the cache type matching the destination - like SnowflakeCache for landing data in Snowflake. + +**6. Can PyAirbyte import a connector from a local directory that has python project files, or does it have to be pip install** +Yes, PyAirbyte can use any local install that has a CLI - and will automatically find connectors by name if they are on PATH. + +## Changelog and Release Notes + +For a version history and list of all changes, please see our [GitHub Releases](https://github.com/airbytehq/PyAirbyte/releases) page. diff --git a/packages/airbyte-slim/airbyte b/packages/airbyte-slim/airbyte new file mode 120000 index 000000000..5a7003a8f --- /dev/null +++ b/packages/airbyte-slim/airbyte @@ -0,0 +1 @@ +../../airbyte \ No newline at end of file diff --git a/packages/airbyte-slim/pyproject.toml b/packages/airbyte-slim/pyproject.toml new file mode 100644 index 000000000..5ae77499f --- /dev/null +++ b/packages/airbyte-slim/pyproject.toml @@ -0,0 +1,70 @@ +[build-system] +requires = ["hatchling", "uv-dynamic-versioning"] +build-backend = "hatchling.build" + +[project] +name = "airbyte-slim" +description = "PyAirbyte slim distribution" +readme = "README.md" +authors = [ + { name = "Airbyte", email = "contact@airbyte.io" }, +] +requires-python = ">=3.14,<3.15" +dynamic = ["version"] +dependencies = [ + "airbyte-api>=0.53.0,<1.0", + "airbyte-protocol-models-pdv2>=0.13.0,<1.0", + "cryptography>=44.0.0,<45.0.0", + "google-auth>=2.27.0,<3.0", + "google-cloud-secret-manager>=2.17.0,<3.0", + "jsonschema>=3.2.0,<5.0", + "orjson>=3.10,<4.0", + "overrides>=7.4.0,<8.0", + "pydantic>=2.0,<3.0", + "pydantic-core", + "python-dotenv>=1.0.1,<2.0", + "python-ulid>=3.0.0,<4.0", + "pyyaml>=6.0.2,<7.0", + "requests!=3.32.0", + "rich>=13.7.0,<14.0", + "structlog>=24.4.0,<25.0", + "typing-extensions", + "uuid7>=0.1.0,<1.0", + "uv>=0.5.0,<0.9.0", +] + +[tool.hatch.version] +source = "uv-dynamic-versioning" + +[tool.uv-dynamic-versioning] +vcs = "git" +style = "pep440" +fallback-version = "0.0.0" +format = "{base}a{distance}" +format-jinja = "{% if distance == 0 %}{{ base }}{% else %}{{ base }}a{{ distance }}{% endif %}" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["airbyte"] +exclude = [ + "airbyte/caches", + "airbyte/cli", + "airbyte/destinations", + "airbyte/mcp", + "airbyte/shared/sql_processor.py", + "airbyte/sources", + "airbyte/_processors/sql", +] + +[tool.hatch.build.targets.sdist] +exclude = [ + "airbyte/caches", + "airbyte/cli", + "airbyte/destinations", + "airbyte/mcp", + "airbyte/shared/sql_processor.py", + "airbyte/sources", + "airbyte/_processors/sql", +] diff --git a/pyproject.toml b/pyproject.toml index 466a1aaf0..e8aef02a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -209,7 +209,7 @@ coverage-html = { shell = "coverage html -d htmlcov && open htmlcov/index.html" coverage-reset = { shell = "coverage erase" } check = { shell = "ruff check . && pyrefly check && pytest --collect-only -qq" } -build-slim = { cmd = "scripts/build_slim_package.uv", help = "Build the airbyte-slim package into dist/" } +build-slim = { cmd = "uv build packages/airbyte-slim --out-dir dist", help = "Build the airbyte-slim package into dist/" } docs-generate = {env = {PDOC_ALLOW_EXEC = "1"}, cmd = "python -m docs.generate run"} docs-preview = {shell = "poe docs-generate && open docs/generated/index.html"} diff --git a/scripts/build_slim_package.uv b/scripts/build_slim_package.uv deleted file mode 100755 index 9a0836806..000000000 --- a/scripts/build_slim_package.uv +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env -S uv run --script -# Copyright (c) 2026 Airbyte, Inc., all rights reserved. -# ruff: noqa: EXE003 -# /// script -# requires-python = ">=3.10,<3.15" -# dependencies = ["tomli>=2.0,<3.0", "tomli-w", "uv-dynamic-versioning>=0.8.2,<1.0"] -# /// -"""Build the `airbyte-slim` distribution from the shared PyAirbyte source tree.""" - -from __future__ import annotations - -import argparse -import shutil -import subprocess -from pathlib import Path -from typing import TypeAlias, cast - -import tomli as tomllib -import tomli_w - - -REPO_ROOT = Path(__file__).resolve().parents[1] -PYPROJECT_PATH = REPO_ROOT / "pyproject.toml" -TomlValue: TypeAlias = str | int | float | bool | list["TomlValue"] | dict[str, "TomlValue"] | None -SLIM_EXCLUDED_PACKAGES = [ - "airbyte/caches", - "airbyte/cli", - "airbyte/destinations", - "airbyte/mcp", - "airbyte/shared/sql_processor.py", - "airbyte/sources", - "airbyte/_processors/sql", -] -SAFE_BUILD_PARENT = REPO_ROOT / "build" - - -def _load_pyproject() -> dict[str, TomlValue]: - return tomllib.loads(PYPROJECT_PATH.read_text()) - - -def _build_slim_pyproject(full_pyproject: dict[str, TomlValue]) -> dict[str, TomlValue]: - slim_pyproject = dict(full_pyproject) - dependency_groups = cast(dict[str, TomlValue], full_pyproject["dependency-groups"]) - slim_dependencies = cast(list[TomlValue], dependency_groups["slim"]) - - project = dict(cast(dict[str, TomlValue], full_pyproject["project"])) - project["name"] = "airbyte-slim" - project["requires-python"] = ">=3.14,<3.15" - project["dependencies"] = slim_dependencies - project.pop("scripts", None) - slim_pyproject["project"] = project - - tool = cast(dict[str, TomlValue], full_pyproject["tool"]) - hatch = dict(cast(dict[str, TomlValue], tool["hatch"])) - build = dict(cast(dict[str, TomlValue], hatch["build"])) - targets = dict(cast(dict[str, TomlValue], build["targets"])) - wheel = dict(cast(dict[str, TomlValue], targets["wheel"])) - wheel["packages"] = ["airbyte"] - wheel["exclude"] = SLIM_EXCLUDED_PACKAGES - targets["wheel"] = wheel - sdist = dict(cast(dict[str, TomlValue], targets.get("sdist", {}))) - sdist["exclude"] = SLIM_EXCLUDED_PACKAGES - targets["sdist"] = sdist - build["targets"] = targets - hatch["build"] = build - slim_pyproject["tool"] = {**tool, "hatch": hatch} - - slim_pyproject.pop("dependency-groups", None) - return slim_pyproject - - -def _copy_project(build_dir: Path) -> None: - resolved_build_dir = build_dir.resolve() - safe_build_parent = SAFE_BUILD_PARENT.resolve() - if ( - resolved_build_dir == Path("/") - or resolved_build_dir == REPO_ROOT - or resolved_build_dir.parent != safe_build_parent - ): - raise ValueError(f"Refusing to delete unsafe build_dir: {resolved_build_dir}") - - if resolved_build_dir.exists(): - shutil.rmtree(resolved_build_dir) - - ignore = shutil.ignore_patterns( - ".git", - ".mypy_cache", - ".pytest_cache", - ".ruff_cache", - ".venv", - ".venv-*", - "build", - "dist", - "__pycache__", - "*.pyc", - ) - shutil.copytree(REPO_ROOT, resolved_build_dir, ignore=ignore) - - -def build_airbyte_slim(dist_dir: Path, build_dir: Path) -> None: - """Build the slim wheel and sdist artifacts.""" - dist_dir = dist_dir.resolve() - build_dir = build_dir.resolve() - slim_pyproject = _build_slim_pyproject(_load_pyproject()) - _copy_project(build_dir) - - build_pyproject = build_dir / "pyproject.toml" - build_pyproject.write_bytes(tomli_w.dumps(slim_pyproject).encode()) - - dist_dir.mkdir(parents=True, exist_ok=True) - subprocess.run( - ["uv", "build", "--out-dir", str(dist_dir)], - cwd=build_dir, - check=True, - ) - - -def main() -> None: - """Build the `airbyte-slim` package.""" - parser = argparse.ArgumentParser() - parser.add_argument( - "--dist-dir", - type=Path, - default=REPO_ROOT / "dist", - help="Directory for generated distribution artifacts.", - ) - parser.add_argument( - "--build-dir", - type=Path, - default=REPO_ROOT / "build" / "airbyte-slim", - help="Temporary directory used to stage the slim build.", - ) - args = parser.parse_args() - - build_airbyte_slim(dist_dir=args.dist_dir, build_dir=args.build_dir) - - -if __name__ == "__main__": - main()