diff --git a/airbyte/_connector_base.py b/airbyte/_connector_base.py index 0e8a66a40..7497adc94 100644 --- a/airbyte/_connector_base.py +++ b/airbyte/_connector_base.py @@ -134,7 +134,7 @@ def get_config(self) -> dict[str, Any]: If secrets are passed by reference (`secret_reference::*`), this will return the raw config dictionary without secrets hydrated. - """ + """ # noqa: DOC501 if self._config_dict is None: raise exc.AirbyteConnectorConfigurationMissingError( connector_name=self.name, @@ -259,7 +259,7 @@ def print_config_spec( Otherwise, it will be printed to the console. stderr: If True, print to stderr instead of stdout. This is useful when we want to print the spec to the console but not interfere with other output. - """ + """ # noqa: DOC501 if output_file and stderr: raise exc.PyAirbyteInputError( message="You can set output_file or stderr but not both.", @@ -332,7 +332,7 @@ def check(self) -> None: * execute the connector with check --config * Listen to the messages and return the first AirbyteCatalog that comes along. * Make sure the subprocess is killed when the function returns. - """ + """ # noqa: DOC501 with as_temp_files([self._hydrated_config]) as [config_file]: try: for msg in self._execute(["check", "--config", config_file]): diff --git a/airbyte/_executors/base.py b/airbyte/_executors/base.py index c3ad4e0c8..f94318daa 100644 --- a/airbyte/_executors/base.py +++ b/airbyte/_executors/base.py @@ -75,7 +75,7 @@ def _stream_from_subprocess( in a separate thread while output is read concurrently. This avoids a potential deadlock where the subprocess blocks on stdout (buffer full) while we're waiting for input to finish before reading stdout. - """ + """ # noqa: DOC501 input_thread: Thread | None = None exception_holder = ExceptionHolder() if isinstance(stdin, AirbyteMessageIterator): @@ -180,7 +180,7 @@ def __init__( """Initialize a connector executor. The 'name' param is required if 'metadata' is None. - """ + """ # noqa: DOC501 if not name and not metadata: raise exc.PyAirbyteInternalError(message="Either name or metadata must be provided.") diff --git a/airbyte/_executors/docker.py b/airbyte/_executors/docker.py index a364e0de3..54ebdebbf 100644 --- a/airbyte/_executors/docker.py +++ b/airbyte/_executors/docker.py @@ -41,7 +41,7 @@ def ensure_installation( """Ensure that the connector executable can be found. The auto_fix parameter is ignored for this executor type. - """ + """ # noqa: DOC501 _ = auto_fix try: assert ( diff --git a/airbyte/_executors/local.py b/airbyte/_executors/local.py index 786b690ec..fd296b75e 100644 --- a/airbyte/_executors/local.py +++ b/airbyte/_executors/local.py @@ -36,7 +36,7 @@ def ensure_installation( """Ensure that the connector executable can be found. The auto_fix parameter is ignored for this executor type. - """ + """ # noqa: DOC501 _ = auto_fix try: self.execute(["spec"]) diff --git a/airbyte/_executors/noop.py b/airbyte/_executors/noop.py index 02ad5cea7..f62e2eb11 100644 --- a/airbyte/_executors/noop.py +++ b/airbyte/_executors/noop.py @@ -70,7 +70,7 @@ def execute( """Execute a command and return an iterator of STDOUT lines. Only the 'spec' command is supported. Other commands will raise an error. - """ + """ # noqa: DOC501 _ = stdin, suppress_stderr # Unused if args == ["spec"]: diff --git a/airbyte/_executors/python.py b/airbyte/_executors/python.py index 57a5e0094..a37ec2939 100644 --- a/airbyte/_executors/python.py +++ b/airbyte/_executors/python.py @@ -49,7 +49,7 @@ def __init__( - False: Use Docker instead (handled by factory) - Path: Use interpreter at this path or interpreter name/command - str: Use uv-managed Python version (semver patterns like "3.12", "3.11.5") - """ + """ # noqa: DOC501 super().__init__(name=name, metadata=metadata, target_version=target_version) if not pip_url and metadata and not metadata.pypi_package_name: @@ -115,7 +115,7 @@ def install(self) -> None: """Install the connector in a virtual environment. After installation, the installed version will be stored in self.reported_version. - """ + """ # noqa: DOC501 if not ( self.use_python is None or self.use_python is True @@ -208,7 +208,7 @@ def get_installed_version( In the venv, we run the following: > python -c "from importlib.metadata import version; print(version(''))" - """ + """ # noqa: DOC501 if not recheck and self.reported_version: return self.reported_version @@ -258,7 +258,7 @@ def ensure_installation( Note: Version verification is not supported for connectors installed from a local path. - """ + """ # noqa: DOC501 # Store the installed version (or None if not installed) if not self.reported_version: self.reported_version = self.get_installed_version() diff --git a/airbyte/_executors/util.py b/airbyte/_executors/util.py index 7a30a1bf1..72b40905b 100644 --- a/airbyte/_executors/util.py +++ b/airbyte/_executors/util.py @@ -55,7 +55,7 @@ def _try_get_manifest_connector_files( - `PyAirbyteInputError`: If `source_name` is `None`. - `AirbyteConnectorInstallationError`: If the manifest cannot be downloaded or parsed, or if components.zip cannot be downloaded or extracted (excluding 404 errors). - """ + """ # noqa: DOC501 if source_name is None: raise exc.PyAirbyteInputError( message="Param 'source_name' is required.", @@ -136,7 +136,7 @@ def _get_local_executor( local_executable: Path | str | Literal[True], version: str | None, ) -> Executor: - """Get a local executor for a connector.""" + """Get a local executor for a connector.""" # noqa: DOC501 if version: raise exc.PyAirbyteInputError( message="Param 'version' is not supported when 'local_executable' is set." @@ -188,7 +188,7 @@ def get_connector_executor( # noqa: PLR0912, PLR0913, PLR0914, PLR0915, C901 # """This factory function creates an executor for a connector. For documentation of each arg, see the function `airbyte.sources.util.get_source()`. - """ + """ # noqa: DOC501 install_method_count = sum( [ bool(local_executable), diff --git a/airbyte/_message_iterators.py b/airbyte/_message_iterators.py index e6efd5a11..9d8637921 100644 --- a/airbyte/_message_iterators.py +++ b/airbyte/_message_iterators.py @@ -130,7 +130,7 @@ def from_str_buffer(cls, buffer: IO[str]) -> AirbyteMessageIterator: """Create a iterator that reads messages from a buffer.""" def generator() -> Generator[AirbyteMessage, None, None]: - """Yields AirbyteMessage objects read from STDIN.""" + """Yields AirbyteMessage objects read from STDIN.""" # noqa: DOC501 while True: next_line: str | None = next(buffer, None) # Read the next line from STDIN if next_line is None: diff --git a/airbyte/_processors/sql/bigquery.py b/airbyte/_processors/sql/bigquery.py index bcb0ea8ba..874da6b2c 100644 --- a/airbyte/_processors/sql/bigquery.py +++ b/airbyte/_processors/sql/bigquery.py @@ -229,7 +229,7 @@ def _table_exists( """Return true if the given table exists. We override the default implementation because BigQuery is very slow at scanning tables. - """ + """ # noqa: DOC501 client = self.sql_config.get_vendor_client() table_id = f"{self.sql_config.project_name}.{self.sql_config.dataset_name}.{table_name}" try: @@ -286,7 +286,7 @@ def _swap_temp_table_with_final_table( For example, BigQuery expects this format: ALTER TABLE my_schema.my_old_table_name RENAME TO my_new_table_name; - """ + """ # noqa: DOC501 if final_table_name is None: raise exc.PyAirbyteInternalError(message="Arg 'final_table_name' cannot be None.") if temp_table_name is None: diff --git a/airbyte/_processors/sql/snowflake.py b/airbyte/_processors/sql/snowflake.py index b9efb4de9..2187072d2 100644 --- a/airbyte/_processors/sql/snowflake.py +++ b/airbyte/_processors/sql/snowflake.py @@ -51,7 +51,7 @@ class SnowflakeConfig(SqlConfig): data_retention_time_in_days: int | None = None def _validate_authentication_config(self) -> None: - """Validate that authentication configuration is correct.""" + """Validate that authentication configuration is correct.""" # noqa: DOC501 auth_methods = { "password": self.password is not None, "private_key": self.private_key is not None, @@ -82,7 +82,7 @@ def _validate_authentication_config(self) -> None: ) def _get_private_key_content(self) -> bytes: - """Get the private key content from either private_key or private_key_path.""" + """Get the private key content from either private_key or private_key_path.""" # noqa: DOC501 if self.private_key: return str(self.private_key).encode("utf-8") if self.private_key_path: @@ -217,7 +217,7 @@ def _write_files_to_new_table( stream_name: str, batch_id: str, ) -> str: - """Write files to a new table.""" + """Write files to a new table.""" # noqa: DOC501 temp_table_name = self._create_table_for_loading( stream_name=stream_name, batch_id=batch_id, diff --git a/airbyte/_util/api_util.py b/airbyte/_util/api_util.py index e3f0a9eb5..5c80da559 100644 --- a/airbyte/_util/api_util.py +++ b/airbyte/_util/api_util.py @@ -62,7 +62,7 @@ def _validate_pagination_params( *, limit: int | None, ) -> None: - """Validate common pagination parameters.""" + """Validate common pagination parameters.""" # noqa: DOC501 if limit is not None and limit <= 0: raise PyAirbyteInputError(message="`limit` must be greater than 0.") @@ -255,7 +255,7 @@ def get_workspace( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> models.WorkspaceResponse: - """Get a workspace object.""" + """Get a workspace object.""" # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( api_root=api_root, client_id=client_id, @@ -295,7 +295,7 @@ def create_workspace( organization_id: str | None = None, region_id: str | None = None, ) -> models.WorkspaceResponse: - """Create a workspace.""" + """Create a workspace.""" # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( api_root=api_root, client_id=client_id, @@ -342,7 +342,7 @@ def rename_workspace( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> models.WorkspaceResponse: - """Rename a workspace.""" + """Rename a workspace.""" # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( api_root=api_root, client_id=client_id, @@ -405,7 +405,7 @@ def permanently_delete_workspace( PyAirbyteInputError: If safe mode is True and the workspace name does not meet the safety requirements. AirbyteWorkspaceNotEmptyError: If the workspace contains connections. - """ + """ # noqa: DOC501 if safe_mode: if workspace_name is None: workspace_info = get_workspace( @@ -486,7 +486,7 @@ def list_connections( name_filter: Callable[[str], bool] | None = None, limit: int | None = None, ) -> list[models.ConnectionResponse]: - """List connections.""" + """List connections.""" # noqa: DOC501 if name is not None and name_filter: raise PyAirbyteInputError(message="You can provide name or name_filter, but not both.") _validate_pagination_params(limit=limit) @@ -555,7 +555,7 @@ def list_workspaces( name_filter: Callable[[str], bool] | None = None, limit: int | None = None, ) -> list[models.WorkspaceResponse]: - """List workspaces.""" + """List workspaces.""" # noqa: DOC501 if name is not None and name_filter: raise PyAirbyteInputError(message="You can provide name or name_filter, but not both.") _validate_pagination_params(limit=limit) @@ -623,7 +623,7 @@ def list_sources( name_filter: Callable[[str], bool] | None = None, limit: int | None = None, ) -> list[models.SourceResponse]: - """List sources.""" + """List sources.""" # noqa: DOC501 if name is not None and name_filter: raise PyAirbyteInputError(message="You can provide name or name_filter, but not both.") _validate_pagination_params(limit=limit) @@ -690,7 +690,7 @@ def list_destinations( name_filter: Callable[[str], bool] | None = None, limit: int | None = None, ) -> list[models.DestinationResponse]: - """List destinations.""" + """List destinations.""" # noqa: DOC501 if name is not None and name_filter: raise PyAirbyteInputError(message="You can provide name or name_filter, but not both.") _validate_pagination_params(limit=limit) @@ -762,7 +762,7 @@ def get_connection( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> models.ConnectionResponse: - """Get a connection.""" + """Get a connection.""" # noqa: DOC501 _ = workspace_id # Not used (yet) airbyte_instance = get_airbyte_server_instance( client_id=client_id, @@ -812,7 +812,7 @@ def run_connection( If block is True, this will block until the connection is finished running. If raise_on_failure is True, this will raise an exception if the connection fails. - """ + """ # noqa: DOC501 _ = workspace_id # Not used (yet) airbyte_instance = get_airbyte_server_instance( client_id=client_id, @@ -876,7 +876,7 @@ def get_job_logs( # noqa: PLR0913 # Too many arguments - needed for auth flexi Returns: A list of JobResponse objects. - """ + """ # noqa: DOC501 _validate_pagination_params(limit=limit) airbyte_instance = get_airbyte_server_instance( client_id=client_id, @@ -974,7 +974,7 @@ def get_job_info( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> models.JobResponse: - """Get a job.""" + """Get a job.""" # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( client_id=client_id, client_secret=client_secret, @@ -1017,7 +1017,7 @@ def create_source( """Create a source connector instance. Either `definition_id` or `config[sourceType]` must be provided. - """ + """ # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( client_id=client_id, client_secret=client_secret, @@ -1054,7 +1054,7 @@ def get_source( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> models.SourceResponse: - """Get a connection.""" + """Get a connection.""" # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( client_id=client_id, client_secret=client_secret, @@ -1108,7 +1108,7 @@ def delete_source( Raises: PyAirbyteInputError: If safe_mode is True and the source name does not meet the safety requirements. - """ + """ # noqa: DOC501 _ = workspace_id # Not used (yet) if safe_mode: @@ -1184,7 +1184,7 @@ def patch_source( Returns: Updated SourceResponse object - """ + """ # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( client_id=client_id, client_secret=client_secret, @@ -1249,7 +1249,7 @@ def create_destination( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> models.DestinationResponse: - """Get a connection.""" + """Get a connection.""" # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( client_id=client_id, client_secret=client_secret, @@ -1290,7 +1290,7 @@ def get_destination( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> models.DestinationResponse: - """Get a connection.""" + """Get a connection.""" # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( client_id=client_id, client_secret=client_secret, @@ -1362,7 +1362,7 @@ def delete_destination( Raises: PyAirbyteInputError: If safe_mode is True and the destination name does not meet the safety requirements. - """ + """ # noqa: DOC501 _ = workspace_id # Not used (yet) if safe_mode: @@ -1439,7 +1439,7 @@ def patch_destination( Returns: Updated DestinationResponse object - """ + """ # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( client_id=client_id, client_secret=client_secret, @@ -1555,7 +1555,7 @@ def get_connection_by_name( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> models.ConnectionResponse: - """Get a connection.""" + """Get a connection.""" # noqa: DOC501 connections = list_connections( workspace_id=workspace_id, api_root=api_root, @@ -1626,7 +1626,7 @@ def delete_connection( Raises: PyAirbyteInputError: If safe_mode is True and the connection name does not meet the safety requirements. - """ + """ # noqa: DOC501 if safe_mode: if connection_name is None: connection_info = get_connection( @@ -1709,7 +1709,7 @@ def patch_connection( # noqa: PLR0913 # Too many arguments Returns: Updated ConnectionResponse object - """ + """ # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( client_id=client_id, client_secret=client_secret, @@ -1863,7 +1863,7 @@ def check_connector( - /v1/sources/check_connection: https://github.com/airbytehq/airbyte-platform-internal/blob/10bb92e1745a282e785eedfcbed1ba72654c4e4e/oss/airbyte-api/server-api/src/main/openapi/config.yaml#L1409 - /v1/destinations/check_connection: https://github.com/airbytehq/airbyte-platform-internal/blob/10bb92e1745a282e785eedfcbed1ba72654c4e4e/oss/airbyte-api/server-api/src/main/openapi/config.yaml#L1995 - """ + """ # noqa: DOC501 _ = workspace_id # Not used (yet) json_result = _make_config_api_request( @@ -1909,7 +1909,7 @@ def validate_yaml_manifest( Returns: Tuple of (is_valid, error_message) - """ + """ # noqa: DOC501 if not isinstance(manifest, dict): error = "Manifest must be a dictionary" if raise_on_error: @@ -1943,7 +1943,7 @@ def create_custom_yaml_source_definition( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> models.DeclarativeSourceDefinitionResponse: - """Create a custom YAML source definition.""" + """Create a custom YAML source definition.""" # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( api_root=api_root, client_id=client_id, @@ -1978,7 +1978,7 @@ def list_custom_yaml_source_definitions( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> list[models.DeclarativeSourceDefinitionResponse]: - """List all custom YAML source definitions in a workspace.""" + """List all custom YAML source definitions in a workspace.""" # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( api_root=api_root, client_id=client_id, @@ -2016,7 +2016,7 @@ def get_custom_yaml_source_definition( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> models.DeclarativeSourceDefinitionResponse: - """Get a specific custom YAML source definition.""" + """Get a specific custom YAML source definition.""" # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( api_root=api_root, client_id=client_id, @@ -2057,7 +2057,7 @@ def update_custom_yaml_source_definition( client_secret: SecretString | None, bearer_token: SecretString | None, ) -> models.DeclarativeSourceDefinitionResponse: - """Update a custom YAML source definition.""" + """Update a custom YAML source definition.""" # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( api_root=api_root, client_id=client_id, @@ -2117,7 +2117,7 @@ def delete_custom_yaml_source_definition( Raises: PyAirbyteInputError: If safe_mode is True and the connector name does not meet the safety requirements. - """ + """ # noqa: DOC501 if safe_mode: definition_info = get_custom_yaml_source_definition( workspace_id=workspace_id, @@ -2335,7 +2335,7 @@ def list_organizations_for_user( Returns: List of OrganizationResponse objects containing organization_id, organization_name, email - """ + """ # noqa: DOC501 airbyte_instance = get_airbyte_server_instance( api_root=api_root, client_id=client_id, @@ -2550,7 +2550,7 @@ def replace_connection_state( Returns: Dictionary containing the updated ConnectionState object. - """ + """ # noqa: DOC501 try: return _make_config_api_request( path="/state/create_or_update_safe", diff --git a/airbyte/_util/destination_smoke_tests.py b/airbyte/_util/destination_smoke_tests.py index 8990d6021..6905483a2 100644 --- a/airbyte/_util/destination_smoke_tests.py +++ b/airbyte/_util/destination_smoke_tests.py @@ -158,7 +158,7 @@ def get_smoke_test_source( `custom_scenarios_file` is an optional path to a JSON or YAML file containing additional scenario definitions. Each scenario should have `name`, `json_schema`, and optionally `records` and `primary_key`. - """ + """ # noqa: DOC501 # Normalize empty list to "fast" (default) if isinstance(scenarios, list) and not scenarios: scenarios = "fast" diff --git a/airbyte/_util/name_normalizers.py b/airbyte/_util/name_normalizers.py index ad615a572..ee58c0ef7 100644 --- a/airbyte/_util/name_normalizers.py +++ b/airbyte/_util/name_normalizers.py @@ -66,7 +66,7 @@ def normalize(name: str) -> str: # pyrefly: ignore[bad-override] # pyrefly dec - "Average Sales (#)" -> "average_sales____" - "+1" -> "_1" - "-1" -> "_1" - """ + """ # noqa: DOC501 result = name # Replace all non-alphanumeric characters with underscores. diff --git a/airbyte/_writers/file_writers.py b/airbyte/_writers/file_writers.py index d8d034544..2050b4ed5 100644 --- a/airbyte/_writers/file_writers.py +++ b/airbyte/_writers/file_writers.py @@ -167,7 +167,7 @@ def process_record_message( """Write a record to the cache. This method is called for each record message, before the batch is written. - """ + """ # noqa: DOC501 stream_name = record_msg.stream batch_handle: BatchHandle @@ -212,7 +212,7 @@ def _write_airbyte_message_stream( This is not implemented for file writers, as they should be wrapped by another writer that handles state tracking and other logic. - """ + """ # noqa: DOC501 _ = stdin, catalog_provider, write_strategy, state_writer, progress_tracker raise exc.PyAirbyteInternalError from NotImplementedError( "File writers should be wrapped by another AirbyteWriterInterface." diff --git a/airbyte/caches/_state_backend.py b/airbyte/caches/_state_backend.py index c61833d78..aa1f92ea2 100644 --- a/airbyte/caches/_state_backend.py +++ b/airbyte/caches/_state_backend.py @@ -211,7 +211,7 @@ def get_state_provider( refresh: bool = True, destination_name: str | None = None, ) -> StateProviderBase: - """Return the state provider.""" + """Return the state provider.""" # noqa: DOC501 if destination_name and table_prefix: raise PyAirbyteInputError( message="Both 'destination_name' and 'table_prefix' cannot be set at the same time." diff --git a/airbyte/caches/_utils/_cache_to_dest.py b/airbyte/caches/_utils/_cache_to_dest.py index 2826d92e2..5142af09b 100644 --- a/airbyte/caches/_utils/_cache_to_dest.py +++ b/airbyte/caches/_utils/_cache_to_dest.py @@ -37,7 +37,7 @@ def cache_to_destination_configuration( cache: CacheBase, ) -> api_util.DestinationConfiguration: - """Get the destination configuration from the cache.""" + """Get the destination configuration from the cache.""" # noqa: DOC501 conversion_fn_map: dict[str, Callable[[Any], api_util.DestinationConfiguration]] = { "BigQueryCache": bigquery_cache_to_destination_configuration, "bigquery": bigquery_cache_to_destination_configuration, diff --git a/airbyte/caches/_utils/_dest_to_cache.py b/airbyte/caches/_utils/_dest_to_cache.py index a5b4700dc..4dc73f874 100644 --- a/airbyte/caches/_utils/_dest_to_cache.py +++ b/airbyte/caches/_utils/_dest_to_cache.py @@ -55,7 +55,7 @@ def destination_to_cache( *, schema_name: str | None = None, ) -> CacheBase: - """Get the destination configuration from the cache.""" + """Get the destination configuration from the cache.""" # noqa: DOC501 conversion_fn_map: dict[str, Callable[[Any], CacheBase]] = { "bigquery": bigquery_destination_to_cache, "duckdb": duckdb_destination_to_cache, @@ -181,7 +181,7 @@ def duckdb_destination_to_cache( def motherduck_destination_to_cache( destination_configuration: DestinationDuckdb | dict[str, Any], ) -> MotherDuckCache: - """Create a new MotherDuck cache from the destination configuration.""" + """Create a new MotherDuck cache from the destination configuration.""" # noqa: DOC501 if isinstance(destination_configuration, dict): filtered = { k: v @@ -203,7 +203,7 @@ def motherduck_destination_to_cache( def postgres_destination_to_cache( destination_configuration: DestinationPostgres | dict[str, Any], ) -> PostgresCache: - """Create a new Postgres cache from the destination configuration.""" + """Create a new Postgres cache from the destination configuration.""" # noqa: DOC501 if isinstance(destination_configuration, dict): # Strip dispatch keys before constructing the model object. filtered = { @@ -235,7 +235,7 @@ def snowflake_destination_to_cache( We may have to inject credentials, because they are obfuscated when config is returned from the REST API. - """ + """ # noqa: DOC501 if isinstance(destination_configuration, dict): filtered = { k: v diff --git a/airbyte/caches/base.py b/airbyte/caches/base.py index 2683577d4..5732d6498 100644 --- a/airbyte/caches/base.py +++ b/airbyte/caches/base.py @@ -209,7 +209,7 @@ def run_sql_query( Returns: List of dictionaries representing the query results - """ + """ # noqa: DOC501 # Execute the SQL within a connection context to ensure the connection stays open # while we fetch the results sql_text = text(sql_query) if isinstance(sql_query, str) else sql_query diff --git a/airbyte/caches/util.py b/airbyte/caches/util.py index 79b820137..5b5ab87df 100644 --- a/airbyte/caches/util.py +++ b/airbyte/caches/util.py @@ -55,7 +55,7 @@ def new_local_cache( Cache files are stored in the `.cache` directory, relative to the current working directory. - """ + """ # noqa: DOC501 if cache_name: if " " in cache_name: raise exc.PyAirbyteInputError( @@ -135,7 +135,7 @@ def get_colab_cache( drive_name="My Company Drive", ) ``` - """ + """ # noqa: DOC501 try: from google.colab import drive # noqa: PLC0415 # type: ignore[reportMissingImports] except ImportError: diff --git a/airbyte/cli/pyab.py b/airbyte/cli/pyab.py index 04173d39d..4afac97d0 100644 --- a/airbyte/cli/pyab.py +++ b/airbyte/cli/pyab.py @@ -138,7 +138,7 @@ def _resolve_config( config: str, ) -> dict[str, Any]: - """Resolve the configuration file into a dictionary.""" + """Resolve the configuration file into a dictionary.""" # noqa: DOC501 def _inject_secrets(config_dict: dict[str, Any]) -> None: """Inject secrets into the configuration dictionary.""" @@ -231,7 +231,7 @@ def _resolve_source_job( all streams will be selected. If not provided, all streams will be selected. pip_url: Optional. A location from which to install the connector. use_python: Optional. Python interpreter specification. - """ + """ # noqa: DOC501 config_dict = _resolve_config(config) if config else None streams_list: str | list[str] = streams or "*" if isinstance(streams, str) and streams != "*": @@ -313,7 +313,7 @@ def _resolve_destination_job( config: The path to a configuration file for the named source or destination. pip_url: Optional. A location from which to install the connector. use_python: Optional. Python interpreter specification. - """ + """ # noqa: DOC501 config_dict = _resolve_config(config) if config else {} use_python_parsed = _parse_use_python(use_python) @@ -397,7 +397,7 @@ def validate( If 'config' is provided, we will also run a `check` on the connector with the provided config. - """ + """ # noqa: DOC501 if not connector: raise PyAirbyteInputError( message="No connector provided.", @@ -496,7 +496,7 @@ def benchmark( If a source is being benchmarked, you can provide a configuration file or a job definition file to run the source job. - """ + """ # noqa: DOC501 if source and destination: raise PyAirbyteInputError( message="For benchmarking, source or destination can be provided, but not both.", diff --git a/airbyte/cloud/_credentials.py b/airbyte/cloud/_credentials.py index a842f6c73..2330e9e04 100644 --- a/airbyte/cloud/_credentials.py +++ b/airbyte/cloud/_credentials.py @@ -58,7 +58,7 @@ def from_auth( When `env_vars` is True (default), environment variables are checked as a fallback after explicit inputs. - """ + """ # noqa: DOC501 resolved_bearer_token = _first_value( str(bearer_token) if bearer_token is not None else None, _env_value(BEARER_TOKEN_ENV_VAR, CLOUD_BEARER_TOKEN_ENV_VAR) if env_vars else None, diff --git a/airbyte/cloud/client.py b/airbyte/cloud/client.py index ced3c7ba7..8826faa34 100644 --- a/airbyte/cloud/client.py +++ b/airbyte/cloud/client.py @@ -122,7 +122,7 @@ def _from_credentials(cls, credentials: _AirbyteCredentials) -> CloudClient: ) def get_workspace(self, workspace_id: str | None = None) -> CloudWorkspace: - """Create a `CloudWorkspace` using this client's credentials.""" + """Create a `CloudWorkspace` using this client's credentials.""" # noqa: DOC501 resolved_workspace_id = workspace_id or self._credentials.workspace_id if not resolved_workspace_id: raise exc.PyAirbyteInputError( @@ -233,7 +233,7 @@ def list_workspaces( name_filter: Callable[[str], bool] | None = None, limit: int | None = None, ) -> list[CloudWorkspaceInfo]: - """List workspaces available to this client.""" + """List workspaces available to this client.""" # noqa: DOC501 if organization_id is not None or self.organization_id is not None: resolved_organization_id = organization_id or self.organization_id if not resolved_organization_id: @@ -291,7 +291,7 @@ def get_organization( *, organization_name: str | None = None, ) -> CloudOrganization: - """Resolve an organization by ID or exact name.""" + """Resolve an organization by ID or exact name.""" # noqa: DOC501 resolved_organization_id = organization_id or self.organization_id if resolved_organization_id and organization_name: raise exc.PyAirbyteInputError( diff --git a/airbyte/cloud/client_config.py b/airbyte/cloud/client_config.py index 30755704a..fa70a440b 100644 --- a/airbyte/cloud/client_config.py +++ b/airbyte/cloud/client_config.py @@ -93,7 +93,7 @@ class CloudClientConfig: """The Config API root URL.""" def __post_init__(self) -> None: - """Validate credentials and ensure secrets are properly wrapped.""" + """Validate credentials and ensure secrets are properly wrapped.""" # noqa: DOC501 # Wrap secrets in SecretString if they aren't already if self.client_id is not None: self.client_id = SecretString(self.client_id) diff --git a/airbyte/cloud/connections.py b/airbyte/cloud/connections.py index 528b91de9..708a555e1 100644 --- a/airbyte/cloud/connections.py +++ b/airbyte/cloud/connections.py @@ -477,7 +477,7 @@ def import_raw_state( Raises: AirbyteConnectionSyncActiveError: If a sync is currently running on this connection (HTTP 423). Wait for the sync to complete before retrying. - """ + """ # noqa: DOC501 api_state: dict[str, Any] if isinstance(connection_state, list): if not _is_protocol_state_format(connection_state): diff --git a/airbyte/cloud/connectors.py b/airbyte/cloud/connectors.py index 622e57ac9..ce803efaf 100644 --- a/airbyte/cloud/connectors.py +++ b/airbyte/cloud/connectors.py @@ -162,7 +162,7 @@ def check( A `CheckResult` object containing the result. The object is truthy if the check was successful and falsy otherwise. The error message is available in the `error_message` or by converting the object to a string. - """ + """ # noqa: DOC501 result = api_util.check_connector( workspace_id=self.workspace.workspace_id, connector_type=self.connector_type, @@ -794,7 +794,7 @@ def deploy_source( Returns: A `CloudSource` object representing the newly created source. - """ + """ # noqa: DOC501 if self.definition_type != "yaml": raise NotImplementedError( "Only YAML custom source definitions can be used to deploy new sources. " diff --git a/airbyte/cloud/sync_results.py b/airbyte/cloud/sync_results.py index e2a88729d..e631b52e3 100644 --- a/airbyte/cloud/sync_results.py +++ b/airbyte/cloud/sync_results.py @@ -171,7 +171,7 @@ def created_at(self) -> datetime: return ab_datetime_parse(timestamp) def _get_attempt_data(self) -> dict[str, Any]: - """Get attempt data from the provided attempt data.""" + """Get attempt data from the provided attempt data.""" # noqa: DOC501 if self._attempt_data is None: raise ValueError( "Attempt data not provided. SyncAttempt should be created via " @@ -309,7 +309,7 @@ def records_synced(self) -> int: @property def start_time(self) -> datetime: - """Return the start time of the sync job in UTC.""" + """Return the start time of the sync job in UTC.""" # noqa: DOC501 try: return ab_datetime_parse(self._fetch_latest_job_info().start_time) except (ValueError, TypeError) as e: @@ -374,7 +374,7 @@ def raise_failure_status( method will raise a `AirbyteConnectionSyncError`. Otherwise, do nothing. - """ + """ # noqa: DOC501 if not refresh_status and self._latest_job_info: latest_status = self._latest_job_info.status else: @@ -395,7 +395,7 @@ def wait_for_completion( raise_timeout: bool = True, raise_failure: bool = False, ) -> JobStatusEnum: - """Wait for a job to finish running.""" + """Wait for a job to finish running.""" # noqa: DOC501 start_time = time.time() while True: latest_status = self.get_job_status() diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index 58f5539e7..cc344e95e 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -117,7 +117,7 @@ def __init__( config_api_root: str | None = None, bearer_token: str | SecretString | None = None, ) -> None: - """Validate and initialize credentials.""" + """Validate and initialize credentials.""" # noqa: DOC501 env_vars = not (client_id or client_secret or bearer_token) credentials = _AirbyteCredentials.from_auth( workspace_id=workspace_id, @@ -383,7 +383,7 @@ def deploy_source( unique: Whether to require a unique name. If `True`, duplicate names are not allowed. Defaults to `True`. random_name_suffix: Whether to append a random suffix to the name. - """ + """ # noqa: DOC501 source_config_dict = source._hydrated_config.copy() # noqa: SLF001 (non-public API) source_config_dict["sourceType"] = source.name.replace("source-", "") @@ -431,7 +431,7 @@ def deploy_destination( unique: Whether to require a unique name. If `True`, duplicate names are not allowed. Defaults to `True`. random_name_suffix: Whether to append a random suffix to the name. - """ + """ # noqa: DOC501 if isinstance(destination, Destination): destination_conf_dict = destination._hydrated_config.copy() # noqa: SLF001 (non-public API) destination_conf_dict["destinationType"] = destination.name.replace("destination-", "") @@ -482,7 +482,7 @@ def permanently_delete_source( source: The source ID or CloudSource object to delete safe_mode: If True, requires the source name to contain "delete-me" or "deleteme" (case insensitive) to prevent accidental deletion. Defaults to True. - """ + """ # noqa: DOC501 if not isinstance(source, (str, CloudSource)): raise exc.PyAirbyteInputError( message="Invalid source type.", @@ -515,7 +515,7 @@ def permanently_delete_destination( destination: The destination ID or CloudDestination object to delete safe_mode: If True, requires the destination name to contain "delete-me" or "deleteme" (case insensitive) to prevent accidental deletion. Defaults to True. - """ + """ # noqa: DOC501 if not isinstance(destination, (str, CloudDestination)): raise exc.PyAirbyteInputError( message="Invalid destination type.", @@ -558,7 +558,7 @@ def deploy_connection( CloudDestination object. table_prefix: Optional. The table prefix to use when syncing to the destination. selected_streams: The selected stream names to sync within the connection. - """ + """ # noqa: DOC501 if not selected_streams: raise exc.PyAirbyteInputError( guidance="You must provide `selected_streams` when creating a connection." @@ -607,7 +607,7 @@ def permanently_delete_connection( safe_mode: If True, requires the connection name to contain "delete-me" or "deleteme" (case insensitive) to prevent accidental deletion. Defaults to True. Also applies to cascade deletes. - """ + """ # noqa: DOC501 if connection is None: raise ValueError("No connection ID provided.") diff --git a/airbyte/destinations/base.py b/airbyte/destinations/base.py index 0c540ffd7..44a53ddaf 100644 --- a/airbyte/destinations/base.py +++ b/airbyte/destinations/base.py @@ -157,7 +157,7 @@ def write( # noqa: PLR0912, PLR0915 # Too many arguments/statements If the cache has tracked state, this will be used for the sync. Otherwise, if there is a known destination state, the destination-specific state will be used. If neither are available, a full refresh will be performed. - """ + """ # noqa: DOC501 if not isinstance(source_data, ReadResult | Source): raise exc.PyAirbyteInputError( message="Invalid source_data type for `source_data` arg.", @@ -316,7 +316,7 @@ def _write_airbyte_message_stream( state_writer: StateWriterBase | None = None, progress_tracker: ProgressTracker, ) -> None: - """Read from the connector and write to the cache.""" + """Read from the connector and write to the cache.""" # noqa: DOC501 # Run optional validation step if state_writer is None: state_writer = StdOutStateWriter() diff --git a/airbyte/mcp/_arg_resolvers.py b/airbyte/mcp/_arg_resolvers.py index cb4f8df5c..5118fbcb1 100644 --- a/airbyte/mcp/_arg_resolvers.py +++ b/airbyte/mcp/_arg_resolvers.py @@ -40,7 +40,7 @@ def resolve_list_of_strings(value: str | list[str] | set[str] | None) -> list[st Args: value: A string or list of strings. - """ + """ # noqa: DOC501 if value is None: return None @@ -88,7 +88,7 @@ def resolve_connector_config( # noqa: PLR0912 ValueError: If JSON parsing fails or a provided input is invalid We reject hardcoded secrets in a config dict if we detect them. - """ + """ # noqa: DOC501 config_dict: dict[str, Any] = {} if config is None and config_file is None and config_secret_name is None: diff --git a/airbyte/mcp/_config.py b/airbyte/mcp/_config.py index 00925a0af..f4b2cc793 100644 --- a/airbyte/mcp/_config.py +++ b/airbyte/mcp/_config.py @@ -23,7 +23,7 @@ def _load_dotenv_file(dotenv_path: Path | str) -> None: - """Load environment variables from a .env file.""" + """Load environment variables from a .env file.""" # noqa: DOC501 if isinstance(dotenv_path, str): dotenv_path = Path(dotenv_path) if not dotenv_path.exists(): diff --git a/airbyte/mcp/_tool_utils.py b/airbyte/mcp/_tool_utils.py index bead0fdca..4796af218 100644 --- a/airbyte/mcp/_tool_utils.py +++ b/airbyte/mcp/_tool_utils.py @@ -116,7 +116,7 @@ def check_guid_created_in_session(guid: str) -> None: Args: guid: The GUID to check - """ + """ # noqa: DOC501 if AIRBYTE_CLOUD_MCP_SAFE_MODE and guid not in _GUIDS_CREATED_IN_SESSION: raise SafeModeError( f"Cannot perform destructive operation on '{guid}': " diff --git a/airbyte/mcp/cloud.py b/airbyte/mcp/cloud.py index 0beab1b41..217307dc7 100644 --- a/airbyte/mcp/cloud.py +++ b/airbyte/mcp/cloud.py @@ -256,7 +256,7 @@ def _get_cloud_workspace( The ctx parameter provides access to MCP config values that are resolved from HTTP headers or environment variables based on the config args defined in server.py. - """ + """ # noqa: DOC501 resolved_workspace_id = workspace_id or get_mcp_config(ctx, MCP_CONFIG_WORKSPACE_ID) if not resolved_workspace_id: raise PyAirbyteInputError( @@ -1077,7 +1077,7 @@ def get_cloud_sync_logs( ), ], ) -> LogReadResult: - """Get the logs from a sync job attempt on Airbyte Cloud.""" + """Get the logs from a sync job attempt on Airbyte Cloud.""" # noqa: DOC501 # Validate that line_offset and from_tail are not both set if line_offset is not None and from_tail: raise PyAirbyteInputError( @@ -1752,7 +1752,7 @@ def update_custom_source_definition( Updates the manifest and/or testing values for an existing custom source definition. At least one of manifest_yaml, testing_values, or testing_values_secret_name must be provided. - """ + """ # noqa: DOC501 check_guid_created_in_session(definition_id) workspace: CloudWorkspace = _get_cloud_workspace(ctx, workspace_id) @@ -1844,7 +1844,7 @@ def permanently_delete_custom_source_definition( Note: Only YAML (declarative) connectors are currently supported. Docker-based custom sources are not yet available. - """ + """ # noqa: DOC501 check_guid_created_in_session(definition_id) workspace: CloudWorkspace = _get_cloud_workspace(ctx, workspace_id) definition = workspace.get_custom_source_definition( @@ -1901,7 +1901,7 @@ def permanently_delete_cloud_source( The provided name must match the actual name of the source for the operation to proceed. This is a safety measure to ensure you are deleting the correct resource. - """ + """ # noqa: DOC501 check_guid_created_in_session(source_id) workspace: CloudWorkspace = _get_cloud_workspace(ctx) source = workspace.get_source(source_id=source_id) @@ -1957,7 +1957,7 @@ def permanently_delete_cloud_destination( The provided name must match the actual name of the destination for the operation to proceed. This is a safety measure to ensure you are deleting the correct resource. - """ + """ # noqa: DOC501 check_guid_created_in_session(destination_id) workspace: CloudWorkspace = _get_cloud_workspace(ctx) destination = workspace.get_destination(destination_id=destination_id) @@ -2032,7 +2032,7 @@ def permanently_delete_cloud_connection( The provided name must match the actual name of the connection for the operation to proceed. This is a safety measure to ensure you are deleting the correct resource. - """ + """ # noqa: DOC501 check_guid_created_in_session(connection_id) workspace: CloudWorkspace = _get_cloud_workspace(ctx) connection = workspace.get_connection(connection_id=connection_id) @@ -2417,7 +2417,7 @@ def update_cloud_connection( At least one setting must be provided. The 'cron_expression' and 'manual_schedule' parameters are mutually exclusive. - """ + """ # noqa: DOC501 check_guid_created_in_session(connection_id) # Validate that at least one setting is provided diff --git a/airbyte/mcp/interactive/_registry_ui.py b/airbyte/mcp/interactive/_registry_ui.py index 54969e95f..e148612b5 100644 --- a/airbyte/mcp/interactive/_registry_ui.py +++ b/airbyte/mcp/interactive/_registry_ui.py @@ -140,7 +140,7 @@ def show_connectors_list( ), ] = 0, ) -> ToolResult: - """Show an interactive public connector catalog from the OSS registry.""" + """Show an interactive public connector catalog from the OSS registry.""" # noqa: DOC501 if limit < 0: raise exc.PyAirbyteInputError( message="Limit parameter must be non-negative.", diff --git a/airbyte/mcp/interactive/_shared_models.py b/airbyte/mcp/interactive/_shared_models.py index 90a0f22d1..7b2f85665 100644 --- a/airbyte/mcp/interactive/_shared_models.py +++ b/airbyte/mcp/interactive/_shared_models.py @@ -22,7 +22,7 @@ def precedence(self) -> int: @classmethod def parse(cls, value: str) -> SupportLevel: - """Parse a support level keyword or legacy integer precedence value.""" + """Parse a support level keyword or legacy integer precedence value.""" # noqa: DOC501 try: return cls(value) except ValueError: @@ -52,7 +52,7 @@ class ConnectorType(str, Enum): @classmethod def parse(cls, value: str) -> ConnectorType: - """Parse a connector type value.""" + """Parse a connector type value.""" # noqa: DOC501 try: return cls(value) except ValueError: diff --git a/airbyte/mcp/local.py b/airbyte/mcp/local.py index 0e6fab984..05c98bc66 100644 --- a/airbyte/mcp/local.py +++ b/airbyte/mcp/local.py @@ -72,7 +72,7 @@ def _get_mcp_source( install_if_missing: bool = True, manifest_path: str | Path | None, ) -> Source: - """Get the MCP source for a connector.""" + """Get the MCP source for a connector.""" # noqa: DOC501 if manifest_path: override_execution_mode = "yaml" elif override_execution_mode == "auto" and is_docker_installed(): diff --git a/airbyte/records.py b/airbyte/records.py index e28109088..fe284b09a 100644 --- a/airbyte/records.py +++ b/airbyte/records.py @@ -275,7 +275,7 @@ def __setitem__(self, key: str, value: Any) -> None: # noqa: ANN401 super().__setitem__(index_case_key, value) def __delitem__(self, key: str) -> None: - """Delete the item with the given key.""" + """Delete the item with the given key.""" # noqa: DOC501 try: super().__delitem__(key) except KeyError: diff --git a/airbyte/registry.py b/airbyte/registry.py index 1d6f5745b..959e4658d 100644 --- a/airbyte/registry.py +++ b/airbyte/registry.py @@ -263,7 +263,7 @@ def get_connector_metadata(name: str) -> ConnectorMetadata | None: """Check the cache for the connector. If the cache is empty, populate by calling update_cache. - """ + """ # noqa: DOC501 registry_url = _get_registry_url() if _is_registry_disabled(registry_url): @@ -299,7 +299,7 @@ def get_available_connectors( Args: install_type: The type of installation for the connector. Defaults to `InstallType.INSTALLABLE`. - """ + """ # noqa: DOC501 if install_type is None or install_type == InstallType.INSTALLABLE: # Filter for installable connectors (default behavior). if is_docker_installed(): diff --git a/airbyte/results.py b/airbyte/results.py index 687573047..d00fd2b6c 100644 --- a/airbyte/results.py +++ b/airbyte/results.py @@ -56,7 +56,7 @@ def __init__( self._processed_streams = processed_streams def __getitem__(self, stream: str) -> CachedDataset: - """Return the cached dataset for a given stream name.""" + """Return the cached dataset for a given stream name.""" # noqa: DOC501 if stream not in self._processed_streams: raise KeyError(stream) diff --git a/airbyte/secrets/base.py b/airbyte/secrets/base.py index aa0ba911e..052a5e3ed 100644 --- a/airbyte/secrets/base.py +++ b/airbyte/secrets/base.py @@ -86,7 +86,7 @@ def __bool__(self) -> bool: return True def parse_json(self) -> dict: - """Parse the secret string as JSON.""" + """Parse the secret string as JSON.""" # noqa: DOC501 try: return json.loads(self) except json.JSONDecodeError as ex: @@ -107,7 +107,7 @@ def validate( v: Any, # noqa: ANN401 # Must allow `Any` to match Pydantic signature info: ValidationInfo, ) -> SecretString: - """Validate the input value is valid as a secret string.""" + """Validate the input value is valid as a secret string.""" # noqa: DOC501 _ = info # Unused if not isinstance(v, str): raise exc.PyAirbyteInputError( diff --git a/airbyte/secrets/google_gsm.py b/airbyte/secrets/google_gsm.py index 8228793e1..68d65fd63 100644 --- a/airbyte/secrets/google_gsm.py +++ b/airbyte/secrets/google_gsm.py @@ -107,7 +107,7 @@ def __init__( You can provide either the path to the credentials file or the JSON contents of the credentials file. If both are provided, a `PyAirbyteInputError` will be raised. - """ + """ # noqa: DOC501 if credentials_path and credentials_json: raise exc.PyAirbyteInputError( guidance=("You can provide `credentials_path` or `credentials_json` but not both."), @@ -283,7 +283,7 @@ def fetch_connector_secret( Returns: GSMSecretHandle: A handle for the matching secret. - """ + """ # noqa: DOC501 results: Iterable[GSMSecretHandle] = self.fetch_connector_secrets(connector_name) try: result = next(iter(results)) diff --git a/airbyte/secrets/hydration.py b/airbyte/secrets/hydration.py index 9945da550..42b90627b 100644 --- a/airbyte/secrets/hydration.py +++ b/airbyte/secrets/hydration.py @@ -91,7 +91,7 @@ def _walk_dict( @lru_cache def _get_global_secrets_mask() -> list[str]: - """Get the list of properties to mask from the spec mask file.""" + """Get the list of properties to mask from the spec mask file.""" # noqa: DOC501 if AIRBYTE_OFFLINE_MODE: # In offline mode, we cannot fetch the global mask keys. # We return an empty list to avoid masking any keys. diff --git a/airbyte/shared/catalog_providers.py b/airbyte/shared/catalog_providers.py index 5c9c0a810..a3c486e70 100644 --- a/airbyte/shared/catalog_providers.py +++ b/airbyte/shared/catalog_providers.py @@ -82,7 +82,7 @@ def get_configured_stream_info( self, stream_name: str, ) -> ConfiguredAirbyteStream: - """Return the column definitions for the given stream.""" + """Return the column definitions for the given stream.""" # noqa: DOC501 if not self.configured_catalog: raise exc.PyAirbyteInternalError( message="Cannot get stream JSON schema without a catalog.", @@ -147,7 +147,7 @@ def get_primary_keys( self, stream_name: str, ) -> list[str]: - """Return the primary keys for the given stream.""" + """Return the primary keys for the given stream.""" # noqa: DOC501 pks = self.get_configured_stream_info(stream_name).primary_key if not pks: return [] @@ -185,7 +185,7 @@ def resolve_write_method( stream_name: str, write_strategy: WriteStrategy, ) -> WriteMethod: - """Return the write method for the given stream.""" + """Return the write method for the given stream.""" # noqa: DOC501 has_pks: bool = bool(self.get_primary_keys(stream_name)) has_incremental_key: bool = bool(self.get_cursor_key(stream_name)) if write_strategy == WriteStrategy.MERGE and not has_pks: diff --git a/airbyte/shared/sql_processor.py b/airbyte/shared/sql_processor.py index add3e6592..ba981b800 100644 --- a/airbyte/shared/sql_processor.py +++ b/airbyte/shared/sql_processor.py @@ -231,7 +231,7 @@ def __init__( temp_dir: Path | None = None, temp_file_cleanup: bool, ) -> None: - """Create a new SQL processor.""" + """Create a new SQL processor.""" # noqa: DOC501 if not temp_dir and not file_writer: raise exc.PyAirbyteInternalError( message="Either `temp_dir` or `file_writer` must be provided.", @@ -313,7 +313,7 @@ def process_airbyte_messages( """Process a stream of Airbyte messages. This method assumes that the catalog is already registered with the processor. - """ + """ # noqa: DOC501 if not isinstance(write_strategy, WriteStrategy): raise exc.AirbyteInternalError( # pyrefly: ignore[missing-attribute] message="Invalid `write_strategy` argument. Expected instance of WriteStrategy.", @@ -532,7 +532,7 @@ def _get_table_by_name( To prevent unnecessary round-trips to the database, the table is cached after the first query. To ignore the cache and force a refresh, set 'force_refresh' to True. - """ + """ # noqa: DOC501 if force_refresh and shallow_okay: raise exc.PyAirbyteInternalError( message="Cannot force refresh and use shallow query at the same time." @@ -763,7 +763,7 @@ def write_stream_data( TODO: Add a dedupe step here to remove duplicates from the temp table. Some sources will send us duplicate records within the same stream, although this is a fairly rare edge case we can ignore in V1. - """ + """ # noqa: DOC501 if write_method and write_strategy and write_strategy != WriteStrategy.AUTO: raise exc.PyAirbyteInternalError( message=( @@ -858,7 +858,7 @@ def finalizing_batches( self._finalized_state_messages[stream_name] += state_messages_to_finalize def _execute_sql(self, sql: str | TextClause | Executable) -> CursorResult: - """Execute the given SQL statement.""" + """Execute the given SQL statement.""" # noqa: DOC501 if isinstance(sql, str): sql = text(sql) @@ -894,7 +894,7 @@ def _write_files_to_new_table( This is a generic implementation, which can be overridden by subclasses to improve performance. - """ + """ # noqa: DOC501 temp_table_name = self._create_table_for_loading(stream_name, batch_id) for file_path in files: dataframe = pd.read_json(file_path, lines=True) @@ -988,7 +988,7 @@ def _write_temp_table_to_final_table( final_table_name: str, write_method: WriteMethod, ) -> None: - """Write the temp table into the final table using the provided write strategy.""" + """Write the temp table into the final table using the provided write strategy.""" # noqa: DOC501 if write_method == WriteMethod.REPLACE: # Note: No need to check for schema compatibility # here, because we are fully replacing the table. @@ -1068,7 +1068,7 @@ def _swap_temp_table_with_final_table( This implementation requires MERGE support in the SQL DB. Databases that do not support this syntax can override this method. - """ + """ # noqa: DOC501 if final_table_name is None: raise exc.PyAirbyteInternalError(message="Arg 'final_table_name' cannot be None.") if temp_table_name is None: @@ -1130,7 +1130,7 @@ def _get_column_by_name(self, table: str | Table, column_name: str) -> Column: """Return the column object for the given column name. This method is case-insensitive. - """ + """ # noqa: DOC501 if isinstance(table, str): table = self._get_table_by_name(table) try: diff --git a/airbyte/shared/state_providers.py b/airbyte/shared/state_providers.py index a39e43152..3c78714e6 100644 --- a/airbyte/shared/state_providers.py +++ b/airbyte/shared/state_providers.py @@ -45,7 +45,7 @@ def stream_state_artifacts( This is just a type guard around the private variable `_stream_state_artifacts` and the cast to `AirbyteStreamState` objects. - """ + """ # noqa: DOC501 if self._state_message_artifacts is None: raise exc.PyAirbyteInternalError(message="No state artifacts were declared.") @@ -62,7 +62,7 @@ def state_message_artifacts( """Return all state artifacts. This is just a type guard around the private variable `_state_message_artifacts`. - """ + """ # noqa: DOC501 result = self._state_message_artifacts if result is None: raise exc.PyAirbyteInternalError(message="No state artifacts were declared.") @@ -98,7 +98,7 @@ def get_stream_state( stream_name: str, not_found: AirbyteStateMessage | Literal["raise"] | None = "raise", ) -> AirbyteStateMessage: - """Return the state message for the specified stream name.""" + """Return the state message for the specified stream name.""" # noqa: DOC501 for state_message in self.state_message_artifacts: if ( state_message.stream.stream_descriptor.name # pyrefly: ignore[missing-attribute] diff --git a/airbyte/sources/base.py b/airbyte/sources/base.py index 0a07950a6..8e1609cf6 100644 --- a/airbyte/sources/base.py +++ b/airbyte/sources/base.py @@ -247,7 +247,7 @@ def select_streams(self, streams: str | list[str]) -> None: streams: A list of stream names to select. If set to "*", all streams will be selected. Currently, if this is not set, all streams will be read. - """ + """ # noqa: DOC501 if self._config_dict is None: self._to_be_selected_streams = streams self._log_warning_preselected_stream(streams) @@ -308,7 +308,7 @@ def _discover(self) -> AirbyteCatalog: - execute the connector with discover --config - Listen to the messages and return the first AirbyteCatalog that comes along. - Make sure the subprocess is killed when the function returns. - """ + """ # noqa: DOC501 with as_temp_files([self._hydrated_config]) as [config_file]: for msg in self._execute(["discover", "--config", config_file]): if msg.type == Type.CATALOG and msg.catalog: @@ -338,7 +338,7 @@ def _get_spec(self, *, force_refresh: bool = False) -> ConnectorSpecification: * execute the connector with spec * Listen to the messages and return the first AirbyteCatalog that comes along. * Make sure the subprocess is killed when the function returns. - """ + """ # noqa: DOC501 if force_refresh or self._spec is None: for msg in self._execute(["spec"]): if msg.type == Type.SPEC and msg.spec: @@ -432,7 +432,7 @@ def get_configured_catalog( If force_full_refresh is True, streams will be configured with full_refresh sync mode when supported by the stream. Otherwise, incremental sync mode is used when supported. - """ + """ # noqa: DOC501 selected_streams: list[str] = [] if streams is None: selected_streams = self._selected_stream_names or self.get_available_streams() @@ -488,7 +488,7 @@ def _get_sync_mode(stream: AirbyteStream) -> SyncMode: ) def get_stream_json_schema(self, stream_name: str) -> dict[str, Any]: - """Return the JSON Schema spec for the specified stream name.""" + """Return the JSON Schema spec for the specified stream name.""" # noqa: DOC501 catalog: AirbyteCatalog = self.discovered_catalog found: list[AirbyteStream] = [ stream for stream in catalog.streams if stream.name == stream_name @@ -541,7 +541,7 @@ def get_records( * execute the connector with read --config --catalog * Listen to the messages and return the first AirbyteRecordMessages that come along. * Make sure the subprocess is killed when the function returns. - """ + """ # noqa: DOC501 stop_event = stop_event or threading.Event() configured_catalog = self.get_configured_catalog(streams=[stream]) if len(configured_catalog.streams) == 0: @@ -861,7 +861,7 @@ def read( running the connector. This can be helpful in debugging, when you want to send configurations to the connector that otherwise might be rejected by JSON Schema validation rules. - """ + """ # noqa: DOC501 cache = cache or get_default_cache() progress_tracker = ProgressTracker( source=self, @@ -928,7 +928,7 @@ def _read_to_cache( # noqa: PLR0913 # Too many arguments skip_validation: bool = False, progress_tracker: ProgressTracker, ) -> ReadResult: - """Internal read method.""" + """Internal read method.""" # noqa: DOC501 if write_strategy == WriteStrategy.REPLACE and not force_full_refresh: warnings.warn( message=( diff --git a/airbyte/sources/util.py b/airbyte/sources/util.py index 42372ed03..f334fce4c 100644 --- a/airbyte/sources/util.py +++ b/airbyte/sources/util.py @@ -161,7 +161,7 @@ def get_benchmark_source( Returns: Source: The source object for benchmarking. - """ + """ # noqa: DOC501 if isinstance(num_records, str): try: num_records = int(Decimal(num_records.replace("_", ""))) diff --git a/airbyte/strategies.py b/airbyte/strategies.py index e55b4d9a8..e14447ea6 100644 --- a/airbyte/strategies.py +++ b/airbyte/strategies.py @@ -78,7 +78,7 @@ class WriteMethod(str, Enum): @property def destination_sync_mode(self) -> DestinationSyncMode: - """Convert the write method to a destination sync mode.""" + """Convert the write method to a destination sync mode.""" # noqa: DOC501 if self == WriteMethod.MERGE: return DestinationSyncMode.append_dedup diff --git a/airbyte/types.py b/airbyte/types.py index 9ff8e997d..7c9958d12 100644 --- a/airbyte/types.py +++ b/airbyte/types.py @@ -40,7 +40,7 @@ def _get_airbyte_type( # noqa: PLR0911 # Too many return statements """Get the airbyte type and subtype from a JSON schema property definition. Subtype is only used for array types. Otherwise, subtype will return None. - """ + """ # noqa: DOC501 airbyte_type = cast("str", json_schema_property_def.get("airbyte_type", None)) if airbyte_type: return airbyte_type, None diff --git a/airbyte/validate.py b/airbyte/validate.py index 4324edd2a..201930884 100644 --- a/airbyte/validate.py +++ b/airbyte/validate.py @@ -60,7 +60,7 @@ def _run_subprocess_and_raise_on_failure(args: list[str]) -> None: def full_tests(connector_name: str, sample_config: str) -> None: - """Run full tests on the connector.""" + """Run full tests on the connector.""" # noqa: DOC501 print("Creating source and validating spec and version...") source = ab.get_source( connector_name, @@ -118,7 +118,7 @@ def run() -> None: def validate(connector_dir: str, sample_config: str, *, validate_install_only: bool) -> None: - """Validate a connector.""" + """Validate a connector.""" # noqa: DOC501 # read metadata.yaml metadata_path = Path(connector_dir) / "metadata.yaml" metadata = yaml.safe_load(Path(metadata_path).read_text(encoding="utf-8"))["data"] diff --git a/docs/generate.py b/docs/generate.py index 8990ebfb3..31dde84b3 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -39,7 +39,7 @@ def _regenerate_mcp_markdown() -> None: lives under `scripts/` (not on `sys.path`), and a static import would also trip `deptry` into flagging `generate_mcp_markdown` as a missing external dependency. - """ + """ # noqa: DOC501 script = pathlib.Path(__file__).parent.parent / "scripts" / "generate_mcp_markdown.py" if not script.exists(): print(f"[docs-generate] MCP markdown generator not found at {script}; skipping.") diff --git a/scripts/generate_mcp_markdown.py b/scripts/generate_mcp_markdown.py index 78ba75127..b9f62f7a9 100755 --- a/scripts/generate_mcp_markdown.py +++ b/scripts/generate_mcp_markdown.py @@ -101,7 +101,7 @@ def _run_fastmcp_inspect(server_spec: str, report_path: Path) -> dict[str, Any]: - """Invoke `fastmcp inspect` and return the parsed JSON report.""" + """Invoke `fastmcp inspect` and return the parsed JSON report.""" # noqa: DOC501 fastmcp_bin = shutil.which("fastmcp") if fastmcp_bin is None: raise RuntimeError( @@ -555,7 +555,7 @@ def _prepare_output_dir(output: Path) -> Path: footgun where preparing a resolved dir but writing via the raw `output` would silently target a different, non-existent path when cwd differs from the repo root. - """ + """ # noqa: DOC501 resolved = _resolve_output_dir(output) if resolved == _REPO_ROOT or not resolved.is_relative_to(_REPO_ROOT): raise RuntimeError( diff --git a/tests/conftest.py b/tests/conftest.py index eb96e4c9c..df3583ed5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -166,7 +166,7 @@ def new_postgres_db(): This fixture will start a new PostgreSQL container before the tests run and stop it after the tests are done. The host of the PostgreSQL database will be returned to the tests. - """ + """ # noqa: DOC501 client = docker.from_env() try: client.images.get(PYTEST_POSTGRES_IMAGE) diff --git a/tests/integration_tests/cloud/test_cloud_sync.py b/tests/integration_tests/cloud/test_cloud_sync.py index 3989c147f..eafcc0986 100644 --- a/tests/integration_tests/cloud/test_cloud_sync.py +++ b/tests/integration_tests/cloud/test_cloud_sync.py @@ -69,7 +69,7 @@ def test_deploy_and_run_connection( *, use_docker: bool, ) -> None: - """Test deploying a source and cache to a workspace as a new connection.""" + """Test deploying a source and cache to a workspace as a new connection.""" # noqa: DOC501 source = ab.get_source( "source-faker", config={"count": 100}, diff --git a/tests/integration_tests/test_source_faker_integration.py b/tests/integration_tests/test_source_faker_integration.py index e7b2da03e..bcf87de19 100644 --- a/tests/integration_tests/test_source_faker_integration.py +++ b/tests/integration_tests/test_source_faker_integration.py @@ -257,7 +257,7 @@ def test_merge_insert_not_supported_for_duckdb( duckdb_cache: DuckDBCache, mocker: pytest_mock.MockFixture, ) -> None: - """Confirm that duckdb does not support merge insert natively""" + """Confirm that duckdb does not support merge insert natively""" # noqa: DOC501 if DuckDBSqlProcessor.supports_merge_insert: return # Skip this test if the cache supports merge-insert. @@ -287,7 +287,7 @@ def test_merge_insert_not_supported_for_postgres( new_postgres_cache: PostgresCache, mocker: pytest_mock.MockFixture, ): - """Confirm that postgres does not support merge insert natively""" + """Confirm that postgres does not support merge insert natively""" # noqa: DOC501 # TODO - This test keeps getting skipped, investigate why. # It appears to be due to the fixture `new_postgres_cache` not detecting docker properly. if PostgresSqlProcessor.supports_merge_insert: