Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion airbyte/_connector_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,8 @@ def _execute(
"stream": "",
"data": {},
"emitted_at": 1234567890,
# "namespace": "", # We're knowingly omitting this to keep perf impact low.
# Omitting "namespace" key to keep perf impact low.
# "namespace": "", # noqa: ERA001
},
}
)
Expand Down
6 changes: 3 additions & 3 deletions airbyte/_util/document_rendering.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ class DocumentRenderer(BaseModel):

# TODO: Add primary key and cursor key support:
# https://github.com/airbytehq/pyairbyte/issues/319
# primary_key_properties: list[str]
# cursor_property: str | None
# primary_key_properties: list[str] # noqa: ERA001
# cursor_property: str | None # noqa: ERA001

def render_document(self, record: dict[str, Any]) -> Document:
"""Render a record as a document.
Expand Down Expand Up @@ -85,7 +85,7 @@ def render_document(self, record: dict[str, Any]) -> Document:
)

return Document(
# id=doc_id, # TODD: Add support for primary key and doc ID generation.
# id=doc_id, # TODD: Add support for primary key and doc ID generation. # noqa: ERA001
content=content,
metadata={key: record[key] for key in self.metadata_properties},
)
Expand Down
2 changes: 1 addition & 1 deletion airbyte/_util/temp_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def as_temp_files(files_contents: list[dict | str]) -> Generator[list[str], Any,
Path(temp_file.name).chmod(stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)

# Don't close the file yet (breaks Windows)
# temp_file.close()
# temp_file.close() # noqa: ERA001
temp_files.append(temp_file)
yield [file.name for file in temp_files]
finally:
Expand Down
2 changes: 1 addition & 1 deletion airbyte/caches/motherduck.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def get_sql_alchemy_url(self) -> SecretString:
return SecretString(
f"duckdb:///md:{self.database}?motherduck_token={self.api_key}"
# Not sure why this doesn't work. We have to override later in the flow.
# f"&schema={self.schema_name}"
# f"&schema={self.schema_name}" # noqa: ERA001
)

@overrides
Expand Down
2 changes: 1 addition & 1 deletion airbyte/cli/pyab.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,7 @@ def _resolve_destination_job(
use_python=use_python_parsed,
)

# else: # Treat the destination as a name.
# else: # Treat the destination as a name. # noqa: ERA001

return get_destination(
name=destination,
Expand Down
2 changes: 1 addition & 1 deletion airbyte/cloud/_connection_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ def _normalize_state_to_protocol(
}
]

# state_type == "stream"
# state_type == "stream" # noqa: ERA001
if not parsed.stream_state:
return []

Expand Down
2 changes: 1 addition & 1 deletion airbyte/cloud/workspaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -435,7 +435,7 @@ def deploy_destination(
if isinstance(destination, Destination):
destination_conf_dict = destination._hydrated_config.copy() # noqa: SLF001 (non-public API)
destination_conf_dict["destinationType"] = destination.name.replace("destination-", "")
# raise ValueError(destination_conf_dict)
# raise ValueError(destination_conf_dict) # noqa: ERA001
else:
destination_conf_dict = destination.copy()
if "destinationType" not in destination_conf_dict:
Expand Down
2 changes: 1 addition & 1 deletion airbyte/mcp/cloud.py
Original file line number Diff line number Diff line change
Expand Up @@ -2506,7 +2506,7 @@ def get_connection_artifact(
return {"ERROR": "No state is set for this connection (stateType: not_set)"}
return state

# artifact_type == "catalog"
# artifact_type == "catalog" # noqa: ERA001
catalog = connection.dump_raw_catalog()
if catalog is None:
return {"ERROR": "No catalog found for this connection"}
Expand Down
2 changes: 1 addition & 1 deletion airbyte/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ def default_install_type(self) -> InstallType:
if InstallType.PYTHON in self.install_types:
return InstallType.PYTHON

# Else: Java or Docker
# Else: Java or Docker # noqa: ERA001
return InstallType.DOCKER


Expand Down
2 changes: 1 addition & 1 deletion airbyte/sources/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ def get_benchmark_source(
return get_source(
name="source-e2e-test",
docker_image=True,
# docker_image="airbyte/source-e2e-test:latest",
# docker_image="airbyte/source-e2e-test:latest", # noqa: ERA001
config={
"type": "BENCHMARK",
"schema": "FIVE_STRING_COLUMNS",
Expand Down
2 changes: 1 addition & 1 deletion airbyte/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ def to_sql_type( # noqa: PLR0911 # Too many return statements
except SQLTypeConversionError:
print(f"Could not determine airbyte type from JSON schema: {json_schema_property_def}")
except KeyError:
# pyrefly: ignore[unbound-name]
# pyrefly: ignore[unbound-name] # noqa: ERA001
print(f"Could not find SQL type for airbyte type: {airbyte_type}")
else:
# No exceptions were raised, so we can return the SQL type.
Expand Down
2 changes: 1 addition & 1 deletion docs/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def run() -> None:
# individual tools / prompts / resources show up in the left nav. This
# monkey-patches the module-level `markdown_extensions` dict because pdoc
# 16's `configure()` does not expose markdown extension options.
# pyrefly: ignore[unsupported-operation]
# pyrefly: ignore[unsupported-operation] # noqa: ERA001
pdoc.render_helpers.markdown_extensions["toc"] = {"depth": 3}

pdoc.render.configure(
Expand Down
2 changes: 1 addition & 1 deletion examples/run_bigquery_destination.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def main() -> None:
)
write_result = destination.write(
source,
# cache=False, # Toggle comment to test with/without caching
# cache=False, # Toggle comment to test with/without caching # noqa: ERA001
)


Expand Down
2 changes: 1 addition & 1 deletion examples/run_file_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,5 @@
)
source.check()

# print(list(source.get_records("pokemon")))
# print(list(source.get_records("pokemon"))) # noqa: ERA001
source.read(cache=ab.new_local_cache("poke"))
4 changes: 2 additions & 2 deletions examples/run_get_documents_from_github.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ def main() -> None:
title_property="title",
content_properties=["body"],
metadata_properties=["state", "url", "number"],
# primary_key_properties=["id"],
# cursor_property="updated_at",
# primary_key_properties=["id"], # noqa: ERA001
# cursor_property="updated_at", # noqa: ERA001
render_metadata=True,
):
rich.print(rich.markdown.Markdown(str(doc) + "\n\n" + str("-" * 40)))
Expand Down
2 changes: 1 addition & 1 deletion examples/run_pokeapi.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,5 @@
)
source.check()

# print(list(source.get_records("pokemon")))
# print(list(source.get_records("pokemon"))) # noqa: ERA001
source.read(cache=ab.new_local_cache("poke"))
2 changes: 1 addition & 1 deletion examples/run_sync_to_destination_from_read_result.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def get_my_destination() -> ab.Destination:
},
docker_image="airbyte/destination-duckdb:latest",
# OR:
# pip_url="git+https://github.com/airbytehq/airbyte.git#subdirectory=airbyte-integrations/connectors/destination-duckdb",
# pip_url="git+https://github.com/airbytehq/airbyte.git#subdirectory=airbyte-integrations/connectors/destination-duckdb", # noqa: ERA001
)


Expand Down
2 changes: 1 addition & 1 deletion examples/run_sync_to_destination_w_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def get_my_destination() -> ab.Destination:
},
docker_image=True,
# OR:
# pip_url="git+https://github.com/airbytehq/airbyte.git#subdirectory=airbyte-integrations/connectors/destination-duckdb",
# pip_url="git+https://github.com/airbytehq/airbyte.git#subdirectory=airbyte-integrations/connectors/destination-duckdb", # noqa: ERA001
)


Expand Down
2 changes: 1 addition & 1 deletion examples/run_sync_to_destination_wo_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def get_my_destination() -> ab.Destination:
},
docker_image="airbyte/destination-duckdb:latest",
# OR:
# pip_url="git+https://github.com/airbytehq/airbyte.git#subdirectory=airbyte-integrations/connectors/destination-duckdb",
# pip_url="git+https://github.com/airbytehq/airbyte.git#subdirectory=airbyte-integrations/connectors/destination-duckdb", # noqa: ERA001
)


Expand Down
6 changes: 3 additions & 3 deletions tests/integration_tests/cloud/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ def deployable_dummy_source(*, use_docker: bool) -> Source:
config={
"count": 100,
},
# install_if_missing=False,
# install_if_missing=False, # noqa: ERA001
docker_image=use_docker,
)

Expand Down Expand Up @@ -150,8 +150,8 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
"""
deployable_destination_fixtures: dict[str, str] = {
# Ordered by priority (fastest first)
# "MotherDuck": "new_motherduck_destination",
# "Postgres": "new_remote_postgres_cache",
# "MotherDuck": "new_motherduck_destination", # noqa: ERA001
# "Postgres": "new_remote_postgres_cache", # noqa: ERA001
"BigQuery": "new_bigquery_destination",
"Snowflake": "new_snowflake_destination",
}
Expand Down
2 changes: 1 addition & 1 deletion tests/integration_tests/cloud/test_cloud_api_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ def test_get_bearer_token(
"connector_id, connector_type, expect_success",
[
("f45dd701-d1f0-4e8e-97c4-2b89c40ac928", "source", True),
# ("......-....-....-............", "destination", True),
# ("......-....-....-............", "destination", True), # noqa: ERA001
],
)
def test_check_connector(
Expand Down
4 changes: 2 additions & 2 deletions tests/integration_tests/cloud/test_cloud_sql_reads.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ def test_read_from_deployed_connection(
assert len(data_as_list) == 100

# TODO: Fails on BigQuery: https://github.com/airbytehq/PyAirbyte/issues/165
# pandas_df = dataset.to_pandas()
# pandas_df = dataset.to_pandas() # noqa: ERA001

pandas_df = pd.DataFrame(data_as_list)

Expand Down Expand Up @@ -196,7 +196,7 @@ def test_read_from_previous_job(
assert len(data_as_list) == 100

# TODO: Fails on BigQuery: https://github.com/airbytehq/PyAirbyte/issues/165
# pandas_df = dataset.to_pandas()
# pandas_df = dataset.to_pandas() # noqa: ERA001

pandas_df = pd.DataFrame(data_as_list)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def new_duckdb_destination_executor() -> Executor:
return get_connector_executor(
name="destination-duckdb",
docker_image="airbyte/destination-duckdb:latest",
# pip_url="git+https://github.com/airbytehq/airbyte.git#subdirectory=airbyte-integrations/connectors/destination-duckdb",
# pip_url="git+https://github.com/airbytehq/airbyte.git#subdirectory=airbyte-integrations/connectors/destination-duckdb", # noqa: ERA001
)


Expand Down
2 changes: 1 addition & 1 deletion tests/integration_tests/test_all_cache_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def test_faker_read(
assert sum(1 for _ in arrow_dataset.to_batches()) == FAKER_SCALE_A / 10

# TODO: Uncomment this line after resolving https://github.com/airbytehq/PyAirbyte/issues/165
# assert len(result["users"].to_pandas()) == FAKER_SCALE_A
# assert len(result["users"].to_pandas()) == FAKER_SCALE_A # noqa: ERA001


@pytest.mark.requires_creds
Expand Down
2 changes: 1 addition & 1 deletion tests/integration_tests/test_bigquery_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def test_bigquery_props(
new_bigquery_cache: ab.BigQueryCache,
) -> None:
"""Test that the BigQueryCache properties are set correctly."""
# assert new_bigquery_cache.credentials_path.endswith(".json")
# assert new_bigquery_cache.credentials_path.endswith(".json") # noqa: ERA001
assert new_bigquery_cache.dataset_name == new_bigquery_cache.schema_name, (
"Dataset name should be the same as schema name."
)
Expand Down
2 changes: 1 addition & 1 deletion tests/integration_tests/test_source_test_fixture.py
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ def test_dataset_list_and_len(

# Test the lazy dataset implementation
lazy_dataset = source.get_records("stream1")
# assert len(stream_1) == 2 # This is not supported by the lazy dataset
# assert len(stream_1) == 2 # This is not supported by the lazy dataset # noqa: ERA001
lazy_dataset_list = list(lazy_dataset)
# Make sure counts are correct
assert len(list(lazy_dataset_list)) == 2
Expand Down
10 changes: 5 additions & 5 deletions tests/integration_tests/test_state_handling.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,15 +175,15 @@ def test_destination_state(
e2e_test_destination: ab.Destination,
) -> None:
"""Test destination state handling."""
# config_a = source_faker_seed_a.get_config()
# config_a["always_updated"] = False # disable ensuring new `updated_at` timestamps
# source_faker_seed_a.set_config(config_a)
# config_a = source_faker_seed_a.get_config() # noqa: ERA001
# config_a["always_updated"] = False # disable ensuring new `updated_at` timestamps # noqa: ERA001
# source_faker_seed_a.set_config(config_a) # noqa: ERA001

cache = ab.new_local_cache("aj_test05")

source_faker_seed_a.select_streams(["products", "users"])
read_result = source_faker_seed_a.read(cache)
# assert read_result.processed_records == NUM_PRODUCTS + FAKER_SCALE_A * 2
# assert read_result.processed_records == NUM_PRODUCTS + FAKER_SCALE_A * 2 # noqa: ERA001

cache_state_provider = cache.get_state_provider("source-faker")
assert cache_state_provider.known_stream_names == {
Expand All @@ -198,7 +198,7 @@ def test_destination_state(
read_result,
state_cache=cache,
)
# assert write_result.processed_records == NUM_PRODUCTS + FAKER_SCALE_A * 2
# assert write_result.processed_records == NUM_PRODUCTS + FAKER_SCALE_A * 2 # noqa: ERA001
write_result_state_provider = write_result.get_state_provider()
assert write_result_state_provider.known_stream_names == {
"users",
Expand Down
Loading