From b3c60b39f52ba948f6ba3d44801a2507c777215e Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Fri, 17 Jul 2026 00:20:01 +0200 Subject: [PATCH 1/6] Router: Don't signal `scheme://bucket-name/file-glob` as deprecated --- src/dlt_filesystem/source/router.py | 8 ++------ tests/dlt_filesystem/test_source_hints.py | 6 ++---- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/dlt_filesystem/source/router.py b/src/dlt_filesystem/source/router.py index 5aee57ea9..d6da5a664 100644 --- a/src/dlt_filesystem/source/router.py +++ b/src/dlt_filesystem/source/router.py @@ -1,4 +1,3 @@ -import warnings from typing import Any, Dict, Optional, Tuple, TypeAlias from urllib.parse import ParseResult, parse_qsl, urlparse @@ -41,12 +40,9 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]: table = table.strip() host = uri.netloc.strip() + # Form: scheme://bucket-name/file-glob + # Note: This form was previously slated for deprecation. if table == "" or uri.path.strip() != "": - warnings.warn( - f"Using the form '{uri.scheme}://bucket-name/file-glob' is deprecated and will be removed in future versions.", - DeprecationWarning, - stacklevel=2, - ) return host, uri.path.lstrip("/") table_uri = urlparse(table) diff --git a/tests/dlt_filesystem/test_source_hints.py b/tests/dlt_filesystem/test_source_hints.py index dbe86f27b..10e7f83fe 100644 --- a/tests/dlt_filesystem/test_source_hints.py +++ b/tests/dlt_filesystem/test_source_hints.py @@ -79,12 +79,11 @@ def test_s3_threads_hints(table: str, expected_hints: dict[str, str]): def test_s3_threads_hints_from_uri_path_form(): - """The deprecated `s3://bucket/path#frag` URI-path form carries the fragment in - parsed_uri.fragment; blob_hints reconstructs it so hints still thread.""" + """The `s3://bucket/path#frag` form carries the fragment in + URI-path's `fragment`; blob_hints reconstructs it so hints still thread.""" with ( patch(RESOURCE_FOR_READER) as rfr, patch("s3fs.S3FileSystem"), - pytest.warns(DeprecationWarning), ): S3Source().dlt_source( "s3://bucket/book.xlsx?access_key_id=KEY&secret_access_key=SECRET#sheet_name=foo", @@ -100,7 +99,6 @@ def test_s3_blob_hints_track_the_loaded_file_when_both_forms_given(): with ( patch(RESOURCE_FOR_READER) as rfr, patch("s3fs.S3FileSystem"), - pytest.warns(DeprecationWarning), ): S3Source().dlt_source( "s3://bucket/loaded.xlsx?access_key_id=KEY&secret_access_key=SECRET#sheet_name=uri", From 78d8145403c6a2b1adcef865dc480a077577b045 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Fri, 17 Jul 2026 00:19:29 +0200 Subject: [PATCH 2/6] Tests: Improve parameterized test output --- tests/dlt_filesystem/test_source_remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dlt_filesystem/test_source_remote.py b/tests/dlt_filesystem/test_source_remote.py index 49653b707..511c919ce 100644 --- a/tests/dlt_filesystem/test_source_remote.py +++ b/tests/dlt_filesystem/test_source_remote.py @@ -36,7 +36,7 @@ class URITestCase: ] -@pytest.mark.parametrize("test_case", test_cases) +@pytest.mark.parametrize("test_case", test_cases, ids=[case.uri for case in test_cases]) def test_parse_uri(test_case: URITestCase): """Parsing a source URI splits it into the expected bucket and file glob.""" uri = urlparse(test_case.uri) From 49a43bc4009055370766bcbbed28c48a98edf2be Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Fri, 17 Jul 2026 00:19:09 +0200 Subject: [PATCH 3/6] Tests: Validate initialization of existing remote filesystems --- tests/main/filesystem/test_remote_read.py | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 tests/main/filesystem/test_remote_read.py diff --git a/tests/main/filesystem/test_remote_read.py b/tests/main/filesystem/test_remote_read.py new file mode 100644 index 000000000..fe189e458 --- /dev/null +++ b/tests/main/filesystem/test_remote_read.py @@ -0,0 +1,62 @@ +from urllib.parse import urlparse + +import pytest + +from omniload.core.factory import SourceDestinationFactory + +# A collection of filesystem source URIs without table parameter. +URIS = [ + # TODO: Review if account name is not already provided in the hostname per `acme`. + "abfss://schrott@acme.dfs.core.windows.net/path/to/data.parquet?account_name=acme&account_key=secret", + "adls://schrott@acme.dfs.core.windows.net/path/to/data.parquet?account_name=acme&account_key=secret", + "az://schrott@acme.dfs.core.windows.net/path/to/data.parquet?account_name=acme&account_key=secret", + # TODO: Mock `gs` backend. + # ValueError: Provided token is either not valid, or expired. + "gs://table-bucket-name/path/to/data.parquet?credentials_path=/path/to/service-account.json", + # FIXME: KeyError: 'refresh_token' + "gs://table-bucket-name/path/to/data.parquet?credentials_base64=eyJrZXkiOiAidmFsdWUifQ==", + "s3://bucket/path/to/data.parquet?access_key_id=foo&secret_access_key=bar", + "sftp://username:password@example.com:2222/path/to/data.parquet", +] + + +@pytest.mark.parametrize("source_uri", URIS) +def test_init_generic_filesystems(source_uri): + """Initialize all available filesystem implementations without table parameter""" + parsed_uri = urlparse(source_uri) + if parsed_uri.scheme in ["gs", "sftp"]: + pytest.skip(f"{parsed_uri.scheme}:// needs monkeypatching to make it testable") + factory = SourceDestinationFactory(source_uri, "file://") + source = factory.get_source() + dlt_source = source.dlt_source( + uri=source_uri, + # TODO: AzureSource.dlt_source() missing 1 required positional argument: 'table' + table="", + ) + assert dlt_source.name == "read_parquet" + assert dlt_source.section == "readers" + assert dlt_source._parent.name == "filesystem" + assert dlt_source._parent.section == "adapter" + + +def test_init_http_filesystem(): + """Initialize HTTP filesystem implementation without table parameter""" + source_uri = "http://example.org/path/to/data.parquet" + factory = SourceDestinationFactory(source_uri, "file://") + source = factory.get_source() + dlt_source = source.dlt_source( + uri=source_uri, + # TODO: Make `table` parameter optional. + # AzureSource.dlt_source() missing 1 required positional argument: 'table' + table="", + ) + assert dlt_source.name == "http_source" + assert dlt_source.section == "adapter" + + +def test_init_unknown_filesystem(): + """Initialize unknown filesystem implementation""" + factory = SourceDestinationFactory("unknown://", "file://") + with pytest.raises(ValueError) as exc_info: + factory.get_source() + assert exc_info.match("Unsupported source scheme: unknown") From 2e14f64d1ce621c60ee6c27790f1db4a8a817eb6 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Fri, 17 Jul 2026 02:22:31 +0200 Subject: [PATCH 4/6] Core: Raise `NotImplementedError` when using unknown pipeline elements --- src/omniload/core/factory.py | 6 ++++-- tests/main/filesystem/test_remote_read.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/omniload/core/factory.py b/src/omniload/core/factory.py index 8f6eb6f2a..6c265b894 100644 --- a/src/omniload/core/factory.py +++ b/src/omniload/core/factory.py @@ -45,13 +45,15 @@ def get_source(self) -> SourceProtocol: elif self.source_scheme in self.sources: return self.sources[self.source_scheme]() else: - raise ValueError(f"Unsupported source scheme: {self.source_scheme}") + raise NotImplementedError( + f"Unsupported source scheme: {self.source_scheme}" + ) def get_destination(self) -> DestinationProtocol: """Build the destination connector for the parsed destination scheme.""" if self.destination_scheme in self.destinations: return self.destinations[self.destination_scheme]() else: - raise ValueError( + raise NotImplementedError( f"Unsupported destination scheme: {self.destination_scheme}" ) diff --git a/tests/main/filesystem/test_remote_read.py b/tests/main/filesystem/test_remote_read.py index fe189e458..900ad7a6f 100644 --- a/tests/main/filesystem/test_remote_read.py +++ b/tests/main/filesystem/test_remote_read.py @@ -57,6 +57,6 @@ def test_init_http_filesystem(): def test_init_unknown_filesystem(): """Initialize unknown filesystem implementation""" factory = SourceDestinationFactory("unknown://", "file://") - with pytest.raises(ValueError) as exc_info: + with pytest.raises(NotImplementedError) as exc_info: factory.get_source() assert exc_info.match("Unsupported source scheme: unknown") From 7b4e5be4236b7a4124338801701097c7b6cc1041 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Fri, 17 Jul 2026 00:53:48 +0200 Subject: [PATCH 5/6] Filesystem: Add read support for more fsspec-based filesystems - Add support for Databricks files, Dropbox, FTP, Google Drive, HDFS, OCI, OneDrive, OSS, R2, SharePoint, SMB, WebDAV, WebHDFS. - Filesystem documentation improvements across the board. --- .gitleaksignore | 5 + docs/changelog.md | 2 + docs/conf.py | 9 +- docs/supported-sources/azure-blob-storage.md | 165 ----------- .../azure-data-lake-storage.md | 49 ---- docs/supported-sources/azure-storage.md | 235 ++++++++++++++++ docs/supported-sources/cbor.md | 5 +- docs/supported-sources/databricks-files.md | 144 ++++++++++ ...{databricks.md => databricks-warehouse.md} | 11 +- docs/supported-sources/dropbox.md | 49 ++++ docs/supported-sources/filesystem.md | 34 ++- docs/supported-sources/ftp.md | 93 +++++++ .../supported-sources/google-cloud-storage.md | 9 +- docs/supported-sources/google-drive.md | 126 +++++++++ docs/supported-sources/hdfs.md | 93 +++++++ docs/supported-sources/msgpack.md | 5 +- docs/supported-sources/nextcloud.md | 40 +++ docs/supported-sources/ods.md | 5 +- docs/supported-sources/onedrive.md | 110 ++++++++ docs/supported-sources/oracle-oci.md | 94 +++++++ docs/supported-sources/oss.md | 99 +++++++ docs/supported-sources/r2.md | 76 +++++ docs/supported-sources/s3.md | 259 ++++++++++-------- docs/supported-sources/sftp.md | 55 ++-- docs/supported-sources/sharepoint.md | 117 ++++++++ docs/supported-sources/smb.md | 110 ++++++++ docs/supported-sources/webdav.md | 104 +++++++ docs/supported-sources/webhdfs.md | 119 ++++++++ docs/supported-sources/xlsx.md | 5 +- docs/supported-sources/xml.md | 5 +- docs/supported-sources/yaml.md | 5 +- pyproject.toml | 22 +- src/dlt_filesystem/error.py | 2 +- src/dlt_filesystem/source/adapter.py | 40 +-- src/dlt_filesystem/source/api.py | 15 - src/dlt_filesystem/source/base.py | 28 ++ src/dlt_filesystem/source/core.py | 82 ++++++ src/dlt_filesystem/source/error.py | 2 + .../source/fsspec/databricks.py | 74 +++++ src/dlt_filesystem/source/fsspec/dropbox.py | 42 +++ src/dlt_filesystem/source/fsspec/ftp.py | 37 +++ src/dlt_filesystem/source/fsspec/gdrive.py | 48 ++++ src/dlt_filesystem/source/fsspec/hdfs.py | 49 ++++ .../source/{impl => fsspec}/local.py | 2 +- src/dlt_filesystem/source/fsspec/oci.py | 51 ++++ src/dlt_filesystem/source/fsspec/onedrive.py | 13 + src/dlt_filesystem/source/fsspec/oss.py | 53 ++++ src/dlt_filesystem/source/fsspec/r2.py | 30 ++ .../source/fsspec/sharepoint.py | 49 ++++ src/dlt_filesystem/source/fsspec/smb.py | 54 ++++ src/dlt_filesystem/source/fsspec/webdav.py | 65 +++++ src/dlt_filesystem/source/fsspec/webhdfs.py | 55 ++++ src/dlt_filesystem/source/impl/remote.py | 114 ++++---- src/dlt_filesystem/source/model.py | 144 +++++++++- src/dlt_filesystem/source/router.py | 23 +- src/dlt_filesystem/util/fsspec.py | 109 ++++++++ src/dlt_filesystem/util/python.py | 63 ++++- src/dlt_filesystem/util/web.py | 6 + src/omniload/core/registry.py | 30 +- tests/assets/privatekey-fingerprint.txt | 1 + tests/assets/privatekey.pem | 10 + tests/dlt_filesystem/format/test_bson.py | 2 +- tests/dlt_filesystem/format/test_cbor.py | 2 +- tests/dlt_filesystem/format/test_msgpack.py | 2 +- tests/dlt_filesystem/format/test_xml.py | 2 +- tests/dlt_filesystem/format/test_yaml.py | 2 +- tests/dlt_filesystem/test_source_hints.py | 4 +- .../dlt_filesystem/test_source_incremental.py | 67 +++-- tests/dlt_filesystem/test_source_local.py | 4 +- .../main/filesystem/test_file_incremental.py | 2 +- tests/main/filesystem/test_local_read.py | 2 +- tests/main/filesystem/test_remote_read.py | 139 +++++++++- 72 files changed, 3201 insertions(+), 577 deletions(-) delete mode 100644 docs/supported-sources/azure-blob-storage.md delete mode 100644 docs/supported-sources/azure-data-lake-storage.md create mode 100644 docs/supported-sources/azure-storage.md create mode 100644 docs/supported-sources/databricks-files.md rename docs/supported-sources/{databricks.md => databricks-warehouse.md} (87%) create mode 100644 docs/supported-sources/dropbox.md create mode 100644 docs/supported-sources/ftp.md create mode 100644 docs/supported-sources/google-drive.md create mode 100644 docs/supported-sources/hdfs.md create mode 100644 docs/supported-sources/nextcloud.md create mode 100644 docs/supported-sources/onedrive.md create mode 100644 docs/supported-sources/oracle-oci.md create mode 100644 docs/supported-sources/oss.md create mode 100644 docs/supported-sources/r2.md create mode 100644 docs/supported-sources/sharepoint.md create mode 100644 docs/supported-sources/smb.md create mode 100644 docs/supported-sources/webdav.md create mode 100644 docs/supported-sources/webhdfs.md delete mode 100644 src/dlt_filesystem/source/api.py create mode 100644 src/dlt_filesystem/source/core.py create mode 100644 src/dlt_filesystem/source/fsspec/databricks.py create mode 100644 src/dlt_filesystem/source/fsspec/dropbox.py create mode 100644 src/dlt_filesystem/source/fsspec/ftp.py create mode 100644 src/dlt_filesystem/source/fsspec/gdrive.py create mode 100644 src/dlt_filesystem/source/fsspec/hdfs.py rename src/dlt_filesystem/source/{impl => fsspec}/local.py (98%) create mode 100644 src/dlt_filesystem/source/fsspec/oci.py create mode 100644 src/dlt_filesystem/source/fsspec/onedrive.py create mode 100644 src/dlt_filesystem/source/fsspec/oss.py create mode 100644 src/dlt_filesystem/source/fsspec/r2.py create mode 100644 src/dlt_filesystem/source/fsspec/sharepoint.py create mode 100644 src/dlt_filesystem/source/fsspec/smb.py create mode 100644 src/dlt_filesystem/source/fsspec/webdav.py create mode 100644 src/dlt_filesystem/source/fsspec/webhdfs.py create mode 100644 src/dlt_filesystem/util/fsspec.py create mode 100644 src/dlt_filesystem/util/web.py create mode 100644 tests/assets/privatekey-fingerprint.txt create mode 100644 tests/assets/privatekey.pem diff --git a/.gitleaksignore b/.gitleaksignore index c3dc60f30..2004f47bc 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -2,6 +2,11 @@ omniload/src/telemetry/event.py:generic-api-key:17 omniload/src/testdata/fakebqcredentials.json:private-key:5 docs/supported-sources/shopify.md:generic-api-key:26 docs/supported-sources/smartsheets.md:generic-api-key:38 +tests/assets/privatekey.pem:private-key:1 +tests/main/filesystem/test_remote_read.py:generic-api-key:43 +tests/main/filesystem/test_remote_read.py:generic-api-key:55 +tests/main/filesystem/test_remote_read.py:generic-api-key:56 +tests/main/filesystem/test_remote_read.py:generic-api-key:58 tests/saas/test_google_analytics.py:generic-api-key:45 tests/saas/test_google_analytics.py:generic-api-key:51 tests/saas/test_google_analytics.py:generic-api-key:58 diff --git a/docs/changelog.md b/docs/changelog.md index 072400e55..433419e2e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,8 @@ ## in progress - Filesystem: Migrated local filesystem access to Apache Arrow. +- Filesystem: Added support for reading from + Dropbox, FTP, HDFS, OCI, OneDrive, OSS, R2, SharePoint, SMB, WebDAV. ## 2026/07/16 v0.7.0 diff --git a/docs/conf.py b/docs/conf.py index 511f08832..2f5c471cf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -73,13 +73,7 @@ # -- Intersphinx ---------------------------------------------------------- -intersphinx_mapping = { - "crash": ("https://cratedb.com/docs/crate/crash/en/latest/", None), - "cloud": ("https://cratedb.com/docs/cloud/en/latest/", None), - "croud": ("https://cratedb.com/docs/cloud/cli/en/latest/", None), - "guide": ("https://cratedb.com/docs/guide/", None), - "influxio": ("https://influxio.readthedocs.io/", None), -} +intersphinx_mapping = {} linkcheck_ignore = [ r"https://pulse.internetsociety.org/", r"https://www.g2.com/", @@ -101,6 +95,7 @@ r"https://github.com/", r"https://web.archive.org/", r"https://images.minimus.io/", + r"https://medium.com/", ] linkcheck_anchors_ignore_for_url = [ r"https://developers.zoom.us/", diff --git a/docs/supported-sources/azure-blob-storage.md b/docs/supported-sources/azure-blob-storage.md deleted file mode 100644 index ef7b1e245..000000000 --- a/docs/supported-sources/azure-blob-storage.md +++ /dev/null @@ -1,165 +0,0 @@ -# Azure Blob Storage - -[Azure Blob Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) is Microsoft's object storage service for the cloud, optimized for storing large amounts of unstructured data. - -`omniload` supports Azure Blob Storage as both a data source and destination. The same backend also serves Azure Data Lake Storage Gen2; see [Azure Data Lake Storage](azure-data-lake-storage.md). - -## URI Format - -The URI for connecting to Azure Blob Storage is structured as follows: - -```text -az://?account_name=&account_key= -``` - -**URI Parameters:** - -* `account_name`: Your Azure storage account name (required). -* `account_key`: Your storage account access key (account-key auth). -* `sas_token`: A Shared Access Signature token (SAS auth, alternative to `account_key`). -* `tenant_id`, `client_id`, `client_secret`: Azure AD service-principal credentials (service-principal auth). All three are required together. -* `account_host`: Custom storage endpoint host (optional, for sovereign clouds or Azurite). -* `layout`: Layout template (optional, destination only). - -Supply **one** authentication mode: an `account_key`, a `sas_token`, or the full service-principal triplet (`tenant_id` + `client_id` + `client_secret`). Supplying both an account key/SAS and service-principal fields is rejected as ambiguous, and a partial service-principal triplet reports the missing field. - -::: warning -Account keys are base64 (containing `+`, `/`, `=`) and SAS tokens embed their own `&` and `=` characters. **URL-encode credential values** in the URI (`+` becomes `%2B`, `/` becomes `%2F`, `=` becomes `%3D`, `&` becomes `%26`). Unencoded values are mangled when the query string is parsed. -::: - -The `--source-table` parameter specifies the container and file pattern using the following format: - -``` -/ -``` - -## Setting up an Azure Blob Storage Integration - -To integrate `omniload` with Azure Blob Storage, you need a storage account and one of the supported credentials. For guidance on obtaining an account key or SAS token, refer to the Microsoft documentation on [managing storage account access keys](https://learn.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage) and [shared access signatures](https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview). Service-principal credentials come from an [Azure AD app registration](https://learn.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal). - -Once you have your credentials, you can configure the `az://` URI. The container name and file glob pattern are specified in the `--source-table` argument. - -### Example: Loading data from Azure Blob Storage - -Let's assume the following details: -* `account_name`: `mystorageacct` -* `account_key`: `dGVzdA==` -* Container name: `my-container` -* Path to files within the container: `students/students_details.csv` - -The following command demonstrates how to copy data from the specified Azure location to a DuckDB database (the account key is URL-encoded, so `==` becomes `%3D%3D`): - -```sh -omniload ingest \ - --source-uri 'az://?account_name=mystorageacct&account_key=dGVzdA%3D%3D' \ - --source-table 'my-container/students/students_details.csv' \ - --dest-uri duckdb:///azure_data.duckdb \ - --dest-table 'processed_students.student_details' -``` - -This command will create a table named `student_details` within the `processed_students` schema (or equivalent grouping) in the DuckDB database file located at `azure_data.duckdb`. - -### Example: Uploading data to Azure Blob Storage - -For this example, we'll assume that: -* `records.db` is a duckdb database. -* It has a table called `public.users`. -* The Azure credentials are the same as the example above. - -The following command demonstrates how to copy data from a local duckdb database to Azure Blob Storage: - -```sh -omniload ingest \ - --source-uri 'duckdb:///records.db' \ - --source-table 'public.users' \ - --dest-uri 'az://?account_name=mystorageacct&account_key=dGVzdA%3D%3D' \ - --dest-table 'my-container/records' -``` - -This will result in a file structure like the following: -``` -my-container/ -└── records - ├── _dlt_loads - ├── _dlt_pipeline_state - ├── _dlt_version - └── users - └── ..parquet -``` - -The value of `load_id` and `file_id` is determined at runtime. The default layout creates a folder with the same table name as the source and places the data inside a parquet file. This layout is configurable using the `layout` parameter. See the [available layout placeholders](https://dlthub.com/docs/dlt-ecosystem/destinations/filesystem#available-layout-placeholders) for the full list. - -### Authenticating with a SAS token - -```sh -omniload ingest \ - --source-uri 'az://?account_name=mystorageacct&sas_token=sv%3D2023-01-03%26ss%3Db%26sig%3DaBcD1234%253D' \ - --source-table 'my-container/data.csv' \ - --dest-uri 'duckdb:///local.duckdb' \ - --dest-table 'public.my_data' -``` - -### Authenticating with a service principal - -```sh -omniload ingest \ - --source-uri 'az://?account_name=mystorageacct&tenant_id=&client_id=&client_secret=' \ - --source-table 'my-container/data.csv' \ - --dest-uri 'duckdb:///local.duckdb' \ - --dest-table 'public.my_data' -``` - -### File Glob Pattern Examples: - -::: warning -Glob patterns only apply when loading data from Azure Blob Storage as source. -::: - -The `` in the `--source-table` argument allows for flexible file selection. Here are some common patterns and their descriptions: - -| Pattern | Description | -| :--------------------------------------------- | :----------------------------------------------------------------------------------------------------------------- | -| `container/**/*.csv` | Retrieves all CSV files recursively from `az://container`. | -| `container/*.csv` | Retrieves all CSV files located at the root level of `az://container`. | -| `container/myFolder/**/*.jsonl` | Retrieves all JSONL files recursively from the `myFolder` directory and its subdirectories in `az://container`. | -| `container/myFolder/mySubFolder/users.parquet` | Retrieves the specific `users.parquet` file from the `myFolder/mySubFolder/` path in `az://container`. | -| `container/employees.jsonl` | Retrieves the `employees.jsonl` file located at the root level of `az://container`. | - -### Working with compressed files - -`omniload` automatically detects and handles gzipped files in your container. You can load data from compressed files with the `.gz` extension without any additional configuration. - -For example, to load data from a gzipped CSV file: - -```sh -omniload ingest \ - --source-uri 'az://?account_name=mystorageacct&account_key=dGVzdA%3D%3D' \ - --source-table 'my-container/logs/event-data.csv.gz' \ - --dest-uri duckdb:///compressed_data.duckdb \ - --dest-table 'logs.events' -``` - -### File type hinting - -If your files are properly encoded but lack the correct file extension (BSON, CSV, JSONL, or Parquet), you can provide a file type hint to inform `omniload` about the format of the files. This is done by appending a fragment identifier (`#format`) to the end of the path in your `--source-table` parameter. - -For example, if you have JSONL-formatted log files stored in Azure with a non-standard extension: - -``` ---source-table "my-container/logs/event-data#jsonl" -``` - -Supported format hints include: -- `#bson` - For BSON (MongoDB dump) files. See [BSON](bson.md). -- `#csv` - For comma-separated values files with headers -- `#csv_headless` - For CSV files without headers -- `#jsonl` - For line-delimited JSON files -- `#parquet` - For Parquet format files - -::: tip -File type hinting works with `gzip` compressed files as well. -::: - -## Azure Data Lake Storage Gen2 - -The `adls://` and `abfss://` schemes are aliases for the same backend and accept identical parameters. Use them when you think in terms of ADLS Gen2. See [Azure Data Lake Storage](azure-data-lake-storage.md). diff --git a/docs/supported-sources/azure-data-lake-storage.md b/docs/supported-sources/azure-data-lake-storage.md deleted file mode 100644 index d49bc50f2..000000000 --- a/docs/supported-sources/azure-data-lake-storage.md +++ /dev/null @@ -1,49 +0,0 @@ -# Azure Data Lake Storage - -[Azure Data Lake Storage Gen2](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction) is a set of capabilities built on Azure Blob Storage for big-data analytics. A Gen2 account is a storage account with a hierarchical namespace enabled; it shares the same underlying service and API as Azure Blob Storage. - -`omniload` supports Azure Data Lake Storage Gen2 as both a data source and destination through the `adls://` and `abfss://` schemes. - -## URI Format - -```text -adls://?account_name=&account_key= -``` - -`abfss://` is accepted as well and behaves identically; it is the dominant Gen2 scheme in the Spark, Databricks, and Synapse ecosystems: - -```text -abfss://?account_name=&account_key= -``` - -Both schemes are aliases of [Azure Blob Storage](azure-blob-storage.md): they share the same `adlfs` backend, accept the same URI parameters and authentication modes (account key, SAS token, service principal), and resolve to the same `az://` bucket internally. A Gen2 account differs from a plain Blob account only by having its hierarchical namespace enabled, so no separate configuration is required. - -**URI Parameters, authentication, and credential URL-encoding are identical to Azure Blob Storage.** See the [Azure Blob Storage](azure-blob-storage.md) page for the full parameter table, the auth-mode rules, and the important note on URL-encoding account keys and SAS tokens. - -The `--source-table` parameter specifies the container (filesystem) and file pattern: - -``` -/ -``` - -## Example: Loading data from ADLS Gen2 - -```sh -omniload ingest \ - --source-uri 'adls://?account_name=mystorageacct&account_key=dGVzdA%3D%3D' \ - --source-table 'my-filesystem/events/2024/*.parquet' \ - --dest-uri duckdb:///adls_data.duckdb \ - --dest-table 'analytics.events' -``` - -## Example: Uploading data to ADLS Gen2 - -```sh -omniload ingest \ - --source-uri 'duckdb:///records.db' \ - --source-table 'public.users' \ - --dest-uri 'abfss://?account_name=mystorageacct&account_key=dGVzdA%3D%3D' \ - --dest-table 'my-filesystem/users' -``` - -For glob patterns, compressed-file handling, file type hinting, and the full set of examples, see [Azure Blob Storage](azure-blob-storage.md). diff --git a/docs/supported-sources/azure-storage.md b/docs/supported-sources/azure-storage.md new file mode 100644 index 000000000..763ae22e0 --- /dev/null +++ b/docs/supported-sources/azure-storage.md @@ -0,0 +1,235 @@ +(azure-storage)= + +# Azure Storage + +[Azure Blob Storage] is Microsoft's object storage service for the cloud, +optimized for storing large amounts of unstructured data. Resources are +addressed using the `az://` URL scheme. + +[Azure Data Lake Storage Gen2] is a set of capabilities built on Azure Blob +Storage for big-data analytics. A Gen2 account is a storage account with a +hierarchical namespace enabled; it shares the same underlying service and +API as Azure Blob Storage, and uses the `adls://` and `abfss://` URL schemes. + +`omniload` supports both Azure Blob Storage and Azure Data Lake Storage as +both data source and destination. + +## URI format + +The URI for connecting to Azure Blob Storage is structured as follows. +```text +az://?account_name=&account_key= +``` + +The URIs for connecting to Azure Data Lake Storage are structured as follows. +```text +adls://?account_name=&account_key= +``` +`abfss://` is accepted as well and behaves identically; it is the dominant +Gen2 scheme in the Spark, Databricks, and Synapse ecosystems. +```text +abfss://?account_name=&account_key= +``` + +## URI parameters + +:account_name: + Your Azure storage account name (required). + +:account_key: + Your storage account access key (account-key auth). + +:sas_token: + A Shared Access Signature token (SAS auth), alternatively to `account_key`. + +:tenant_id, client_id, client_secret: + Azure AD service-principal credentials (service-principal auth). All three are required together. + +:account_host: + Custom storage endpoint host (optional, for sovereign clouds or Azurite). + +:layout: + Layout template (optional, destination only). + +Supply **one** authentication mode: an `account_key`, a `sas_token`, or the +full service-principal triplet (`tenant_id` + `client_id` + `client_secret`). +Supplying both an account key/SAS and service-principal fields is rejected as +ambiguous, and a partial service-principal triplet reports the missing field. + +:::{warning} +Account keys are base64 (containing `+`, `/`, `=`) and SAS tokens embed their +own `&` and `=` characters. You must URL-encode credential values in the URI +(`+` becomes `%2B`, `/` becomes `%2F`, `=` becomes `%3D`, `&` becomes `%26`). +Unencoded values are mangled when the query string is parsed. +::: + +:::{note} +A Gen2 account differs from a plain Blob account only by having its +hierarchical namespace enabled, so no separate configuration is required. +::: + +## Set up an Azure Storage integration + +To integrate `omniload` with Azure Blob or Data Lake Storage, you need a +storage account and one of the supported credentials. For guidance on +obtaining an account key or SAS token, refer to the Microsoft documentation +on [managing storage account access keys] and [shared access signatures]. +Service-principal credentials come from an [Azure AD app registration]. + +## Authenticate with SAS token + +```sh +omniload ingest \ + --source-uri 'az://?account_name=mystorageacct&sas_token=sv%3D2023-01-03%26ss%3Db%26sig%3DaBcD1234%253D' \ + --source-table 'my-container/data.csv' \ + --dest-uri 'duckdb:///local.duckdb' \ + --dest-table 'public.my_data' +``` + +## Authenticate with service principal + +```sh +omniload ingest \ + --source-uri 'az://?account_name=mystorageacct&tenant_id=&client_id=&client_secret=' \ + --source-table 'my-container/data.csv' \ + --dest-uri 'duckdb:///local.duckdb' \ + --dest-table 'public.my_data' +``` + +## File glob patterns + +The `--source-table` parameter specifies the Azure Storage container name +and the file glob pattern using the following format. + +```text +/ +``` + +The `` allows for flexible file selection. Here are some +common patterns and their descriptions. + +| Pattern | Description | +| :--------------------------------------------- | :----------------------------------------------------------------------------------------------------------------- | +| `container/**/*.csv` | Retrieves all CSV files recursively from `az://container`. | +| `container/*.csv` | Retrieves all CSV files located at the root level of `az://container`. | +| `container/myFolder/**/*.jsonl` | Retrieves all JSONL files recursively from the `myFolder` directory and its subdirectories in `az://container`. | +| `container/myFolder/mySubFolder/users.parquet` | Retrieves the specific `users.parquet` file from the `myFolder/mySubFolder/` path in `az://container`. | +| `container/employees.jsonl` | Retrieves the `employees.jsonl` file located at the root level of `az://container`. | + +## File type hinting + +If your files are properly encoded but lack the correct file extension (BSON, +CSV, JSONL, or Parquet), you can provide a file type hint to inform `omniload` +about the format of the files. This is done by appending a fragment identifier +(`#format`) to the end of the path in your `--source-table` parameter. + +For example, if you have JSONL-formatted log files stored in Azure with a +non-standard extension, use an URI pattern like this. + +```text +--source-table "my-container/logs/event-data#jsonl" +``` + +See also the full list of {ref}`file-formats` and their type hints. + +:::{tip} +File type hinting works with `gzip` compressed files as well. +::: + +## Compressed files + +`omniload` automatically detects and handles gzipped files in your container. You can load data from compressed files with the `.gz` extension without any additional configuration. + +For example, to load data from a gzipped CSV file: + +```sh +omniload ingest \ + --source-uri 'az://?account_name=mystorageacct&account_key=dGVzdA%3D%3D' \ + --source-table 'my-container/logs/event-data.csv.gz' \ + --dest-uri duckdb:///compressed_data.duckdb \ + --dest-table 'logs.events' +``` + +## Examples + +### Load data from Azure Blob Storage + +Let's assume the following details: +* `account_name`: `mystorageacct` +* `account_key`: `dGVzdA==` +* Container name: `my-container` +* Path to files within the container: `path/to/data.csv` + +The following command demonstrates how to copy data from the specified Azure location to a DuckDB database (the account key is URL-encoded, so `==` becomes `%3D%3D`): + +```sh +omniload ingest \ + --source-uri 'az://?account_name=mystorageacct&account_key=dGVzdA%3D%3D' \ + --source-table 'my-container/path/to/data.csv' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + +### Upload data to Azure Blob Storage + +For this example, we'll assume that: +* `records.db` is a duckdb database. +* It has a table called `public.users`. +* The Azure credentials are the same as the example above. + +The following command demonstrates how to copy data from a local duckdb database to Azure Blob Storage: + +```sh +omniload ingest \ + --source-uri 'duckdb:///records.db' \ + --source-table 'public.users' \ + --dest-uri 'az://?account_name=mystorageacct&account_key=dGVzdA%3D%3D' \ + --dest-table 'my-container/records' +``` + +This will result in a file structure like the following: +```text +my-container/ +└── records + ├── _dlt_loads + ├── _dlt_pipeline_state + ├── _dlt_version + └── users + └── ..parquet +``` + +The value of `load_id` and `file_id` is determined at runtime. The default +layout creates a folder with the same table name as the source and places +the data inside a parquet file. This layout is configurable using the +`layout` parameter. See the [available layout placeholders] for the full list. + +### Load data from ADLS Gen2 + +```sh +omniload ingest \ + --source-uri 'adls://?account_name=mystorageacct&account_key=dGVzdA%3D%3D' \ + --source-table 'my-filesystem/events/2024/*.parquet' \ + --dest-uri duckdb:///adls_data.duckdb \ + --dest-table 'analytics.events' +``` + +### Upload data to ADLS Gen2 + +```sh +omniload ingest \ + --source-uri 'duckdb:///records.db' \ + --source-table 'public.users' \ + --dest-uri 'abfss://?account_name=mystorageacct&account_key=dGVzdA%3D%3D' \ + --dest-table 'my-filesystem/users' +``` + + +[available layout placeholders]: https://dlthub.com/docs/dlt-ecosystem/destinations/filesystem#available-layout-placeholders +[Azure AD app registration]: https://learn.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal +[Azure Blob Storage]: https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction +[Azure Data Lake Storage Gen2]: https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction +[managing storage account access keys]: https://learn.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage +[shared access signatures]: https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview diff --git a/docs/supported-sources/cbor.md b/docs/supported-sources/cbor.md index 5f6e907be..9d41d9946 100644 --- a/docs/supported-sources/cbor.md +++ b/docs/supported-sources/cbor.md @@ -38,9 +38,8 @@ time, so if you control the writer, emit a single top-level array of records. CBOR is available on every source that goes through the shared file readers: -- Local files: [`file://`](file.md) -- [`s3://`](s3.md), [`gs://`](google-cloud-storage.md), [Azure blob storage](azure-blob-storage.md) -- [`sftp://`](sftp.md) +- Local files: {ref}`file` +- Remote files: {ref}`s3`, {ref}`gcs`, {ref}`azure-storage`, {ref}`sftp`, ... Remote reads go through the source's own fsspec handle, so they reuse its existing authentication (no separate CBOR storage configuration). A file is read as CBOR when its diff --git a/docs/supported-sources/databricks-files.md b/docs/supported-sources/databricks-files.md new file mode 100644 index 000000000..05bb97da3 --- /dev/null +++ b/docs/supported-sources/databricks-files.md @@ -0,0 +1,144 @@ +(dbfs)= +(databricks-files)= + +# Databricks files + +[Databricks] is a platform for big data analytics and artificial intelligence. +`omniload` supports access to files on Databricks volumes and workspaces as a +source. + +- Unity Catalog Volumes +- Workspace files +- Legacy DBFS (Databricks File System) + +## URI format + +The URI for connecting to Databricks files is structured as follows. + +```text +dbfs:/Volumes/catalog/schema/volume/path/to/data.parquet +``` +```text +dbfs:/Workspace/path/to/data.parquet +``` + +## URI parameters + +:host: + See `Databricks native authentication `_. + +:account_id: + See `Databricks native authentication `_. + +:token: + See `Databricks native authentication `_. + +:username: + See `Databricks native authentication `_. + +:password: + See `Databricks native authentication `_. + +:client_id: + (Service principal OAuth only) The client ID you were assigned when creating your service principal. + +:client_secret: + (Service principal OAuth only) The client secret you generated when creating your service principal. + +:azure_workspace_resource_id: + See `Azure native authentication `_. + +:azure_client_secret: + See `Azure native authentication `_. + +:azure_client_id: + See `Azure native authentication `_. + +:azure_tenant_id: + See `Azure native authentication `_. + +:azure_environment: + See `Azure native authentication `_. + +:auth_type: + See `Additional configuration options `_. + +:cluster_id: + The ID of the cluster to connect. See `Compute configuration for Databricks Connect `_. + +:profile: + See `Overriding .databrickscfg `_. + +:config_file: + See `Overriding .databrickscfg `_. + +:debug_headers: + See `Additional configuration options `_. + Type: `bool`. Default: `false`. + +:debug_truncate_bytes: + See `Additional configuration options `_. + Type: `int`. + +:volume_fs_max_read_concurrency: + The maximum number of concurrent file read operations on a Unity Catalog Volume file. + Type: `int`. Default: `24`. + +:volume_fs_min_read_block_size: + The minimum data size to read for each read operation on a Unity Catalog Volume file. + Type: `int`. Default: `1048576` (1 MB). + +:volume_fs_max_read_block_size: + The maximum data size to read for each read operation on a Unity Catalog Volume file. + Type: `int`. Default: `4194304` (4 MB). + +:volume_fs_max_write_concurrency: + The maximum number of concurrent file write operations on a Unity Catalog Volume file. + Type: `int`. Default: `24`. + +:volume_fs_min_write_block_size: + The minimum data size to write for each write operation on a Unity Catalog Volume file. + Type: `int`. Default: `5242880` (5 MB). + +:volume_fs_max_write_block_size: + The maximum data size to write for each write operation on a Unity Catalog Volume file. + Type: `int`. Default: `16777216` (16 MB). + +:volume_min_multipart_upload_size: + The minimum file size to use multipart upload for uploading files to Unity Catalog Volume. + Type: `int`. Default: `5242880` (5 MB). + +:volume_fs_connection_pool_size: + The maximum number of connections in the aiohttp connection pool for the Unity Catalog Volume file system. + Type: `int`. Default: `100`. + +:use_local_fs_in_workspace: + Access files from the local file system rather than the remote Databricks API when running within a Databricks workspace. + Type: `bool`. Default: `true`. + +:verbose_debug_log: + Whether to enable verbose debug logging for file system operations. + Type: `bool`. Default: `false`. + +## Authentication + +The Databricks connector `fsspec-databricks` uses Databricks Unified +Authentication provided by the Databricks Python SDK. You can find information +about supported authentication parameters and environment variables in the +[Databricks Python SDK documentation]. + +## Example: Load Parquet file from Databricks into DuckDB + +```sh +omniload ingest \ + --source-uri 'dbfs:/Volumes/catalog/schema/volume/path/to/data.parquet' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + + +[Databricks]: https://www.databricks.com/ +[Databricks Python SDK documentation]: https://databricks-sdk-py.readthedocs.io/en/latest/authentication.html diff --git a/docs/supported-sources/databricks.md b/docs/supported-sources/databricks-warehouse.md similarity index 87% rename from docs/supported-sources/databricks.md rename to docs/supported-sources/databricks-warehouse.md index ca5570548..edfac2080 100644 --- a/docs/supported-sources/databricks.md +++ b/docs/supported-sources/databricks-warehouse.md @@ -1,7 +1,9 @@ -# Databricks -Databricks is a platform for big data analytics and artificial intelligence. +(databricks-warehouse)= -omniload supports Databricks as both a source and destination. +# Databricks warehouse + +[Databricks] is a platform for big data analytics and artificial intelligence. +`omniload` supports the Databricks SQL warehouse as both a source and destination. ## URI format @@ -42,3 +44,6 @@ To set up OAuth M2M authentication: You can read more about Databricks OAuth M2M authentication [here](https://docs.databricks.com/en/dev-tools/auth/oauth-m2m.html). The same URI structure can be used both for sources and destinations. You can read more about SQLAlchemy's Databricks dialect [here](https://docs.databricks.com/en/dev-tools/sqlalchemy.html). + + +[Databricks]: https://www.databricks.com/ diff --git a/docs/supported-sources/dropbox.md b/docs/supported-sources/dropbox.md new file mode 100644 index 000000000..2a18c93ba --- /dev/null +++ b/docs/supported-sources/dropbox.md @@ -0,0 +1,49 @@ +(dropbox)= + +# Dropbox + +[Dropbox] is a file hosting service that offers cloud storage, file +synchronization, personal cloud, and client software. +`omniload` supports Dropbox as a data source. + +## URI format + +The URI for connecting to Dropbox is structured as follows. + +```text +dropbox://path/to/data.parquet?token=secret +``` + +## URI parameters + +:token: + Generated key by adding a dropbox app in the user dropbox account. + +## Authentication + +To integrate `omniload` with Dropbox, you need to authenticate with the +Dropbox API using an access token. +See [generate an access token for your own account]. + +## Example: Load CSV file from Dropbox into DuckDB + +```sh +omniload ingest \ + --source-uri 'dropbox://?token=secret' \ + --source-table 'path/to/data.csv' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + +:::{tip} +Here, instead of defining the remote resource exclusively per source URI +using its `` component, the `--source-table` option can specify the +base directory on the server where `omniload` should start looking for files. +::: + + +[Dropbox]: https://www.dropbox.com/dropbox +[generate an access token for your own account]: https://dropbox.tech/developers/generate-an-access-token-for-your-own-account diff --git a/docs/supported-sources/filesystem.md b/docs/supported-sources/filesystem.md index 0c1c446fe..782f264f9 100644 --- a/docs/supported-sources/filesystem.md +++ b/docs/supported-sources/filesystem.md @@ -38,18 +38,30 @@ Supported formats for write operations are currently CSV, JSONL, and Parquet. ## Supported filesystems -| Name | Description | Protocol scheme | -|:---------------------|:----------------------------------------|:----------------| -| {ref}`Local ` | Local and mounted filesystems | file:// | -| [Amazon S3] | S3 and compatible filesystems | s3:// | -| [Google GCS] | Google Cloud Storage | gs:// | -| [Azure Blob Storage] | Azure Blob Storage | az:// | -| [SFTP] | Simple File Transfer Protocol (RFC 913) | sftp:// | - +| Name | Description | Protocol scheme | Read | Write | +|:--------------------------|:--------------------------------------------------|:--------------------------------|:-----|:------| +| {ref}`Local files ` | Files on local and mounted filesystems | file:// | ✅ | ✅ | +| {ref}`azure-storage` | Azure Blob and Data Lake Storage | az://, adls://, abfss:// | ✅ | ✅ | +| {ref}`dbfs` | Databricks files on volumes and workspaces | dbfs:// | ✅ | ❌ | +| {ref}`dropbox` | Dropbox | dropbox:// | ✅ | ❌ | +| {ref}`s3` | Amazon S3 and compatible filesystems | s3:// | ✅ | ✅ | +| {ref}`ftp` | File transfer protocol (FTP) | ftp:// | ✅ | ❌ | +| {ref}`gcs` | Google Cloud Storage | gs:// | ✅ | ✅ | +| {ref}`gdrive` | Google Drive | gdrive:// | ✅ | ❌ | +| {ref}`hdfs` | Hadoop distributed file system | hdfs:// | ✅ | ❌ | +| {ref}`oracle-oci` | Oracle Cloud Infrastructure Object Storage | oci:// | ✅ | ❌ | +| {ref}`onedrive` | Microsoft OneDrive | onedrive:// | ✅ | ❌ | +| {ref}`oss` | Alibaba Object Storage Service (OSS) | oss:// | ✅ | ❌ | +| {ref}`r2` | Cloudflare R2 | r2:// | ✅ | ❌ | +| {ref}`sharepoint` | Microsoft SharePoint | sharepoint:// | ✅ | ❌ | +| {ref}`sftp` | SSH File Transfer Protocol | sftp:// | ✅ | ❌ | +| {ref}`webdav` | Web Distributed Authoring and Versioning (WebDAV) | http+webdav://, https+webdav:// | ✅ | ❌ | +| {ref}`webhdfs` | WebHDFS REST API for Hadoop | webhdfs:// | ✅ | ❌ | :::{note} `omniload` supports read and write operations on both local and remote filesystems. -See {ref}`file:// destination ` for write support. +For some filesystems, write support has not been unlocked yet, but we expect it to +land during the upcoming releases. ::: ## Incremental file selection @@ -262,13 +274,9 @@ guarantee you get. are dropped silently. Validate file integrity upstream if partial loads would be a problem. -[Amazon S3]: https://aws.amazon.com/ -[Azure Blob Storage]: https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction [CSV]: https://en.wikipedia.org/wiki/Comma-separated_values -[Google GCS]: https://cloud.google.com/storage [iterabledata]: https://pypi.org/project/iterabledata/ [JSONL]: https://en.wikipedia.org/wiki/JSON_streaming#JSONL [Parquet]: https://en.wikipedia.org/wiki/Apache_Parquet [polars.read_csv]: https://docs.pola.rs/api/python/stable/reference/api/polars.read_csv.html [polars.read_excel]: https://docs.pola.rs/api/python/stable/reference/api/polars.read_excel.html -[SFTP]: https://en.wikipedia.org/wiki/File_Transfer_Protocol#Simple_File_Transfer_Protocol diff --git a/docs/supported-sources/ftp.md b/docs/supported-sources/ftp.md new file mode 100644 index 000000000..4e3fb0839 --- /dev/null +++ b/docs/supported-sources/ftp.md @@ -0,0 +1,93 @@ +(ftp)= + +# FTP + +The [File Transfer Protocol (FTP)] is a standard communication protocol used +for the transfer of computer files from a server to a client over a computer +network. +`omniload` supports FTP as a data source. + +## URI format + +The URI for connecting to an FTP server is structured as follows. + +```text +ftp://:@intranet.example.org/path/to/data.parquet?tls=tls +``` + +## URI parameters + +:host: + The remote server name/ip to connect to. + +:port: + Port to connect with. + Type: `int`. Default: `21`. + +:username: + The username for authenticating (optional). + +:password: + The password for authenticating (optional). + +:acct: + Some servers also need an "account" string for authentication. + +:block_size: + The read-ahead or write buffer size in bytes. + Type: `int`. Default: `65536`. + +:tempdir: + Directory on remote to put temporary files when in a transaction. + +:timeout: + Timeout of the FTP connection in seconds. + Type: `int`. Default: `30`. + +:encoding: + Encoding to use for directories and filenames in FTP connection. + Default: `utf-8`. + +:tls: + Enable FTP-TLS for secure connections. + Type: `bool` or `str`. Default: `False`. + Accepted values are: + + - `false`: Use plain FTP (default). + - `true`: Use explicit TLS (FTPS with AUTH TLS command). + - `tls`: Auto-negotiate the highest protocol. + - `tlsv1`: TLS v1.0 + - `tlsv1_1`: TLS v1.1 + - `tlsv1_2`: TLS v1.2 + +## Authentication + +Authentication will be anonymous if username/password credentials are not +provided. + +## Examples + +To integrate `omniload` with an FTP server, you need the server's +hostname, port, a valid username, and a password. + +### Load CSV data from FTP into DuckDB + +```sh +omniload ingest \ + --source-uri 'ftp://:@intranet.example.org?tls=tls' \ + --source-table '/path/to/data.csv' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + +:::{tip} +Here, instead of defining the remote resource exclusively per source URI +using its `` component, the `--source-table` option can specify the +base directory on the server where `omniload` should start looking for files. +::: + + +[File Transfer Protocol (FTP)]: https://en.wikipedia.org/wiki/File_Transfer_Protocol diff --git a/docs/supported-sources/google-cloud-storage.md b/docs/supported-sources/google-cloud-storage.md index be264743a..cd889d44c 100644 --- a/docs/supported-sources/google-cloud-storage.md +++ b/docs/supported-sources/google-cloud-storage.md @@ -1,6 +1,8 @@ -# Google Cloud Storage +(gcs)= -[Google Cloud Storage](https://cloud.google.com/storage?hl=en) is an online file storage web service for storing and accessing data on Google Cloud Platform infrastructure. The service combines the performance and scalability of Google's cloud with advanced security and sharing capabilities. It is an Infrastructure as a Service (IaaS), comparable to Amazon S3. +# GCS + +[Google Cloud Storage (GCS)] is an online file storage web service for storing and accessing data on Google Cloud Platform infrastructure. The service combines the performance and scalability of Google's cloud with advanced security and sharing capabilities. It is an Infrastructure as a Service (IaaS), comparable to Amazon S3. `omniload` supports Google Cloud Storage as both a data source and destination. @@ -194,3 +196,6 @@ omniload ingest \ --dest-uri "duckdb:///local.db" \ --dest-table "public.raw_data" ``` + + +[Google Cloud Storage (GCS)]: https://cloud.google.com/storage?hl=en diff --git a/docs/supported-sources/google-drive.md b/docs/supported-sources/google-drive.md new file mode 100644 index 000000000..0bf5b2392 --- /dev/null +++ b/docs/supported-sources/google-drive.md @@ -0,0 +1,126 @@ +(gdrive)= +(google-drive)= + +# Google Drive + +[Google Drive] is a secure cloud storage platform for seamless file sharing +and enhanced collaboration. +`omniload` supports access to files on Google Drive as a source. + +## URI format + +The URI for connecting to Google Drive files is structured as follows. + +```text +gdrive://path/to/data.parquet?token=anon +``` + +## URI parameters + +:root_file_id: + If you have a share, drive or folder ID to treat as the FS root, enter + it here. Otherwise, you will get your default drive + Default: None. + +:token: + One of "anon", "browser", "cache", "service_account". Using "browser" will prompt a URL to + be put in a browser, and cache the response for future use with token="cache". + "browser" will remove any previously cached token file, if it exists. + +:access: + One of `full_control` or `read_only`. + +:spaces: + Category of files to search; can be `drive`, `appDataFolder` or `photos`. + Of these, only the first is general. + +:creds: + Required just for "service_account" token, a dict containing the service account + credentials obtained in GCP console. The dict content is the same as the json file + downloaded from GCP console. See also [service account keys]. + This credential can be useful when integrating with other GCP services, and when you + don't want the user to be prompted to authenticate. + The files need to be shared with the service account email address, that can be found + in the json file. + Type: `dict`. Use JSON to encode the dictionary. + +:auth_kwargs: + Additional keyword arguments passed to the authentication backend + (`pydata_google_auth.get_user_credentials` for user OAuth, or + `service_account.Credentials.from_service_account_info` for service + accounts). For headless or remote environments where a local callback + server is unavailable, pass `use_local_webserver=False` to request a + token via the console. + Type: `dict`. Use JSON to encode the dictionary. + +:use_local_webserver: + Type: `bool`. Default: `true`. + +## Authentication + +There are several methods to authenticate with Google Drive. + +### 1. Service account credentials + +In this method, you provide a dictionary containing the service account +credentials obtained in the GCP console. The dictionary content is the +same as the JSON file downloaded from the GCP console. See also +[service account keys]. + +This credential can be useful when integrating with other GCP services, +and when you don't want the user to be prompted to authenticate. + +Example: +```text +?token=service_account&creds=%7B%22type%22%3A%22service_account%22%2C%22project_id%22%3A%22my-project%22%2C%22private_key_id%22%3A%22key-id%22%2C%22private_key%22%3A%22-----BEGIN%20PRIVATE%20KEY-----%5Cn...%5Cn-----END%20PRIVATE%20KEY-----%5Cn%22%2C%22client_email%22%3A%22omniload%40my-project.iam.gserviceaccount.com%22%2C%22client_id%22%3A%221234567890%22%2C%22token_uri%22%3A%22https%3A%2F%2Foauth2.googleapis.com%2Ftoken%22%7D +``` + +In this example, the JSON object from a downloaded Google service-account +key is percent-encoded as one `creds` query value. + +#### 2. OAuth with user credentials + +A browser will be opened to complete the OAuth authentication flow. +Afterwards, the access token will be stored locally, and you can reuse +it in subsequent sessions. + +Example: +```text +?token=browser +``` + +On headless or remote machines (SSH sessions, containers, CI, and similar +environments), you may not be able to bind a local callback server or open +a browser on the same host. In that case, pass `use_local_webserver: False` +in `auth_kwargs` to request a token via the console. + +Example: +```text +?token=browser&use_local_webserver=false +``` + +#### 3. Anonymous (read-only) access + +If you want to interact with files that are shared publicly ("anyone with +the link"), then you do not need to authenticate to Google Drive. + +Example: +```text +?token=anon +``` + +## Example: Load Parquet file from Google Drive into DuckDB + +```sh +omniload ingest \ + --source-uri 'gdrive://path/to/data.parquet?token=anon' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + + +[Google Drive]: https://workspace.google.com/products/drive/ +[service account keys]: https://docs.cloud.google.com/iam/docs/service-account-creds#key-types diff --git a/docs/supported-sources/hdfs.md b/docs/supported-sources/hdfs.md new file mode 100644 index 000000000..52051d741 --- /dev/null +++ b/docs/supported-sources/hdfs.md @@ -0,0 +1,93 @@ +(hdfs)= + +# HDFS + +[Hadoop distributed file system (HDFS)] is a distributed, scalable, and +portable file system written in Java for the Hadoop framework. + +`omniload` supports HDFS as a data source. + +## URI Format + +The URI for connecting to HDFS is structured as follows. + +```text +hdfs://example.com:8020/path/to/data.parquet?user=test +``` + +## URI components + +:host: + HDFS host to connect to. Set to "default" for `fs.defaultFS` from + `core-site.xml`. + +:port: + HDFS port to connect to. + Type: `int`. Default: `8020`. + +:user: + Username when connecting to HDFS; None implies login user. + +:replication: + Number of copies each block will have. + Type: `int`. Default: `3`. + +:buffer_size: + The size of the temporary read and write buffer in bytes. + `0` means no buffering will happen. + Type: `int`. Default: `0`. + +:block_size: + The block size in bytes. `None` means the default configuration + for HDFS, a typical block size is 128 MB. + Type: `int`. Default: `None`. + +:kerb_ticket: + The path to the Kerberos ticket cache. + +:extra_conf: + Optional extra key/value pairs for configuration; will override any + `hdfs-site.xml` properties. + Type: `dict`. Use JSON to encode the dictionary. + +## Examples + +To integrate `omniload` with HDFS, you need the server's hostname (endpoint) +and valid credentials. + +### Load Parquet data from HDFS into DuckDB + +The following command demonstrates how to copy data from a specified HDFS +location into a DuckDB database. + +```sh +omniload ingest \ + --source-uri 'hdfs://example.com:8020/?user=test' \ + --source-table 'path/to/data.parquet' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + +:::{tip} +Here, instead of defining the remote resource exclusively per source URI +using its `` component, the resource location is specified using the +separate `--source-table` option. Both addressing variants are supported equally. +::: + +## Backlog + +:::{todo} +PyArrow comes with bindings to the Hadoop File System, however you must +still [configure it properly]. In this spirit, because no packaging +efforts were poured into this, the HDFS connector can not be expected +to work out of the box, for example when using the omniload OCI image. +Please [create an issue] to ping us about any improvement needs. +::: + + +[configure it properly]: https://arrow.apache.org/docs/python/filesystems.html#hadoop-distributed-file-system-hdfs +[create an issue]: https://github.com/panodata/omniload/issues +[Hadoop distributed file system (HDFS)]: https://en.wikipedia.org/wiki/Apache_Hadoop#HDFS diff --git a/docs/supported-sources/msgpack.md b/docs/supported-sources/msgpack.md index 687148b0a..2c02f4562 100644 --- a/docs/supported-sources/msgpack.md +++ b/docs/supported-sources/msgpack.md @@ -30,9 +30,8 @@ reader; see {ref}`file-format-routing` about how omniload chooses a reader per f MessagePack is available on every source that goes through the shared file readers: -- Local files: [`file://`](file.md) -- [`s3://`](s3.md), [`gs://`](google-cloud-storage.md), [Azure blob storage](azure-blob-storage.md) -- [`sftp://`](sftp.md) +- Local files: {ref}`file` +- Remote files: {ref}`s3`, {ref}`gcs`, {ref}`azure-storage`, {ref}`sftp`, ... Remote reads go through the source's own fsspec handle, so they reuse its existing authentication (no separate MessagePack storage configuration). A file is read as diff --git a/docs/supported-sources/nextcloud.md b/docs/supported-sources/nextcloud.md new file mode 100644 index 000000000..eeaedf2a6 --- /dev/null +++ b/docs/supported-sources/nextcloud.md @@ -0,0 +1,40 @@ +(nextcloud)= + +# Nextcloud + +[Nextcloud Files] is a secure cloud storage and file sharing software for easy +sync, sharing and collaboration on your files. +`omniload` supports Nextcloud as a data source. + +## URI format + +The URI for connecting to Nextcloud is structured as follows. +```text +https+webdav://:@cloud.example.org/remote.php/webdav +``` + +## URI parameters + +The Nextcloud connected uses the {ref}`webdav` connector, so all of its +URL parameters are also available here. + +## Authentication + +To integrate `omniload` with Nextcloud, you need to authenticate like you +do with any HTTP server. + +## Example: Load CSV file from Nextcloud into DuckDB + +```sh +omniload ingest \ + --source-uri 'https+webdav://:@cloud.example.org/remote.php/webdav' \ + --source-table 'path/to/data.csv' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + + +[Nextcloud Files]: https://nextcloud.com/files/ diff --git a/docs/supported-sources/ods.md b/docs/supported-sources/ods.md index 2ace4809d..06804e1c2 100644 --- a/docs/supported-sources/ods.md +++ b/docs/supported-sources/ods.md @@ -10,9 +10,8 @@ used by [OpenOffice], [LibreOffice], and other spreadsheet applications. OpenOffice and LibreOffice ODS files can be accessed on every source that goes through the shared file readers: -- Local files: [`file://`](file.md) -- [`s3://`](s3.md), [`gs://`](google-cloud-storage.md), [Azure blob storage](azure-blob-storage.md) -- [`sftp://`](sftp.md) +- Local files: {ref}`file` +- Remote files: {ref}`s3`, {ref}`gcs`, {ref}`azure-storage`, {ref}`sftp`, ... A file is read as ODS when its extension is `.ods` (optionally `.ods.gz`), or when an explicit `#ods` {ref}`format hint ` is appended. diff --git a/docs/supported-sources/onedrive.md b/docs/supported-sources/onedrive.md new file mode 100644 index 000000000..fc810fa58 --- /dev/null +++ b/docs/supported-sources/onedrive.md @@ -0,0 +1,110 @@ +(onedrive)= + +# OneDrive + +[Microsoft OneDrive] is a cloud storage service that lets you store, +share, and collaborate on files. You can connect your OneDrive for +Business instance as a data source for your managed knowledge base +to crawl the personal drives of users in your Microsoft 365 tenant. +`omniload` supports OneDrive as a data source. + +## URI format + +The URI for connecting to OneDrive is structured as follows. + +```text +onedrive:///path/to/data.xlsx?client_id=1d2befad-2f22-4124-a779-b147dfeca342&tenant_id=6b337423-f504-4060-a91b-e9eaaf782609&client_secret=abc~xyz789EXAMPLE_foo +``` + +## URI parameters + +:drive_id: + The ID of the OneDrive drive. If provided, enables single-site mode. + +:client_id: + OAuth2 client ID. Can also be set via MSGRAPHFS_CLIENT_ID + or AZURE_CLIENT_ID environment variables. + +:tenant_id: + OAuth2 tenant ID. Can also be set via MSGRAPHFS_TENANT_ID + or AZURE_TENANT_ID environment variables. + +:client_secret: + OAuth2 client secret. Can also be set via MSGRAPHFS_CLIENT_SECRET + or AZURE_CLIENT_SECRET environment variables. + +:site_name: + The name of the OneDrive site. If provided with drive_name, + enables single-site mode. + +:drive_name: + The name of the OneDrive drive/library (e.g., "Documents", + "CustomLibrary"). If provided with `site_name`, enables + single-site mode. + +:url_path: + URL-style path specification (e.g., "msgd://TestSite/Documents"). + If provided, extracts `site_name` and `drive_name` from the URL. + URL parameters override direct `site_name`/`drive_name` parameters. + +:oauth2_client_params: + Parameters for the OAuth2 client. If not provided, will be built + from `client_id`, `tenant_id`, `client_secret`. + Type: `dict`. Use JSON to encode the dictionary. + +:use_recycle_bin: + If True, deleted files are moved to recycle bin. Default is False. + Truthy values are `"true", "yes", "on", "y", "t", "1"`. + Falsy values are `"false", "no", "off", "n", "f", "0"`. + +:::{note} +Access works unified for both OneDrive sites and drives. +The module handles both single-site/drive operations and multi-site +operations based on the parameters provided during initialization: + +- Single-site mode: When `site_name` + `drive_name` or `drive_id` are provided +- Multi-site mode: When neither `site_name` + `drive_name` nor `drive_id` are provided + +Multi-site mode can handle URL-based paths that specify +the site and drive dynamically (e.g., `msgd://SiteA/DriveB/file.txt`). +::: + +## Authentication + +The OneDrive connector uses OAuth 2.0 to authenticate with the +Microsoft Graph API. [Microsoft Graph] is a protected API gateway for +accessing data in Microsoft cloud services like [Microsoft Entra ID], +Microsoft 365, OneDrive, or SharePoint. It is protected by the +[Microsoft identity platform], which authorizes and verifies that an +app is authorized to call Microsoft Graph. + +Please get familiar with relevant concepts to configure OAuth 2.0 +authentication properly, see also [authentication and authorization basics] +and [set up OAuth 2.0 authentication for OneDrive] tutorial by AWS. + +## Example: Load CSV file from OneDrive into DuckDB + +```sh +omniload ingest \ + --source-uri 'onedrive://?client_id=1d2befad-2f22-4124-a779-b147dfeca342&tenant_id=6b337423-f504-4060-a91b-e9eaaf782609&client_secret=abc~xyz789EXAMPLE_foo' \ + --source-table '//path/to/data.csv' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + +:::{tip} +Here, instead of defining the remote resource exclusively per source URI +using its `` component, the `--source-table` option can specify the +base directory on the server where `omniload` should start looking for files. +::: + + +[authentication and authorization basics]: https://learn.microsoft.com/en-us/graph/auth/auth-concepts +[Microsoft Entra ID]: https://en.wikipedia.org/wiki/Microsoft_Entra_ID +[Microsoft Graph]: https://learn.microsoft.com/en-us/graph/ +[Microsoft identity platform]: https://learn.microsoft.com/en-us/entra/identity-platform/v2-overview +[Microsoft OneDrive]: https://en.wikipedia.org/wiki/OneDrive +[Set up OAuth 2.0 authentication for OneDrive]: https://docs.aws.amazon.com/bedrock/latest/userguide/kb-managed-onedrive-oauth2-setup.html diff --git a/docs/supported-sources/oracle-oci.md b/docs/supported-sources/oracle-oci.md new file mode 100644 index 000000000..266484f2a --- /dev/null +++ b/docs/supported-sources/oracle-oci.md @@ -0,0 +1,94 @@ +(oracle-oci)= + +# OCI + +[Oracle Cloud Infrastructure Object Storage (OCI)] is an internet-scale, +high-performance storage platform that offers reliable and cost-efficient +data durability provided by Oracle. The Object Storage service can store +an unlimited amount of unstructured data of any content type. + +`omniload` supports OCI as a data source. + +## URI format + +The URI for connecting to OCI is structured as follows. +```text +oci://@//data.parquet?iam_type=api_key&config={"user":"ocid1.user.oc1..24g4uzg","region":"us-ashburn-1","tenancy":"ocid1.tenancy.oc1..23423r3","key_file":"/path/to/key.pem","fingerprint":"06:8c:ce:5b:4a:b5:53:d4:f8:e0:d2:58:63:1c:8d:d2"} +``` + +## URI parameters + +:config: + Config for the connection to OCI, see [SDK and CLI Configuration File]. + If a dict, it should be returned from `oci.config.from_file`. + If a str, it should be the location of the config file. + If None, user should have a "resource principal" configured environment. + If resource principal is not available, use instance principal. + Use JSON to encode the dictionary. + Type: `dict` or `str`. Default: None. + +:signer: + A signer from the OCI sdk. More info: oci.auth.signers + Type: `oci.auth.signers`. Default: None. + +:profile: + The profile to use from the config, if the config is passed in. + +:iam_type: + The IAM Auth principal type to use. + Values can be one of "api_key", "resource_principal", "instance_principal", + "oke_principal", "unknown_signer". + +:compartment_id: + The OCID of the compartment to scope the authorization to. If not + provided, the tenancy's root compartment will be used. + +:region: + The region identifier that the client should connect to. + Regions can be found here: + https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm + +:block_size: + The block size in bytes. Default: `5242880` (`5MB`). + +:config_kwargs: + Dictionary of parameters passed to the OCI Client upon connection. + Use JSON to encode the dictionary. + Type: `dict`. Default: None. + +:oci_additional_kwargs: + Dictionary of parameters that are used when calling OCI api + methods. Typically used for things like "retry_strategy". + Use JSON to encode the dictionary. + Type: `dict`. Default: None. + +:more...: + Other parameters for OCI session. + This includes default parameters for tenancy, namespace, and region. + +## Set up an OCI integration + +To integrate `omniload` with Oracle Cloud Infrastructure Object Storage, +you need a storage account and one of the supported credentials. You can +start with the [free tier] and later [upgrade your account]. + +## Examples + +### Load data from OCI + +```shell +omniload ingest \ + --source-uri 'oci://bucket@namespace?iam_type=api_key&config={"user":"ocid1.user.oc1..24g4uzg","region":"us-ashburn-1","tenancy":"ocid1.tenancy.oc1..23423r3","key_file":"/path/to/key.pem","fingerprint":"06:8c:ce:5b:4a:b5:53:d4:f8:e0:d2:58:63:1c:8d:d2"}' \ + --source-table '/prefix/data.parquet' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + + +[free tier]: https://docs.oracle.com/en-us/iaas/Content/FreeTier/freetier.htm +[Oracle Cloud Infrastructure Object Storage (OCI)]: https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/objectstorageoverview.htm +[SDK and CLI Configuration File]: https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm +[upgrade your account]: https://docs.oracle.com/en-us/iaas/Content/Billing/Tasks/changingpaymentmethod.htm diff --git a/docs/supported-sources/oss.md b/docs/supported-sources/oss.md new file mode 100644 index 000000000..45d7ce194 --- /dev/null +++ b/docs/supported-sources/oss.md @@ -0,0 +1,99 @@ +(oss)= + +# OSS + +[Object Storage Service (OSS)] is a scalable cloud storage service offered by +Alibaba Cloud. It is a fully managed object storage service to store and +access any amount of data from anywhere. For more information, see the +[OSS getting started FAQ]. + +`omniload` supports OSS as a data source. + +## URI Format + +The URI for connecting to OSS is structured as follows. Either access the resource +anonymously, use key/secret credentials, or a security token for authentication. +In this form, the remote resource is addressed exclusively using a single +parameter, an URI, encoding all required options. + +:Anonymous access: + ```text + oss://bucket/path/to/data.parquet?endpoint=http://oss-cn-hangzhou.aliyuncs.com/ + ``` +:Authenticate with key and secret: + ```text + oss://bucket/path/to/data.parquet?endpoint=https://oss-me-east-1.aliyuncs.com/&key=foo&secret=bar + ``` +:Authenticate with security token: + ```text + oss://bucket/path/to/data.parquet?endpoint=https://oss-me-east-1.aliyuncs.com/&token=foobar + ``` + +## URI components + +:\: + The path to the resource can reference one or multiple files using + file globbing. + +:endpoint: + OSS endpoint location that can be changed after the initialization. + Can alternatively be defined using the `OSS_ENDPOINT` environment variable. + Examples: `http://oss-cn-hangzhou.aliyuncs.com` or `https://oss-me-east-1.aliyuncs.com`. + +:key: + If not anonymous, use this access key ID, when specified. + +:secret: + If not anonymous, use this secret access key, when specified. + +:token: + If not anonymous, use this security token, when specified. + +:block_size: + The block size in bytes. Default: `5242880` (`5MB`). + +:cache_type: + The cache type value used for `open()`. + Set to `none` to disable caching. + The default value is `readahead`. + For other cache types, please refer to the `fsspec` package. + +## Set up an OSS integration + +Before you use Alibaba Cloud OSS, make sure you have registered an Alibaba +Cloud account. For instructions, see [create an Alibaba Cloud account]. +After you create an Alibaba Cloud account, [activate OSS]. + +## Examples + +To integrate `omniload` with OSS, you need the server's hostname (endpoint) +and valid credentials. + +### Load CSV data from OSS into DuckDB + +The following command demonstrates how to copy data from a specified OSS +location into a DuckDB database. + +```sh +omniload ingest \ + --source-uri 'oss://?endpoint=http://oss-cn-hangzhou.aliyuncs.com/' \ + --source-table 'path/to/data.parquet' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + +:::{tip} +Here, instead of defining the remote resource exclusively per source URI +using its `` component, the bucket name and the file glob pattern +are specified using the separate `--source-table` option. Both addressing +variants are supported equally. +::: + + +[activate OSS]: https://oss.console.alibabacloud.com/overview +[create an Alibaba Cloud account]: https://account.alibabacloud.com/register/intl_register.htm +[Object Storage Service (OSS)]: https://www.alibabacloud.com/en/product/object-storage-service +[OSS getting started FAQ]: https://www.alibabacloud.com/help/en/oss/faq-15 diff --git a/docs/supported-sources/r2.md b/docs/supported-sources/r2.md new file mode 100644 index 000000000..1f709ac33 --- /dev/null +++ b/docs/supported-sources/r2.md @@ -0,0 +1,76 @@ +(r2)= + +# R2 + +[Cloudflare R2] is a scalable, durable, affordable and managed object storage +drop-in replacement for existing S3 workflows. + +`omniload` supports R2 as a data source. + +## URI Format + +The URI for connecting to R2 is structured as follows. + +```text +r2://bucket/path/to/data.parquet?endpoint_url=https://YOUR_ACCOUNT_ID.r2.cloudflarestorage.com&access_key_id=YOUR_ACCESS_KEY&secret_access_key=YOUR_SECRET_ACCESS_KEY +``` + +## URI parameters + +:endpoint_url: + The endpoint URL to connect to. + +:access_key_id: + Your access key ID. + +:secret_access_key: + Your secret access key. + +## Set up an R2 integration + +Before you use Cloudflare R2, make sure you have registered an account. +Then, [generate an API token] to serve as the Access Key for usage with +existing S3-compatible SDKs or XML APIs. + +## Examples + +To integrate `omniload` with R2, you need your account id, access key, +and secret. + +### Load Parquet data from R2 into DuckDB + +The following command demonstrates how to copy data from a specified R2 +location into a DuckDB database. + +```sh +omniload ingest \ + --source-uri 'r2://?endpoint_url=https://YOUR_ACCOUNT_ID.r2.cloudflarestorage.com&access_key_id=YOUR_ACCESS_KEY&secret_access_key=YOUR_SECRET_ACCESS_KEY' \ + --source-table 'my_bucket/path/to/data.parquet' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + +:::{tip} +Here, instead of defining the remote resource exclusively per source URI +using its `` component, the bucket name and the file glob pattern +are specified using the separate `--source-table` option. Both addressing +variants are supported equally. +::: + +## Backlog + +:::{todo} + +- The `endpoint_url` doesn't need to be conveyed fully. It is enough to let + the user supply the `account_id` parameter in the connection URL. + +- The [temporary credentials API] is not covered yet. +::: + + +[Cloudflare R2]: https://www.cloudflare.com/products/r2/ +[generate an API token]: https://developers.cloudflare.com/r2/api/tokens/ +[temporary credentials API]: https://developers.cloudflare.com/r2/examples/authenticate-r2-temp-credentials/ diff --git a/docs/supported-sources/s3.md b/docs/supported-sources/s3.md index 72f574011..2505dc514 100644 --- a/docs/supported-sources/s3.md +++ b/docs/supported-sources/s3.md @@ -1,110 +1,45 @@ +(s3)= + # S3 -Amazon Simple Storage Service (S3) is a scalable cloud storage service offered by Amazon Web Services (AWS). It allows users to store and retrieve extensive amounts of data from anywhere on the web. +[Amazon S3] (Simple Storage Service) is a scalable cloud storage service offered by +Amazon Web Services (AWS). It allows users to store and retrieve extensive amounts +of data from anywhere on the web. `omniload` supports Amazon S3 as both a data source and destination. -## URI Format +## URI format -The URI for connecting to Amazon S3 is structured as follows: +The URI for connecting to Amazon S3 is structured as follows. ```text s3://?access_key_id=&secret_access_key= ``` -**URI Parameters:** - -* `access_key_id`: Your AWS access key ID. -* `secret_access_key`: Your AWS secret access key. -* `endpoint_url`: URL of an S3-Compatible API Server (optional, for S3-compatible storage like Minio) -* `layout`: Layout template (optional, destination only) - -These credentials are required to authenticate and authorize access to your S3 buckets. - -The `--source-table` parameter specifies the S3 bucket and file pattern using the following format: - -``` -/ -``` - -## Setting up an S3 Integration - -To integrate `omniload` with Amazon S3, you need an `access_key_id` and a `secret_access_key`. For guidance on obtaining these credentials, refer to the dltHub documentation on [AWS credentials](https://dlthub.com/docs/dlt-ecosystem/verified-sources/filesystem#get-credentials). +## URI parameters -Once you have your credentials, you can configure the S3 URI. The `bucket_name` and `path_to_files` (file glob pattern) are specified in the `--source-table` argument. +:access_key_id: + Your AWS access key ID. -### Example: Loading data from S3 +:secret_access_key: + Your AWS secret access key. -Let's assume the following details: -* `access_key_id`: `AKC3YOW7E` -* `secret_access_key`: `XCtkpL5B` -* S3 bucket name: `my_bucket` -* Path to files within the bucket: `students/students_details.csv` - -The following command demonstrates how to copy data from the specified S3 location to a DuckDB database: - -```sh -omniload ingest \ - --source-uri 's3://?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \ - --source-table 'my_bucket/students/students_details.csv' \ - --dest-uri duckdb:///s3_data.duckdb \ - --dest-table 'processed_students.student_details' -``` - -This command will create a table named `student_details` within the `processed_students` schema (or equivalent grouping) in the DuckDB database file located at `s3_data.duckdb`. - -### Example: Uploading data to S3 -For this, example we'll assume that: -* `records.db` is a duckdb database. -* has a table called `public.users`. -* the S3 credentials are the same as the example above. - -The following command demonstrates how to copy data from a local duckdb database to S3: -```sh -omniload ingest \ - --source-uri 'duckdb:///records.db' \ - --source-table 'public.users' \ - --dest-uri 's3://?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \ - --dest-table 'my_bucket/records' -``` - -This will result in a file structure like the following: -``` -my_bucket/ -└── records - ├── _dlt_loads - ├── _dlt_pipeline_state - ├── _dlt_version - └── users - └── ..parquet -``` +:endpoint_url: + URL of an S3-Compatible API Server (optional, for S3-compatible storage like MinIO). -The value of `load_id` and `file_id` is determined at runtime. The default layout creates a folder with the same table name as the source and places the data inside a parquet file. This layout is configurable using the `layout` parameter. +:layout: + Layout template (optional, destination only) -For example, if you would like to create a parquet file with the same name as the source table (as opposed to a folder) you can set `layout` to `{table_name}.{ext}` in the commandline above: +These credentials are required to authenticate and authorize access to your S3 buckets. -```sh -omniload ingest \ - --source-uri 'duckdb:///records.db' \ - --source-table 'public.users' \ - --dest-uri 's3://?layout={table_name}.{ext}&access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \ - --dest-table 'my_bucket/records' -``` +## Authentication -Result: -``` -my_bucket/ -└── records - ├── _dlt_loads - ├── _dlt_pipeline_state - ├── _dlt_version - └── users.parquet -``` +To integrate `omniload` with Amazon S3, you need an `access_key_id` and a `secret_access_key`. +For guidance on obtaining these credentials, refer to the dltHub documentation on [AWS credentials]. -List of available Layout variables is available [here](https://dlthub.com/docs/dlt-ecosystem/destinations/filesystem#available-layout-placeholders) +## S3-compatible object stores -### Working with S3-Compatible object stores -`omniload` supports S3-compatible storage services like [Minio](https://min.io/), Digital Ocean [Spaces](https://www.digitalocean.com/products/spaces) and Cloudflare [R2](https://developers.cloudflare.com/r2/). You can set the `endpoint_url` in your URI to read from or write to these object stores. +`omniload` supports S3-compatible storage services like [MinIO](https://min.io/), Digital Ocean [Spaces](https://www.digitalocean.com/products/spaces) and Cloudflare [R2](https://developers.cloudflare.com/r2/). You can set the `endpoint_url` in your URI to read from or write to these object stores. For example, if you're running Minio on `localhost:9000`, you can read data from it: ```sh @@ -124,13 +59,17 @@ omniload ingest \ --dest-table 'my_bucket/records' ``` -### File Glob Pattern Examples: +## File glob patterns -::: warning -Glob patterns only apply when loading data from S3 as source. -::: +The `--source-table` parameter specifies the S3 bucket and file glob pattern +using the following format. -The `` in the `--source-table` argument allows for flexible file selection. Here are some common patterns and their descriptions: +```text +/ +``` + +The `` allows for flexible file selection. Here are some +common patterns and their descriptions. | Pattern | Description | | :------------------------------------------ | :--------------------------------------------------------------------------------------------------------- | @@ -140,8 +79,30 @@ The `` in the `--source-table` argument allows for flexible f | `bucket/myFolder/mySubFolder/users.parquet` | Retrieves the specific `users.parquet` file from the `myFolder/mySubFolder/` path in `s3://bucket`. | | `bucket/employees.jsonl` | Retrieves the `employees.jsonl` file located at the root level of the `s3://bucket`. | +## File type hinting -### Working with compressed files +If your files are properly encoded but lack the correct file extension (BSON, CSV, JSONL, or Parquet), you can provide a file type hint to inform `omniload` about the format of the files. This is done by appending a fragment identifier (`#format`) to the end of the path in your `--source-table` parameter. + +For example, if you have JSONL-formatted log files stored in S3 with a non-standard extension: + +```text +--source-table "my_bucket/logs/event-data#jsonl" +``` + +This tells `omniload` to process the files as JSONL, regardless of their actual extension. + +Supported format hints include: +- `#bson` - For BSON (MongoDB dump) files. See [BSON](bson.md). +- `#csv` - For comma-separated values files with headers +- `#csv_headless` - For CSV files without headers +- `#jsonl` - For line-delimited JSON files +- `#parquet` - For Parquet format files + +:::{tip} +File type hinting works with `gzip` compressed files as well. +::: + +## Compressed files `omniload` automatically detects and handles gzipped files in your S3 bucket. You can load data from compressed files with the `.gz` extension without any additional configuration. @@ -165,32 +126,10 @@ omniload ingest \ --dest-table 'logs.events' ``` -### File type hinting - -If your files are properly encoded but lack the correct file extension (BSON, CSV, JSONL, or Parquet), you can provide a file type hint to inform `omniload` about the format of the files. This is done by appending a fragment identifier (`#format`) to the end of the path in your `--source-table` parameter. - -For example, if you have JSONL-formatted log files stored in S3 with a non-standard extension: - -``` ---source-table "my_bucket/logs/event-data#jsonl" -``` - -This tells `omniload` to process the files as JSONL, regardless of their actual extension. - -Supported format hints include: -- `#bson` - For BSON (MongoDB dump) files. See [BSON](bson.md). -- `#csv` - For comma-separated values files with headers -- `#csv_headless` - For CSV files without headers -- `#jsonl` - For line-delimited JSON files -- `#parquet` - For Parquet format files - -::: tip -File type hinting works with `gzip` compressed files as well. -::: +## CSV files without headers -### CSV files without headers - -For CSV files that don't have a header row, use the `#csv_headless` format hint. You can optionally provide column names using the `--columns` flag: +For CSV files that don't have a header row, use the `#csv_headless` format +hint. You can optionally provide column names using the `--columns` flag. ```sh # With custom column names @@ -202,7 +141,8 @@ omniload ingest \ --dest-table 'public.raw_data' ``` -If no column names are provided, columns will be automatically named `unknown_col_0`, `unknown_col_1`, etc.: +If no column names are provided, columns will be automatically named +`unknown_col_0`, `unknown_col_1`, etc. ```sh # Without column names (auto-generated) @@ -211,4 +151,83 @@ omniload ingest \ --source-table 'my_bucket/data/raw-data.csv#csv_headless' \ --dest-uri duckdb:///local.duckdb \ --dest-table 'public.raw_data' -``` \ No newline at end of file +``` + +## Examples + +### Load data from S3 + +Let's assume the following details: +* `access_key_id`: `YOUR_ACCESS_KEY_ID` +* `secret_access_key`: `YOUR_SECRET_ACCESS_KEY` +* S3 bucket name: `YOUR_BUCKET_NAME` +* Path to files within the bucket: `path/to/data.csv` + +The following command demonstrates how to copy data from the specified S3 location to a DuckDB database: + +```sh +omniload ingest \ + --source-uri 's3://?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \ + --source-table 'YOUR_BUCKET_NAME/path/to/data.csv' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + +### Upload data to S3 + +For this, example we'll assume that: +* `records.db` is a duckdb database. +* has a table called `public.users`. +* the S3 credentials are the same as the example above. + +The following command demonstrates how to copy data from a local duckdb database to S3: +```sh +omniload ingest \ + --source-uri 'duckdb:///records.db' \ + --source-table 'public.users' \ + --dest-uri 's3://?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \ + --dest-table 'my_bucket/records' +``` + +This will result in a file structure like the following: +```text +my_bucket/ +└── records + ├── _dlt_loads + ├── _dlt_pipeline_state + ├── _dlt_version + └── users + └── ..parquet +``` + +The value of `load_id` and `file_id` is determined at runtime. The default layout creates a folder with the same table name as the source and places the data inside a parquet file. This layout is configurable using the `layout` parameter. + +For example, if you would like to create a parquet file with the same name as the source table (as opposed to a folder) you can set `layout` to `{table_name}.{ext}` in the commandline above: + +```sh +omniload ingest \ + --source-uri 'duckdb:///records.db' \ + --source-table 'public.users' \ + --dest-uri 's3://?layout={table_name}.{ext}&access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \ + --dest-table 'my_bucket/records' +``` + +Result: +```text +my_bucket/ +└── records + ├── _dlt_loads + ├── _dlt_pipeline_state + ├── _dlt_version + └── users.parquet +``` + +See also list of [available layout placeholders]. + + +[Amazon S3]: https://aws.amazon.com/ +[available layout placeholders]: https://dlthub.com/docs/dlt-ecosystem/destinations/filesystem#available-layout-placeholders +[AWS credentials]: https://dlthub.com/docs/dlt-ecosystem/verified-sources/filesystem#get-credentials diff --git a/docs/supported-sources/sftp.md b/docs/supported-sources/sftp.md index e651c8070..751151075 100644 --- a/docs/supported-sources/sftp.md +++ b/docs/supported-sources/sftp.md @@ -1,43 +1,60 @@ +(sftp)= + # SFTP -SFTP (SSH File Transfer Protocol) is a secure file transfer protocol that runs over the SSH protocol. It provides a secure way to transfer files between a local and a remote computer. +The SSH File Transfer Protocol ([SFTP]) is a secure file transfer protocol +that runs over the SSH protocol. It provides a secure way to transfer files +between a local and a remote computer. `omniload` supports SFTP as a data source. ## URI Format -The URI for connecting to an SFTP server is structured as follows: +The URI for connecting to an SFTP server is structured as follows. ```text -sftp://:@: +sftp://:@:/path/to/data.parquet ``` -## URI Components: -- `username`: The username for the SFTP server. -- `password`: The password for the SFTP server. -- `host`: The hostname or IP address of the SFTP server. -- `port`: The port number of the SFTP server (defaults to 22 if not specified). +## URI components + +:host: + The hostname or IP address of the SFTP server. + +:port: + The port number of the SFTP server (defaults to 22 if not specified). + +:username: + The username for the SFTP server. -## Setting up an SFTP Integration +:password: + The password for the SFTP server. -To integrate `omniload` with an SFTP server, you need the server's hostname, port, a valid username, and a password. +## Examples -Once you have your credentials, you can load data to desired destaination. +To integrate `omniload` with an SFTP server, you need the server's +hostname, port, a valid username, and a password. -### Example: Loading data from SFTP TO DUCKDB +### Load CSV data from SFTP into DuckDB ```sh omniload ingest \ - --source-uri 'sftp://myuser:MySecretPassword123@sftp.example.com' \ - --source-table 'user.csv' \ - --dest-uri duckdb:///sftp_data.duckdb \ - --dest-table 'dest.users_deatils' + --source-uri 'sftp://YOUR_USERNAME:YOUR_PASSWORD@sftp.example.com' \ + --source-table 'path/to/data.csv' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' ``` -sftp - +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. -The `--source-table` specifies `/path/to/directory` The base directory on the server where `omniload` should start looking for files. +sftp +:::{tip} +Here, instead of defining the remote resource exclusively per source URI +using its `` component, the `--source-table` option can specify the +base directory on the server where `omniload` should start looking for files. +::: +[SFTP]: https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol diff --git a/docs/supported-sources/sharepoint.md b/docs/supported-sources/sharepoint.md new file mode 100644 index 000000000..9155cb9a4 --- /dev/null +++ b/docs/supported-sources/sharepoint.md @@ -0,0 +1,117 @@ +(sharepoint)= + +# SharePoint + +[Microsoft SharePoint] is a collaborative web-based service for working on +documents, web pages, web sites, lists, and more, mostly used for building +corporate intranets. You can connect your SharePoint Online instance as a +data source for your managed knowledge base to crawl files and pages from +one or more SharePoint sites. +`omniload` supports SharePoint as a data source. + +## URI format + +The URI for connecting to SharePoint is structured as follows. + +```text +sharepoint:////path/to/data.xlsx?client_id=1d2befad-2f22-4124-a779-b147dfeca342&tenant_id=6b337423-f504-4060-a91b-e9eaaf782609&client_secret=abc~xyz789EXAMPLE_foo +``` + +## URI parameters + +:drive_id: + The ID of the SharePoint drive. If provided, enables single-site mode. + +:client_id: + OAuth2 client ID. Can also be set via MSGRAPHFS_CLIENT_ID + or AZURE_CLIENT_ID environment variables. + +:tenant_id: + OAuth2 tenant ID. Can also be set via MSGRAPHFS_TENANT_ID + or AZURE_TENANT_ID environment variables. + +:client_secret: + OAuth2 client secret. Can also be set via MSGRAPHFS_CLIENT_SECRET + or AZURE_CLIENT_SECRET environment variables. + +:site_name: + The name of the SharePoint site. If provided with drive_name, + enables single-site mode. + +:drive_name: + The name of the SharePoint drive/library (e.g., "Documents", + "CustomLibrary"). If provided with `site_name`, enables + single-site mode. + +:url_path: + URL-style path specification (e.g., "msgd://TestSite/Documents"). + If provided, extracts `site_name` and `drive_name` from the URL. + URL parameters override direct `site_name`/`drive_name` parameters. + +:oauth2_client_params: + Parameters for the OAuth2 client. If not provided, will be built + from `client_id`, `tenant_id`, `client_secret`. + Type: `dict`. Use JSON to encode the dictionary. + +:use_recycle_bin: + If True, deleted files are moved to recycle bin. Default is False. + Truthy values are `"true", "yes", "on", "y", "t", "1"`. + Falsy values are `"false", "no", "off", "n", "f", "0"`. + +:::{note} +Access works unified for both SharePoint sites and drives. +The module handles both single-site/drive operations and multi-site +operations based on the parameters provided during initialization: + +- Single-site mode: When `site_name` + `drive_name` or `drive_id` are provided +- Multi-site mode: When neither `site_name` + `drive_name` nor `drive_id` are provided + +Multi-site mode can handle URL-based paths that specify +the site and drive dynamically (e.g., `msgd://SiteA/DriveB/file.txt`). +::: + +## Authentication + +The SharePoint connector uses OAuth 2.0 to authenticate with the +Microsoft Graph API. [Microsoft Graph] is a protected API gateway for +accessing data in Microsoft cloud services like [Microsoft Entra ID], +Microsoft 365, OneDrive, or SharePoint. It is protected by the +[Microsoft identity platform], which authorizes and verifies that an +app is authorized to call Microsoft Graph. + +Please get familiar with relevant concepts to configure OAuth 2.0 +authentication properly, see also [authentication and authorization +basics] and [set up OAuth 2.0 authentication for SharePoint] tutorial +by AWS and other resources. + +- [Mastering File Access in SharePoint with OAuth 2.0: A Comprehensive Guide] +- [Understanding Microsoft Entra ID and OAuth 2.0 in the context of SharePoint Online modern development] + +## Example: Load CSV file from SharePoint into DuckDB + +```sh +omniload ingest \ + --source-uri 'sharepoint://?client_id=1d2befad-2f22-4124-a779-b147dfeca342&tenant_id=6b337423-f504-4060-a91b-e9eaaf782609&client_secret=abc~xyz789EXAMPLE_foo' \ + --source-table '//path/to/data.csv' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + +:::{tip} +Here, instead of defining the remote resource exclusively per source URI +using its `` component, the `--source-table` option can specify the +base directory on the server where `omniload` should start looking for files. +::: + + +[authentication and authorization basics]: https://learn.microsoft.com/en-us/graph/auth/auth-concepts +[Microsoft Entra ID]: https://en.wikipedia.org/wiki/Microsoft_Entra_ID +[Mastering File Access in SharePoint with OAuth 2.0: A Comprehensive Guide]: https://medium.com/@pavithrasainath7/mastering-file-access-in-sharepoint-with-oauth-2-0-a-comprehensive-guide-0a6b2d53736a +[Microsoft Graph]: https://learn.microsoft.com/en-us/graph/ +[Microsoft identity platform]: https://learn.microsoft.com/en-us/entra/identity-platform/v2-overview +[Microsoft SharePoint]: https://en.wikipedia.org/wiki/SharePoint +[Set up OAuth 2.0 authentication for SharePoint]: https://docs.aws.amazon.com/bedrock/latest/userguide/kb-managed-sharepoint-oauth2-setup.html +[Understanding Microsoft Entra ID and OAuth 2.0 in the context of SharePoint Online modern development]: https://learn.microsoft.com/en-us/sharepoint/dev/sp-add-ins-modernize/understanding-aad-and-oauth-for-spo-modern diff --git a/docs/supported-sources/smb.md b/docs/supported-sources/smb.md new file mode 100644 index 000000000..9178ec41e --- /dev/null +++ b/docs/supported-sources/smb.md @@ -0,0 +1,110 @@ +(smb)= + +# SMB + +[Server Message Block (SMB)] is a communication protocol used to share files, +printers, serial ports, and miscellaneous communications between nodes on a +network, mostly used by Microsoft Windows operating systems. + +`omniload` supports SMB as a data source for reading files on Microsoft +Windows Server Shares. + +## URI Format + +The URI for connecting to SMB is structured as follows. Authentication +either works transparently using NTLM or Kerberos, or by using +username/password credentials. + +```text +smb://workgroup;user:password@server.example.org:445/path/to/data.parquet +``` + +## URI parameters + +:host: + The remote server name or IP address to connect to. + +:port: + Port to connect with. Usually `445`, sometimes `139`. + Type: `int`. Default: `445`. + +:username: + Username to connect with. + Required when not using Kerberos authentication. + +:password: + User's password on the server, if using username. + +:timeout: + Connection timeout in seconds. + Type: `int`. + +:encrypt: + Whether to force encryption or not. Once this has been set to `True` + the session cannot be changed back to `False`. + Type: `bool`. Default: `False`. + +:share_access: + Specifies the default access mode applied to file `open` operations + performed with this file system object. + This affects whether other processes can concurrently open a handle + to the same file. + + - `None`: exclusively locks the file until closed (default). + - `r`: Allow other handles to be opened with read access. + - `w`: Allow other handles to be opened with write access. + - `d`: Allow other handles to be opened with delete access. + +:register_session_retries: + Number of retries to register a session with the server. Retries are not performed + for authentication errors, as they are considered as invalid credentials and not network + issues. If set to negative value, no register attempts will be performed. + Type: `int`. Default: `4`. + +:register_session_retry_wait: + Time in seconds to wait between each retry. Number must be non-negative. + Type: `int`. Default: `1`. + +:register_session_retry_factor: + Base factor for the wait time between each retry. The wait time + is calculated using the exponential function. For `factor=1` all wait times + will be equal to `register_session_retry_wait`. For any number of retries, + the last wait time will be equal to `register_session_retry_wait` and for retries>1 + the first wait time will be equal to `register_session_retry_wait / factor`. + Number must be equal to or greater than `1`. The optimal factor is `10`. + Type: `int`. Default: `10`. + +:auto_mkdir: + Whether, when opening a file, the directory containing it should + be created (if it doesn't already exist). + Type: `bool`. Default: `false`. + +## Examples + +To integrate `omniload` with SMB, you need the SMB connection address +to connect to. + +### Load Parquet data from SMB into DuckDB + +The following command demonstrates how to copy data from a specified SMB +location into a DuckDB database. + +```sh +omniload ingest \ + --source-uri 'smb://workgroup;user:password@server.example.org:445' \ + --source-table 'path/to/data.parquet' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + +:::{tip} +Here, instead of defining the remote resource exclusively per source URI +using its `` component, the file glob pattern is specified using the +separate `--source-table` option. Both addressing variants are supported equally. +::: + + +[Server Message Block (SMB)]: https://en.wikipedia.org/wiki/Server_Message_Block diff --git a/docs/supported-sources/webdav.md b/docs/supported-sources/webdav.md new file mode 100644 index 000000000..d7425310f --- /dev/null +++ b/docs/supported-sources/webdav.md @@ -0,0 +1,104 @@ +(webdav)= + +# WebDAV + +[WebDAV] (Web Distributed Authoring and Versioning) is a set of extensions to +the Hypertext Transfer Protocol (HTTP) allowing user agents to collaboratively +edit content directly in an HTTP server, which may act as a file server. +`omniload` supports WebDAV as a data source. + +## URI format + +The URI for connecting to WebDAV is structured as follows. + +```text +https+webdav://:@www.example.org/path/to/data.parquet +``` + +## URI parameters + +:retry: Disable retry on the HTTP client. When enabled, some well-known + errors are handled and retried a few times with backoff. + Type: `bool`. Default: `true`. + +:chunk_size: + The chunk size in bytes. + Type: `int`. Default: `4194304` (`4 MB`). + +:headers: + HTTP headers to include when sending requests. + Type: `dict`. Use JSON to encode the dictionary. + +:cookies: + Cookie items to include when sending requests. + Type: `dict`. Use JSON to encode the dictionary. + +:verify: + SSL certificates used to verify the identity of requested + hosts. Can be any of: + + - `true`: Use default CA bundle. + - `false`: Disable verification. + - Path to an SSL certificate file. + + Default: `true`. + +:cert: + An SSL certificate used by the requested host to + authenticate the client. + Either a path to an SSL certificate file, + or two-tuple of (certificate file, key file), + or a three-tuple of (certificate file, key file, password). + Note: Tuple decoding is not implemented yet. + +:proxies: + A mapping of proxy keys to proxy URLs. + Type: `dict`. Use JSON to encode the dictionary. + +:timeout: + The socket timeout in seconds. + Type: `float`. + +:max_redirects: + The maximum number of redirect responses that should be followed. + Type: `int`. + +:trust_env: + Enable or disable usage of environment variables for configuration. + Type: `bool`. Default: `true`. + +## Authentication + +To integrate `omniload` with WebDAV, you need to authenticate like you +do with any HTTP server. + +:::{note} +The module currently forwards parameters for HTTP Basic authentication +using username/password credentials. In theory, all [authentication types +supported by HTTPX] can be unlocked. Please [create an issue] to let us +know about your needs. +::: + +## Example: Load CSV file from WebDAV into DuckDB + +```sh +omniload ingest \ + --source-uri 'https+webdav://:@www.example.org' \ + --source-table 'path/to/data.csv' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + +:::{tip} +Here, instead of defining the remote resource exclusively per source URI +using its `` component, the `--source-table` option can specify the +base directory on the server where `omniload` should start looking for files. +::: + + +[authentication types supported by HTTPX]: https://www.python-httpx.org/advanced/authentication/ +[create an issue]: https://github.com/panodata/omniload/issues +[WebDAV]: https://en.wikipedia.org/wiki/WebDAV diff --git a/docs/supported-sources/webhdfs.md b/docs/supported-sources/webhdfs.md new file mode 100644 index 000000000..137050690 --- /dev/null +++ b/docs/supported-sources/webhdfs.md @@ -0,0 +1,119 @@ +(webhdfs)= + +# WebHDFS + +[WebHDFS] provides a complete FileSystem interface for HDFS over HTTP. +Supports also HttpFS gateways. +`omniload` supports WebHDFS as a data source. + +## URI Format + +The URI for connecting to WebHDFS is structured as follows. Either access the resource +anonymously, use key/secret credentials, or a security token for authentication. +In this form, the remote resource is addressed exclusively using a single +parameter, an URI, encoding all required options. + +```text +webhdfs://host:9870/path +``` + +## URI parameters + +:host: str + Name-node address. + +:port: int + Port for webHDFS server. + +:kerberos: + Whether to authenticate with kerberos for this connection. + Type: `bool`. Default: `false`. + +:token: + If given, use this token on every call to authenticate. A user + and user-proxy may be encoded in the token and should not be also + given. + +:user: + If given, assert the user name to connect with. + +:password: + If given, assert the password to use for basic auth. If password + is provided, user must be provided also. + +:proxy_to: + If given, the user has the authority to proxy, and this value is + the user in who's name actions are taken. + +:kerberos_options: + Any extra arguments for HTTPKerberosAuth, see [kerberos_.py]. + Type: `dict`. Use JSON to encode the dictionary. + +:data_proxy: + Map host names / data-node addresses `host->data_proxy[host]`. + This can be necessary if the HDFS cluster is behind a proxy, + running on Docker or otherwise has a mismatch between the + host-names given by the name-node and the address by which to + refer to them from the client. + Type: `dict`. Use JSON to encode the dictionary. + +:use_https: + Whether to connect to the Name-node using HTTPS instead of HTTP. + Type: `bool`. Default: `false`. + +:session_cert: + Path to a certificate file, or tuple of (cert, key) files to use + for the conversation. + Remark: Decoding tuples is not implemented yet. + +:session_verify: + Whether to verify the conversation, or the path to a certificate file + for doing so. + Type: `str|bool`. Default: `true`. + +## Authentication + +Four authentication mechanisms are supported. + +:insecure: + No authentication is performed, and the user is assumed to be whoever they + say they are (parameter `user`), or a predefined value such as "dr.who" + if not given. + +:spnego: + When kerberos authentication is enabled, authentication is negotiated by + [requests_kerberos]. + This establishes a session based on existing kinit login and/or + specified principal/password; parameters are passed with `kerberos_options`. + +:token: + Uses an existing Hadoop delegation token from another secured + service. Indeed, this client can also generate such tokens when + not insecure. Note that tokens expire, but can be renewed (by a + previously specified user) and may allow for proxying. + +:basic-auth: + Used when both parameter `user` and parameter `password` are provided. + +## Examples + +### Load Parquet data from WebHDFS into DuckDB + +The following command demonstrates how to copy data from a specified OSS +location into a DuckDB database. + +```sh +omniload ingest \ + --source-uri 'webhdfs://host:9870/path' \ + --source-table 'path/to/data.parquet' \ + --dest-uri 'duckdb:///demo.duckdb' \ + --dest-table 'testdrive.data' +``` + +Running the command creates a table named `data` within the `testdrive` +schema in the DuckDB database file located at `demo.duckdb`. + + +[kerberos_.py]: https://github.com/requests/requests-kerberos/blob/master/requests_kerberos/kerberos_.py +[requests_kerberos]: https://github.com/requests/requests-kerberos +[WebHDFS]: https://hadoop.apache.org/docs/r1.0.4/webhdfs.html diff --git a/docs/supported-sources/xlsx.md b/docs/supported-sources/xlsx.md index b32ed6ba6..6f84611b1 100644 --- a/docs/supported-sources/xlsx.md +++ b/docs/supported-sources/xlsx.md @@ -10,9 +10,8 @@ XLSX is currently supported for read operations only. Excel XLSX files can be accessed on every source that goes through the shared file readers: -- Local files: [`file://`](file.md) -- [`s3://`](s3.md), [`gs://`](google-cloud-storage.md), [Azure blob storage](azure-blob-storage.md) -- [`sftp://`](sftp.md) +- Local files: {ref}`file` +- Remote files: {ref}`s3`, {ref}`gcs`, {ref}`azure-storage`, {ref}`sftp`, ... A file is read as XLSX when its extension is `.xlsx` (optionally `.xlsx.gz`), or when an explicit `#xlsx` {ref}`format hint ` is appended. diff --git a/docs/supported-sources/xml.md b/docs/supported-sources/xml.md index 5cfa2755d..0df8613d8 100644 --- a/docs/supported-sources/xml.md +++ b/docs/supported-sources/xml.md @@ -48,9 +48,8 @@ local name, so `#tagname=product` matches both `` and ``. W XML is available on every source that goes through the shared file readers: -- Local files: [`file://`](file.md) -- [`s3://`](s3.md), [`gs://`](google-cloud-storage.md), [Azure blob storage](azure-blob-storage.md) -- [`sftp://`](sftp.md) +- Local files: {ref}`file` +- Remote files: {ref}`s3`, {ref}`gcs`, {ref}`azure-storage`, {ref}`sftp`, ... Remote reads go through the source's own fsspec handle, so they reuse its existing authentication (no separate XML storage configuration). A file is read as XML when its extension diff --git a/docs/supported-sources/yaml.md b/docs/supported-sources/yaml.md index 86bbb8e6c..38d03777e 100644 --- a/docs/supported-sources/yaml.md +++ b/docs/supported-sources/yaml.md @@ -57,9 +57,8 @@ An empty file loads zero rows; a malformed document raises rather than loading p YAML is available on every source that goes through the shared file readers: -- Local files: [`file://`](file.md) -- [`s3://`](s3.md), [`gs://`](google-cloud-storage.md), [Azure blob storage](azure-blob-storage.md) -- [`sftp://`](sftp.md) +- Local files: {ref}`file` +- Remote files: {ref}`s3`, {ref}`gcs`, {ref}`azure-storage`, {ref}`sftp`, ... Remote reads go through the source's own fsspec handle, so they reuse its existing authentication (no separate YAML storage configuration). A file is read as YAML when its diff --git a/pyproject.toml b/pyproject.toml index d8b1d53b2..746199d09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,6 @@ classifiers = [ ] dynamic = [ "version" ] dependencies = [ - "adlfs", "aiohttp<4", "asana>=5,<6", "click<8.5", @@ -91,8 +90,6 @@ dependencies = [ "elasticsearch<10", "facebook-business<26", "flatten-json<0.2", - "fsspec>2021.9", - "gcsfs", "google-ads<32", "google-analytics-data<0.24", "google-api-python-client<3", @@ -102,6 +99,7 @@ dependencies = [ "influxdb-client<2", "mq-bridge-py>=0.3.2,<0.4", "mysql-connector-python<10", + "omniload[filesystem]", "oracledb<5", "orjson<4", "pendulum>=3,<4", @@ -118,7 +116,6 @@ dependencies = [ "python-quickbooks<0.10", "requests<3", "rudder-sdk-python<2.2", - "s3fs", "simple-salesforce<2", "smartsheet-python-sdk<5", "snowflake-connector-python>=3.18,<5", @@ -146,6 +143,22 @@ optional-dependencies.develop = [ "ty==0.0.63", "validate-pyproject<1", ] +optional-dependencies.filesystem = [ + "adlfs", + "dropboxdrivefs", + "fsspec>=2024.6.0", + "fsspec-databricks", + "gcsfs", + "gdrive-fsspec", + "msgraphfs", + "ocifs", + "ossfs", + "paramiko", + "s3fs", + "smbprotocol; platform_system=='Linux'", + "smbprotocol[kerberos]; platform_system!='Linux'", + "webdav4[fsspec]", +] optional-dependencies.full = [ "omniload[ibm-db,iterable,odbc,oracle]" ] optional-dependencies.ibm-db = [ "ibm-db<4; platform_machine!='aarch64'", @@ -181,6 +194,7 @@ optional-dependencies.test = [ "pytest<10", "pytest-asyncio<2", "pytest-cov<8", + "pytest-mock<4", "pytest-xdist[psutil]<4", "testcontainers[azurite,mysql,postgres]>=4.9.1,<4.16", "types-requests<3", diff --git a/src/dlt_filesystem/error.py b/src/dlt_filesystem/error.py index 699cb4754..0c2505ae0 100644 --- a/src/dlt_filesystem/error.py +++ b/src/dlt_filesystem/error.py @@ -9,6 +9,6 @@ def __init__(self, option, connector): class InvalidBlobTableError(Exception): def __init__(self, source): super().__init__( - f"Invalid source table for {source} " + f"Invalid source table for: {source}. " "Ensure that the table is in the format {bucket-name}/{file glob}" ) diff --git a/src/dlt_filesystem/source/adapter.py b/src/dlt_filesystem/source/adapter.py index afd5a31e0..3432d8f0e 100644 --- a/src/dlt_filesystem/source/adapter.py +++ b/src/dlt_filesystem/source/adapter.py @@ -17,7 +17,6 @@ from typing import Iterator, List, Optional, Tuple, Union import dlt -from dlt.extract import DltSource from dlt.sources import DltResource from dlt.sources.credentials import FileSystemCredentials from dlt.sources.filesystem import FileItem, FileItemDict, fsspec_filesystem, glob_files @@ -39,7 +38,7 @@ read_yaml, ) -from .model import FilesystemConfigurationResource, FilesystemReference +from .model import FilesystemConfigurationResource @dlt.source(_impl_cls=ReadersSource, spec=FilesystemConfigurationResource) @@ -133,40 +132,3 @@ def filesystem( files_chunk = [] if files_chunk: yield files_chunk - - -def resource_for_reader(ref: FilesystemReference) -> DltSource | DltResource: - """Build the filesystem reader resource named by ``ref.reader_name``. - - Threads ``column_types`` into ``read_csv_headless`` and per-URI reader hints (e.g. XML's - ``#tagname``) into a hint-consuming reader; every other reader is selected as-is. - """ - - # Establish filesystem and reader elements. - filesystem_resource = filesystem( - ref.bucket_url, - ref.fs, - file_glob=ref.file_glob, - extract_content=False, - ) - if ref.filesystem_incremental: - filesystem_resource = filesystem_resource.with_name( - ref.incremental_resource_name - ) - filesystem_resource.apply_hints( - incremental=dlt.sources.incremental("modification_date") - ) - all_readers = readers( - ref.bucket_url, ref.fs, file_glob=ref.file_glob - ).with_resources(ref.reader_name) - reader = all_readers.selected_resources[ref.reader_name] - - # Apply parameter bindings for certain readers. - if ref.reader_name == "read_csv_headless": - column_names = list(ref.column_types.keys()) if ref.column_types else None - reader = reader.bind(column_names=column_names, **ref.hints) - else: - reader = reader.bind(**ref.hints) - - # Connect and propagate elements. - return filesystem_resource | reader diff --git a/src/dlt_filesystem/source/api.py b/src/dlt_filesystem/source/api.py deleted file mode 100644 index 14cd0782c..000000000 --- a/src/dlt_filesystem/source/api.py +++ /dev/null @@ -1,15 +0,0 @@ -from dlt_filesystem.source.impl.local import LocalFilesystemSource -from dlt_filesystem.source.impl.remote import ( - AzureSource, - GCSSource, - S3Source, - SFTPSource, -) - -__all__ = [ - "AzureSource", - "GCSSource", - "LocalFilesystemSource", - "S3Source", - "SFTPSource", -] diff --git a/src/dlt_filesystem/source/base.py b/src/dlt_filesystem/source/base.py index 4c8a7fb97..b372ea8ae 100644 --- a/src/dlt_filesystem/source/base.py +++ b/src/dlt_filesystem/source/base.py @@ -1,3 +1,7 @@ +from typing import Union +from urllib.parse import urlparse + + class FilesystemSource: """Shared capabilities for the filesystem-family sources. @@ -24,3 +28,27 @@ def honours_run_disposition(self) -> bool: def supports_filesystem_incremental(self) -> bool: """Return whether the source supports file-level mtime selection.""" return True + + @staticmethod + def endpoint_namespace(endpoint: Union[str, None], default: str) -> str: + """ + Return a normalized endpoint identity without credentials or query values. + It is used for incremental loading based on file modification times. + + # TODO: Remove `default` argument again? + """ + if not endpoint: + return default + + parsed = urlparse(endpoint if "://" in endpoint else f"//{endpoint}") + host = parsed.hostname + if not host: + return default + + host = host.lower() + if ":" in host: + host = f"[{host}]" + if parsed.port is not None: + host = f"{host}:{parsed.port}" + + return f"{host}{parsed.path.rstrip('/')}" diff --git a/src/dlt_filesystem/source/core.py b/src/dlt_filesystem/source/core.py new file mode 100644 index 000000000..8190fcb49 --- /dev/null +++ b/src/dlt_filesystem/source/core.py @@ -0,0 +1,82 @@ +from typing import Union + +import dlt +from dlt.extract import DltResource, DltSource +from fsspec import AbstractFileSystem + +from dlt_filesystem.source.adapter import filesystem, readers +from dlt_filesystem.source.error import UnsupportedEndpointError +from dlt_filesystem.source.format.registry import supported_file_format_message +from dlt_filesystem.source.model import FilesystemLocator, FilesystemReference +from dlt_filesystem.source.router import determine_endpoint + + +def resource_for_reader(ref: FilesystemReference) -> Union[DltSource, DltResource]: + """Build the filesystem reader resource named by ``ref.reader_name``. + + Threads ``column_types`` into ``read_csv_headless`` and per-URI reader hints (e.g. XML's + ``#tagname``) into a hint-consuming reader; every other reader is selected as-is. + """ + + # Establish filesystem and reader elements. + filesystem_resource = filesystem( + ref.bucket_url, + ref.fs, + file_glob=ref.file_glob, + extract_content=False, + ) + if ref.filesystem_incremental: + filesystem_resource = filesystem_resource.with_name( + ref.incremental_resource_name + ) + filesystem_resource.apply_hints( + incremental=dlt.sources.incremental("modification_date") + ) + all_readers = readers( + ref.bucket_url, ref.fs, file_glob=ref.file_glob + ).with_resources(ref.reader_name) + reader = all_readers.selected_resources[ref.reader_name] + + # Apply parameter bindings for certain readers. + # TODO: Can this be generalized? Why not always loop in column_names into reader hints? + if ref.reader_name == "read_csv_headless": + column_names = list(ref.column_types.keys()) if ref.column_types else None + reader = reader.bind(column_names=column_names, **ref.hints) + else: + reader = reader.bind(**ref.hints) + + # Connect and propagate elements. + return filesystem_resource | reader + + +def infer_resource( + fs: AbstractFileSystem, locator: FilesystemLocator +) -> Union[DltSource, DltResource]: + """ + Infer dlt resource from fsspec filesystem, with reader. + """ + + # Decode into base url and url path / file glob, and apply sanity checks. + locator.validate() + + # TODO: Naming things: Rename `determine_endpoint` to `infer_reader`. + try: + endpoint = determine_endpoint(locator.path, locator.file_glob) + except UnsupportedEndpointError: + raise ValueError(supported_file_format_message(locator.name)) from None + + # TODO: FilesystemLocator and FilesystemReference are somewhat redundant now. Refactor! + # => Bundle fs, locator and reader into another data class , then feed that to + # `resource_for_reader`. + return resource_for_reader( + FilesystemReference( + fs=fs, + bucket_url=locator.bucket_url, + file_glob=locator.file_glob, + reader_name=endpoint, + hints=locator.hints, + # TODO: Can `column_types` be looped into reader|writer hints instead? + # We believe it represents a special case handling for `csv_headless`. + column_types=locator.options.params.get("column_types"), + ) + ) diff --git a/src/dlt_filesystem/source/error.py b/src/dlt_filesystem/source/error.py index 017b03190..88802a7ba 100644 --- a/src/dlt_filesystem/source/error.py +++ b/src/dlt_filesystem/source/error.py @@ -10,6 +10,8 @@ class MissingDecoderError(UnsupportedEndpointError): absent. Carries the exact ``pip install`` target so the message is actionable. """ + pass + class MissingReaderOptionError(ValueError): """A file reader needs a per-URI ``#key=value`` hint that was not supplied. diff --git a/src/dlt_filesystem/source/fsspec/databricks.py b/src/dlt_filesystem/source/fsspec/databricks.py new file mode 100644 index 000000000..c33ffa56b --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/databricks.py @@ -0,0 +1,74 @@ +from typing import Type + +from fsspec import AbstractFileSystem + +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import infer_resource +from dlt_filesystem.source.model import FilesystemLocator +from dlt_filesystem.util.python import ( + cast_to_bool, + cast_to_int, + cast_to_list, +) + + +class DatabricksSource(FilesystemSource): + """ + Access files on Databricks Unity Catalog Volumes, Workspace files, and Legacy DBFS (Databricks File System). + """ + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + from fsspec_databricks import DatabricksFileSystem + + return DatabricksFileSystem + + def dlt_source(self, uri: str, table: str, **kwargs): + + # TODO: It looks like this is generic code that could be refactored already + # if it's common amongst different implementations. + if kwargs.get("incremental_key"): + raise ValueError( + "Databricks takes care of incrementality on its own, you should not provide incremental_key" + ) + + # Bundle essential information to infer filesystem wrapper. + locator = FilesystemLocator( + name="Databricks", + fs_class=self.fs_class, + uri=uri, + path=table, + accept_no_bucket_name=True, + accept_no_host_name=True, + ) + + # Decode individual options (type casting, default values, sanity checks). + fs_kwargs = locator.options.fs_kwargs + fs_kwargs.update(kwargs) + cast_to_int( + fs_kwargs, + [ + "debug_truncate_bytes", + "volume_fs_max_read_concurrency", + "volume_fs_min_read_block_size", + "volume_fs_max_read_block_size", + "volume_fs_max_write_concurrency", + "volume_fs_min_write_block_size", + "volume_fs_max_write_block_size", + "volume_min_multipart_upload_size", + "volume_fs_connection_pool_size", + ], + ) + cast_to_list(fs_kwargs, ["scopes"]) + cast_to_bool( + fs_kwargs, + [ + "debug_headers", + "use_local_fs_in_workspace", + "verbose_debug_log", + ], + ) + + # Create filesystem and dlt resource wrapper. + fs = self.fs_class(**fs_kwargs) + return infer_resource(fs=fs, locator=locator) diff --git a/src/dlt_filesystem/source/fsspec/dropbox.py b/src/dlt_filesystem/source/fsspec/dropbox.py new file mode 100644 index 000000000..96fdff6ed --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/dropbox.py @@ -0,0 +1,42 @@ +from typing import Type + +from fsspec import AbstractFileSystem + +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import infer_resource +from dlt_filesystem.source.model import FilesystemLocator + + +class DropboxSource(FilesystemSource): + """ + Access files on Dropbox. + + https://github.com/fsspec/dropboxdrivefs + """ + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + from dropboxdrivefs import DropboxDriveFileSystem + + return DropboxDriveFileSystem + + def dlt_source(self, uri: str, table: str, **kwargs): + + # TODO: Is this applicable for Dropbox and friends at all? + if kwargs.get("incremental_key"): + raise ValueError( + "Dropbox takes care of incrementality on its own, you should not provide incremental_key" + ) + + # Bundle essential information to infer filesystem wrapper. + locator = FilesystemLocator( + name="Dropbox", fs_class=self.fs_class, uri=uri, path=table + ) + + # Decode individual options (type casting, default values, sanity checks). Schema: + fs_kwargs = locator.options.fs_kwargs + fs_kwargs.update(kwargs) + + # Create filesystem and dlt resource wrapper. + fs = self.fs_class(**fs_kwargs) + return infer_resource(fs=fs, locator=locator) diff --git a/src/dlt_filesystem/source/fsspec/ftp.py b/src/dlt_filesystem/source/fsspec/ftp.py new file mode 100644 index 000000000..f6dc08567 --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/ftp.py @@ -0,0 +1,37 @@ +from dlt_filesystem.error import MissingConnectorOption +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import infer_resource +from dlt_filesystem.source.model import FilesystemLocator +from dlt_filesystem.util.python import asbool, cast_to_int + + +class FTPSource(FilesystemSource): + """Access files on FTP servers.""" + + def dlt_source(self, uri: str, table: str, **kwargs): + + from fsspec.implementations.ftp import FTPFileSystem + + # Bundle essential information to infer filesystem wrapper. + locator = FilesystemLocator( + name="FTP", fs_class=FTPFileSystem, uri=uri, path=table, default_port=21 + ) + + # Decode individual options (type casting, default values, sanity checks). + fs_kwargs = locator.options.fs_kwargs + fs_kwargs.update(kwargs) + if "host" not in fs_kwargs or not fs_kwargs["host"]: + raise MissingConnectorOption("host", "FTP") + fs_kwargs["port"] = fs_kwargs.get("port", locator.default_port) + # Cast values to `int`. + cast_to_int(fs_kwargs, ["block_size", "port", "timeout"]) + # Type casting for special parameters. + if "tls" in fs_kwargs: + try: + fs_kwargs["tls"] = asbool(fs_kwargs["tls"]) + except ValueError: + pass + + # Create filesystem and dlt resource wrapper. + fs = FTPFileSystem(**fs_kwargs) + return infer_resource(fs=fs, locator=locator) diff --git a/src/dlt_filesystem/source/fsspec/gdrive.py b/src/dlt_filesystem/source/fsspec/gdrive.py new file mode 100644 index 000000000..ceac8258f --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/gdrive.py @@ -0,0 +1,48 @@ +from typing import Type + +from fsspec import AbstractFileSystem + +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import infer_resource +from dlt_filesystem.source.model import FilesystemLocator +from dlt_filesystem.util.python import apply_alias, cast_to_bool, cast_to_dict + + +class GoogleDriveSource(FilesystemSource): + """ + Access files on Google Drive. + """ + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + from gdrive_fsspec import GoogleDriveFileSystem + + return GoogleDriveFileSystem + + def dlt_source(self, uri: str, table: str, **kwargs): + + # TODO: It looks like this is generic code that could be refactored already + # if it's common amongst different implementations. + if kwargs.get("incremental_key"): + raise ValueError( + "GoogleDrive takes care of incrementality on its own, you should not provide incremental_key" + ) + + # Bundle essential information to infer filesystem wrapper. + locator = FilesystemLocator( + name="GoogleDrive", + fs_class=self.fs_class, + uri=uri, + path=table, + ) + + # Decode individual options (type casting, default values, sanity checks). + fs_kwargs = locator.options.fs_kwargs + fs_kwargs.update(kwargs) + apply_alias(fs_kwargs, "auth", "auth_kwargs") + cast_to_dict(fs_kwargs, ["creds", "auth_kwargs"]) + cast_to_bool(fs_kwargs, ["use_local_webserver"]) + + # Create filesystem and dlt resource wrapper. + fs = self.fs_class(**fs_kwargs) + return infer_resource(fs=fs, locator=locator) diff --git a/src/dlt_filesystem/source/fsspec/hdfs.py b/src/dlt_filesystem/source/fsspec/hdfs.py new file mode 100644 index 000000000..2c6dd6edf --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/hdfs.py @@ -0,0 +1,49 @@ +from typing import Type + +from fsspec import AbstractFileSystem + +from dlt_filesystem.error import MissingConnectorOption +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import infer_resource +from dlt_filesystem.source.model import FilesystemLocator +from dlt_filesystem.util.python import apply_alias, cast_to_dict, cast_to_int + + +class HDFSSource(FilesystemSource): + """ + Access files on HDFS via Arrow. + https://arrow.apache.org/docs/python/generated/pyarrow.fs.HadoopFileSystem.html + """ + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + from fsspec.implementations.arrow import HadoopFileSystem + + return HadoopFileSystem + + def dlt_source(self, uri: str, table: str, **kwargs): + if kwargs.get("incremental_key"): + raise ValueError( + "HDFS takes care of incrementality on its own, you should not provide incremental_key" + ) + + # Bundle essential information to infer filesystem wrapper. + locator = FilesystemLocator( + name="HDFS", fs_class=self.fs_class, uri=uri, path=table, default_port=8020 + ) + + # Decode individual options (type casting, default values, sanity checks). + fs_kwargs = locator.options.fs_kwargs + fs_kwargs.update(kwargs) + if "host" not in fs_kwargs or not fs_kwargs["host"]: + raise MissingConnectorOption("host", "HDFS") + fs_kwargs["port"] = fs_kwargs.get("port", locator.default_port) + apply_alias(fs_kwargs, "block_size", "default_block_size") + cast_to_int( + fs_kwargs, ["port", "replication", "buffer_size", "default_block_size"] + ) + cast_to_dict(fs_kwargs, ["extra_conf"]) + + # Create filesystem and dlt resource wrapper. + fs = self.fs_class(**fs_kwargs) + return infer_resource(fs=fs, locator=locator) diff --git a/src/dlt_filesystem/source/impl/local.py b/src/dlt_filesystem/source/fsspec/local.py similarity index 98% rename from src/dlt_filesystem/source/impl/local.py rename to src/dlt_filesystem/source/fsspec/local.py index 3ce691681..356b6197f 100644 --- a/src/dlt_filesystem/source/impl/local.py +++ b/src/dlt_filesystem/source/fsspec/local.py @@ -82,7 +82,7 @@ def dlt_source(self, uri: str, table: str, **kwargs): fs = ArrowFSWrapper(LocalFileSystem()) - from dlt_filesystem.source.adapter import resource_for_reader + from dlt_filesystem.source.core import resource_for_reader from dlt_filesystem.source.model import FilesystemReference # Pass the plain absolute directory (not a hand-built file:// URL). dlt's diff --git a/src/dlt_filesystem/source/fsspec/oci.py b/src/dlt_filesystem/source/fsspec/oci.py new file mode 100644 index 000000000..69d074755 --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/oci.py @@ -0,0 +1,51 @@ +from typing import Type + +from fsspec import AbstractFileSystem + +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import infer_resource +from dlt_filesystem.source.model import FilesystemLocator +from dlt_filesystem.util.python import apply_alias, cast_to_dict, cast_to_int + + +class OCISource(FilesystemSource): + """ + Access files on Oracle Cloud Infrastructure Object Storage (OCI). + + https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/objectstorageoverview.htm + https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm + """ + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + from ocifs import OCIFileSystem + + return OCIFileSystem + + def dlt_source(self, uri: str, table: str, **kwargs): + + if kwargs.get("incremental_key"): + raise ValueError( + "OCI takes care of incrementality on its own, you should not provide incremental_key" + ) + + # Bundle essential information to infer filesystem wrapper. + locator = FilesystemLocator( + name="OCI", fs_class=self.fs_class, uri=uri, path=table + ) + + # Decode individual options (type casting, default values, sanity checks). Schema: + # {"default_block_size": int, "config": dict, "config_kwargs": dict, "oci_additional_kwargs": dict} + fs_kwargs = locator.options.fs_kwargs + fs_kwargs.update(kwargs) + + # Decode dict-typed `config`, `config_kwargs`, `oci_additional_kwargs` from JSON. + cast_to_dict(fs_kwargs, ["config", "config_kwargs", "oci_additional_kwargs"]) + # The `default_` prefix seems unnecessary. Let's make it optional by using an alias. + apply_alias(fs_kwargs, "block_size", "default_block_size") + # Convert to integers. + cast_to_int(fs_kwargs, ["default_block_size"]) + + # Create filesystem and dlt resource wrapper. + fs = self.fs_class(**fs_kwargs) + return infer_resource(fs=fs, locator=locator) diff --git a/src/dlt_filesystem/source/fsspec/onedrive.py b/src/dlt_filesystem/source/fsspec/onedrive.py new file mode 100644 index 000000000..348107905 --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/onedrive.py @@ -0,0 +1,13 @@ +from dlt_filesystem.source.fsspec.sharepoint import SharePointSource + + +class OneDriveSource(SharePointSource): + """ + Access files on Microsoft SharePoint or OneDrive. + + https://github.com/acsone/msgraphfs + """ + + @property + def fs_name(self): + return "OneDrive" diff --git a/src/dlt_filesystem/source/fsspec/oss.py b/src/dlt_filesystem/source/fsspec/oss.py new file mode 100644 index 000000000..69b673870 --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/oss.py @@ -0,0 +1,53 @@ +from typing import Type + +from fsspec import AbstractFileSystem + +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import infer_resource +from dlt_filesystem.source.model import FilesystemLocator +from dlt_filesystem.util.python import apply_alias, cast_to_int + + +class OSSSource(FilesystemSource): + """ + Access files on Alibaba Cloud Object Storage Service (OSS). + https://www.alibabacloud.com/en/product/object-storage-service + """ + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + import ossfs + + return ossfs.OSSFileSystem + + def dlt_source(self, uri: str, table: str, **kwargs): + + # TODO: It looks like this is generic code that could be refactored already + # if it's common amongst different implementations. Breaking out the + if kwargs.get("incremental_key"): + raise ValueError( + "OSS takes care of incrementality on its own, you should not provide incremental_key" + ) + + # Bundle essential information to infer filesystem wrapper. + locator = FilesystemLocator( + name="OSS", fs_class=self.fs_class, uri=uri, path=table + ) + + # Decode individual options (type casting, default values, sanity checks). + fs_kwargs = locator.options.fs_kwargs + fs_kwargs.update(kwargs) + apply_alias(fs_kwargs, "block_size", "default_block_size") + apply_alias(fs_kwargs, "cache_type", "default_cache_type") + cast_to_int(fs_kwargs, ["default_block_size"]) + + # TODO: BaseOSSFileSystem accepts `default_cache_type` as a `str` type with + # a choice of different values. The default value is `readahead`, and + # setting `none` is possible. For all other values, the inline + # documentation refers to the `fsspec` documentation. Let's harvest + # relevant details and add them to the parameter data model. + # No demo implementation here. + + # Create filesystem and dlt resource wrapper. + fs = self.fs_class(**fs_kwargs) + return infer_resource(fs=fs, locator=locator) diff --git a/src/dlt_filesystem/source/fsspec/r2.py b/src/dlt_filesystem/source/fsspec/r2.py new file mode 100644 index 000000000..49c3dcdfe --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/r2.py @@ -0,0 +1,30 @@ +from typing import Type + +from fsspec import AbstractFileSystem + +from dlt_filesystem.source.impl.remote import S3CompatibleSource + + +class R2Source(S3CompatibleSource): + """ + Access files on Cloudflare R2. + + R2 is compatible with Amazon S3. + https://github.com/panodata/omniload/issues/163 + """ + + @property + def fs_name(self) -> str: + return "R2" + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + if hasattr(self, "_r2_fs_class"): + return self._r2_fs_class + import s3fs + + class R2FileSystem(s3fs.S3FileSystem): + protocol = "r2" + + self._r2_fs_class = R2FileSystem + return self._r2_fs_class diff --git a/src/dlt_filesystem/source/fsspec/sharepoint.py b/src/dlt_filesystem/source/fsspec/sharepoint.py new file mode 100644 index 000000000..26e8e9f6b --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/sharepoint.py @@ -0,0 +1,49 @@ +from typing import Type + +from fsspec import AbstractFileSystem + +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import infer_resource +from dlt_filesystem.source.model import FilesystemLocator +from dlt_filesystem.util.python import cast_to_bool, cast_to_dict + + +class SharePointSource(FilesystemSource): + """ + Access files on Microsoft SharePoint or OneDrive. + + https://github.com/acsone/msgraphfs + """ + + @property + def fs_name(self): + return "SharePoint" + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + from msgraphfs import MSGDriveFS + + return MSGDriveFS + + def dlt_source(self, uri: str, table: str, **kwargs): + + # TODO: Is this applicable for Dropbox and friends at all? + if kwargs.get("incremental_key"): + raise ValueError( + f"{self.fs_name} takes care of incrementality on its own, you should not provide incremental_key" + ) + + # Bundle essential information to infer filesystem wrapper. + locator = FilesystemLocator( + name=self.fs_name, fs_class=self.fs_class, uri=uri, path=table + ) + + # Decode individual options (type casting, default values, sanity checks). Schema: + fs_kwargs = locator.options.fs_kwargs + fs_kwargs.update(kwargs) + cast_to_dict(fs_kwargs, ["oauth2_client_params"]) + cast_to_bool(fs_kwargs, ["use_recycle_bin"]) + + # Create filesystem and dlt resource wrapper. + fs = self.fs_class(**fs_kwargs) + return infer_resource(fs=fs, locator=locator) diff --git a/src/dlt_filesystem/source/fsspec/smb.py b/src/dlt_filesystem/source/fsspec/smb.py new file mode 100644 index 000000000..00cc8cbd4 --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/smb.py @@ -0,0 +1,54 @@ +from typing import Type + +from fsspec import AbstractFileSystem + +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import infer_resource +from dlt_filesystem.source.model import FilesystemLocator +from dlt_filesystem.util.python import cast_to_bool, cast_to_int + + +class SMBSource(FilesystemSource): + """ + Access files on Microsoft Windows Server Shares. + + https://en.wikipedia.org/wiki/Server_Message_Block + """ + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + from fsspec.implementations.smb import SMBFileSystem + + return SMBFileSystem + + def dlt_source(self, uri: str, table: str, **kwargs): + + # TODO: Is this applicable for Dropbox and friends at all? + if kwargs.get("incremental_key"): + raise ValueError( + "SMB takes care of incrementality on its own, you should not provide incremental_key" + ) + + # Bundle essential information to infer filesystem wrapper. + locator = FilesystemLocator( + name="SMB", fs_class=self.fs_class, uri=uri, path=table + ) + + # Decode individual options (type casting, default values, sanity checks). Schema: + fs_kwargs = locator.options.fs_kwargs + fs_kwargs.update(kwargs) + cast_to_int( + fs_kwargs, + [ + "port", + "timeout", + "register_session_retries", + "register_session_retry_wait", + "register_session_retry_factor", + ], + ) + cast_to_bool(fs_kwargs, ["encrypt", "auto_mkdir"]) + + # Create filesystem and dlt resource wrapper. + fs = self.fs_class(**fs_kwargs) + return infer_resource(fs=fs, locator=locator) diff --git a/src/dlt_filesystem/source/fsspec/webdav.py b/src/dlt_filesystem/source/fsspec/webdav.py new file mode 100644 index 000000000..b27c359ed --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/webdav.py @@ -0,0 +1,65 @@ +from typing import Type + +from fsspec import AbstractFileSystem + +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import infer_resource +from dlt_filesystem.source.model import FilesystemLocator +from dlt_filesystem.util.python import asbool, cast_to_bool, cast_to_dict, cast_to_int + + +class WebdavSource(FilesystemSource): + """ + Access files on WebDAV. + + https://skshetry.github.io/webdav4/ + https://en.wikipedia.org/wiki/WebDAV + """ + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + from webdav4.fsspec import WebdavFileSystem + + return WebdavFileSystem + + def dlt_source(self, uri: str, table: str, **kwargs): + + # This adjustment is specific to WebDAV. + # omniload uses the `https+webdav://`, but fsspec uses `https://`. + uri = uri.replace("+webdav", "").replace("+dav", "") + + # TODO: Is this applicable for Dropbox and friends at all? + if kwargs.get("incremental_key"): + raise ValueError( + "WebDAV takes care of incrementality on its own, you should not provide incremental_key" + ) + + # Bundle essential information to infer filesystem wrapper. + locator = FilesystemLocator( + name="WebDAV", fs_class=self.fs_class, uri=uri, path=table + ) + + # Decode individual arguments. + fs_kwargs = locator.options.fs_kwargs + fs_kwargs.update(kwargs) + cast_to_bool(fs_kwargs, ["retry", "trust_env"]) + cast_to_dict(fs_kwargs, ["headers", "cookies", "proxies"]) + cast_to_int(fs_kwargs, ["chunk_size", "max_redirects"]) + if "verify" in fs_kwargs: + try: + fs_kwargs["verify"] = asbool(fs_kwargs["verify"]) + except ValueError: + pass + + # Extract authentication credentials. + auth = None + if "username" in fs_kwargs: + auth = (fs_kwargs.pop("username"), fs_kwargs.pop("password", None)) + + # Downstream implementation does not accept those kwargs. + fs_kwargs.pop("host", None) + fs_kwargs.pop("port", None) + + # Create filesystem and dlt resource wrapper. + fs = self.fs_class(base_url=uri, auth=auth, **fs_kwargs) + return infer_resource(fs=fs, locator=locator) diff --git a/src/dlt_filesystem/source/fsspec/webhdfs.py b/src/dlt_filesystem/source/fsspec/webhdfs.py new file mode 100644 index 000000000..f1b33f43c --- /dev/null +++ b/src/dlt_filesystem/source/fsspec/webhdfs.py @@ -0,0 +1,55 @@ +from typing import Type + +from fsspec import AbstractFileSystem + +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import infer_resource +from dlt_filesystem.source.model import FilesystemLocator +from dlt_filesystem.util.python import ( + apply_alias, + cast_to_bool, + cast_to_dict, + cast_to_int, +) + + +class WebHDFSSource(FilesystemSource): + """ + Access files on WebHDFS. + """ + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + + from fsspec.implementations.webhdfs import WebHDFS + + return WebHDFS + + def dlt_source(self, uri: str, table: str, **kwargs): + + # TODO: It looks like this is generic code that could be refactored already + # if it's common amongst different implementations. + if kwargs.get("incremental_key"): + raise ValueError( + "WebHDFS takes care of incrementality on its own, you should not provide incremental_key" + ) + + # Bundle essential information to infer filesystem wrapper. + locator = FilesystemLocator( + name="WebHDFS", + fs_class=self.fs_class, + uri=uri, + path=table, + ) + + # Decode individual options (type casting, default values, sanity checks). + fs_kwargs = locator.options.fs_kwargs + fs_kwargs.update(kwargs) + apply_alias(fs_kwargs, "kerberos_options", "kerb_kwargs") + cast_to_int(fs_kwargs, ["port"]) + cast_to_bool(fs_kwargs, ["kerberos", "use_https", "session_verify"]) + cast_to_dict(fs_kwargs, ["kerb_kwargs", "data_proxy"]) + + # Create filesystem and dlt resource wrapper. + fs = self.fs_class(**fs_kwargs) + return infer_resource(fs=fs, locator=locator) diff --git a/src/dlt_filesystem/source/impl/remote.py b/src/dlt_filesystem/source/impl/remote.py index 33dfeb6f7..6dd7984ce 100644 --- a/src/dlt_filesystem/source/impl/remote.py +++ b/src/dlt_filesystem/source/impl/remote.py @@ -1,6 +1,7 @@ import base64 import json -from typing import Any, Dict, Type +from abc import abstractmethod +from typing import TYPE_CHECKING, Any, Dict, Type from urllib.parse import parse_qs, urlparse from fsspec import AbstractFileSystem @@ -12,29 +13,12 @@ from dlt_filesystem.source.router import ( blob_hints, determine_endpoint, - parse_fragment, parse_uri, ) from dlt_filesystem.util.auth import AzureBlobAuth, parse_azure_blob_auth - -def _endpoint_namespace(endpoint: str | None, default: str) -> str: - """Return a normalized endpoint identity without credentials or query values.""" - if not endpoint: - return default - - parsed = urlparse(endpoint if "://" in endpoint else f"//{endpoint}") - host = parsed.hostname - if not host: - return default - - host = host.lower() - if ":" in host: - host = f"[{host}]" - if parsed.port is not None: - host = f"{host}:{parsed.port}" - - return f"{host}{parsed.path.rstrip('/')}" +if TYPE_CHECKING: + from fsspec import AbstractFileSystem class GCSSource(FilesystemSource): @@ -90,13 +74,13 @@ def dlt_source(self, uri: str, table: str, **kwargs): try: endpoint: str = determine_endpoint(table, path_to_file) except UnsupportedEndpointError: - raise ValueError(supported_file_format_message("GCS")) + raise ValueError(supported_file_format_message("GCS")) from None except Exception as e: raise ValueError( f"Failed to parse endpoint from path: {path_to_file}" ) from e - from dlt_filesystem.source.adapter import resource_for_reader + from dlt_filesystem.source.core import resource_for_reader from dlt_filesystem.source.model import FilesystemReference return resource_for_reader( @@ -113,32 +97,53 @@ def dlt_source(self, uri: str, table: str, **kwargs): ) -class S3Source(FilesystemSource): - """dlt source for Amazon S3""" +class S3CompatibleSource(FilesystemSource): + """ + Access S3 and compatible filesystems. + + TODO: Forward more parameters than `access_key_id` and `secret_access_key` + (key/secret/endpoint_url) only, like `region`. + """ + + @property + @abstractmethod + def fs_name(self) -> str: + raise NotImplementedError("Need to implement abstract property") + + @property + def fs_class(self) -> Type["AbstractFileSystem"]: + import s3fs + + return s3fs.S3FileSystem + + @property + def fs_protocol(self) -> str: + if isinstance(self.fs_class.protocol, (list, tuple)): + return self.fs_class.protocol[0] + return self.fs_class.protocol def dlt_source(self, uri: str, table: str, **kwargs): if kwargs.get("incremental_key"): raise ValueError( - "S3 takes care of incrementality on its own, you should not provide incremental_key" + f"{self.fs_name} takes care of incrementality on its own, " + f"you should not provide incremental_key" ) parsed_uri = urlparse(uri) source_fields = parse_qs(parsed_uri.query) access_key_id = source_fields.get("access_key_id") if not access_key_id: - raise ValueError("access_key_id is required to connect to S3") + raise MissingConnectorOption("access_key_id", self.fs_name) secret_access_key = source_fields.get("secret_access_key") if not secret_access_key: - raise ValueError("secret_access_key is required to connect to S3") + raise MissingConnectorOption("secret_access_key", self.fs_name) bucket_name, path_to_file = parse_uri(parsed_uri, table) if not bucket_name or not path_to_file: - raise InvalidBlobTableError("S3") + raise InvalidBlobTableError(self.fs_name) - bucket_url = f"s3://{bucket_name}/" - - import s3fs + bucket_url = f"{self.fs_protocol}://{bucket_name}/" endpoint_url = source_fields.get("endpoint_url") fs_kwargs: dict = { @@ -148,18 +153,18 @@ def dlt_source(self, uri: str, table: str, **kwargs): if endpoint_url: fs_kwargs["endpoint_url"] = endpoint_url[0] - fs = s3fs.S3FileSystem(**fs_kwargs) + fs = self.fs_class(**fs_kwargs) try: endpoint: str = determine_endpoint(table, path_to_file) except UnsupportedEndpointError: - raise ValueError(supported_file_format_message("S3")) + raise ValueError(supported_file_format_message(self.fs_name)) from None except Exception as e: raise ValueError( f"Failed to parse endpoint from path: {path_to_file}" ) from e - from dlt_filesystem.source.adapter import resource_for_reader + from dlt_filesystem.source.core import resource_for_reader from dlt_filesystem.source.model import FilesystemReference return resource_for_reader( @@ -168,7 +173,7 @@ def dlt_source(self, uri: str, table: str, **kwargs): bucket_url=bucket_url, file_glob=path_to_file, reader_name=endpoint, - storage_namespace=f"s3:{_endpoint_namespace(endpoint_url[0] if endpoint_url else None, 'aws')}", + storage_namespace=f"s3:{self.endpoint_namespace(endpoint_url[0] if endpoint_url else None, 'aws')}", filesystem_incremental=kwargs.get("filesystem_incremental", False), hints=blob_hints(parsed_uri, table), column_types=kwargs.get("column_types"), @@ -176,6 +181,12 @@ def dlt_source(self, uri: str, table: str, **kwargs): ) +class S3Source(S3CompatibleSource): + @property + def fs_name(self) -> str: + return "S3" + + def _azure_kwargs(auth: AzureBlobAuth): """Return AzureBlobAuth information as dictionary. @@ -248,13 +259,13 @@ def dlt_source(self, uri: str, table: str, **kwargs): try: endpoint: str = determine_endpoint(table, path_to_file) except UnsupportedEndpointError: - raise ValueError(supported_file_format_message("Azure")) + raise ValueError(supported_file_format_message("Azure")) from None except Exception as e: raise ValueError( f"Failed to parse endpoint from path: {path_to_file}" ) from e - from dlt_filesystem.source.adapter import resource_for_reader + from dlt_filesystem.source.core import resource_for_reader from dlt_filesystem.source.model import FilesystemReference return resource_for_reader( @@ -265,7 +276,7 @@ def dlt_source(self, uri: str, table: str, **kwargs): reader_name=endpoint, storage_namespace=( f"azure:{(auth.account_name or '').lower()}:" - f"{_endpoint_namespace(auth.account_host, 'azure-public')}" + f"{self.endpoint_namespace(auth.account_host, 'azure-public')}" ), filesystem_incremental=kwargs.get("filesystem_incremental", False), hints=blob_hints(parsed_uri, table), @@ -275,11 +286,13 @@ def dlt_source(self, uri: str, table: str, **kwargs): class SFTPSource(FilesystemSource): + """Access files on SFTP servers.""" + def dlt_source(self, uri: str, table: str, **kwargs): parsed_uri = urlparse(uri) host = parsed_uri.hostname if not host: - raise MissingConnectorOption("host", "SFTP URI") + raise MissingConnectorOption("host", "SFTP") port = parsed_uri.port or 22 username = parsed_uri.username password = parsed_uri.password @@ -301,33 +314,34 @@ def dlt_source(self, uri: str, table: str, **kwargs): raise ConnectionError( f"Failed to connect or authenticate to sftp server {host}:{port}. Error: {e}" ) from e - bucket_url = f"sftp://{host}:{port}" - table_path, _, hints = parse_fragment(table) - if table_path.startswith("/"): - file_glob = table_path - else: - file_glob = f"/{table_path}" + bucket_name, path_to_file = parse_uri(parsed_uri, table) + if not bucket_name or not path_to_file: + raise InvalidBlobTableError("SFTP") + + bucket_url = f"sftp://{host}:{port}" try: - endpoint = determine_endpoint(table, file_glob) + endpoint = determine_endpoint(table, path_to_file) except UnsupportedEndpointError: raise ValueError(supported_file_format_message("SFTP")) from None except Exception as e: - raise ValueError(f"Failed to parse endpoint from path: {table}") from e + raise ValueError( + f"Failed to parse endpoint from path: {path_to_file}" + ) from e - from dlt_filesystem.source.adapter import resource_for_reader + from dlt_filesystem.source.core import resource_for_reader from dlt_filesystem.source.model import FilesystemReference return resource_for_reader( FilesystemReference( fs=fs, bucket_url=bucket_url, - file_glob=file_glob, + file_glob=path_to_file, reader_name=endpoint, storage_namespace=(f"sftp:{host.lower()}:{port}:{username or ''}"), filesystem_incremental=kwargs.get("filesystem_incremental", False), - hints=hints, + hints=blob_hints(parsed_uri, table), column_types=kwargs.get("column_types"), ) ) diff --git a/src/dlt_filesystem/source/model.py b/src/dlt_filesystem/source/model.py index 2a1283d0c..c97952add 100644 --- a/src/dlt_filesystem/source/model.py +++ b/src/dlt_filesystem/source/model.py @@ -1,7 +1,9 @@ import hashlib import json +from copy import deepcopy from dataclasses import dataclass, field -from typing import Any, Optional, Type, Union +from typing import Any, Dict, Optional, Type, Union +from urllib.parse import parse_qs from dlt.common.configuration import configspec, resolve_type from dlt.common.configuration.specs import CredentialsConfiguration @@ -9,6 +11,10 @@ from dlt.common.storages.configuration import FileSystemCredentials from fsspec import AbstractFileSystem +from dlt_filesystem.error import InvalidBlobTableError +from dlt_filesystem.util.fsspec import infer_storage_options +from dlt_filesystem.util.web import shrink_qs_dict + @configspec class FilesystemConfigurationResource(FilesystemConfiguration): @@ -27,9 +33,141 @@ def resolve_credentials_type(self) -> Type[CredentialsConfiguration]: ] +@dataclass +class FilesystemOptions: + """Bundle filesystem options from URL.""" + + address: Dict[str, Any] = field(default_factory=dict) + params: Dict[str, Any] = field(default_factory=dict) + + @property + def fs_kwargs(self) -> Dict[str, Any]: + """Return inferred storage options modulo `protocol` and `path` fields.""" + # FIXME: Review using URL options as a baseline. + response = deepcopy(self.address) + # Remove certain options like `_get_kwargs_from_urls` is doing it. + response.pop("path", None) + response.pop("protocol", None) + response.update(self.params) + return response + + +@dataclass +class FilesystemLocator: + """A full filesystem information locator.""" + + # FIXME: Get rid of inline imports by applying another round of refactoring. + + name: str + fs_class: Type[AbstractFileSystem] + uri: str + path: str + default_port: Optional[int] = None + address: Dict[str, str] = field(default_factory=dict) + options: FilesystemOptions = field(default_factory=FilesystemOptions) + accept_no_bucket_name: Optional[bool] = False + accept_no_host_name: Optional[bool] = False + + def __post_init__(self): + """Decode fundamental options right away.""" + self.read_options() + + def read_options(self) -> "FilesystemLocator": + """ + Destructure input URL as a baseline for fsspec kwargs. + + Let's use the fsspec approach of decoding + URIs, based on `fsspec.utils.infer_storage_options`. + """ + + self.address = infer_storage_options(self.uri) + + # URL query parameters. + params = shrink_qs_dict(parse_qs(self.address.pop("url_query", ""))) + # TODO: Why not compute hints right here instead of doing it at runtime? + self.address.pop("url_fragment", None) + + # Reader or writer hints. + self.options = FilesystemOptions( + address=self.address, + params=params, + ) + return self + + def validate(self): + """Decode into base url and url path / file glob, and apply sanity checks.""" + if not self.bucket_name or not self.file_glob: + if self.accept_no_bucket_name: + return + # TODO: Rename exception. + raise InvalidBlobTableError(self.name) + + @property + def bucket_url(self) -> str: + """URL without credentials and path.""" + + if self.accept_no_host_name: + return self.uri + + address = self.options.address + if "host" in address and ("port" in address or self.default_port is not None): + return f"{address['protocol']}://{address['host']}:{address.get('port', self.default_port)}" + elif "host" in address: + return f"{address['protocol']}://{address['host']}" + + # dlt will fail per `verify_bucket_url()` when no netloc or path is given: + # dlt.common.configuration.exceptions.ConfigurationValueError: File `path` + # and `netloc` are missing. Field `bucket_url` of `FilesystemClientConfiguration` + # must contain valid url with a path or host:password component. + # When that happens, try to borrow a hostname from other suitable parameters + # like `endpoint`. + else: + surrogate_host_port = self.options.params.get("endpoint") + if not surrogate_host_port: + raise ValueError("dlt needs bucket_url to include netloc and path") + return f"{address['protocol']}://{surrogate_host_port}" + + @property + def bucket_name(self) -> str: + """URL component that describes the bucket name.""" + # FIXME: Inline imports! + from dlt_filesystem.source.router import parse_uri + + bucket_name, _ = parse_uri(self.uri, self.path) + return bucket_name + + @property + def file_glob(self) -> str: + """URL path component that describes the file glob.""" + # FIXME: Inline imports! + from dlt_filesystem.source.router import parse_uri + + _, file_glob = parse_uri(self.uri, self.path) + return file_glob + + @property + def hints(self) -> Dict[str, Any]: + """ + Destructure reader or writer hints from URL fragment. + + Let's use the omniload approach of decoding + URL fragments, because it handles a few edge cases, also taking + the URL path into consideration. + + TODO: Refactor inline imports! + """ + from urllib.parse import urlparse + + from dlt_filesystem.source.router import blob_hints + + parsed_uri = urlparse(self.uri) + return blob_hints(parsed_uri, self.path) + + @dataclass class FilesystemReference: - """Bundle the arguments needed by `resource_for_reader` to build a resource. + """ + Bundle the arguments needed by `resource_for_reader` to build a resource. Args: fs (AbstractFilesystem): fsspec filesystem instance. @@ -47,6 +185,8 @@ class FilesystemReference: key a reader looks up is that reader's contract; no reader consumes hints yet, so this is currently populated but unread. column_types (dict[str, Any], optional): Column name to type mapping, e.g. used by `read_csv_headless`. + + TODO: Zap into / synchronize with the new `FilesystemLocator`? """ fs: AbstractFileSystem diff --git a/src/dlt_filesystem/source/router.py b/src/dlt_filesystem/source/router.py index d6da5a664..ce00c6cc6 100644 --- a/src/dlt_filesystem/source/router.py +++ b/src/dlt_filesystem/source/router.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, TypeAlias +from typing import Any, Dict, Optional, Tuple, TypeAlias, Union from urllib.parse import ParseResult, parse_qsl, urlparse import dlt @@ -8,6 +8,7 @@ from dlt.extract import DltResource from fsspec import AbstractFileSystem +from dlt_filesystem.source.error import UnsupportedEndpointError from dlt_filesystem.source.format.registry import ( FORMAT_TO_READER, reader_for_format, @@ -17,7 +18,7 @@ FileGlob: TypeAlias = str -def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]: +def parse_uri(uri: Union[ParseResult, str], table: str) -> Tuple[BucketName, FileGlob]: """ parse the URI of a blob storage and return the bucket name and the file glob. @@ -37,6 +38,9 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]: The first form is the preferred method. Other forms are supported but discouraged. """ + if isinstance(uri, str): + uri = urlparse(uri) + table = table.strip() host = uri.netloc.strip() @@ -172,7 +176,7 @@ def blob_hints(parsed_uri: ParseResult, table: str) -> Dict[str, str]: def determine_endpoint(table: str, path: str) -> str: """ - determines the endpoint/method to use for reading data from a blob source + Find the designated reader method from either `table` or URL `path` component. """ _, file_format = split_format_hint(table) @@ -180,9 +184,16 @@ def determine_endpoint(table: str, path: str) -> str: return reader_for_format(file_format) try: - return parse_endpoint(path) - except Exception as e: - raise ValueError(f"Failed to parse endpoint from path: {path}") from e + return parse_endpoint(table) + except Exception: + try: + return parse_endpoint(path) + except UnsupportedEndpointError: + raise + except Exception as e: + raise ValueError( + f"Failed to parse endpoint from table '{table}' or path '{path}'" + ) from e def fsspec_from_resource(filesystem_instance: DltResource) -> AbstractFileSystem: diff --git a/src/dlt_filesystem/util/fsspec.py b/src/dlt_filesystem/util/fsspec.py new file mode 100644 index 000000000..a1560423a --- /dev/null +++ b/src/dlt_filesystem/util/fsspec.py @@ -0,0 +1,109 @@ +import re +from typing import Any +from urllib.parse import urlsplit + +from fsspec.utils import update_storage_options + + +def infer_storage_options( + urlpath: str, inherit_storage_options: dict[str, Any] | None = None +) -> dict[str, Any]: + """Infer storage options from URL path and merge it with existing storage + options. + + TODO: This is a copy of `fsspec.utils.infer_storage_options` with two + adjustments. If this can land upstream, let's remove the function + again. + - The regex: r"^[a-zA-Z0-9+]+://". + https://github.com/fsspec/filesystem_spec/pull/2085 + - Don't suppress parsing http/https urls. + + Parameters + ---------- + urlpath: str or unicode + Either local absolute file path or URL (hdfs://namenode:8020/file.csv) + inherit_storage_options: dict (optional) + Its contents will get merged with the inferred information from the + given path + + Returns + ------- + Storage options dict. + + Examples + -------- + >>> infer_storage_options('/mnt/datasets/test.csv') # doctest: +SKIP + {"protocol": "file", "path", "/mnt/datasets/test.csv"} + >>> infer_storage_options( + ... 'hdfs://username:pwd@node:123/mnt/datasets/test.csv?q=1', + ... inherit_storage_options={'extra': 'value'}, + ... ) # doctest: +SKIP + {"protocol": "hdfs", "username": "username", "password": "pwd", + "host": "node", "port": 123, "path": "/mnt/datasets/test.csv", + "url_query": "q=1", "extra": "value"} + """ + + # Discover Windows paths including disk name in this special case. + is_filesystem = re.match(r"^[a-zA-Z]:[\\/]", urlpath) + + # Discover URI according to RFC 3986: Scheme names consist of a + # sequence of characters beginning with a letter and followed by + # any combination of letters, digits, plus ("+"), period ("."), + # or hyphen ("-"). + # https://datatracker.ietf.org/doc/html/rfc3986#section-3.1 + is_uri = re.match(r"^[a-zA-Z0-9+.-]+://", urlpath) + + if is_filesystem and not is_uri: + return {"protocol": "file", "path": urlpath} + + parsed_path = urlsplit(urlpath) + protocol = parsed_path.scheme or "file" + if parsed_path.fragment: + path = "#".join([parsed_path.path, parsed_path.fragment]) + else: + path = parsed_path.path + if protocol == "file": + # Special case parsing file protocol URL on Windows according to: + # https://msdn.microsoft.com/en-us/library/jj710207.aspx + windows_path = re.match(r"^/([a-zA-Z])[:|]([\\/].*)$", path) + if windows_path: + drive, path = windows_path.groups() + path = f"{drive}:{path}" + + # Within omniload, we _want_ to parse, to create fewer anomalies. + # Specifically, the WebDAV connector needs it because it uses the + # `http` protocol scheme. + """ + if protocol in ["http", "https"]: + # for HTTP, we don't want to parse, as requests will anyway + return {"protocol": protocol, "path": urlpath} + """ + + options: dict[str, Any] = {"protocol": protocol, "path": path} + + if parsed_path.netloc: + # Parse `hostname` from netloc manually because `parsed_path.hostname` + # lowercases the hostname which is not always desirable (e.g. in S3): + # https://github.com/dask/dask/issues/1417 + options["host"] = parsed_path.netloc.rsplit("@", 1)[-1].rsplit(":", 1)[0] + + if protocol in ("s3", "s3a", "gcs", "gs"): + options["path"] = options["host"] + options["path"] + else: + options["host"] = options["host"] + if parsed_path.port: + options["port"] = parsed_path.port + if parsed_path.username: + options["username"] = parsed_path.username + if parsed_path.password: + options["password"] = parsed_path.password + + if parsed_path.query: + options["url_query"] = parsed_path.query + if parsed_path.fragment: + options["url_fragment"] = parsed_path.fragment + + if inherit_storage_options: + update_storage_options(options, inherit_storage_options) + + return options diff --git a/src/dlt_filesystem/util/python.py b/src/dlt_filesystem/util/python.py index caf5a552d..eb7a1770f 100644 --- a/src/dlt_filesystem/util/python.py +++ b/src/dlt_filesystem/util/python.py @@ -3,7 +3,7 @@ import logging import types import typing -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, List, Optional _TRUE_VALUES = {"true", "1", "yes", "on"} _FALSE_VALUES = {"false", "0", "no", "off"} @@ -86,3 +86,64 @@ def _cast_value(value: str, target_type: Any) -> Any: return json.loads(value) except (json.JSONDecodeError, TypeError): return value + + +# from paste.deploy.converters +def asbool(obj: Any) -> bool: + """From `sqlalchemy.util.langhelpers`""" + if isinstance(obj, str): + obj = obj.strip().lower() + if obj in ["true", "yes", "on", "y", "t", "1"]: + return True + elif obj in ["false", "no", "off", "n", "f", "0"]: + return False + else: + raise ValueError("String is not true/false: %r" % obj) + return bool(obj) + + +def cast_to_int(data: Dict[str, Any], names: List[str]) -> Dict[str, Any]: + """Cast dictionary values to integers.""" + for field_name in names: + if field_name in data: + data[field_name] = int(data[field_name]) + return data + + +def cast_to_float(data: Dict[str, Any], names: List[str]) -> Dict[str, Any]: + """Cast dictionary values to floats.""" + for field_name in names: + if field_name in data: + data[field_name] = float(data[field_name]) + return data + + +def cast_to_bool(data: Dict[str, Any], names: List[str]) -> Dict[str, Any]: + """Cast dictionary values to booleans.""" + for field_name in names: + if field_name in data: + data[field_name] = asbool(data[field_name]) + return data + + +def cast_to_dict(data: Dict[str, Any], names: List[str]) -> Dict[str, Any]: + """Cast dictionary values from JSON.""" + for field_name in names: + if field_name in data: + data[field_name] = json.loads(data[field_name]) + return data + + +def cast_to_list(data: Dict[str, Any], names: List[str]) -> Dict[str, Any]: + """Cast list values from JSON.""" + for field_name in names: + if field_name in data: + data[field_name] = json.loads(data[field_name]) + return data + + +def apply_alias(data: Dict[str, Any], name: str, effective_name: str) -> Dict[str, Any]: + """Apply aliasing to dictionary keys.""" + if name in data: + data[effective_name] = data.pop(name) + return data diff --git a/src/dlt_filesystem/util/web.py b/src/dlt_filesystem/util/web.py new file mode 100644 index 000000000..341289050 --- /dev/null +++ b/src/dlt_filesystem/util/web.py @@ -0,0 +1,6 @@ +from typing import Any, Dict + + +def shrink_qs_dict(data: Dict[str, Any]) -> Dict[str, Any]: + """Let's only use the first element when decoding URL query parameters.""" + return {key: value[0] for key, value in data.items()} diff --git a/src/omniload/core/registry.py b/src/omniload/core/registry.py index 39af9a652..21c9b6a5e 100644 --- a/src/omniload/core/registry.py +++ b/src/omniload/core/registry.py @@ -1,3 +1,4 @@ +# ruff: noqa: E501 from omniload.core.model import LazyRegistry SQL_SOURCE_SCHEMES = [ @@ -32,9 +33,9 @@ sources: LazyRegistry = LazyRegistry( { - "abfss": "dlt_filesystem.source.api:AzureSource", + "abfss": "dlt_filesystem.source.impl.remote:AzureSource", "adjust": "omniload.source.adjust.api:AdjustSource", - "adls": "dlt_filesystem.source.api:AzureSource", + "adls": "dlt_filesystem.source.impl.remote:AzureSource", "airtable": "omniload.source.airtable.api:AirtableSource", "allium": "omniload.source.allium.api:AlliumSource", "amqp+mqb": "omniload.source.mqbridge.api:MqBridgeSource", @@ -46,7 +47,7 @@ "asana": "omniload.source.asana.api:AsanaSource", "attio": "omniload.source.attio.api:AttioSource", "aws+mqb": "omniload.source.mqbridge.api:MqBridgeSource", - "az": "dlt_filesystem.source.api:AzureSource", + "az": "dlt_filesystem.source.impl.remote:AzureSource", "bruin": "omniload.source.bruin.api:BruinSource", "chess": "omniload.source.chess.api:ChessSource", "clickup": "omniload.source.clickup.api:ClickupSource", @@ -54,26 +55,33 @@ "csv": "omniload.source.csv.api:LocalCsvSource", "cursor": "omniload.source.cursor.api:CursorSource", "customerio": "omniload.source.customer_io.api:CustomerIoSource", + "dbfs": "dlt_filesystem.source.fsspec.databricks:DatabricksSource", "docebo": "omniload.source.docebo.api:DoceboSource", + "dropbox": "dlt_filesystem.source.fsspec.dropbox:DropboxSource", "dune": "omniload.source.dune.api:DuneSource", "dynamodb": "omniload.source.dynamodb.api:DynamoDBSource", "elasticsearch": "omniload.source.elasticsearch.api:ElasticsearchSource", "facebookads": "omniload.source.facebook_ads.api:FacebookAdsSource", - "file": "dlt_filesystem.source.api:LocalFilesystemSource", + "file": "dlt_filesystem.source.fsspec.local:LocalFilesystemSource", "fireflies": "omniload.source.fireflies.api:FirefliesSource", "fluxx": "omniload.source.fluxx.api:FluxxSource", "frankfurter": "omniload.source.frankfurter.api:FrankfurterSource", "freshdesk": "omniload.source.freshdesk.api:FreshdeskSource", + "ftp": "dlt_filesystem.source.fsspec.ftp:FTPSource", "fundraiseup": "omniload.source.fundraiseup.api:FundraiseupSource", + "gdrive": "dlt_filesystem.source.fsspec.gdrive:GoogleDriveSource", "github": "omniload.source.github.api:GitHubSource", "googleads": "omniload.source.google_ads.api:GoogleAdsSource", "googleanalytics": "omniload.source.google_analytics.api:GoogleAnalyticsSource", "gorgias": "omniload.source.gorgias.api:GorgiasSource", - "gs": "dlt_filesystem.source.api:GCSSource", + "gs": "dlt_filesystem.source.impl.remote:GCSSource", "gsheets": "omniload.source.google_sheets.api:GoogleSheetsSource", + "hdfs": "dlt_filesystem.source.fsspec.hdfs:HDFSSource", "hostaway": "omniload.source.hostaway.api:HostawaySource", "http": "omniload.source.http.api:HttpSource", "https": "omniload.source.http.api:HttpSource", + "http+webdav": "dlt_filesystem.source.fsspec.webdav:WebdavSource", + "https+webdav": "dlt_filesystem.source.fsspec.webdav:WebdavSource", "hubspot": "omniload.source.hubspot.api:HubspotSource", "ibmmq+mqb": "omniload.source.mqbridge.api:MqBridgeSource", "indeed": "omniload.source.indeed.api:IndeedSource", @@ -95,8 +103,12 @@ "mongodb": "omniload.source.mongodb.api:MongoDbSource", "mongodb+srv": "omniload.source.mongodb.api:MongoDbSource", "mqtt+mqb": "omniload.source.mqbridge.api:MqBridgeSource", + "msgd": "dlt_filesystem.source.fsspec.sharepoint:SharePointSource", "nats+mqb": "omniload.source.mqbridge.api:MqBridgeSource", "notion": "omniload.source.notion.api:NotionSource", + "oci": "dlt_filesystem.source.fsspec.oci:OCISource", + "onedrive": "dlt_filesystem.source.fsspec.onedrive:OneDriveSource", + "oss": "dlt_filesystem.source.fsspec.oss:OSSSource", "personio": "omniload.source.personio.api:PersonioSource", "phantombuster": "omniload.source.phantombuster.api:PhantombusterSource", "pinterest": "omniload.source.pinterest.api:PinterestSource", @@ -104,20 +116,24 @@ "plusvibeai": "omniload.source.plusvibeai.api:PlusVibeAISource", "primer": "omniload.source.primer.api:PrimerSource", "quickbooks": "omniload.source.quickbooks.api:QuickBooksSource", + "r2": "dlt_filesystem.source.fsspec.r2:R2Source", "redditads": "omniload.source.reddit_ads.api:RedditAdsSource", "revenuecat": "omniload.source.revenuecat.api:RevenueCatSource", - "s3": "dlt_filesystem.source.api:S3Source", + "s3": "dlt_filesystem.source.impl.remote:S3Source", "salesforce": "omniload.source.salesforce.api:SalesforceSource", - "sftp": "dlt_filesystem.source.api:SFTPSource", + "sftp": "dlt_filesystem.source.impl.remote:SFTPSource", + "sharepoint": "dlt_filesystem.source.fsspec.sharepoint:SharePointSource", "shopify": "omniload.source.shopify.api:ShopifySource", "slack": "omniload.source.slack.api:SlackSource", "smartsheet": "omniload.source.smartsheets.api:SmartsheetSource", + "smb": "dlt_filesystem.source.fsspec.smb:SMBSource", "snapchatads": "omniload.source.snapchat_ads.api:SnapchatAdsSource", "socrata": "omniload.source.socrata.api:SocrataSource", "solidgate": "omniload.source.solidgate.api:SolidgateSource", "stripe": "omniload.source.stripe.api:StripeAnalyticsSource", "tiktok": "omniload.source.tiktok_ads.api:TikTokSource", "trustpilot": "omniload.source.trustpilot.api:TrustpilotSource", + "webhdfs": "dlt_filesystem.source.fsspec.webhdfs:WebHDFSSource", "wise": "omniload.source.wise.api:WiseSource", "zendesk": "omniload.source.zendesk.api:ZendeskSource", "zeromq+mqb": "omniload.source.mqbridge.api:MqBridgeSource", diff --git a/tests/assets/privatekey-fingerprint.txt b/tests/assets/privatekey-fingerprint.txt new file mode 100644 index 000000000..c76a829bb --- /dev/null +++ b/tests/assets/privatekey-fingerprint.txt @@ -0,0 +1 @@ +51:b0:25:05:4b:01:98:4f:43:a0:2c:12:c3:93:c7:46 \ No newline at end of file diff --git a/tests/assets/privatekey.pem b/tests/assets/privatekey.pem new file mode 100644 index 000000000..265a9e59a --- /dev/null +++ b/tests/assets/privatekey.pem @@ -0,0 +1,10 @@ +-----BEGIN PRIVATE KEY----- +MIIBVgIBADANBgkqhkiG9w0BAQEFAASCAUAwggE8AgEAAkEA48pyUEQdRCof4aHP +LCIEb6JHFn+44i5CEAlhuNGTOk83xuCxSPebIw/7Pnbzknv6oVdV5fSrmVsZ4mP7 +Zy5FZwIDAQABAkB2NB+Nt0rYjGNu2mB/LkfPBg6NhkmSVR7C45tqJJaZsncYX7bi +qEhemm5Ee8D4gyu8vXhPvLfioW0CfVr8qh1BAiEA+vWspYNPXV94UF2Pzb9L4qFj ++x8o/hhXq0vfYHw9xhMCIQDoXaXelqF5RVkPovr6y4I0EELtzB4QJqv+XdUE87F9 +3QIhAIELALqe2Zl2tOQGKCKwwfGH8WQ4cpitpa7UNclkOVN3AiEAwmdHFJjVc45P +WsOyUwzcXA6W0DiLllukXeXHCKORhYkCIQCEFR1GWse3LCBNKIHCgK24cEalv695 +wNhHCIKYdmzUnw== +-----END PRIVATE KEY----- diff --git a/tests/dlt_filesystem/format/test_bson.py b/tests/dlt_filesystem/format/test_bson.py index 250fa810d..cafbc2a44 100644 --- a/tests/dlt_filesystem/format/test_bson.py +++ b/tests/dlt_filesystem/format/test_bson.py @@ -13,9 +13,9 @@ from bson.min_key import MinKey from dlt.common.utils import map_nested_values_in_place -from dlt_filesystem.source.api import LocalFilesystemSource from dlt_filesystem.source.format.bson_codec import convert_bson_objs from dlt_filesystem.source.format.readers import read_bson +from dlt_filesystem.source.fsspec.local import LocalFilesystemSource from dlt_filesystem.testing.writer import write_bson OID = "507f1f77bcf86cd799439011" diff --git a/tests/dlt_filesystem/format/test_cbor.py b/tests/dlt_filesystem/format/test_cbor.py index 2a813258f..0d6772bd2 100644 --- a/tests/dlt_filesystem/format/test_cbor.py +++ b/tests/dlt_filesystem/format/test_cbor.py @@ -17,13 +17,13 @@ import pytest -from dlt_filesystem.source.api import LocalFilesystemSource from dlt_filesystem.source.error import MissingDecoderError from dlt_filesystem.source.format.iterable_codec import ( FORMAT_TO_ITERABLE, read_via_iterable, ) from dlt_filesystem.source.format.readers import read_cbor +from dlt_filesystem.source.fsspec.local import LocalFilesystemSource from dlt_filesystem.testing.stub import ( FileItemStub, NonSeekableItem, diff --git a/tests/dlt_filesystem/format/test_msgpack.py b/tests/dlt_filesystem/format/test_msgpack.py index 0e86bae4d..05c31a832 100644 --- a/tests/dlt_filesystem/format/test_msgpack.py +++ b/tests/dlt_filesystem/format/test_msgpack.py @@ -18,7 +18,6 @@ import pytest -from dlt_filesystem.source.api import LocalFilesystemSource from dlt_filesystem.source.error import MissingDecoderError from dlt_filesystem.source.format.iterable_codec import ( FORMAT_TO_ITERABLE, @@ -26,6 +25,7 @@ read_via_iterable, ) from dlt_filesystem.source.format.readers import read_msgpack +from dlt_filesystem.source.fsspec.local import LocalFilesystemSource from dlt_filesystem.testing.stub import ( FileItemStub, NonSeekableItem, diff --git a/tests/dlt_filesystem/format/test_xml.py b/tests/dlt_filesystem/format/test_xml.py index cad227970..72c7e00ec 100644 --- a/tests/dlt_filesystem/format/test_xml.py +++ b/tests/dlt_filesystem/format/test_xml.py @@ -16,7 +16,6 @@ import pytest -from dlt_filesystem.source.api import LocalFilesystemSource from dlt_filesystem.source.error import ( MissingDecoderError, MissingReaderOptionError, @@ -26,6 +25,7 @@ read_via_iterable, ) from dlt_filesystem.source.format.readers import read_xml +from dlt_filesystem.source.fsspec.local import LocalFilesystemSource from dlt_filesystem.testing.stub import ( FileItemStub, NonSeekableItem, diff --git a/tests/dlt_filesystem/format/test_yaml.py b/tests/dlt_filesystem/format/test_yaml.py index a5079167b..a21fc74d1 100644 --- a/tests/dlt_filesystem/format/test_yaml.py +++ b/tests/dlt_filesystem/format/test_yaml.py @@ -16,12 +16,12 @@ import pytest -from dlt_filesystem.source.api import LocalFilesystemSource from dlt_filesystem.source.error import MissingDecoderError from dlt_filesystem.source.format.iterable_codec import ( FORMAT_TO_ITERABLE, ) from dlt_filesystem.source.format.readers import read_yaml +from dlt_filesystem.source.fsspec.local import LocalFilesystemSource from dlt_filesystem.testing.stub import ( FileItemStub, NonSeekableItem, diff --git a/tests/dlt_filesystem/test_source_hints.py b/tests/dlt_filesystem/test_source_hints.py index 10e7f83fe..f4f9a2956 100644 --- a/tests/dlt_filesystem/test_source_hints.py +++ b/tests/dlt_filesystem/test_source_hints.py @@ -10,10 +10,10 @@ import pytest -from dlt_filesystem.source.impl.local import LocalFilesystemSource +from dlt_filesystem.source.fsspec.local import LocalFilesystemSource from dlt_filesystem.source.impl.remote import GCSSource, S3Source, SFTPSource -RESOURCE_FOR_READER = "dlt_filesystem.source.adapter.resource_for_reader" +RESOURCE_FOR_READER = "dlt_filesystem.source.core.resource_for_reader" GCS_URI = "gs://?credentials_base64=e30K" # base64 for "{}" S3_URI = "s3://?access_key_id=KEY&secret_access_key=SECRET" diff --git a/tests/dlt_filesystem/test_source_incremental.py b/tests/dlt_filesystem/test_source_incremental.py index 6d327be9f..3accef456 100644 --- a/tests/dlt_filesystem/test_source_incremental.py +++ b/tests/dlt_filesystem/test_source_incremental.py @@ -1,3 +1,4 @@ +import base64 import re from typing import Any from unittest.mock import patch @@ -7,14 +8,15 @@ from fsspec.implementations.arrow import ArrowFSWrapper from pyarrow.fs import LocalFileSystem -from dlt_filesystem.source.adapter import readers, resource_for_reader -from dlt_filesystem.source.impl.local import LocalFilesystemSource +from dlt_filesystem.source.adapter import readers +from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import resource_for_reader +from dlt_filesystem.source.fsspec.local import LocalFilesystemSource from dlt_filesystem.source.impl.remote import ( AzureSource, GCSSource, S3Source, SFTPSource, - _endpoint_namespace, ) from dlt_filesystem.source.model import FilesystemReference @@ -123,14 +125,14 @@ def _captured_reference(call) -> FilesystemReference: def test_filesystem_sources_thread_incremental_identity_without_auth_material(tmp_path): local_path = tmp_path / "local" / "*.csv" - with patch("dlt_filesystem.source.adapter.resource_for_reader") as build: + with patch("dlt_filesystem.source.core.resource_for_reader") as build: LocalFilesystemSource().dlt_source( f"file://{local_path}", "", filesystem_incremental=True ) local = _captured_reference(build) with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as build, + patch("dlt_filesystem.source.core.resource_for_reader") as build, patch("gcsfs.GCSFileSystem"), ): GCSSource().dlt_source( @@ -141,7 +143,7 @@ def test_filesystem_sources_thread_incremental_identity_without_auth_material(tm gcs = _captured_reference(build) with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as build, + patch("dlt_filesystem.source.core.resource_for_reader") as build, patch("s3fs.S3FileSystem"), ): S3Source().dlt_source( @@ -152,7 +154,7 @@ def test_filesystem_sources_thread_incremental_identity_without_auth_material(tm s3 = _captured_reference(build) with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as build, + patch("dlt_filesystem.source.core.resource_for_reader") as build, patch("adlfs.AzureBlobFileSystem"), ): AzureSource().dlt_source( @@ -163,7 +165,7 @@ def test_filesystem_sources_thread_incremental_identity_without_auth_material(tm azure = _captured_reference(build) with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as build, + patch("dlt_filesystem.source.core.resource_for_reader") as build, patch("fsspec.filesystem"), ): SFTPSource().dlt_source( @@ -190,7 +192,7 @@ def test_filesystem_sources_thread_incremental_identity_without_auth_material(tm def test_endpoint_namespace_excludes_credentials_and_query_values(): assert ( - _endpoint_namespace( + FilesystemSource.endpoint_namespace( "https://user:password@MINIO.example:9000/storage/?token=secret", "default", ) @@ -199,49 +201,55 @@ def test_endpoint_namespace_excludes_credentials_and_query_values(): def test_endpoint_namespace_normalizes_bare_host_and_url_forms(): - assert _endpoint_namespace("account.blob.example", "default") == ( - _endpoint_namespace("https://account.blob.example", "default") + assert FilesystemSource.endpoint_namespace("account.blob.example", "default") == ( + FilesystemSource.endpoint_namespace("https://account.blob.example", "default") ) def test_endpoint_namespace_brackets_ipv6_hosts(): assert ( - _endpoint_namespace("https://[::1]:9000/bucket", "default") + FilesystemSource.endpoint_namespace("https://[::1]:9000/bucket", "default") == "[::1]:9000/bucket" ) def test_endpoint_namespace_falls_back_when_hostname_is_missing(): - assert _endpoint_namespace("/just/a/path", "default") == "default" + assert FilesystemSource.endpoint_namespace("/just/a/path", "default") == "default" -def test_auth_rotation_does_not_change_incremental_resource_names(): +def test_auth_rotation_does_not_change_incremental_resource_names(mocker): + gs_credentials_first = base64.b64encode( + b'{"client_id":"foo","client_secret":"bar","refresh_token":"baz"}' + ).decode("ascii") + gs_credentials_second = base64.b64encode( + b'{"client_id":"qux","client_secret":"quux","refresh_token":"corge"}' + ).decode("ascii") with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as first_build, - patch("gcsfs.GCSFileSystem"), + patch("dlt_filesystem.source.core.resource_for_reader") as first_build, ): - GCSSource().dlt_source("gs://?credentials_base64=e30K", "bucket/*.csv") + GCSSource().dlt_source( + f"gs://?credentials_base64={gs_credentials_first}", "bucket/*.csv" + ) with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as second_build, - patch("gcsfs.GCSFileSystem"), + patch("dlt_filesystem.source.core.resource_for_reader") as second_build, ): - GCSSource().dlt_source("gs://?credentials_base64=e30=", "bucket/*.csv") + GCSSource().dlt_source( + f"gs://?credentials_base64={gs_credentials_second}", "bucket/*.csv" + ) assert ( _captured_reference(first_build).incremental_resource_name == _captured_reference(second_build).incremental_resource_name ) with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as first_build, - patch("s3fs.S3FileSystem"), + patch("dlt_filesystem.source.core.resource_for_reader") as first_build, ): S3Source().dlt_source( "s3://?access_key_id=OLD&secret_access_key=OLD_SECRET", "bucket/*.csv", ) with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as second_build, - patch("s3fs.S3FileSystem"), + patch("dlt_filesystem.source.core.resource_for_reader") as second_build, ): S3Source().dlt_source( "s3://?access_key_id=NEW&secret_access_key=NEW_SECRET", @@ -253,7 +261,7 @@ def test_auth_rotation_does_not_change_incremental_resource_names(): ) with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as first_build, + patch("dlt_filesystem.source.core.resource_for_reader") as first_build, patch("adlfs.AzureBlobFileSystem"), ): AzureSource().dlt_source( @@ -261,7 +269,7 @@ def test_auth_rotation_does_not_change_incremental_resource_names(): "container/*.csv", ) with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as second_build, + patch("dlt_filesystem.source.core.resource_for_reader") as second_build, patch("adlfs.AzureBlobFileSystem"), ): AzureSource().dlt_source( @@ -273,14 +281,13 @@ def test_auth_rotation_does_not_change_incremental_resource_names(): == _captured_reference(second_build).incremental_resource_name ) + mocker.patch("fsspec.implementations.sftp.SFTPFileSystem._connect") with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as first_build, - patch("fsspec.filesystem"), + patch("dlt_filesystem.source.core.resource_for_reader") as first_build, ): SFTPSource().dlt_source("sftp://user:OLD_SECRET@sftp.example:22", "/*.csv") with ( - patch("dlt_filesystem.source.adapter.resource_for_reader") as second_build, - patch("fsspec.filesystem"), + patch("dlt_filesystem.source.core.resource_for_reader") as second_build, ): SFTPSource().dlt_source("sftp://user:NEW_SECRET@sftp.example:22", "/*.csv") assert ( diff --git a/tests/dlt_filesystem/test_source_local.py b/tests/dlt_filesystem/test_source_local.py index f38683735..e48682868 100644 --- a/tests/dlt_filesystem/test_source_local.py +++ b/tests/dlt_filesystem/test_source_local.py @@ -4,8 +4,8 @@ import pytest from dlt_filesystem.error import MissingConnectorOption -from dlt_filesystem.source.api import LocalFilesystemSource from dlt_filesystem.source.format.registry import supported_file_format_message +from dlt_filesystem.source.fsspec.local import LocalFilesystemSource from dlt_filesystem.source.impl.util import _is_absolute_local, _url_path_to_local from dlt_filesystem.source.model import FilesystemReference @@ -26,7 +26,7 @@ def fake_reader(ref: FilesystemReference): captured.update(ref.__dict__) return "SENTINEL" - with patch("dlt_filesystem.source.adapter.resource_for_reader", fake_reader): + with patch("dlt_filesystem.source.core.resource_for_reader", fake_reader): result = LocalFilesystemSource().dlt_source(uri, table, **kwargs) assert result == "SENTINEL" diff --git a/tests/main/filesystem/test_file_incremental.py b/tests/main/filesystem/test_file_incremental.py index d8463d705..3cf601ada 100644 --- a/tests/main/filesystem/test_file_incremental.py +++ b/tests/main/filesystem/test_file_incremental.py @@ -9,8 +9,8 @@ from fsspec.implementations.arrow import ArrowFSWrapper from pyarrow.fs import LocalFileSystem -from dlt_filesystem.source.adapter import resource_for_reader from dlt_filesystem.source.base import FilesystemSource +from dlt_filesystem.source.core import resource_for_reader from dlt_filesystem.source.model import FilesystemReference from omniload import ValidationError, run_ingest from omniload.core.factory import SourceDestinationFactory diff --git a/tests/main/filesystem/test_local_read.py b/tests/main/filesystem/test_local_read.py index 8eb70bdc6..026d4a55b 100644 --- a/tests/main/filesystem/test_local_read.py +++ b/tests/main/filesystem/test_local_read.py @@ -1,4 +1,4 @@ -from dlt_filesystem.source.impl.local import LocalFilesystemSource +from dlt_filesystem.source.fsspec.local import LocalFilesystemSource from omniload.core.factory import SourceDestinationFactory diff --git a/tests/main/filesystem/test_remote_read.py b/tests/main/filesystem/test_remote_read.py index 900ad7a6f..7d58573b8 100644 --- a/tests/main/filesystem/test_remote_read.py +++ b/tests/main/filesystem/test_remote_read.py @@ -1,43 +1,154 @@ +import os +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Optional from urllib.parse import urlparse import pytest +from fsspec.implementations.memory import MemoryFileSystem from omniload.core.factory import SourceDestinationFactory + +@dataclass +class Item: + uri: str + table: Optional[str] = None + + +# Create private key file in PKCS8 format and compute fingerprint. +""" +openssl genrsa -out test.pem 512 +openssl pkcs8 -nocrypt -topk8 -in test.pem -out test-pkcs8.pem +openssl rsa -pubout -outform DER -in test-pkcs8.pem | openssl md5 -c +""" +assets_dir = Path(__file__).resolve().parents[2] / "assets" + +private_key_file = (assets_dir / "privatekey.pem").as_posix() +private_key_fingerprint = ( + (assets_dir / "privatekey-fingerprint.txt").read_text().strip() +) + # A collection of filesystem source URIs without table parameter. URIS = [ # TODO: Review if account name is not already provided in the hostname per `acme`. "abfss://schrott@acme.dfs.core.windows.net/path/to/data.parquet?account_name=acme&account_key=secret", "adls://schrott@acme.dfs.core.windows.net/path/to/data.parquet?account_name=acme&account_key=secret", "az://schrott@acme.dfs.core.windows.net/path/to/data.parquet?account_name=acme&account_key=secret", - # TODO: Mock `gs` backend. - # ValueError: Provided token is either not valid, or expired. + "dbfs:/Volumes/catalog/schema/volume/path/to/data.parquet", + "dbfs:/Workspace/path/to/data.parquet", + # TODO: Review note on the README at https://github.com/fsspec/dropboxdrivefs: + # > Use `dropbox:///folder1/folder2/etc`. Yes, with three /// ! What happen if not, for some reasons + # > the dropbox api will remove everything before the first / in the path keep only what is after. + # Currently, we are using two slashes, because otherwise the machinery fails. In this + # spirit, it is essential to run a few cycles of user testing. + "dropbox://path/to/data.parquet?token=secret", + Item( + uri="ftp://username:password@intranet.example.org/path/to/data.parquet?tls=tls", + table="", + ), + # TODO: Two FTP tests currently don't work. Why? + # Item( + # uri="ftp://username:password@intranet.example.org?tls=tls", + # table="/path/to/data.parquet", + # ), + "gdrive://path/to/data.parquet?token=anon", "gs://table-bucket-name/path/to/data.parquet?credentials_path=/path/to/service-account.json", - # FIXME: KeyError: 'refresh_token' - "gs://table-bucket-name/path/to/data.parquet?credentials_base64=eyJrZXkiOiAidmFsdWUifQ==", + "gs://table-bucket-name/path/to/data.parquet?credentials_base64=eyJjbGllbnRfaWQiOiAiZm9vIiwgImNsaWVudF9zZWNyZXQiOiAiYmFyIiwgInJlZnJlc2hfdG9rZW4iOiAiYW55dGhpbmcifQ==", + Item( + uri="hdfs://example.com:8020/path/to/data.parquet?user=test", + table="", + ), + Item( + uri="hdfs://example.com:8020?user=test", + table="path/to/data.parquet", + ), + Item( + uri="http+webdav://public.example.org/path/to/data.parquet", + table="", + ), + Item( + uri="https+webdav://username:password@cloud.example.org:4443/remote.php/webdav", + table="path/to/data.parquet", + ), + "msgd://site_name/drive_name/path/to/data.parquet?client_id=1d2befad-2f22-4124-a779-b147dfeca342&tenant_id=6b337423-f504-4060-a91b-e9eaaf782609&client_secret=abc~xyz789EXAMPLE_foo", + f'oci://bucket@namespace/prefix/path/to/data.parquet?iam_type=api_key&config={{"user":"ocid1.user.oc1..24g4uzg","region":"us-ashburn-1","tenancy":"ocid1.tenancy.oc1..23423r3","key_file":"{private_key_file}","fingerprint":"{private_key_fingerprint}"}}', + "onedrive://drive_name/path/to/data.parquet?client_id=1d2befad-2f22-4124-a779-b147dfeca342&tenant_id=6b337423-f504-4060-a91b-e9eaaf782609&client_secret=abc~xyz789EXAMPLE_foo", + "oss://bucket/path/to/data.parquet?endpoint=http://oss-cn-hangzhou.aliyuncs.com/&key=foo&secret=bar", + "oss://bucket/path/to/data.parquet?endpoint=https://oss-me-east-1.aliyuncs.com/&token=foobar", + Item( + uri="oss://?endpoint=https://oss-me-east-1.aliyuncs.com/&token=foobar", + table="bucket/path/to/data.parquet", + ), + # TODO: Consolidate R2 and S3 to just use `key` and `secret`, like the fsspec implementations are + # doing it across the board, and like the OSS wrapper already started inheriting it. + "r2://bucket/path/to/data.parquet?access_key_id=foo&secret_access_key=bar", "s3://bucket/path/to/data.parquet?access_key_id=foo&secret_access_key=bar", - "sftp://username:password@example.com:2222/path/to/data.parquet", + Item( + uri="sftp://username:password@intranet.example.org:2222", + table="/path/to/data.parquet", + ), + "sftp://username:password@intranet.example.org:2222/path/to/data.parquet", + "sharepoint://site_name/drive_name/path/to/data.parquet?client_id=1d2befad-2f22-4124-a779-b147dfeca342&tenant_id=6b337423-f504-4060-a91b-e9eaaf782609&client_secret=abc~xyz789EXAMPLE_foo", + "smb://workgroup;user:password@server.example.org:445/path/to/data.parquet", + Item( + uri="webhdfs://host:9870/endpoint", + table="path/to/data.parquet", + ), ] -@pytest.mark.parametrize("source_uri", URIS) -def test_init_generic_filesystems(source_uri): +@pytest.mark.parametrize("source_uri", URIS, ids=[str(item) for item in URIS]) +def test_init_generic_filesystems(source_uri, mocker): """Initialize all available filesystem implementations without table parameter""" - parsed_uri = urlparse(source_uri) - if parsed_uri.scheme in ["gs", "sftp"]: - pytest.skip(f"{parsed_uri.scheme}:// needs monkeypatching to make it testable") - factory = SourceDestinationFactory(source_uri, "file://") + if isinstance(source_uri, Item): + uri = source_uri.uri + table = source_uri.table + else: + uri = source_uri + table = "" + parsed_uri = urlparse(uri) + + # Testing a few modules has problems on Windows. + no_dbfs = parsed_uri.scheme == "dbfs" and sys.version_info < (3, 11) + no_hdfs = parsed_uri.scheme == "hdfs" and sys.version_info < (3, 11) + no_oci = parsed_uri.scheme == "oci" and sys.platform == "win32" + if no_dbfs or no_hdfs or no_oci: + pytest.skip(f"{parsed_uri.scheme}:// fails testing on this test matrix slot") + + # Apply monkeypatching to make a few filesystem implementations ready for unit testing. + + # GCS is fine with this environment variable being mocked. + mocker.patch.dict(os.environ, {"FETCH_RAW_TOKEN_EXPIRY": "false"}) + + # Must patch the whole class, because can't patch details which are immutable. + mocker.patch("pyarrow.fs.HadoopFileSystem", MemoryFileSystem) + + # It's enough to mock the `_connect` method with SFTP and SMB. + mocker.patch("fsspec.implementations.sftp.SFTPFileSystem._connect") + mocker.patch("fsspec.implementations.smb.SMBFileSystem._connect") + + # For FTP, let's mock the low-level libraries. + mocker.patch("ftplib.FTP") + mocker.patch("ftplib.FTP_TLS") + + factory = SourceDestinationFactory(uri, "file://") source = factory.get_source() dlt_source = source.dlt_source( - uri=source_uri, - # TODO: AzureSource.dlt_source() missing 1 required positional argument: 'table' - table="", + uri=uri, + # TODO: Make `table` argument optional. + # AzureSource.dlt_source() missing 1 required positional argument: 'table' + table=table, ) assert dlt_source.name == "read_parquet" assert dlt_source.section == "readers" assert dlt_source._parent.name == "filesystem" assert dlt_source._parent.section == "adapter" + # Remark: Unfortunately can't inspect the fsspec instance, + # because there is no reference to it. + def test_init_http_filesystem(): """Initialize HTTP filesystem implementation without table parameter""" From 3b9b008510347af0dad5e8f670b2a7768fc0ee2b Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sat, 25 Jul 2026 23:21:36 +0200 Subject: [PATCH 6/6] Filesystem: Increase test coverage for error cases and utilities --- src/dlt_filesystem/source/fsspec/ftp.py | 5 + src/dlt_filesystem/source/impl/remote.py | 6 + tests/dlt_filesystem/test_util.py | 43 +++++++ tests/main/filesystem/test_remote_read.py | 143 ++++++++++++++++++++-- 4 files changed, 189 insertions(+), 8 deletions(-) create mode 100644 tests/dlt_filesystem/test_util.py diff --git a/src/dlt_filesystem/source/fsspec/ftp.py b/src/dlt_filesystem/source/fsspec/ftp.py index f6dc08567..f68dc1eb0 100644 --- a/src/dlt_filesystem/source/fsspec/ftp.py +++ b/src/dlt_filesystem/source/fsspec/ftp.py @@ -10,6 +10,11 @@ class FTPSource(FilesystemSource): def dlt_source(self, uri: str, table: str, **kwargs): + if kwargs.get("incremental_key"): + raise ValueError( + "FTP takes care of incrementality on its own, you should not provide incremental_key" + ) + from fsspec.implementations.ftp import FTPFileSystem # Bundle essential information to infer filesystem wrapper. diff --git a/src/dlt_filesystem/source/impl/remote.py b/src/dlt_filesystem/source/impl/remote.py index 6dd7984ce..67f94be31 100644 --- a/src/dlt_filesystem/source/impl/remote.py +++ b/src/dlt_filesystem/source/impl/remote.py @@ -289,6 +289,12 @@ class SFTPSource(FilesystemSource): """Access files on SFTP servers.""" def dlt_source(self, uri: str, table: str, **kwargs): + + if kwargs.get("incremental_key"): + raise ValueError( + "SFTP takes care of incrementality on its own, you should not provide incremental_key" + ) + parsed_uri = urlparse(uri) host = parsed_uri.hostname if not host: diff --git a/tests/dlt_filesystem/test_util.py b/tests/dlt_filesystem/test_util.py new file mode 100644 index 000000000..4329f82d8 --- /dev/null +++ b/tests/dlt_filesystem/test_util.py @@ -0,0 +1,43 @@ +from dlt_filesystem.util.python import ( + apply_alias, + asbool, + cast_to_bool, + cast_to_float, + cast_to_list, +) + + +def test_asbool(): + """Validate the `asbool` utility function.""" + assert asbool("true") is True + assert asbool("yes") is True + assert asbool("false") is False + assert asbool("no") is False + + +def test_cast_to_bool(): + """Validate the `cast_to_bool` utility function.""" + data = {"foo": "1"} + cast_to_bool(data, ["foo"]) + assert data["foo"] is True + + +def test_cast_to_float(): + """Validate the `cast_to_float` utility function.""" + data = {"foo": "42.42"} + cast_to_float(data, ["foo"]) + assert data["foo"] == 42.42 + + +def test_cast_to_list(): + """Validate the `cast_to_list` utility function.""" + data = {"foo": '["bar"]'} + cast_to_list(data, ["foo"]) + assert data["foo"] == ["bar"] + + +def test_apply_alias(): + """Validate the `apply_alias` utility function.""" + data = {"foo": "bar"} + apply_alias(data, "foo", "effective") + assert data["effective"] == "bar" diff --git a/tests/main/filesystem/test_remote_read.py b/tests/main/filesystem/test_remote_read.py index 7d58573b8..ab69a9726 100644 --- a/tests/main/filesystem/test_remote_read.py +++ b/tests/main/filesystem/test_remote_read.py @@ -8,6 +8,7 @@ import pytest from fsspec.implementations.memory import MemoryFileSystem +from dlt_filesystem.error import MissingConnectorOption from omniload.core.factory import SourceDestinationFactory @@ -48,7 +49,7 @@ class Item: uri="ftp://username:password@intranet.example.org/path/to/data.parquet?tls=tls", table="", ), - # TODO: Two FTP tests currently don't work. Why? + # TODO: More than one FTP test currently doesn't work due to mocking issues. Please resolve. # Item( # uri="ftp://username:password@intranet.example.org?tls=tls", # table="/path/to/data.parquet", @@ -99,9 +100,47 @@ class Item: ] -@pytest.mark.parametrize("source_uri", URIS, ids=[str(item) for item in URIS]) -def test_init_generic_filesystems(source_uri, mocker): - """Initialize all available filesystem implementations without table parameter""" +URIS_UNKNOWN_FORMAT = [ + "az://schrott@acme.dfs.core.windows.net/path/to/data.unknown?account_name=acme&account_key=secret", + "dbfs:/Workspace/path/to/data.unknown", + "dropbox://path/to/data.unknown?token=secret", + # TODO: More than one FTP test currently doesn't work due to mocking issues. Please resolve. + # Item( + # uri="ftp://username:password@intranet.example.org/path/to/data.unknown?tls=tls", + # table="", + # ), + "gdrive://path/to/data.unknown?token=anon", + "gs://table-bucket-name/path/to/data.unknown?credentials_path=/path/to/service-account.json", + Item( + uri="hdfs://example.com:8020/path/to/data.unknown?user=test", + table="", + ), + Item( + uri="http+webdav://public.example.org/path/to/data.unknown", + table="", + ), + Item( + uri="https+webdav://username:password@cloud.example.org:4443/remote.php/webdav", + table="path/to/data.unknown", + ), + "msgd://site_name/drive_name/path/to/data.unknown?client_id=1d2befad-2f22-4124-a779-b147dfeca342&tenant_id=6b337423-f504-4060-a91b-e9eaaf782609&client_secret=abc~xyz789EXAMPLE_foo", + f'oci://bucket@namespace/prefix/path/to/data.unknown?iam_type=api_key&config={{"user":"ocid1.user.oc1..24g4uzg","region":"us-ashburn-1","tenancy":"ocid1.tenancy.oc1..23423r3","key_file":"{private_key_file}","fingerprint":"{private_key_fingerprint}"}}', + "onedrive://drive_name/path/to/data.unknown?client_id=1d2befad-2f22-4124-a779-b147dfeca342&tenant_id=6b337423-f504-4060-a91b-e9eaaf782609&client_secret=abc~xyz789EXAMPLE_foo", + "oss://bucket/path/to/data.unknown?endpoint=http://oss-cn-hangzhou.aliyuncs.com/&key=foo&secret=bar", + "r2://bucket/path/to/data.unknown?access_key_id=foo&secret_access_key=bar", + "s3://bucket/path/to/data.unknown?access_key_id=foo&secret_access_key=bar", + "sftp://username:password@intranet.example.org:2222/path/to/data.unknown", + "sharepoint://site_name/drive_name/path/to/data.unknown?client_id=1d2befad-2f22-4124-a779-b147dfeca342&tenant_id=6b337423-f504-4060-a91b-e9eaaf782609&client_secret=abc~xyz789EXAMPLE_foo", + "smb://workgroup;user:password@server.example.org:445/path/to/data.unknown", + Item( + uri="webhdfs://host:9870/endpoint", + table="path/to/data.unknown", + ), +] + + +def decode_uri(source_uri): + """Helper function to decode URI into components.""" if isinstance(source_uri, Item): uri = source_uri.uri table = source_uri.table @@ -109,16 +148,21 @@ def test_init_generic_filesystems(source_uri, mocker): uri = source_uri table = "" parsed_uri = urlparse(uri) + return parsed_uri, uri, table - # Testing a few modules has problems on Windows. + +def check_skip_tests(parsed_uri): + """Testing a few modules has problems on Windows.""" no_dbfs = parsed_uri.scheme == "dbfs" and sys.version_info < (3, 11) no_hdfs = parsed_uri.scheme == "hdfs" and sys.version_info < (3, 11) no_oci = parsed_uri.scheme == "oci" and sys.platform == "win32" if no_dbfs or no_hdfs or no_oci: pytest.skip(f"{parsed_uri.scheme}:// fails testing on this test matrix slot") - # Apply monkeypatching to make a few filesystem implementations ready for unit testing. +@pytest.fixture +def fsspec_mock(mocker): + """Apply monkeypatching to make a few filesystem implementations ready for unit testing.""" # GCS is fine with this environment variable being mocked. mocker.patch.dict(os.environ, {"FETCH_RAW_TOKEN_EXPIRY": "false"}) @@ -133,6 +177,15 @@ def test_init_generic_filesystems(source_uri, mocker): mocker.patch("ftplib.FTP") mocker.patch("ftplib.FTP_TLS") + +@pytest.mark.parametrize("source_uri", URIS, ids=[str(item) for item in URIS]) +def test_touch_filesystems_success(source_uri, fsspec_mock): + """Initialize all available filesystem implementations just a bit""" + parsed_uri, uri, table = decode_uri(source_uri) + + # Testing a few modules has problems on Windows. + check_skip_tests(parsed_uri) + factory = SourceDestinationFactory(uri, "file://") source = factory.get_source() dlt_source = source.dlt_source( @@ -150,7 +203,51 @@ def test_init_generic_filesystems(source_uri, mocker): # because there is no reference to it. -def test_init_http_filesystem(): +@pytest.mark.parametrize( + "source_uri", URIS_UNKNOWN_FORMAT, ids=[str(item) for item in URIS_UNKNOWN_FORMAT] +) +def test_touch_filesystems_unknown_format(source_uri, fsspec_mock): + """Using an unknown file format should raise an error.""" + parsed_uri, uri, table = decode_uri(source_uri) + + # Testing a few modules has problems on Windows. + check_skip_tests(parsed_uri) + + factory = SourceDestinationFactory(uri, "file://") + source = factory.get_source() + with pytest.raises(ValueError) as exc_info: + source.dlt_source( + uri=uri, + # TODO: Make `table` argument optional. + # AzureSource.dlt_source() missing 1 required positional argument: 'table' + table=table, + ) + assert exc_info.match("Source only supports file formats") + + +@pytest.mark.parametrize("source_uri", URIS, ids=[str(item) for item in URIS]) +def test_touch_filesystems_incremental_key(source_uri, fsspec_mock): + """Using the `incremental_key` kwarg should raise an error.""" + # source_uri = "az://schrott@acme.dfs.core.windows.net/path/to/data.parquet?account_name=acme&account_key=secret" + parsed_uri, uri, table = decode_uri(source_uri) + + # Testing a few modules has problems on Windows. + check_skip_tests(parsed_uri) + + factory = SourceDestinationFactory(uri, "file://") + source = factory.get_source() + with pytest.raises(ValueError) as exc_info: + source.dlt_source( + uri=uri, + # TODO: Make `table` argument optional. + # AzureSource.dlt_source() missing 1 required positional argument: 'table' + table=table, + incremental_key="foobar", + ) + assert exc_info.match("you should not provide incremental_key") + + +def test_touch_http_filesystem(): """Initialize HTTP filesystem implementation without table parameter""" source_uri = "http://example.org/path/to/data.parquet" factory = SourceDestinationFactory(source_uri, "file://") @@ -165,9 +262,39 @@ def test_init_http_filesystem(): assert dlt_source.section == "adapter" -def test_init_unknown_filesystem(): +def test_touch_unknown_filesystem(): """Initialize unknown filesystem implementation""" factory = SourceDestinationFactory("unknown://", "file://") with pytest.raises(NotImplementedError) as exc_info: factory.get_source() assert exc_info.match("Unsupported source scheme: unknown") + + +def test_touch_s3_filesystem_without_secret_access_key(): + """Missing a required parameter should raise an error.""" + source_uri = "s3://bucket/path/to/data.parquet?access_key_id=foo" + factory = SourceDestinationFactory(source_uri, "file://") + source = factory.get_source() + with pytest.raises(MissingConnectorOption) as exc_info: + source.dlt_source( + uri=source_uri, + # TODO: Make `table` parameter optional. + # AzureSource.dlt_source() missing 1 required positional argument: 'table' + table="", + ) + assert exc_info.match("secret_access_key is required") + + +def test_touch_sftp_filesystem_without_host(): + """Missing a required parameter should raise an error.""" + source_uri = "sftp://" + factory = SourceDestinationFactory(source_uri, "file://") + source = factory.get_source() + with pytest.raises(MissingConnectorOption) as exc_info: + source.dlt_source( + uri=source_uri, + # TODO: Make `table` parameter optional. + # AzureSource.dlt_source() missing 1 required positional argument: 'table' + table="", + ) + assert exc_info.match("host is required")