diff --git a/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java b/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java index bc72cd8cd57..e9936a0e6cc 100644 --- a/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java +++ b/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java @@ -101,6 +101,7 @@ public enum HopMetadataPropertyType { VFS_MINIO_CONNECTION, VFS_S3_CONNECTION, VFS_WEBDAV_CONNECTION, + VFS_DATABRICKS_CONNECTION, // DATA STREAM DATA_STREAM, diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-connection.png b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-connection.png new file mode 100644 index 00000000000..89acd1be8a3 Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-connection.png differ diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-environment-configuration-file.png b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-environment-configuration-file.png new file mode 100644 index 00000000000..d2525e3c56e Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-environment-configuration-file.png differ diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-execution-perspective-looking-at-spark-execution.png b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-execution-perspective-looking-at-spark-execution.png new file mode 100644 index 00000000000..b0f10adde53 Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-execution-perspective-looking-at-spark-execution.png differ diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-native-spark-pipeline-run-configuration.png b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-native-spark-pipeline-run-configuration.png new file mode 100644 index 00000000000..92012c5340b Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-native-spark-pipeline-run-configuration.png differ diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-vfs-connection-to-volume.png b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-vfs-connection-to-volume.png new file mode 100644 index 00000000000..30b4c7b4c4f Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-vfs-connection-to-volume.png differ diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-volume-caching-execution-information-location.png b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-volume-caching-execution-information-location.png new file mode 100644 index 00000000000..674c05425d4 Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/pipeline/spark/databricks-volume-caching-execution-information-location.png differ diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc index 63fca1915d4..86fb13bc8e9 100644 --- a/docs/hop-user-manual/modules/ROOT/nav.adoc +++ b/docs/hop-user-manual/modules/ROOT/nav.adoc @@ -396,6 +396,7 @@ under the License. * xref:vfs.adoc[Virtual File System] ** xref:vfs/aws-s3-vfs.adoc[Amazon Web Services S3] ** xref:vfs/azure-blob-storage-vfs.adoc[Microsoft Azure Storage] +** xref:vfs/databricks-vfs.adoc[Databricks Volumes / Workspace] ** xref:vfs/dropbox-vfs.adoc[Dropbox] ** xref:vfs/google-cloud-storage-vfs.adoc[Google Cloud Storage] ** xref:vfs/google-drive-vfs.adoc[Google Drive] @@ -455,6 +456,8 @@ under the License. ** xref:metadata-types/azure-authentication.adoc[Azure Connection] ** xref:metadata-types/beam-file-definition.adoc[Beam File Definition] ** xref:metadata-types/cassandra/cassandra-connection.adoc[Cassandra Connection] +** xref:metadata-types/databricks-connection.adoc[Databricks Connection] +** xref:metadata-types/databricks-vfs-connection.adoc[Databricks VFS Connection] ** xref:metadata-types/data-set.adoc[Data Set] ** xref:metadata-types/execution-data-profile.adoc[] ** xref:metadata-types/execution-information-location.adoc[] diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-gui/perspective-execution-information.adoc b/docs/hop-user-manual/modules/ROOT/pages/hop-gui/perspective-execution-information.adoc index 9499b1fd221..89f70207abe 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/hop-gui/perspective-execution-information.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/hop-gui/perspective-execution-information.adoc @@ -31,6 +31,8 @@ The Execution Information perspective provides an overview of execution informat The execution information displayed in this perspective is not collected automatically. You need to specify to which xref:metadata-types/execution-information-location.adoc[location] you want to send execution information in the xref:metadata-types/pipeline-run-config.adoc[Pipeline run configuration] or xref:metadata-types/workflow-run-config.adoc[Workflow run configuration] of your choice. +TIP: Locations can use a VFS URI (for example a Databricks Volume via a named scheme). That lets you inspect native Spark jobs that wrote execution JSON on the cluster from this perspective on your laptop. See xref:pipeline/spark/databricks.adoc#exec-info-on-volume[Execution information on a UC Volume]. + == Navigation You can navigate to it by clicking the image:icons/execution.svg[width="24px"] icon from the vertical perspective toolbar in the Hop GUI. diff --git a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/databricks-connection.adoc b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/databricks-connection.adoc index ba8aee6cce6..99b9b5bf4b8 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/databricks-connection.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/databricks-connection.adoc @@ -53,8 +53,24 @@ Use this metadata type when orchestrating jobs from Hop (for example the upcomin Use **Test connection** in the editor to call the workspace identity endpoint (SCIM *Me*) or fall back to listing jobs. A successful test shows the resolved user name or host. +== VFS + +For Hop file dialogs and VFS-aware transforms, create a separate +xref:metadata-types/databricks-vfs-connection.adoc[Databricks VFS Connection] under **File storage**. +That object’s **name** is the URI scheme; it **references this connection** for host and PAT. + +Example: Jobs connection `prod-workspace`, VFS connection name `dbx-jars`: + +---- +dbx-jars:///Volumes/apache-hop/default/jars/data.csv +---- + +See xref:vfs/databricks-vfs.adoc[Databricks Volumes / Workspace VFS]. + == Related +* xref:metadata-types/databricks-vfs-connection.adoc[Databricks VFS Connection] — named scheme for Volumes / Workspace +* xref:vfs/databricks-vfs.adoc[Databricks Volumes / Workspace VFS] * xref:pipeline/spark/databricks.adoc[Native Spark on Databricks] — Deploy & run, classic clusters, Volumes * xref:workflow/actions/databricks-job-run.adoc[Databricks Job Run] — trigger and wait for Jobs API runs * xref:database/databases/databricks.adoc[Databricks database (JDBC)] — SQL warehouse access diff --git a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/databricks-vfs-connection.adoc b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/databricks-vfs-connection.adoc new file mode 100644 index 00000000000..283c6e40b93 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/databricks-vfs-connection.adoc @@ -0,0 +1,74 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:imagesdir: ../../assets/images/ +:page-pagination: +:description: Named Databricks VFS connection for Unity Catalog Volumes and Workspace files (Files API) + += Databricks VFS Connection + +== Description + +*Databricks VFS Connection* registers a **named VFS scheme** for Unity Catalog Volumes and Workspace files via the Databricks **Files API**. + +* The metadata **Name** is the URI scheme (like xref:metadata-types/minio-connection.adoc[MinIO Connection]). +* **Authentication** is not stored on this object. Select an existing xref:metadata-types/databricks-connection.adoc[Databricks Connection] (workspace host + PAT). + +Example: VFS name `db-volume`, root path `/Volumes/apache-hop/default/testing`, Databricks Connection `prod-workspace`: + +---- +db-volume:///input/customers-1M.txt +---- + +resolves to workspace path `/Volumes/apache-hop/default/testing/input/customers-1M.txt`. + +This is **Hop-side I/O** only (GUI, local engine, classic file transforms). +**Native Spark** Dataset I/O does **not** use Hop VFS schemes — use absolute `/Volumes/…` or cloud URIs (`s3a://…`, …) on the cluster. +Do not use this connection as the primary bulk path for large Spark datasets (workspace Files API is not a high-throughput object-store client). +See xref:vfs/databricks-vfs.adoc[Databricks Volumes / Workspace VFS] and +xref:pipeline/spark/paths-and-filesystems.adoc[Paths and file systems on native Spark]. + +== Options + +[options="header",cols="1,3"] +|=== +|Option |Description + +|Name (URI scheme) +|VFS scheme used in paths: `{name}:///…`. Prefer short names; do not reuse built-in schemes (`file`, `s3`, …). + +|Description +|Optional free-text description. + +|Databricks Connection +|Existing Jobs/workspace connection that supplies host and personal access token. + +|Root path +|Absolute UC Volume or Workspace path that is the **root of this scheme**. Shortens URIs: `{name}:///input` → `{root}/input`. Leave empty if every URI must include full `/Volumes/…` or `/Workspace/…` paths. +|=== + +== Tips + +* Create the **Databricks Connection** first and use **Test connection** there (or on this editor, which validates the referenced connection). +* Set **Root path** to your testing volume (for example `/Volumes/apache-hop/default/testing`) so workflow paths stay short. +* In File → Open, type `{name}:///` to list the root path contents (when root is set). +* One Jobs connection can back several VFS schemes (different root volumes). + +== Related + +* xref:vfs/databricks-vfs.adoc[Databricks Volumes / Workspace VFS] +* xref:metadata-types/databricks-connection.adoc[Databricks Connection] +* xref:pipeline/spark/databricks.adoc[Native Spark on Databricks] diff --git a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-information-location.adoc b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-information-location.adoc index 29b76e5a3d7..3a05175dd2b 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-information-location.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-information-location.adoc @@ -72,6 +72,8 @@ Options: * *Persistence delay*: This is the maximum time to wait before execution information is written to disk, expressed in milliseconds. * *Maximum cache age*: This is the maximum time to keep execution information around in memory, before clearing it out, expressed in milliseconds. It's advised to keep this below the expected duration of your workflow or pipeline. +The root folder can be a local path or a **VFS URI**. For example, on Databricks you can store execution JSON on a Unity Catalog Volume via a xref:metadata-types/databricks-vfs-connection.adoc[Databricks VFS Connection] (`db-volume:///executions`) so a native Spark job on the cluster and the laptop GUI share the same location. Full walkthrough: xref:pipeline/spark/databricks.adoc#exec-info-on-volume[Execution information on a UC Volume]. + === Remote location A remote location allows you to write execution information to a Hop Server. diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/databricks.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/databricks.adoc index c62ef8dabce..2118f2224d1 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/databricks.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/databricks.adoc @@ -16,7 +16,7 @@ under the License. //// [[NativeSparkOnDatabricks]] :imagesdir: ../../../assets/images -:description: Run Apache Hop native Spark (MainSpark JAR) jobs on Databricks — workspace tiers, classic clusters, UC Volumes, compute modes, and Deploy & run lessons learned. +:description: Run Apache Hop native Spark (MainSpark JAR) jobs on Databricks — workspace tiers, classic clusters, UC Volumes, execution information on Volumes, compute modes, and Deploy & run lessons learned. :toc: :toclevels: 3 @@ -37,6 +37,7 @@ A minimal Hop project for this path typically includes: * A workflow with **Databricks Job Run** (Deploy & run) and optional **Databricks Job Wait** * A `work/` folder with the **native-provided** fat jar * Metadata: Databricks Connection, Native Spark pipeline run configuration +* Optionally (for remote execution history from the laptop GUI): Databricks VFS Connection, **Caching File** execution information location on a UC Volume, and an environment config with `DATABRICKS_HOST` / `DATABRICKS_TOKEN` — see <> .Sample `hello-databricks.hpl` in the project explorer (Generate rows → UUID → Dummy) image::pipeline/spark/databricks-pipeline-hello-databricks.png[Hello Databricks sample pipeline in Hop GUI: Generate rows, Random value (uuid), Dummy,width=85%] @@ -176,6 +177,151 @@ If logs still show `POST /api/2.0/dbfs/create` for a `/Volumes/…` path, the ru Job library and MainSpark parameter paths for volumes stay as absolute `/Volumes/…` (no `dbfs:` scheme). +=== Hop VFS for Volumes + +From Hop GUI and local transforms you can use a xref:metadata-types/databricks-vfs-connection.adoc[Databricks VFS Connection] (scheme = metadata name, auth via referenced Databricks Connection) to browse, read, and write the same Files API paths (for example `dbx-jars:///Volumes/apache-hop/default/jars/…`). +That is **Hop-side I/O only** — it does not replace absolute `/Volumes/…` paths on the cluster or in Jobs API library URIs. +See xref:vfs/databricks-vfs.adoc[Databricks Volumes / Workspace VFS]. + +[[exec-info-on-volume]] +=== Execution information on a UC Volume + +You can store Hop **execution information** (pipeline graph, logs, transform metrics, and row samples from an execution data profile) on a **Unity Catalog Volume** and open it from the laptop Hop GUI — even though the pipeline ran on Databricks under `MainSpark`. + +That works because: + +* The **Caching File** location writes one JSON file per top-level execution (plus nested state/samples inside it). +* The root folder can be a **Databricks VFS** URI (for example `db-volume:///executions`). +* On the cluster, MainSpark resolves named VFS schemes from the **exported project metadata** and writes through the **Files API** using the Databricks Connection PAT. +* On the laptop, the same location metadata + environment variables let the xref:hop-gui/perspective-execution-information.adoc[Execution Information Perspective] list and open those remote files. + +This is for **execution metadata** only (logs, metrics, sample rows), not for bulk Spark datasets. Prefer **Caching File** over plain File so writes are batched (fewer Files API round-trips). Native Spark Dataset I/O still uses `/Volumes/…` or cloud URIs — not the Hop VFS scheme. + +==== Pattern overview + +[options="header",cols="1,3"] +|=== +|Piece |Role + +|xref:metadata-types/databricks-connection.adoc[Databricks Connection] +|Workspace host + PAT (Jobs API and Files API auth for VFS) + +|Environment / lifecycle config +|`DATABRICKS_HOST`, `DATABRICKS_TOKEN` (and optional cluster id). Must be active in Hop GUI **and** deployed to the job (`--HopConfigFile` / env-config artifact) + +|xref:metadata-types/databricks-vfs-connection.adoc[Databricks VFS Connection] +|Named scheme (e.g. `db-volume`) + optional root path under a Volume + +|xref:metadata-types/execution-information-location.adoc[Execution Information Location] (**Caching File**) +|Root folder = `{scheme}:///executions` (or another folder under the VFS root) + +|xref:metadata-types/pipeline-run-config.adoc[Pipeline run configuration] (**Native Spark**) +|**Execution information location** = the Caching File location; optional **execution data profile** for sample rows + +|Deploy & run package +|Exports connection, VFS, location, run config, and (when configured) the env config so the **driver** can write the same URI +|=== + +==== 1. Databricks Connection (variables for secrets) + +Create a Jobs/PAT connection. Prefer variables so the same metadata works on the laptop and on the cluster after Deploy & run ships the environment config. + +.Databricks Connection: workspace host and PAT via `${DATABRICKS_HOST}` / encrypted token variable +image::pipeline/spark/databricks-connection.png[Databricks Connection metadata editor with variable-based host and personal access token,width=85%] + +==== 2. Environment configuration file + +Define at least host and token in a Hop **environment** or lifecycle configuration file that you use when opening the project in GUI and when Deploy & run attaches env config for MainSpark: + +---- +DATABRICKS_HOST=https://dbc-….cloud.databricks.com +DATABRICKS_TOKEN=Encrypted … +---- + +Optional: `DATABRICKS_CLUSTER_ID` for existing-cluster deploy. + +.Environment configuration: `DATABRICKS_HOST`, encrypted `DATABRICKS_TOKEN`, optional cluster id +image::pipeline/spark/databricks-environment-configuration-file.png[Hop environment configuration file editor with Databricks host, token, and cluster id variables,width=90%] + +[IMPORTANT] +==== +If the GUI can browse the Volume but the cluster writes a **local** path such as `db-volume:///…` under the driver cwd, the driver did **not** resolve the named VFS scheme or the host/token variables. Confirm: + +* Env config is deployed (`env-config-….json` / `--HopConfigFile`) and contains the same variables. +* Exported **metadata** includes the Databricks Connection, VFS connection, and execution location (Deploy & run package / metadata export). +* The PAT can **write** to the Volume path. +==== + +==== 3. Databricks VFS Connection (scheme + Volume root) + +Create a VFS connection whose **name is the URI scheme**. Point **Root path** at a folder on a UC Volume (create that folder in Catalog Explorer if needed). + +.Databricks VFS Connection: scheme `db-volume`, connection `databricks-free`, root `/Volumes/…/testing` +image::pipeline/spark/databricks-vfs-connection-to-volume.png[Databricks VFS Connection editor with scheme name, Databricks Connection, and Volume root path,width=85%] + +With root path `/Volumes/apache-hop/default/testing`: + +* Hop URI `db-volume:///executions` maps to workspace path `/Volumes/apache-hop/default/testing/executions` +* Leave root empty only if you prefer full paths in every URI (`db-volume:///Volumes/…`) + +See xref:vfs/databricks-vfs.adoc[Databricks Volumes / Workspace VFS]. + +==== 4. Caching File execution information location + +Create an xref:metadata-types/execution-information-location.adoc[Execution Information Location]: + +* **Location type**: **Caching File location** +* **Folder**: the VFS URI, e.g. `db-volume:///executions` +* **Create folder?**: optional; you can also create `executions` on the Volume first +* **Persistence delay**: multi-second (e.g. `5000`) so the driver does not write on every metric tick +* **Maximum cache age**: long enough to cover the pipeline (e.g. hours for long jobs) + +.Caching File location `db-executions` with folder `db-volume:///executions` +image::pipeline/spark/databricks-volume-caching-execution-information-location.png[Execution information location editor: Caching File type and Databricks VFS folder URI,width=85%] + +==== 5. Native Spark run configuration + +On the **Native Spark** pipeline run configuration used by Deploy & run: + +* **Execution information location** = the Caching File location (e.g. `db-executions`) +* **Execution data profile** = optional but recommended (e.g. `first-last`) so the perspective can show sample rows +* Engine type **Native Spark pipeline engine**; fat jar path is used for local `spark-submit` — on Databricks, Deploy & run supplies the library jar separately + +.Native Spark run configuration: execution location `db-executions`, data profile `first-last` +image::pipeline/spark/databricks-native-spark-pipeline-run-configuration.png[Native Spark pipeline run configuration with execution information location and data profile,width=75%] + +==== 6. Run on Databricks, inspect from Hop GUI + +. Deploy & run the pipeline (or workflow with Job Run) with the run configuration above and env config attached. +. Wait until the job finishes (or is far enough along that the cache has flushed at least once). +. In Hop GUI, open the **Execution Information** perspective (`Ctrl-Shift-I`). +. Select the location (e.g. `db-executions`), refresh, and open the pipeline execution. +. Use **Info**, **Log**, **Metrics**, and **Data** tabs — same as a local run. + +.Execution Information perspective: `hello-databricks` from Volume location `db-executions` with cluster log text +image::pipeline/spark/databricks-execution-perspective-looking-at-spark-execution.png[Execution Information perspective showing a Spark pipeline execution loaded from a Databricks Volume,width=95%] + +Typical log lines on a successful remote capture: + +---- +Using execution information location 'db-executions' … +Prepared native Spark pipeline engine with run configuration 'databricks-native' … +Reusing active SparkSession (version=4.1.0, …) +Spark pipeline finished, result row count=… +Registered … transform sample set(s) from Spark executors +---- + +JSON files on the Volume look like `{execution-id}.json` under the mapped folder (e.g. `/Volumes/…/testing/executions/`). + +==== Tips and limits + +* **Time filter** in the perspective defaults to a short window (e.g. `< 1h`). Widen it if the run is older. +* **Parents only** / status filters still apply; turn them off if you expect a child execution. +* Files API is **single-stream HTTP** — fine for execution JSON (tens to a few hundred KB typical); do not use this path for large data landings. +* **Create parent folder** on the Volume before the first run if create-folder is off or list permissions are restricted. +* Laptop and cluster must share the **same** connection name, VFS scheme name, location name, and variable names so metadata exported for Deploy & run matches what the GUI uses. +* Deleting executions from the perspective removes the remote JSON (subject to PAT permissions). + == Compute modes (Deploy & run) The **Cluster / compute** field accepts one of: @@ -531,12 +677,17 @@ Databricks does **not** expose a standard content MD5/SHA for volume files compa * [ ] Cluster / compute = existing classic id **or** `new_cluster` with valid `node_type_id` / `spark_version` * [ ] Job id empty on first create; wait mode and timeout sized for upload + **job cluster cold start** (often 8+ minutes) + pipeline run * [ ] After success: note Job ID, Run ID, run page URL variables; open run in Databricks UI +* [ ] (Optional remote execution history) VFS connection + Caching File location on a Volume; run config points at that location; env vars on driver; GUI Execution Information perspective can list the run — see <> == Related * xref:workflow/actions/databricks-job-run.adoc[Databricks Job Run] * xref:workflow/actions/databricks-job-wait.adoc[Databricks Job Wait] * xref:metadata-types/databricks-connection.adoc[Databricks Connection] +* xref:metadata-types/databricks-vfs-connection.adoc[Databricks VFS Connection] +* xref:vfs/databricks-vfs.adoc[Databricks Volumes / Workspace VFS] +* xref:metadata-types/execution-information-location.adoc[Execution Information Location] +* xref:hop-gui/perspective-execution-information.adoc[Execution Information Perspective] * xref:pipeline/spark/getting-started-with-native-spark.adoc[Getting started with the native Spark engine] * xref:pipeline/pipeline-run-configurations/native-spark-pipeline-engine.adoc[Native Spark run configuration] * xref:pipeline/spark/paths-and-filesystems.adoc[Paths and file systems] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc index d8bfda450bb..a137ff28fb8 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/spark/getting-started-with-native-spark.adoc @@ -518,6 +518,8 @@ How data is produced: If the location is file-based and you need *samples from cluster workers*, the root folder must be *writable from every executor* (shared filesystem or object store). A path that exists only on the driver will still get pipeline/transform *state*, but executor samples will be missing or incomplete. ==== +On **Databricks**, a practical pattern is a **Caching File** location whose root is a named Databricks VFS URI on a UC Volume (driver writes execution JSON via the Files API; laptop GUI opens the same location). Walkthrough: xref:pipeline/spark/databricks.adoc#exec-info-on-volume[Execution information on a UC Volume]. + On `local[*]`, Spark’s Jackson libraries must not shadow Hop’s Jackson under the plugin classloader. The Spark plugin packaging excludes core Jackson jars so sampling and `HopJson` keep working. == How transforms map to Spark diff --git a/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc b/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc index 52031222662..f722d7d10f9 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc @@ -46,6 +46,7 @@ Click the File system name to access more detailed file system documentation. |File System|Description|URI Format |xref:vfs/aws-s3-vfs.adoc[AWS S3]|Provides access to Amazon S3 Buckets|`s3://` |xref:vfs/azure-blob-storage-vfs.adoc[Azure Blob Storage]|Provides access to Azure Blob Storage|`azure://` (alias: `azfs://`) +|xref:vfs/databricks-vfs.adoc[Databricks Volumes / Workspace]|UC Volumes and Workspace files via Files API; scheme from xref:metadata-types/databricks-vfs-connection.adoc[Databricks VFS Connection]|`+:///Volumes/…+` or `+…/Workspace/…+` |xref:vfs/dropbox-vfs.adoc[Dropbox]|Provides access to Dropbox|`dropbox://` |xref:vfs/google-cloud-storage-vfs.adoc[Google Cloud Storage]|Provides access to Google Cloud Storage buckets|`gs://` |xref:vfs/google-drive-vfs.adoc[Google Drive]|Provides access to Google Drive folders|`googledrive://` diff --git a/docs/hop-user-manual/modules/ROOT/pages/vfs/databricks-vfs.adoc b/docs/hop-user-manual/modules/ROOT/pages/vfs/databricks-vfs.adoc new file mode 100644 index 00000000000..a93c7b98380 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/vfs/databricks-vfs.adoc @@ -0,0 +1,109 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// + +:documentationPath: /vfs/ +:language: en_US +:description: Hop VFS for Databricks Unity Catalog Volumes and Workspace files via the Files API + += Databricks Volumes / Workspace VFS + +== Scheme + +Each xref:metadata-types/databricks-vfs-connection.adoc[Databricks VFS Connection] registers as its own VFS scheme. +The **metadata name** is the scheme (same idea as MinIO). + +With a **Root path** set to a volume (recommended for most projects): + +---- +{vfs-connection-name}:///relative/path/under/root +---- + +Example: name `db-volume`, root `/Volumes/apache-hop/default/testing`: + +---- +db-volume:///input/customers-1M.txt +---- + +→ workspace `/Volumes/apache-hop/default/testing/input/customers-1M.txt`. + +Without a root path, URIs must be absolute workspace paths: + +---- +{vfs-connection-name}:///Volumes////path/to/file +{vfs-connection-name}:///Workspace/Users/you@example.com/path/to/file +---- + +Authentication comes from the **Databricks Connection** selected on that VFS metadata (host + PAT). + +Pick a VFS **Name** that does not clash with +https://commons.apache.org/proper/commons-vfs/filesystems.html[built-in VFS schemes] +(`file`, `ftp`, `http`, …) or other plugins (`s3`, `azure`, `gs`, …). + +== What it supports + +Hop talks to the workspace **Files API** (Bearer PAT from the referenced connection): + +* List, create, and delete directories under UC Volumes and Workspace +* Read and write file content (including large jars within Databricks Files API limits, ~5 GiB per PUT) +* Browse from Hop file dialogs and use VFS-aware transforms/actions + +Classic **DBFS** paths (`dbfs:/FileStore/…`) are **not** exposed through this VFS provider. +Prefer `/Volumes/…` on modern workspaces where public DBFS root is disabled. +Deploy & run still supports classic DBFS as a fallback upload API when you use absolute `dbfs:` paths there. + +== Path dialect (Hop vs cluster) + +[IMPORTANT] +==== +**Native Spark does not use Hop VFS** for Dataset I/O (*Spark File Input/Output*, Lake Table PATH). +Those transforms need **Spark/Hadoop URIs** on the cluster (`/Volumes/…`, `s3a://…`, `abfs://…`, …), not named schemes such as `db-volume:///…`. +The same rule as the general xref:vfs.adoc[Virtual File Systems] overview and +xref:pipeline/spark/paths-and-filesystems.adoc[Paths and file systems on native Spark]. + +Hop Databricks VFS URIs are for **Hop-side** I/O only: + +* Hop GUI file dialogs, local engine, classic file transforms/actions +* Convenience copies (configs, samples, modest artifacts) via the workspace **Files API** +* **Execution information** (Caching File location root folder) so MainSpark on the cluster and the laptop GUI share the same Volume path — see xref:pipeline/spark/databricks.adoc#exec-info-on-volume[Execution information on a UC Volume] + +Do **not** use this VFS as the primary path for **large Spark data landings**. +The Files API is a single-stream workspace HTTP upload — fine for small/medium files, not a bulk object-store client. +For multi‑hundred‑MB or multi‑GB datasets, land data with cloud storage tools or cluster-side paths, then point Spark File / Lake transforms at **Spark-visible** URIs. + +Jobs API library and MainSpark parameter paths stay absolute `/Volumes/…` (no Hop VFS scheme). +See xref:pipeline/spark/databricks.adoc[Native Spark on Databricks]. +==== + +== Configuration + +. Create a xref:metadata-types/databricks-connection.adoc[Databricks Connection] (workspace host + PAT). Prefer variables for secrets. +. Create a xref:metadata-types/databricks-vfs-connection.adoc[Databricks VFS Connection]: + ** **Name** = URI scheme (for example `db-volume`) + ** **Databricks Connection** = the Jobs connection from step 1 + ** **Root path** = `/Volumes/catalog/schema/volume` (or a folder under a volume) +. In Hop file dialogs or transforms, enter `{name}:///subdir/…` relative to that root. + +If root path is left empty, every URI must include the full `/Volumes/…` or `/Workspace/…` path. + +== Related + +* xref:metadata-types/databricks-vfs-connection.adoc[Databricks VFS Connection] +* xref:metadata-types/databricks-connection.adoc[Databricks Connection] +* xref:pipeline/spark/databricks.adoc[Native Spark on Databricks] — artifact storage, Files API vs DBFS, xref:pipeline/spark/databricks.adoc#exec-info-on-volume[execution information on a Volume] +* xref:metadata-types/execution-information-location.adoc[Execution Information Location] +* xref:workflow/actions/databricks-job-run.adoc[Databricks Job Run] +* xref:vfs.adoc[Virtual File Systems overview] diff --git a/engine/src/main/java/org/apache/hop/execution/caching/CacheEntry.java b/engine/src/main/java/org/apache/hop/execution/caching/CacheEntry.java index 6fa9d75d86e..43b792e30b7 100644 --- a/engine/src/main/java/org/apache/hop/execution/caching/CacheEntry.java +++ b/engine/src/main/java/org/apache/hop/execution/caching/CacheEntry.java @@ -34,6 +34,7 @@ import org.apache.commons.vfs2.FileObject; import org.apache.hop.core.Const; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.execution.Execution; import org.apache.hop.execution.ExecutionData; @@ -98,13 +99,23 @@ public CacheEntry() { * @throws HopException In case there was an error writing. */ public void writeToDisk(String rootFolder) throws HopException { + writeToDisk(rootFolder, null); + } + + /** + * Write this cache entry to a file under {@code rootFolder}. + * + * @param variables required for named VFS schemes (e.g. Databricks/MinIO connection schemes) + */ + public void writeToDisk(String rootFolder, IVariables variables) throws HopException { String targetFilename = calculateFilename(rootFolder); String filename = targetFilename + ".new"; // Use Hop VFS (not java.io.FileOutputStream): rootFolder is often a VFS URI such as // file:///data/hop-data/executions when resolved from ${HOP_DATA}. FileOutputStream treats // "file://…" as a literal path and fails with FileNotFoundException even when the folder - // was created successfully via VFS. - try (OutputStream os = HopVfs.getOutputStream(filename, false)) { + // was created successfully via VFS. Named schemes (db-volume://) need variables so providers + // load from project metadata. + try (OutputStream os = HopVfs.getOutputStream(filename, false, variables)) { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.writeValue(os, this); } catch (Exception e) { @@ -113,11 +124,11 @@ public void writeToDisk(String rootFolder) throws HopException { // Now delete the old file and rename the new one. // try { - FileObject targetFileObject = HopVfs.getFileObject(targetFilename); + FileObject targetFileObject = HopVfs.getFileObject(targetFilename, variables); if (targetFileObject.exists()) { targetFileObject.delete(); } - FileObject fileObject = HopVfs.getFileObject(filename); + FileObject fileObject = HopVfs.getFileObject(filename, variables); fileObject.moveTo(targetFileObject); } catch (Exception e) { throw new HopException( @@ -130,9 +141,13 @@ public void writeToDisk(String rootFolder) throws HopException { } public void deleteFromDisk(String rootFolder) throws HopException { + deleteFromDisk(rootFolder, null); + } + + public void deleteFromDisk(String rootFolder, IVariables variables) throws HopException { String targetFilename = calculateFilename(rootFolder); try { - FileObject fileObject = HopVfs.getFileObject(targetFilename); + FileObject fileObject = HopVfs.getFileObject(targetFilename, variables); fileObject.delete(); } catch (Exception e) { throw new HopException( diff --git a/engine/src/main/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocation.java b/engine/src/main/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocation.java index 645e0245b51..f02de3d70b2 100644 --- a/engine/src/main/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocation.java +++ b/engine/src/main/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocation.java @@ -126,7 +126,7 @@ public void initialize(IVariables variables, IHopMetadataProvider metadataProvid if (createParentFolder) { try { - FileObject folder = HopVfs.getFileObject(actualRootFolder); + FileObject folder = HopVfs.getFileObject(actualRootFolder, variables); if (!folder.exists()) { folder.createFolder(); } @@ -149,7 +149,7 @@ protected void persistCacheEntry(CacheEntry cacheEntry) throws HopException { mergeChildrenFromDisk(cacheEntry); // Before writing to disk, we calculate some summaries for convenience of other tools. cacheEntry.calculateSummary(); - cacheEntry.writeToDisk(actualRootFolder); + cacheEntry.writeToDisk(actualRootFolder, variables); // Remember when we last wrote to disk cacheEntry.setLastWritten(new Date()); } catch (Exception e) { @@ -193,7 +193,7 @@ private static void mergeMap(Map fromDisk, Map into) { @Override public void deleteCacheEntry(CacheEntry cacheEntry) throws HopException { try { - cacheEntry.deleteFromDisk(actualRootFolder); + cacheEntry.deleteFromDisk(actualRootFolder, variables); } catch (Exception e) { throw new HopException( "Error deleting caching file entry from folder " + actualRootFolder, e); @@ -205,11 +205,11 @@ protected synchronized CacheEntry loadCacheEntry(String executionId) throws HopE CacheEntry entry = new CacheEntry(); entry.setId(executionId); String filename = entry.calculateFilename(actualRootFolder); - if (!HopVfs.fileExists(filename)) { + if (!HopVfs.fileExists(filename, variables)) { return null; } ObjectMapper objectMapper = new ObjectMapper(); - return objectMapper.readValue(HopVfs.getInputStream(filename), CacheEntry.class); + return objectMapper.readValue(HopVfs.getInputStream(filename, variables), CacheEntry.class); } catch (Exception e) { throw new HopException( "Error loading execution information location file for executionId '" + executionId + "'", @@ -237,7 +237,9 @@ protected void retrieveIds( long startDate = ZonedDateTime.of(roughStartDate, ZoneId.systemDefault()).toInstant().toEpochMilli(); long lastModified = file.getContent().getLastModifiedTime(); - if (lastModified < startDate) { + // Some VFS providers (e.g. Databricks Files API before list-cache attach) report + // lastModified as 0 / unknown. Do not pre-skip those — load and apply content filters. + if (lastModified > 0 && lastModified < startDate) { // Skip for performance continue; } @@ -258,7 +260,18 @@ protected void retrieveIds( } if (!ids.contains(new DatedId(id, null))) { - ids.add(new DatedId(id, new Date(file.getContent().getLastModifiedTime()))); + long lastModified = file.getContent().getLastModifiedTime(); + Date dated; + if (lastModified > 0) { + dated = new Date(lastModified); + } else if (entry.getExecution() != null + && entry.getExecution().getExecutionStartDate() != null) { + // VFS providers without mtime: use execution start for sorting + dated = entry.getExecution().getExecutionStartDate(); + } else { + dated = new Date(0L); + } + ids.add(new DatedId(id, dated)); // To add child IDs we need to load the file. // We won't store these in the cache though. @@ -290,7 +303,7 @@ private String getIdFromFileName(FileObject file) { private FileObject[] getAllFileObjects(String actualRootFolder) throws FileSystemException, HopFileException { - FileObject folder = HopVfs.getFileObject(actualRootFolder); + FileObject folder = HopVfs.getFileObject(actualRootFolder, variables); return folder.findFiles( new AllFileSelector() { @Override diff --git a/engine/src/main/java/org/apache/hop/execution/local/FileExecutionInfoLocation.java b/engine/src/main/java/org/apache/hop/execution/local/FileExecutionInfoLocation.java index 71f1103c7d1..81d4106d7af 100644 --- a/engine/src/main/java/org/apache/hop/execution/local/FileExecutionInfoLocation.java +++ b/engine/src/main/java/org/apache/hop/execution/local/FileExecutionInfoLocation.java @@ -124,7 +124,7 @@ public void initialize(IVariables variables, IHopMetadataProvider metadataProvid if (createParentFolder && StringUtils.isNotEmpty(rootFolder)) { String actualRootFolder = variables.resolve(rootFolder); try { - FileObject folder = HopVfs.getFileObject(actualRootFolder); + FileObject folder = HopVfs.getFileObject(actualRootFolder, variables); if (!folder.exists()) { folder.createFolder(); } @@ -162,11 +162,12 @@ public synchronized void registerExecution(Execution execution) throws HopExcept // Create the folder(s) of the parent: // - HopVfs.getFileObject(registrationFileName).getParent().createFolder(); + HopVfs.getFileObject(registrationFileName, variables).getParent().createFolder(); // Write the execution information to disk... // - try (OutputStream outputStream = HopVfs.getOutputStream(registrationFileName, false)) { + try (OutputStream outputStream = + HopVfs.getOutputStream(registrationFileName, false, variables)) { ObjectMapper mapper = HopJson.newMapper(); mapper.writerWithDefaultPrettyPrinter().writeValue(outputStream, execution); } @@ -187,7 +188,7 @@ public synchronized boolean deleteExecution(String executionId) throws HopExcept // Delete the folder and everything in it // - FileObject executionFolder = HopVfs.getFileObject(getSubFolder(executionId)); + FileObject executionFolder = HopVfs.getFileObject(getSubFolder(executionId), variables); for (FileObject child : executionFolder.getChildren()) { child.delete(); } @@ -255,9 +256,9 @@ public synchronized void updateExecutionState(ExecutionState executionState) thr // Create the folder(s) of the parent if needed: // - HopVfs.getFileObject(updateFilename).getParent().createFolder(); + HopVfs.getFileObject(updateFilename, variables).getParent().createFolder(); - try (OutputStream outputStream = HopVfs.getOutputStream(updateFilename, false)) { + try (OutputStream outputStream = HopVfs.getOutputStream(updateFilename, false, variables)) { ObjectMapper mapper = HopJson.newMapper(); mapper.writerWithDefaultPrettyPrinter().writeValue(outputStream, executionState); } @@ -266,7 +267,7 @@ public synchronized void updateExecutionState(ExecutionState executionState) thr // if (saveLoggingToFile) { String logFilename = getLogFilename(executionState); - try (OutputStream outputStream = HopVfs.getOutputStream(logFilename, false)) { + try (OutputStream outputStream = HopVfs.getOutputStream(logFilename, false, variables)) { outputStream.write(loggingText.getBytes(StandardCharsets.UTF_8)); } } @@ -285,10 +286,10 @@ public synchronized ExecutionState getExecutionState(String executionId, boolean throws HopException { try { String updateFilename = getUpdateFilename(executionId); - if (!HopVfs.fileExists(updateFilename)) { + if (!HopVfs.fileExists(updateFilename, variables)) { return null; } - try (InputStream inputStream = HopVfs.getInputStream(updateFilename)) { + try (InputStream inputStream = HopVfs.getInputStream(updateFilename, variables)) { ObjectMapper mapper = HopJson.newMapper(); ExecutionState executionState = mapper.readValue(inputStream, ExecutionState.class); @@ -328,13 +329,13 @@ protected String getExecutionStateLoggingText(ExecutionState executionState, int try { // If there's a separate log file we'll read everything from there. String logFilename = getLogFilename(executionState); - if (HopVfs.fileExists(logFilename)) { + if (HopVfs.fileExists(logFilename, variables)) { // Only read the first part of the file, if a size limit was set. // try (Reader reader = new BufferedReader( new InputStreamReader( - HopVfs.getInputStream(logFilename), StandardCharsets.UTF_8))) { + HopVfs.getInputStream(logFilename, variables), StandardCharsets.UTF_8))) { StringBuilder log = new StringBuilder(); int c; while ((c = reader.read()) != -1 && (sizeLimit <= 0 || sizeLimit > log.length())) { @@ -373,7 +374,7 @@ public synchronized void registerData(ExecutionData data) throws HopException { // String dataFilename = getDataFilename(data); - try (OutputStream outputStream = HopVfs.getOutputStream(dataFilename, false)) { + try (OutputStream outputStream = HopVfs.getOutputStream(dataFilename, false, variables)) { ObjectMapper mapper = HopJson.newMapper(); mapper.writerWithDefaultPrettyPrinter().writeValue(outputStream, data); } @@ -392,7 +393,7 @@ public synchronized List getExecutionIds(boolean includeChildren, int li List subFolders = new ArrayList<>(); - FileObject folder = HopVfs.getFileObject(variables.resolve(rootFolder)); + FileObject folder = HopVfs.getFileObject(variables.resolve(rootFolder), variables); if (!folder.exists()) { return Collections.emptyList(); } @@ -459,7 +460,7 @@ public synchronized List findChildIds( // For a Beam pipeline to find child transforms. // String suffix = CONST_DATA_JSON; - FileObject folderObject = HopVfs.getFileObject(getSubFolder(parentExecutionId)); + FileObject folderObject = HopVfs.getFileObject(getSubFolder(parentExecutionId), variables); // In this folder we have a number of files ending with CONST_DATA_JSON for (FileObject child : folderObject.getChildren()) { @@ -489,7 +490,7 @@ public synchronized Execution getExecution(String executionId) throws HopExcepti try { // Look in the pipeline executions // - try (FileObject folder = HopVfs.getFileObject(getSubFolder(executionId))) { + try (FileObject folder = HopVfs.getFileObject(getSubFolder(executionId), variables)) { if (folder == null || !folder.exists()) { // No Execution info to be found return null; @@ -587,7 +588,7 @@ public synchronized String findParentId(String childId) throws HopException { public synchronized ExecutionData getExecutionData(String parentExecutionId, String executionId) throws HopException { try { - try (FileObject folder = HopVfs.getFileObject(getSubFolder(parentExecutionId))) { + try (FileObject folder = HopVfs.getFileObject(getSubFolder(parentExecutionId), variables)) { if (!folder.exists()) { return null; } diff --git a/plugins/actions/movefiles/src/main/java/org/apache/hop/workflow/actions/movefiles/ActionMoveFiles.java b/plugins/actions/movefiles/src/main/java/org/apache/hop/workflow/actions/movefiles/ActionMoveFiles.java index 82c6d54e451..d2701a495ce 100644 --- a/plugins/actions/movefiles/src/main/java/org/apache/hop/workflow/actions/movefiles/ActionMoveFiles.java +++ b/plugins/actions/movefiles/src/main/java/org/apache/hop/workflow/actions/movefiles/ActionMoveFiles.java @@ -656,7 +656,8 @@ public boolean includeFile(FileSelectInfo info) { "ActionMoveFiles.Error.Exception.MoveProcess", realSourceFilefoldername, destinationFileFolder == null ? "" : destinationFileFolder.toString(), - e.getMessage())); + e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName()), + e); } finally { if (sourceFileFolder != null) { try { @@ -939,13 +940,16 @@ private boolean moveFile( } } } catch (Exception e) { + // Pass Throwable so Hop logs the full stack (including VFS root cause), not only + // e.getMessage(). logError( BaseMessages.getString( PKG, "ActionMoveFiles.Error.Exception.MoveProcessError", sourceFileFolder.toString(), destinationFilename.toString(), - e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName())); + e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName()), + e); updateErrors(); } finally { if (destinationFile != null) { diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/run/MainSpark.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/run/MainSpark.java index 4c0978e2c7c..fbb2bcf0500 100644 --- a/plugins/engines/spark/src/main/java/org/apache/hop/spark/run/MainSpark.java +++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/run/MainSpark.java @@ -19,11 +19,13 @@ import java.io.IOException; import java.io.InputStream; +import java.util.List; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hop.core.Const; import org.apache.hop.core.HopEnvironment; import org.apache.hop.core.config.DescribedVariablesConfigFile; +import org.apache.hop.core.encryption.Encr; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.metadata.SerializableMetadataProvider; import org.apache.hop.core.plugins.IPlugin; @@ -33,7 +35,10 @@ import org.apache.hop.core.variables.Variables; import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.core.xml.XmlHandler; +import org.apache.hop.metadata.api.IHopMetadataProvider; import org.apache.hop.metadata.api.IHopMetadataSerializer; +import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider; +import org.apache.hop.metadata.util.HopMetadataInstance; import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration; import org.apache.hop.pipeline.config.PipelineRunConfiguration; @@ -168,7 +173,18 @@ private static void runPipeline(String[] args) throws Exception { String metadataJson = readFileIntoString(metadataPath, Const.UTF_8); String runConfigName = parsed.getRunConfigName(); - SerializableMetadataProvider metadataProvider = new SerializableMetadataProvider(metadataJson); + SerializableMetadataProvider serializableMetadata = + new SerializableMetadataProvider(metadataJson); + // Publish as the process-wide metadata instance so named VFS plugins (Databricks, MinIO, …) + // load connections from the export — not only from a hop-config metadata folder (empty on + // Databricks drivers). Reset HopVfs so scheme providers are registered against this metadata. + MultiMetadataProvider metadataProvider = + new MultiMetadataProvider( + Encr.getEncoder(), List.of(serializableMetadata), variables); + HopMetadataInstance.setMetadataProvider(metadataProvider); + HopVfs.reset(); + System.out.println( + ">>>>>> Runtime metadata published for named VFS schemes (HopMetadataInstance)"); IHopMetadataSerializer serializer = metadataProvider.getSerializer(PipelineRunConfiguration.class); diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/DatabricksFilesClient.java b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/DatabricksFilesClient.java new file mode 100644 index 00000000000..1b3027d4fc3 --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/DatabricksFilesClient.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.databricks.client; + +import java.io.InputStream; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import org.apache.hop.core.exception.HopException; + +/** + * Databricks workspace Files API surface for UC Volumes and Workspace files ({@code /Volumes/…}, + * {@code /Workspace/…}). Not the legacy DBFS block API. + */ +public interface DatabricksFilesClient extends AutoCloseable { + + /** Upload a local file (overwrite). Path must be absolute Files API path. */ + void upload(Path localFile, String workspacePath) throws HopException; + + /** Upload UTF-8 text (overwrite). Convenience for small sidecars. */ + void uploadText(String workspacePath, String text) throws HopException; + + /** + * Metadata for a remote file. {@link WorkspaceFileMetadata#exists()} is false when missing or not + * a file. + */ + WorkspaceFileMetadata getFileMetadata(String workspacePath) throws HopException; + + /** Download a small remote text file; empty when missing. */ + Optional downloadTextIfExists(String workspacePath) throws HopException; + + /** + * Open a binary download stream for the file. Caller must close the stream. Throws if the path + * does not exist. + */ + InputStream openInputStream(String workspacePath) throws HopException; + + /** List directory contents (all pages). Path must be an existing directory. */ + List listDirectory(String workspacePath) throws HopException; + + /** Create a directory (parents must exist per API rules for the volume). */ + void createDirectory(String workspacePath) throws HopException; + + /** Delete a file. */ + void deleteFile(String workspacePath) throws HopException; + + /** Delete a directory; when {@code recursive}, delete contents first as required by the API. */ + void deleteDirectory(String workspacePath, boolean recursive) throws HopException; + + @Override + void close(); +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/DirectoryEntry.java b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/DirectoryEntry.java new file mode 100644 index 00000000000..926f793ff20 --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/DirectoryEntry.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.databricks.client; + +/** + * One entry from a Databricks Files API directory listing ({@code GET + * /api/2.0/fs/directories{path}}). + */ +public record DirectoryEntry( + String name, String path, boolean directory, long sizeBytes, long lastModifiedEpochMs) { + + public static DirectoryEntry ofFile(String name, String path, long sizeBytes, long lastModified) { + return new DirectoryEntry(name, path, false, Math.max(0L, sizeBytes), lastModified); + } + + public static DirectoryEntry ofDirectory(String name, String path, long lastModified) { + return new DirectoryEntry(name, path, true, -1L, lastModified); + } +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/RestDatabricksFilesClient.java b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/RestDatabricksFilesClient.java new file mode 100644 index 00000000000..1e58d9a1faf --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/RestDatabricksFilesClient.java @@ -0,0 +1,680 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.databricks.client; + +import java.io.InputStream; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.encryption.Encr; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.databricks.metadata.DatabricksConnection; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; + +/** + * Files API client for UC Volumes and Workspace paths ({@code PUT/GET/DELETE /api/2.0/fs/files…}, + * {@code /api/2.0/fs/directories…}). Bearer PAT auth; does not log tokens. + */ +public final class RestDatabricksFilesClient implements DatabricksFilesClient { + + private static final Duration TIMEOUT = Duration.ofSeconds(60); + private static final Duration UPLOAD_TIMEOUT = Duration.ofMinutes(30); + private static final String FILES_PREFIX = "/api/2.0/fs/files"; + private static final String DIRS_PREFIX = "/api/2.0/fs/directories"; + + private final String hostBase; + private final String token; + private final HttpClient httpClient; + private final JSONParser parser = new JSONParser(); + + public RestDatabricksFilesClient(String hostBase, String token, HttpClient httpClient) { + this.hostBase = normalizeHost(hostBase); + this.token = Objects.requireNonNull(token, "token"); + this.httpClient = Objects.requireNonNull(httpClient, "httpClient"); + } + + public static RestDatabricksFilesClient create( + DatabricksConnection connection, IVariables variables) throws HopException { + if (connection == null) { + throw new HopException("Databricks connection is required"); + } + String host = resolve(variables, connection.getHost()); + String token = + Encr.decryptPasswordOptionallyEncrypted(resolve(variables, connection.getToken())); + if (StringUtils.isBlank(host)) { + throw new HopException("Databricks workspace host is required"); + } + if (looksUnresolved(host)) { + throw new HopException( + "Databricks workspace host still contains unresolved variables after resolve: '" + + host + + "'. Set DATABRICKS_HOST (or the variables used in the connection) on the process " + + "that opens the VFS scheme (Hop GUI environment, or MainSpark --HopConfigFile / job env)."); + } + if (StringUtils.isBlank(token)) { + throw new HopException("Databricks personal access token is required"); + } + if (looksUnresolved(token) || token.contains("${")) { + throw new HopException( + "Databricks personal access token still contains unresolved variables. Set DATABRICKS_TOKEN " + + "(or the variables used in the connection) on the process that opens the VFS scheme."); + } + HttpClient client = HttpClient.newBuilder().connectTimeout(TIMEOUT).build(); + return new RestDatabricksFilesClient(host, token, client); + } + + private static boolean looksUnresolved(String value) { + return value != null && (value.contains("${") || value.contains("%%")); + } + + /** Visible for tests with a custom {@link HttpClient}. */ + public static RestDatabricksFilesClient createForTest( + String hostBase, String token, HttpClient httpClient) { + return new RestDatabricksFilesClient(hostBase, token, httpClient); + } + + @Override + public void upload(Path localFile, String workspacePath) throws HopException { + if (localFile == null || !Files.isRegularFile(localFile)) { + throw new HopException("Local file for workspace upload does not exist: " + localFile); + } + String path = requireFilesApiPath(workspacePath); + try { + String encodedPath = encodeFilesApiPath(path); + String url = hostBase + FILES_PREFIX + encodedPath + "?overwrite=true"; + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(UPLOAD_TIMEOUT) + .header("Authorization", "Bearer " + token) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofFile(localFile)) + .build(); + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + int code = response.statusCode(); + String body = response.body() == null ? "" : response.body(); + if (code < 200 || code >= 300) { + throw new HopException( + "Databricks API HTTP " + + code + + " for PUT " + + FILES_PREFIX + + path + + ": " + + sanitizeError(body)); + } + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("Failed to upload " + localFile + " to " + path + " via Files API", e); + } + } + + @Override + public void uploadText(String workspacePath, String text) throws HopException { + if (text == null) { + text = ""; + } + try { + Path tmp = Files.createTempFile("hop-dbx-text-", ".txt"); + try { + Files.writeString(tmp, text, StandardCharsets.UTF_8); + upload(tmp, workspacePath); + } finally { + Files.deleteIfExists(tmp); + } + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("Failed to upload text to " + workspacePath, e); + } + } + + @Override + public WorkspaceFileMetadata getFileMetadata(String workspacePath) throws HopException { + String path = requireFilesApiPath(workspacePath); + try { + String encodedPath = encodeFilesApiPath(path); + String url = hostBase + FILES_PREFIX + encodedPath; + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(TIMEOUT) + .header("Authorization", "Bearer " + token) + .method("HEAD", HttpRequest.BodyPublishers.noBody()) + .build(); + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.discarding()); + int code = response.statusCode(); + if (code == 404) { + return WorkspaceFileMetadata.missing(); + } + if (code < 200 || code >= 300) { + return getFileMetadataViaGetRange(path); + } + long size = contentLength(response.headers().firstValue("Content-Length").orElse(null)); + if (size < 0) { + size = contentLength(response.headers().firstValue("content-length").orElse(null)); + } + if (size < 0) { + return getFileMetadataViaGetRange(path); + } + return WorkspaceFileMetadata.ofFile(size); + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("Failed to get metadata for " + path, e); + } + } + + private WorkspaceFileMetadata getFileMetadataViaGetRange(String absolutePath) + throws HopException { + try { + String encodedPath = encodeFilesApiPath(absolutePath); + String url = hostBase + FILES_PREFIX + encodedPath; + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(TIMEOUT) + .header("Authorization", "Bearer " + token) + .header("Range", "bytes=0-0") + .GET() + .build(); + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()); + int code = response.statusCode(); + if (code == 404) { + return WorkspaceFileMetadata.missing(); + } + if (code != 200 && code != 206) { + String errBody = + response.body() == null ? "" : new String(response.body(), StandardCharsets.UTF_8); + throw new HopException( + "Databricks API HTTP " + + code + + " for GET " + + FILES_PREFIX + + absolutePath + + ": " + + sanitizeError(errBody)); + } + Optional contentRange = response.headers().firstValue("Content-Range"); + if (contentRange.isPresent()) { + String cr = contentRange.get(); + int slash = cr.lastIndexOf('/'); + if (slash > 0 && slash < cr.length() - 1) { + try { + return WorkspaceFileMetadata.ofFile(Long.parseLong(cr.substring(slash + 1).trim())); + } catch (NumberFormatException ignored) { + // fall through + } + } + } + long size = contentLength(response.headers().firstValue("Content-Length").orElse(null)); + if (size >= 0 && code == 200) { + return WorkspaceFileMetadata.ofFile(size); + } + return WorkspaceFileMetadata.missing(); + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("Failed to get metadata for " + absolutePath, e); + } + } + + @Override + public Optional downloadTextIfExists(String workspacePath) throws HopException { + String path = requireFilesApiPath(workspacePath); + try { + String encodedPath = encodeFilesApiPath(path); + String url = hostBase + FILES_PREFIX + encodedPath; + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(TIMEOUT) + .header("Authorization", "Bearer " + token) + .GET() + .build(); + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()); + int code = response.statusCode(); + if (code == 404) { + return Optional.empty(); + } + if (code < 200 || code >= 300) { + String errBody = + response.body() == null ? "" : new String(response.body(), StandardCharsets.UTF_8); + throw new HopException( + "Databricks API HTTP " + + code + + " for GET " + + FILES_PREFIX + + path + + ": " + + sanitizeError(errBody)); + } + byte[] bytes = response.body() == null ? new byte[0] : response.body(); + return Optional.of(new String(bytes, StandardCharsets.UTF_8)); + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("Failed to download " + path, e); + } + } + + @Override + public InputStream openInputStream(String workspacePath) throws HopException { + String path = requireFilesApiPath(workspacePath); + try { + String encodedPath = encodeFilesApiPath(path); + String url = hostBase + FILES_PREFIX + encodedPath; + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(UPLOAD_TIMEOUT) + .header("Authorization", "Bearer " + token) + .GET() + .build(); + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); + int code = response.statusCode(); + if (code == 404) { + response.body().close(); + throw new HopException("File not found: " + path); + } + if (code < 200 || code >= 300) { + String errBody; + try (InputStream err = response.body()) { + errBody = new String(err.readAllBytes(), StandardCharsets.UTF_8); + } + throw new HopException( + "Databricks API HTTP " + + code + + " for GET " + + FILES_PREFIX + + path + + ": " + + sanitizeError(errBody)); + } + return response.body(); + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("Failed to open stream for " + path, e); + } + } + + @Override + public List listDirectory(String workspacePath) throws HopException { + String path = requireFilesApiPath(workspacePath); + List all = new ArrayList<>(); + String pageToken = null; + try { + do { + String encodedPath = encodeFilesApiPath(path); + StringBuilder url = new StringBuilder(hostBase).append(DIRS_PREFIX).append(encodedPath); + if (pageToken != null) { + url.append("?page_token=").append(URLEncoder.encode(pageToken, StandardCharsets.UTF_8)); + } + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url.toString())) + .timeout(TIMEOUT) + .header("Authorization", "Bearer " + token) + .GET() + .build(); + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + int code = response.statusCode(); + String body = response.body() == null ? "" : response.body(); + if (code < 200 || code >= 300) { + throw new HopException( + "Databricks API HTTP " + + code + + " for GET " + + DIRS_PREFIX + + path + + ": " + + sanitizeError(body)); + } + JSONObject json = parseObject(body); + JSONArray contents = (JSONArray) json.get("contents"); + if (contents != null) { + for (Object item : contents) { + if (item instanceof JSONObject entry) { + all.add(parseDirectoryEntry(entry, path)); + } + } + } + Object next = json.get("next_page_token"); + pageToken = + next != null && StringUtils.isNotBlank(next.toString()) ? next.toString() : null; + } while (pageToken != null); + return all; + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("Failed to list directory " + path, e); + } + } + + private static DirectoryEntry parseDirectoryEntry(JSONObject entry, String parentPath) { + String name = entry.get("name") != null ? entry.get("name").toString() : ""; + String entryPath = + entry.get("path") != null + ? entry.get("path").toString() + : (parentPath.endsWith("/") ? parentPath + name : parentPath + "/" + name); + boolean isDir = Boolean.TRUE.equals(entry.get("is_directory")); + long size = 0L; + if (entry.get("file_size") instanceof Number n) { + size = n.longValue(); + } else if (entry.get("file_size") != null) { + try { + size = Long.parseLong(entry.get("file_size").toString()); + } catch (NumberFormatException ignored) { + size = 0L; + } + } + long lastMod = 0L; + if (entry.get("last_modified") instanceof Number n) { + lastMod = n.longValue(); + } else if (entry.get("last_modified") != null) { + try { + lastMod = Long.parseLong(entry.get("last_modified").toString()); + } catch (NumberFormatException ignored) { + lastMod = 0L; + } + } + if (StringUtils.isBlank(name) && StringUtils.isNotBlank(entryPath)) { + int slash = entryPath.lastIndexOf('/'); + name = slash >= 0 ? entryPath.substring(slash + 1) : entryPath; + } + return isDir + ? DirectoryEntry.ofDirectory(name, entryPath, lastMod) + : DirectoryEntry.ofFile(name, entryPath, size, lastMod); + } + + @Override + public void createDirectory(String workspacePath) throws HopException { + String path = requireFilesApiPath(workspacePath); + try { + String encodedPath = encodeFilesApiPath(path); + // Trailing slash required by Files API for directories + if (!encodedPath.endsWith("/")) { + encodedPath = encodedPath + "/"; + } + String url = hostBase + DIRS_PREFIX + encodedPath; + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(TIMEOUT) + .header("Authorization", "Bearer " + token) + .PUT(HttpRequest.BodyPublishers.noBody()) + .build(); + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + int code = response.statusCode(); + String body = response.body() == null ? "" : response.body(); + // 200/201/204 success; some workspaces return 409 if already exists — treat as success + if (code == 409) { + return; + } + if (code < 200 || code >= 300) { + throw new HopException( + "Databricks API HTTP " + + code + + " for PUT " + + DIRS_PREFIX + + path + + ": " + + sanitizeError(body)); + } + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("Failed to create directory " + path, e); + } + } + + @Override + public void deleteFile(String workspacePath) throws HopException { + String path = requireFilesApiPath(workspacePath); + try { + String encodedPath = encodeFilesApiPath(path); + String url = hostBase + FILES_PREFIX + encodedPath; + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(TIMEOUT) + .header("Authorization", "Bearer " + token) + .DELETE() + .build(); + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + int code = response.statusCode(); + String body = response.body() == null ? "" : response.body(); + if (code == 404) { + return; + } + if (code < 200 || code >= 300) { + throw new HopException( + "Databricks API HTTP " + + code + + " for DELETE " + + FILES_PREFIX + + path + + ": " + + sanitizeError(body)); + } + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("Failed to delete file " + path, e); + } + } + + @Override + public void deleteDirectory(String workspacePath, boolean recursive) throws HopException { + String path = requireFilesApiPath(workspacePath); + // Workspace Files API rejects ?recursive=… (FILES_API_UNEXPECTED_QUERY_PARAMETERS). + // Recursive delete: remove children first, then the empty directory. + if (recursive) { + List children = listDirectory(path); + for (DirectoryEntry child : children) { + if (child.directory()) { + deleteDirectory(child.path(), true); + } else { + deleteFile(child.path()); + } + } + } + deleteDirectoryEmpty(path); + } + + /** DELETE {@code /api/2.0/fs/directories{path}/} with no query parameters (empty dir only). */ + private void deleteDirectoryEmpty(String path) throws HopException { + try { + String encodedPath = encodeFilesApiPath(path); + if (!encodedPath.endsWith("/")) { + encodedPath = encodedPath + "/"; + } + String url = hostBase + DIRS_PREFIX + encodedPath; + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(TIMEOUT) + .header("Authorization", "Bearer " + token) + .DELETE() + .build(); + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + int code = response.statusCode(); + String body = response.body() == null ? "" : response.body(); + if (code == 404) { + return; + } + if (code < 200 || code >= 300) { + throw new HopException( + "Databricks API HTTP " + + code + + " for DELETE " + + DIRS_PREFIX + + path + + ": " + + sanitizeError(body)); + } + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("Failed to delete directory " + path, e); + } + } + + @Override + public void close() { + // HttpClient does not require close + } + + /** + * UC Volumes and Workspace files must use the Files API. Classic DBFS roots use the legacy DBFS + * block API (not this client). + */ + public static boolean isFilesApiPath(String absolutePath) { + if (StringUtils.isBlank(absolutePath)) { + return false; + } + String p = absolutePath.trim(); + return p.startsWith("/Volumes/") + || p.equals("/Volumes") + || p.startsWith("/Workspace/") + || p.equals("/Workspace"); + } + + /** + * Normalize workspace paths: strip optional {@code dbfs:} scheme, ensure a leading slash. {@code + * dbfs:/Volumes/…} becomes {@code /Volumes/…}. + */ + public static String normalizeWorkspacePath(String workspacePath) throws HopException { + if (StringUtils.isBlank(workspacePath)) { + throw new HopException("Upload path is required"); + } + String p = workspacePath.trim(); + if (p.startsWith("dbfs:")) { + p = p.substring("dbfs:".length()); + } + if (!p.startsWith("/")) { + p = "/" + p; + } + return p; + } + + static String requireFilesApiPath(String workspacePath) throws HopException { + String path = normalizeWorkspacePath(workspacePath); + if (!isFilesApiPath(path)) { + throw new HopException( + "Path is not a Databricks Files API location (expected /Volumes/… or /Workspace/…): " + + path + + ". Classic DBFS paths are not supported by this client."); + } + return path; + } + + /** + * Encode an absolute workspace path for the Files API URL path (keep {@code /} separators, encode + * each segment). + */ + public static String encodeFilesApiPath(String absolutePath) { + String p = absolutePath.startsWith("/") ? absolutePath : "/" + absolutePath; + String[] parts = p.split("/", -1); + StringBuilder sb = new StringBuilder(); + for (String part : parts) { + if (part.isEmpty()) { + continue; + } + sb.append('/').append(URLEncoder.encode(part, StandardCharsets.UTF_8).replace("+", "%20")); + } + return sb.length() == 0 ? "/" : sb.toString(); + } + + /** Strip likely secrets from error payloads before logging. */ + public static String sanitizeError(String body) { + if (body == null) { + return ""; + } + String trimmed = body.length() > 500 ? body.substring(0, 500) + "…" : body; + return trimmed.replaceAll("(?i)\\b(token|authorization|bearer)\\b\\s*[:=]?\\s*\\S+", "$1=***"); + } + + public static String normalizeHost(String host) { + String h = host.trim(); + if (!h.startsWith("http://") && !h.startsWith("https://")) { + h = "https://" + h; + } + while (h.endsWith("/")) { + h = h.substring(0, h.length() - 1); + } + return h; + } + + private static long contentLength(String header) { + if (StringUtils.isBlank(header)) { + return -1L; + } + try { + return Long.parseLong(header.trim()); + } catch (NumberFormatException e) { + return -1L; + } + } + + private JSONObject parseObject(String body) throws HopException { + try { + Object parsed = parser.parse(body); + if (!(parsed instanceof JSONObject)) { + throw new HopException("Expected JSON object from Databricks API"); + } + return (JSONObject) parsed; + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException("Unable to parse Databricks API response", e); + } + } + + private static String resolve(IVariables variables, String value) { + if (value == null) { + return null; + } + return variables != null ? variables.resolve(value) : value; + } +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/RestDatabricksJobsClient.java b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/RestDatabricksJobsClient.java index 8137b30b477..77161fda3f5 100644 --- a/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/RestDatabricksJobsClient.java +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/client/RestDatabricksJobsClient.java @@ -42,8 +42,9 @@ /** * Databricks Jobs API client using {@link HttpClient} and Jobs REST endpoints (default base {@code - * /api/2.1}). Does not log tokens. Uploads use the Files API for UC Volumes / Workspace paths and - * the legacy DBFS API for classic {@code dbfs:/} roots. + * /api/2.1}). Does not log tokens. Uploads use the Files API for UC Volumes / Workspace paths + * (delegated to {@link RestDatabricksFilesClient}) and the legacy DBFS API for classic {@code + * dbfs:/} roots. */ public final class RestDatabricksJobsClient implements DatabricksJobsClient { @@ -53,13 +54,11 @@ public final class RestDatabricksJobsClient implements DatabricksJobsClient { /** DBFS add-block max is 1 MiB of base64-decoded data. */ private static final int DBFS_BLOCK_BYTES = 1024 * 1024; - /** Files API single PUT supports files up to 5 GiB. */ - private static final String FILES_API_PREFIX = "/api/2.0/fs/files"; - private final String hostBase; private final String apiBase; private final String token; private final HttpClient httpClient; + private final RestDatabricksFilesClient filesClient; private final JSONParser parser = new JSONParser(); public RestDatabricksJobsClient( @@ -68,6 +67,7 @@ public RestDatabricksJobsClient( this.apiBase = normalizeApiBase(apiBase); this.token = Objects.requireNonNull(token, "token"); this.httpClient = Objects.requireNonNull(httpClient, "httpClient"); + this.filesClient = new RestDatabricksFilesClient(this.hostBase, this.token, this.httpClient); } public static RestDatabricksJobsClient create( @@ -83,9 +83,20 @@ public static RestDatabricksJobsClient create( if (StringUtils.isBlank(host)) { throw new HopException("Databricks workspace host is required"); } + if (host.contains("${") || host.contains("%%")) { + throw new HopException( + "Databricks workspace host still contains unresolved variables: '" + + host + + "'. Set DATABRICKS_HOST (or the variables used in the connection) for this process."); + } if (StringUtils.isBlank(token)) { throw new HopException("Databricks personal access token is required"); } + if (token.contains("${") || token.contains("%%")) { + throw new HopException( + "Databricks personal access token still contains unresolved variables. Set DATABRICKS_TOKEN " + + "for this process."); + } HttpClient client = HttpClient.newBuilder().connectTimeout(TIMEOUT).build(); return new RestDatabricksJobsClient(host, apiBase, token, client); } @@ -96,6 +107,11 @@ public static RestDatabricksJobsClient createForTest( return new RestDatabricksJobsClient(hostBase, apiBase, token, httpClient); } + /** Files API client shared for Volume / Workspace paths (deploy + VFS). */ + public RestDatabricksFilesClient files() { + return filesClient; + } + @Override public String testConnection() throws HopException { // Prefer SCIM me when available; fall back to listing one job. @@ -198,7 +214,7 @@ public void uploadToDbfs(Path localFile, String dbfsPath) throws HopException { } String path = normalizeDbfsPath(dbfsPath); if (isFilesApiPath(path)) { - uploadViaFilesApi(localFile, path); + filesClient.upload(localFile, path); } else { uploadViaDbfsApi(localFile, path); } @@ -208,7 +224,7 @@ public void uploadToDbfs(Path localFile, String dbfsPath) throws HopException { public WorkspaceFileMetadata getFileMetadata(String workspacePath) throws HopException { String path = normalizeDbfsPath(workspacePath); if (isFilesApiPath(path)) { - return getFileMetadataFilesApi(path); + return filesClient.getFileMetadata(path); } return getFileMetadataDbfs(path); } @@ -217,13 +233,18 @@ public WorkspaceFileMetadata getFileMetadata(String workspacePath) throws HopExc public Optional downloadTextIfExists(String workspacePath) throws HopException { String path = normalizeDbfsPath(workspacePath); if (isFilesApiPath(path)) { - return downloadTextFilesApi(path); + return filesClient.downloadTextIfExists(path); } return downloadTextDbfs(path); } @Override public void uploadText(String workspacePath, String text) throws HopException { + String path = normalizeDbfsPath(workspacePath); + if (isFilesApiPath(path)) { + filesClient.uploadText(path, text); + return; + } if (text == null) { text = ""; } @@ -231,7 +252,7 @@ public void uploadText(String workspacePath, String text) throws HopException { Path tmp = Files.createTempFile("hop-dbx-text-", ".txt"); try { Files.writeString(tmp, text, StandardCharsets.UTF_8); - uploadToDbfs(tmp, workspacePath); + uploadViaDbfsApi(tmp, path); } finally { Files.deleteIfExists(tmp); } @@ -247,149 +268,7 @@ public void uploadText(String workspacePath, String text) throws HopException { * the legacy DBFS block API. */ static boolean isFilesApiPath(String absolutePath) { - if (StringUtils.isBlank(absolutePath)) { - return false; - } - String p = absolutePath.trim(); - return p.startsWith("/Volumes/") - || p.equals("/Volumes") - || p.startsWith("/Workspace/") - || p.equals("/Workspace"); - } - - /** - * Upload via {@code PUT /api/2.0/fs/files{path}?overwrite=true} (UC Volumes / Workspace). Body is - * raw octets; max ~5 GiB per Databricks Files API. - */ - private void uploadViaFilesApi(Path localFile, String absolutePath) throws HopException { - try { - String encodedPath = encodeFilesApiPath(absolutePath); - String url = hostBase + FILES_API_PREFIX + encodedPath + "?overwrite=true"; - HttpRequest request = - HttpRequest.newBuilder() - .uri(URI.create(url)) - .timeout(UPLOAD_TIMEOUT) - .header("Authorization", "Bearer " + token) - .header("Content-Type", "application/octet-stream") - .PUT(HttpRequest.BodyPublishers.ofFile(localFile)) - .build(); - HttpResponse response = - httpClient.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); - int code = response.statusCode(); - String body = response.body() == null ? "" : response.body(); - if (code < 200 || code >= 300) { - throw new HopException( - "Databricks API HTTP " - + code - + " for PUT " - + FILES_API_PREFIX - + absolutePath - + ": " - + sanitizeError(body)); - } - } catch (HopException e) { - throw e; - } catch (Exception e) { - throw new HopException( - "Failed to upload " + localFile + " to " + absolutePath + " via Files API", e); - } - } - - /** - * Files API get-metadata: HEAD {@code /api/2.0/fs/files{path}} — size from Content-Length (no - * body). - */ - private WorkspaceFileMetadata getFileMetadataFilesApi(String absolutePath) throws HopException { - try { - String encodedPath = encodeFilesApiPath(absolutePath); - String url = hostBase + FILES_API_PREFIX + encodedPath; - HttpRequest request = - HttpRequest.newBuilder() - .uri(URI.create(url)) - .timeout(TIMEOUT) - .header("Authorization", "Bearer " + token) - .method("HEAD", HttpRequest.BodyPublishers.noBody()) - .build(); - HttpResponse response = - httpClient.send(request, HttpResponse.BodyHandlers.discarding()); - int code = response.statusCode(); - if (code == 404) { - return WorkspaceFileMetadata.missing(); - } - if (code < 200 || code >= 300) { - // Some gateways reject HEAD — fall back to GET with range 0-0 for size - return getFileMetadataFilesApiGetHeaders(absolutePath); - } - long size = contentLength(response.headers().firstValue("Content-Length").orElse(null)); - if (size < 0) { - size = contentLength(response.headers().firstValue("content-length").orElse(null)); - } - if (size < 0) { - return getFileMetadataFilesApiGetHeaders(absolutePath); - } - return WorkspaceFileMetadata.ofFile(size); - } catch (HopException e) { - throw e; - } catch (Exception e) { - throw new HopException("Failed to get metadata for " + absolutePath, e); - } - } - - private WorkspaceFileMetadata getFileMetadataFilesApiGetHeaders(String absolutePath) - throws HopException { - try { - String encodedPath = encodeFilesApiPath(absolutePath); - String url = hostBase + FILES_API_PREFIX + encodedPath; - HttpRequest request = - HttpRequest.newBuilder() - .uri(URI.create(url)) - .timeout(TIMEOUT) - .header("Authorization", "Bearer " + token) - .header("Range", "bytes=0-0") - .GET() - .build(); - HttpResponse response = - httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()); - int code = response.statusCode(); - if (code == 404) { - return WorkspaceFileMetadata.missing(); - } - if (code != 200 && code != 206) { - String errBody = - response.body() == null ? "" : new String(response.body(), StandardCharsets.UTF_8); - throw new HopException( - "Databricks API HTTP " - + code - + " for GET " - + FILES_API_PREFIX - + absolutePath - + ": " - + sanitizeError(errBody)); - } - Optional contentRange = response.headers().firstValue("Content-Range"); - if (contentRange.isPresent()) { - // bytes 0-0/12345 - String cr = contentRange.get(); - int slash = cr.lastIndexOf('/'); - if (slash > 0 && slash < cr.length() - 1) { - try { - return WorkspaceFileMetadata.ofFile(Long.parseLong(cr.substring(slash + 1).trim())); - } catch (NumberFormatException ignored) { - // fall through - } - } - } - long size = contentLength(response.headers().firstValue("Content-Length").orElse(null)); - if (size >= 0 && code == 200) { - return WorkspaceFileMetadata.ofFile(size); - } - // Last resort: full GET is too heavy — treat as missing size (force re-upload) - return WorkspaceFileMetadata.missing(); - } catch (HopException e) { - throw e; - } catch (Exception e) { - throw new HopException("Failed to get metadata for " + absolutePath, e); - } + return RestDatabricksFilesClient.isFilesApiPath(absolutePath); } private WorkspaceFileMetadata getFileMetadataDbfs(String path) throws HopException { @@ -414,44 +293,6 @@ private WorkspaceFileMetadata getFileMetadataDbfs(String path) throws HopExcepti } } - private Optional downloadTextFilesApi(String absolutePath) throws HopException { - try { - String encodedPath = encodeFilesApiPath(absolutePath); - String url = hostBase + FILES_API_PREFIX + encodedPath; - HttpRequest request = - HttpRequest.newBuilder() - .uri(URI.create(url)) - .timeout(TIMEOUT) - .header("Authorization", "Bearer " + token) - .GET() - .build(); - HttpResponse response = - httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()); - int code = response.statusCode(); - if (code == 404) { - return Optional.empty(); - } - if (code < 200 || code >= 300) { - String errBody = - response.body() == null ? "" : new String(response.body(), StandardCharsets.UTF_8); - throw new HopException( - "Databricks API HTTP " - + code - + " for GET " - + FILES_API_PREFIX - + absolutePath - + ": " - + sanitizeError(errBody)); - } - byte[] bytes = response.body() == null ? new byte[0] : response.body(); - return Optional.of(new String(bytes, StandardCharsets.UTF_8)); - } catch (HopException e) { - throw e; - } catch (Exception e) { - throw new HopException("Failed to download " + absolutePath, e); - } - } - private Optional downloadTextDbfs(String path) throws HopException { try { // Single-block read: small sidecar files only @@ -480,17 +321,6 @@ private Optional downloadTextDbfs(String path) throws HopException { } } - private static long contentLength(String header) { - if (StringUtils.isBlank(header)) { - return -1L; - } - try { - return Long.parseLong(header.trim()); - } catch (NumberFormatException e) { - return -1L; - } - } - /** Legacy DBFS create / add-block / close for classic {@code dbfs:/} paths. */ private void uploadViaDbfsApi(Path localFile, String path) throws HopException { try { @@ -528,7 +358,7 @@ private void uploadViaDbfsApi(Path localFile, String path) throws HopException { @Override public void close() { - // HttpClient does not require close + filesClient.close(); } /** @@ -536,17 +366,7 @@ public void close() { * dbfs:/Volumes/…} becomes {@code /Volumes/…} so the Files API route is selected. */ static String normalizeDbfsPath(String dbfsPath) throws HopException { - if (StringUtils.isBlank(dbfsPath)) { - throw new HopException("Upload path is required"); - } - String p = dbfsPath.trim(); - if (p.startsWith("dbfs:")) { - p = p.substring("dbfs:".length()); - } - if (!p.startsWith("/")) { - p = "/" + p; - } - return p; + return RestDatabricksFilesClient.normalizeWorkspacePath(dbfsPath); } /** @@ -554,16 +374,7 @@ static String normalizeDbfsPath(String dbfsPath) throws HopException { * each segment). */ static String encodeFilesApiPath(String absolutePath) { - String p = absolutePath.startsWith("/") ? absolutePath : "/" + absolutePath; - String[] parts = p.split("/", -1); - StringBuilder sb = new StringBuilder(); - for (String part : parts) { - if (part.isEmpty()) { - continue; - } - sb.append('/').append(URLEncoder.encode(part, StandardCharsets.UTF_8).replace("+", "%20")); - } - return sb.length() == 0 ? "/" : sb.toString(); + return RestDatabricksFilesClient.encodeFilesApiPath(absolutePath); } private String get(String path) throws HopException { @@ -651,22 +462,11 @@ private static long requireLong(JSONObject json, String key) throws HopException /** Strip likely secrets from error payloads before logging. */ static String sanitizeError(String body) { - if (body == null) { - return ""; - } - String trimmed = body.length() > 500 ? body.substring(0, 500) + "…" : body; - return trimmed.replaceAll("(?i)\\b(token|authorization|bearer)\\b\\s*[:=]?\\s*\\S+", "$1=***"); + return RestDatabricksFilesClient.sanitizeError(body); } static String normalizeHost(String host) { - String h = host.trim(); - if (!h.startsWith("http://") && !h.startsWith("https://")) { - h = "https://" + h; - } - while (h.endsWith("/")) { - h = h.substring(0, h.length() - 1); - } - return h; + return RestDatabricksFilesClient.normalizeHost(host); } static String normalizeApiBase(String apiBase) { diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java index 5f9d235ff2c..12418bf0558 100644 --- a/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java @@ -31,7 +31,9 @@ /** * Workspace connection for the Databricks REST / Jobs API (not JDBC SQL warehouse). Used by - * Databricks Job Run actions and related Jobs API clients. + * Databricks Job Run actions and related Jobs API clients. Also referenced by Databricks VFS + * Connection metadata for Volumes / Workspace file access (host + PAT only — VFS scheme is the VFS + * metadata name, not this connection name). */ @Getter @Setter diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileName.java b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileName.java new file mode 100644 index 00000000000..dfafda49bc1 --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileName.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.provider.AbstractFileName; + +/** Absolute workspace path under a named Databricks connection scheme. */ +public class DatabricksFileName extends AbstractFileName { + + protected DatabricksFileName(final String scheme, final String path, final FileType type) { + super(scheme, path, type); + } + + @Override + public FileName createName(String path, FileType type) { + return new DatabricksFileName(getScheme(), path, type); + } + + @Override + protected void appendRootUri(StringBuilder buffer, boolean addPassword) { + buffer.append(getScheme()); + // Connection is the scheme; path is absolute (/Volumes/… or /Workspace/…). + buffer.append("://"); + } +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileNameParser.java b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileNameParser.java new file mode 100644 index 00000000000..da770d9691f --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileNameParser.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.provider.AbstractFileNameParser; +import org.apache.commons.vfs2.provider.FileNameParser; +import org.apache.commons.vfs2.provider.UriParser; +import org.apache.commons.vfs2.provider.VfsComponentContext; + +/** Parser for {@code connectionName:///Volumes/...} style URIs. */ +public class DatabricksFileNameParser extends AbstractFileNameParser { + private static final DatabricksFileNameParser INSTANCE = new DatabricksFileNameParser(); + + private DatabricksFileNameParser() { + super(); + } + + public static FileNameParser getInstance() { + return INSTANCE; + } + + @Override + public FileName parseUri(VfsComponentContext context, FileName base, String uri) + throws FileSystemException { + StringBuilder name = new StringBuilder(); + + String scheme = UriParser.extractScheme(context.getFileSystemManager().getSchemes(), uri, name); + + UriParser.canonicalizePath(name, 0, name.length(), this); + UriParser.fixSeparators(name); + FileType fileType = UriParser.normalisePath(name); + + return new DatabricksFileName(scheme, name.toString(), fileType); + } +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileObject.java b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileObject.java new file mode 100644 index 00000000000..8196a0a25a1 --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileObject.java @@ -0,0 +1,401 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.vfs2.FileNotFoundException; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.provider.AbstractFileName; +import org.apache.commons.vfs2.provider.AbstractFileObject; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.databricks.client.DatabricksFilesClient; +import org.apache.hop.databricks.client.DirectoryEntry; +import org.apache.hop.databricks.client.RestDatabricksFilesClient; +import org.apache.hop.databricks.client.WorkspaceFileMetadata; +import org.apache.hop.vfs.databricks.DatabricksListCache.ChildInfo; + +/** File object backed by the Databricks Files API (UC Volumes / Workspace). */ +public class DatabricksFileObject extends AbstractFileObject { + + private FileType type = FileType.IMAGINARY; + private long size = -1L; + private long lastModified = 0L; + private List listedChildren; + + protected DatabricksFileObject(final AbstractFileName name, final DatabricksFileSystem fileSystem) + throws FileSystemException { + super(name, fileSystem); + } + + private DatabricksFilesClient client() throws FileSystemException { + return getAbstractFileSystem().getClient(); + } + + /** + * Absolute workspace path for the Files API. When the VFS connection has a root path, logical + * paths such as {@code /input} map under that root. + */ + String workspacePath() { + return DatabricksPathResolver.toWorkspacePath( + getName().getPath(), getAbstractFileSystem().getRootPath()); + } + + @Override + protected void doAttach() throws Exception { + String path = workspacePath(); + if ("/".equals(path)) { + // No root configured — virtual multi-volume root + type = FileType.FOLDER; + size = -1L; + lastModified = 0L; + return; + } + if (!RestDatabricksFilesClient.isFilesApiPath(path)) { + if ("/Volumes".equals(path) || "/Workspace".equals(path)) { + type = FileType.FOLDER; + size = -1L; + lastModified = 0L; + return; + } + type = FileType.IMAGINARY; + size = -1L; + lastModified = 0L; + return; + } + + // Prefer metadata from a recent parent directory listing (includes last_modified). + ChildInfo cached = getAbstractFileSystem().getListCache().get(path); + if (cached != null) { + type = cached.type; + size = cached.size; + lastModified = cached.lastModifiedEpochMs; + if (type == FileType.FOLDER) { + // Keep listedChildren null until doListChildren; type is enough for attach. + } + return; + } + + // Configured scheme root is always a folder (the volume / workspace dir exists or is managed) + if (getAbstractFileSystem().hasRootPath() + && path.equals(getAbstractFileSystem().getRootPath())) { + try { + listedChildren = client().listDirectory(path); + rememberListCache(path, listedChildren); + type = FileType.FOLDER; + size = -1L; + lastModified = 0L; + } catch (HopException e) { + // Still treat as folder so create child works even if list fails (empty / permission) + type = FileType.FOLDER; + size = -1L; + lastModified = 0L; + listedChildren = null; + } + return; + } + + DatabricksFilesClient client = client(); + WorkspaceFileMetadata meta = client.getFileMetadata(path); + if (meta.exists()) { + type = FileType.FILE; + size = meta.sizeBytes(); + // HEAD does not reliably expose last_modified; leave 0 (unknown) unless list cache filled it. + lastModified = 0L; + return; + } + try { + listedChildren = client.listDirectory(path); + rememberListCache(path, listedChildren); + type = FileType.FOLDER; + size = -1L; + lastModified = 0L; + } catch (HopException e) { + type = FileType.IMAGINARY; + size = -1L; + lastModified = 0L; + listedChildren = null; + } + } + + private void rememberListCache(String parentPath, List entries) { + if (entries == null || entries.isEmpty()) { + return; + } + Map map = new LinkedHashMap<>(); + for (DirectoryEntry entry : entries) { + if (entry.path() != null) { + map.put(DatabricksListCache.normalizePath(entry.path()), ChildInfo.fromEntry(entry)); + } + } + getAbstractFileSystem().getListCache().put(parentPath, map); + } + + @Override + protected void doDetach() { + type = FileType.IMAGINARY; + size = -1L; + lastModified = 0L; + listedChildren = null; + } + + @Override + protected FileType doGetType() { + return type; + } + + @Override + protected long doGetContentSize() { + return size; + } + + @Override + protected long doGetLastModifiedTime() { + return lastModified; + } + + @Override + protected String[] doListChildren() throws Exception { + if (type != FileType.FOLDER) { + return null; + } + String path = workspacePath(); + if ("/".equals(path)) { + return new String[] {"Volumes/", "Workspace/"}; + } + if ("/Volumes".equals(path) || "/Workspace".equals(path)) { + return new String[0]; + } + if (listedChildren == null) { + listedChildren = client().listDirectory(path); + rememberListCache(path, listedChildren); + } + return listedChildren.stream() + .map( + e -> { + String n = e.name(); + if (e.directory() && n != null && !n.endsWith("/")) { + return n + "/"; + } + return n; + }) + .toArray(String[]::new); + } + + @Override + protected InputStream doGetInputStream() throws Exception { + if (type != FileType.FILE) { + throw new FileNotFoundException(getName()); + } + return client().openInputStream(workspacePath()); + } + + @Override + protected OutputStream doGetOutputStream(boolean append) throws Exception { + if (append) { + throw new FileSystemException( + "vfs.provider/write-append-not-supported.error", workspacePath()); + } + String path = workspacePath(); + if (!RestDatabricksFilesClient.isFilesApiPath(path)) { + throw new FileSystemException( + "Databricks VFS only supports /Volumes/… and /Workspace/… paths" + + (getAbstractFileSystem().hasRootPath() + ? " (or paths under the connection root path)" + : "") + + ", got workspace path: " + + path + + " from logical path: " + + getName().getPath()); + } + Path temp = Files.createTempFile("hop-dbx-vfs-", ".bin"); + OutputStream fileOut = Files.newOutputStream(temp); + return new FilterOutputStream(fileOut) { + private boolean closed; + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + try { + super.close(); + try { + client().upload(temp, path); + } catch (FileSystemException e) { + throw new IOException(e); + } catch (HopException e) { + throw new IOException(e); + } + try { + type = FileType.FILE; + size = Files.size(temp); + lastModified = System.currentTimeMillis(); + getAbstractFileSystem().getListCache().invalidateParentOf(path); + } catch (IOException ignored) { + type = FileType.FILE; + lastModified = System.currentTimeMillis(); + } + } finally { + try { + Files.deleteIfExists(temp); + } catch (IOException ignored) { + // best effort + } + } + } + }; + } + + @Override + protected void doCreateFolder() throws Exception { + String path = workspacePath(); + // VFS createFile/getOutputStream walks parents. Volume roots and the configured scheme root + // already exist (or cannot be created via Files API) — treat as present. + if (isPreexistingWorkspaceRoot(path, getAbstractFileSystem().getRootPath())) { + type = FileType.FOLDER; + return; + } + if (!RestDatabricksFilesClient.isFilesApiPath(path)) { + throw new FileSystemException( + "Databricks VFS only supports creating folders under /Volumes/… or /Workspace/…, got: " + + path + + " (logical: " + + getName().getPath() + + (getAbstractFileSystem().hasRootPath() + ? ", root: " + getAbstractFileSystem().getRootPath() + : "") + + ")"); + } + client().createDirectory(path); + type = FileType.FOLDER; + listedChildren = null; + lastModified = System.currentTimeMillis(); + getAbstractFileSystem().getListCache().invalidateParentOf(path); + } + + /** + * Paths that are not user-created directories via Files API: filesystem root, {@code /Volumes}, + * {@code /Workspace}, catalog/schema/volume levels, shallow Workspace roots, and the configured + * scheme root itself. + */ + static boolean isPreexistingWorkspaceRoot(String path, String schemeRoot) { + if (StringUtils.isNotBlank(schemeRoot) && schemeRoot.equals(path)) { + return true; + } + if (StringUtils.isBlank(path) || "/".equals(path)) { + return true; + } + if ("/Volumes".equals(path) || "/Workspace".equals(path)) { + return true; + } + // /Volumes/// — volume object must already exist + if (path.startsWith("/Volumes/")) { + // "", "Volumes", catalog, schema, volume → length 5 + return path.split("/", -1).length <= 5; + } + // /Workspace/Users and similar shallow trees are managed by the workspace + if (path.startsWith("/Workspace/")) { + return path.split("/", -1).length <= 3; + } + return false; + } + + @Override + protected void doDelete() throws Exception { + String path = workspacePath(); + if (!RestDatabricksFilesClient.isFilesApiPath(path)) { + throw new FileSystemException("Cannot delete path: " + path); + } + if (getAbstractFileSystem().hasRootPath() + && path.equals(getAbstractFileSystem().getRootPath())) { + throw new FileSystemException("Cannot delete the configured VFS root path: " + path); + } + if (type == FileType.FOLDER) { + client().deleteDirectory(path, true); + } else { + client().deleteFile(path); + } + type = FileType.IMAGINARY; + listedChildren = null; + lastModified = 0L; + getAbstractFileSystem().getListCache().invalidateParentOf(path); + } + + /** + * Files API has no rename endpoint: copy via download+upload then delete the source (same pattern + * as MinIO object rename). Folders are not supported. + */ + @Override + protected void doRename(FileObject newFile) throws Exception { + if (getType() == FileType.FOLDER) { + throw new FileSystemException( + "Databricks VFS does not support renaming folders (move files individually): " + + workspacePath()); + } + if (!(newFile instanceof DatabricksFileObject dest)) { + throw new FileSystemException( + "Cannot rename Databricks VFS file to a different file system: " + newFile); + } + String sourcePath = workspacePath(); + String destPath = dest.workspacePath(); + if (!RestDatabricksFilesClient.isFilesApiPath(sourcePath) + || !RestDatabricksFilesClient.isFilesApiPath(destPath)) { + throw new FileSystemException( + "Rename requires Files API paths under /Volumes or /Workspace (source=" + + sourcePath + + ", dest=" + + destPath + + ")"); + } + if (sourcePath.equals(destPath)) { + return; + } + Path temp = Files.createTempFile("hop-dbx-rename-", ".bin"); + try { + try (InputStream in = client().openInputStream(sourcePath)) { + Files.copy(in, temp, StandardCopyOption.REPLACE_EXISTING); + } + client().upload(temp, destPath); + client().deleteFile(sourcePath); + } catch (HopException e) { + throw new FileSystemException( + "Failed to rename " + sourcePath + " to " + destPath + " via Files API", e); + } finally { + try { + Files.deleteIfExists(temp); + } catch (IOException ignored) { + // best effort + } + } + type = FileType.IMAGINARY; + listedChildren = null; + } +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileProvider.java b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileProvider.java new file mode 100644 index 00000000000..8c3ef213291 --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileProvider.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import java.util.Collection; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.vfs2.Capability; +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileSystem; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemOptions; +import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.databricks.client.RestDatabricksFilesClient; +import org.apache.hop.databricks.metadata.DatabricksConnection; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.metadata.util.HopMetadataInstance; +import org.apache.hop.metadata.util.HopMetadataUtil; +import org.apache.hop.vfs.databricks.metadata.DatabricksVfsConnection; + +/** + * Originating provider for Databricks Files API paths. Bound to a named {@link + * DatabricksVfsConnection} whose name is the VFS scheme; auth is resolved via the referenced {@link + * DatabricksConnection}. Optional root path shortens URIs relative to a volume or workspace folder. + */ +public class DatabricksFileProvider extends AbstractOriginatingFileProvider { + + protected static final Collection CAPABILITIES = + Set.of( + Capability.CREATE, + Capability.DELETE, + Capability.RENAME, + Capability.GET_TYPE, + Capability.LIST_CHILDREN, + Capability.READ_CONTENT, + Capability.URI, + Capability.WRITE_CONTENT, + Capability.GET_LAST_MODIFIED); + + private static final FileSystemOptions DEFAULT_OPTIONS = new FileSystemOptions(); + + private final IVariables variables; + private final DatabricksVfsConnection vfsConnection; + + public DatabricksFileProvider() { + this(null, null); + } + + public DatabricksFileProvider(IVariables variables, DatabricksVfsConnection vfsConnection) { + super(); + this.variables = variables; + this.vfsConnection = vfsConnection; + setFileNameParser(DatabricksFileNameParser.getInstance()); + } + + @Override + public Collection getCapabilities() { + return CAPABILITIES; + } + + @Override + protected FileSystem doCreateFileSystem(FileName rootName, FileSystemOptions fileSystemOptions) + throws FileSystemException { + FileSystemOptions opts = fileSystemOptions != null ? fileSystemOptions : DEFAULT_OPTIONS; + if (vfsConnection == null || variables == null) { + // Default FSM registration path — no live client until a named provider is used. + return new DatabricksFileSystem(rootName, null, opts, ""); + } + try { + DatabricksConnection connection = resolveDatabricksConnection(); + RestDatabricksFilesClient client = RestDatabricksFilesClient.create(connection, variables); + String rootPath = resolveRootPath(); + return new DatabricksFileSystem(rootName, client, opts, rootPath); + } catch (HopException e) { + throw new FileSystemException( + "Unable to create Databricks Files client for VFS connection '" + + vfsConnection.getName() + + "'", + e); + } + } + + private String resolveRootPath() throws HopException { + String raw = vfsConnection.getDefaultBasePath(); + if (StringUtils.isBlank(raw)) { + return ""; + } + return DatabricksPathResolver.normalizeRootPath(variables.resolve(raw)); + } + + private DatabricksConnection resolveDatabricksConnection() throws HopException { + String connName = vfsConnection.getDatabricksConnectionName(); + if (StringUtils.isBlank(connName)) { + throw new HopException( + "Databricks VFS connection '" + + vfsConnection.getName() + + "' has no Databricks Connection selected"); + } + connName = variables.resolve(connName); + IHopMetadataProvider metadataProvider = HopMetadataInstance.getMetadataProvider(); + if (metadataProvider == null) { + metadataProvider = HopMetadataUtil.getStandardHopMetadataProvider(variables); + } + DatabricksConnection connection = + metadataProvider.getSerializer(DatabricksConnection.class).load(connName); + if (connection == null) { + throw new HopException( + "Databricks Connection '" + + connName + + "' not found (referenced by VFS connection '" + + vfsConnection.getName() + + "')"); + } + return connection; + } +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileSystem.java b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileSystem.java new file mode 100644 index 00000000000..52816eb8862 --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksFileSystem.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import java.util.Collection; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.vfs2.Capability; +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemOptions; +import org.apache.commons.vfs2.provider.AbstractFileName; +import org.apache.commons.vfs2.provider.AbstractFileSystem; +import org.apache.hop.databricks.client.DatabricksFilesClient; + +public class DatabricksFileSystem extends AbstractFileSystem { + + private final DatabricksFilesClient client; + + /** Normalized scheme root, or empty when absolute workspace paths are required. */ + private final String rootPath; + + private final DatabricksListCache listCache = new DatabricksListCache(); + + protected DatabricksFileSystem( + final FileName rootName, + final DatabricksFilesClient client, + final FileSystemOptions fileSystemOptions) { + this(rootName, client, fileSystemOptions, ""); + } + + protected DatabricksFileSystem( + final FileName rootName, + final DatabricksFilesClient client, + final FileSystemOptions fileSystemOptions, + final String rootPath) { + super(rootName, null, fileSystemOptions); + this.client = client; + this.rootPath = rootPath == null ? "" : rootPath; + } + + DatabricksListCache getListCache() { + return listCache; + } + + @Override + protected void addCapabilities(final Collection caps) { + caps.addAll(DatabricksFileProvider.CAPABILITIES); + } + + @Override + protected FileObject createFile(final AbstractFileName name) throws FileSystemException { + return new DatabricksFileObject(name, this); + } + + DatabricksFilesClient getClient() throws FileSystemException { + if (client == null) { + throw new FileSystemException( + "Databricks VFS is not bound to a connection. Use a named Databricks VFS Connection as the URI scheme."); + } + return client; + } + + /** Scheme root (e.g. {@code /Volumes/cat/schema/vol}), or blank. */ + String getRootPath() { + return rootPath; + } + + boolean hasRootPath() { + return StringUtils.isNotBlank(rootPath); + } +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksListCache.java b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksListCache.java new file mode 100644 index 00000000000..ef7dfa52b81 --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksListCache.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.vfs2.FileType; +import org.apache.hop.databricks.client.DirectoryEntry; + +/** + * Short-lived cache of directory list results. When a folder is listed via the Files API, child + * paths and their type/size/lastModified are stored so later {@code doAttach()} calls for those + * children can use metadata without an extra HEAD (and so last-modified is available for + * execution-info date filtering). + */ +public class DatabricksListCache { + + public static final long DEFAULT_TTL_MS = 10_000L; + + public static final class ChildInfo { + public final FileType type; + public final long size; + public final long lastModifiedEpochMs; + + public ChildInfo(FileType type, long size, long lastModifiedEpochMs) { + this.type = type; + this.size = size; + this.lastModifiedEpochMs = lastModifiedEpochMs; + } + + static ChildInfo fromEntry(DirectoryEntry entry) { + FileType type = entry.directory() ? FileType.FOLDER : FileType.FILE; + long size = entry.directory() ? -1L : entry.sizeBytes(); + return new ChildInfo(type, size, entry.lastModifiedEpochMs()); + } + } + + private final long ttlMs; + private final ConcurrentHashMap byParentPath = new ConcurrentHashMap<>(); + + public DatabricksListCache() { + this(DEFAULT_TTL_MS); + } + + public DatabricksListCache(long ttlMs) { + this.ttlMs = ttlMs; + } + + private static class CachedList { + final Map byChildPath; + final long expiryMillis; + + CachedList(Map byChildPath, long expiryMillis) { + this.byChildPath = new ConcurrentHashMap<>(byChildPath); + this.expiryMillis = expiryMillis; + } + } + + /** Store list results for a parent workspace path (absolute Files API path). */ + public void put(String parentWorkspacePath, Map childrenByPath) { + if (childrenByPath == null || childrenByPath.isEmpty()) { + return; + } + String key = normalizePath(parentWorkspacePath); + byParentPath.put(key, new CachedList(childrenByPath, System.currentTimeMillis() + ttlMs)); + } + + /** + * Look up cached metadata for a child workspace path. Returns null if missing or the parent list + * cache expired. + */ + public ChildInfo get(String childWorkspacePath) { + String child = normalizePath(childWorkspacePath); + String parent = parentPath(child); + CachedList cached = byParentPath.get(parent); + if (cached == null) { + return null; + } + if (System.currentTimeMillis() > cached.expiryMillis) { + byParentPath.remove(parent, cached); + return null; + } + ChildInfo info = cached.byChildPath.get(child); + if (info != null) { + return info; + } + // Directories may be listed with or without trailing slash + if (!child.endsWith("/")) { + return cached.byChildPath.get(child + "/"); + } + return cached.byChildPath.get(child.substring(0, child.length() - 1)); + } + + public void invalidateParentOf(String workspacePath) { + byParentPath.remove(parentPath(normalizePath(workspacePath))); + } + + public void invalidate(String parentWorkspacePath) { + byParentPath.remove(normalizePath(parentWorkspacePath)); + } + + static String parentPath(String path) { + if (path == null || path.isEmpty() || "/".equals(path)) { + return "/"; + } + String stripped = + path.endsWith("/") && path.length() > 1 ? path.substring(0, path.length() - 1) : path; + int last = stripped.lastIndexOf('/'); + if (last <= 0) { + return "/"; + } + return stripped.substring(0, last); + } + + static String normalizePath(String path) { + if (path == null || path.isEmpty()) { + return "/"; + } + if (path.length() > 1 && path.endsWith("/")) { + return path.substring(0, path.length() - 1); + } + return path; + } +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksPathResolver.java b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksPathResolver.java new file mode 100644 index 00000000000..44552447861 --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksPathResolver.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.databricks.client.RestDatabricksFilesClient; + +/** + * Maps logical VFS paths ({@code scheme:///input}) to absolute Databricks workspace paths under an + * optional scheme root (metadata default base path). + */ +public final class DatabricksPathResolver { + + private DatabricksPathResolver() {} + + /** + * Normalize a configured root path: trim, strip optional {@code dbfs:}, ensure leading slash, no + * trailing slash (except never return empty for a non-blank input). + */ + public static String normalizeRootPath(String rootPath) throws HopException { + if (StringUtils.isBlank(rootPath)) { + return ""; + } + String p = RestDatabricksFilesClient.normalizeWorkspacePath(rootPath); + while (p.length() > 1 && p.endsWith("/")) { + p = p.substring(0, p.length() - 1); + } + if (!RestDatabricksFilesClient.isFilesApiPath(p)) { + throw new HopException( + "Databricks VFS root path must be under /Volumes/… or /Workspace/…, got: " + p); + } + return p; + } + + /** + * Resolve a logical VFS path to a workspace path. + * + * @param logicalPath path from VFS {@link org.apache.commons.vfs2.FileName#getPath()} (e.g. + * {@code /input} or {@code /}) + * @param rootPath normalized scheme root, or empty when URIs must be absolute workspace paths + */ + public static String toWorkspacePath(String logicalPath, String rootPath) { + String logical = logicalPath; + if (StringUtils.isBlank(logical) || "/".equals(logical)) { + logical = "/"; + } else { + while (logical.length() > 1 && logical.endsWith("/")) { + logical = logical.substring(0, logical.length() - 1); + } + if (!logical.startsWith("/")) { + logical = "/" + logical; + } + } + + String root = StringUtils.isBlank(rootPath) ? "" : rootPath.trim(); + if (StringUtils.isBlank(root)) { + return logical; + } + + // Escape hatch: full workspace path in the URI even when a root is configured + if (RestDatabricksFilesClient.isFilesApiPath(logical)) { + return logical; + } + + if ("/".equals(logical)) { + return root; + } + // root has no trailing slash; logical starts with / + return root + logical; + } +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksVfsPlugin.java b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksVfsPlugin.java new file mode 100644 index 00000000000..dd6b7a43292 --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksVfsPlugin.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.vfs2.provider.FileProvider; +import org.apache.hop.core.logging.LogChannel; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.plugin.IVfs; +import org.apache.hop.core.vfs.plugin.VfsPlugin; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.metadata.util.HopMetadataInstance; +import org.apache.hop.metadata.util.HopMetadataUtil; +import org.apache.hop.vfs.databricks.metadata.DatabricksVfsConnection; + +/** + * Registers one Commons VFS scheme per named {@link DatabricksVfsConnection}. Auth is resolved from + * the referenced Databricks Connection. URIs look like {@code + * my-vfs:///Volumes/catalog/schema/volume/file.csv}. + */ +@VfsPlugin( + type = "databricks-vfs-connection", + typeDescription = "Databricks Volumes / Workspace VFS (named connections)", + classLoaderGroup = "vfs-databricks") +public class DatabricksVfsPlugin implements IVfs { + + @Override + public String[] getUrlSchemes() { + // Schemes come from named Databricks VFS Connection metadata. + return new String[] {}; + } + + @Override + public FileProvider getProvider() { + return new DatabricksFileProvider(); + } + + @Override + public Map getProviders(IVariables variables) { + Map providers = new HashMap<>(); + try { + // Prefer the active runtime metadata (GUI project / MainSpark export). Standard folders + // alone are empty on Databricks (no hop-config metadata tree). + IHopMetadataProvider metadataProvider = HopMetadataInstance.getMetadataProvider(); + if (metadataProvider == null) { + metadataProvider = HopMetadataUtil.getStandardHopMetadataProvider(variables); + } + List connections = + metadataProvider.getSerializer(DatabricksVfsConnection.class).loadAll(); + for (DatabricksVfsConnection connection : connections) { + String name = connection.getName(); + if (StringUtils.isEmpty(name)) { + continue; + } + providers.put(name, new DatabricksFileProvider(variables, connection)); + } + } catch (Exception e) { + LogChannel.GENERAL.logError("Unable to load Databricks VFS providers", e); + } + return providers; + } +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnection.java b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnection.java new file mode 100644 index 00000000000..382cc33c984 --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnection.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks.metadata; + +import java.io.Serializable; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.gui.plugin.GuiElementType; +import org.apache.hop.core.gui.plugin.GuiPlugin; +import org.apache.hop.core.gui.plugin.GuiWidgetElement; +import org.apache.hop.databricks.metadata.DatabricksConnection; +import org.apache.hop.metadata.api.HopMetadata; +import org.apache.hop.metadata.api.HopMetadataBase; +import org.apache.hop.metadata.api.HopMetadataCategory; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.HopMetadataPropertyType; +import org.apache.hop.metadata.api.IHopMetadata; + +/** + * Named Databricks Volumes / Workspace VFS endpoint. The metadata name is the VFS + * URI scheme (for example {@code dbx-jars:///Volumes/...}). Authentication comes from a referenced + * {@link DatabricksConnection} (host + PAT), not from credentials stored on this object. + */ +@Getter +@Setter +@GuiPlugin +@HopMetadata( + key = "DatabricksVfsConnectionDefinition", + name = "i18n::DatabricksVfsConnection.Name", + description = "i18n::DatabricksVfsConnection.Description", + image = "databricks-connection.svg", + category = HopMetadataCategory.FILE_STORAGE, + documentationUrl = "/metadata-types/databricks-vfs-connection.html", + hopMetadataPropertyType = HopMetadataPropertyType.VFS_DATABRICKS_CONNECTION) +public class DatabricksVfsConnection extends HopMetadataBase implements Serializable, IHopMetadata { + + private static final String PARENT = DatabricksVfsConnectionEditor.GUI_WIDGETS_PARENT_ID; + + @GuiWidgetElement( + id = "10000-description", + parentId = PARENT, + type = GuiElementType.TEXT, + label = "i18n::DatabricksVfsConnection.Description.Label", + toolTip = "i18n::DatabricksVfsConnection.Description.Tooltip") + @HopMetadataProperty + private String description; + + /** + * Name of the {@link DatabricksConnection} that supplies workspace host and personal access + * token. + */ + @GuiWidgetElement( + id = "10010-databricks-connection", + parentId = PARENT, + type = GuiElementType.METADATA, + metadata = DatabricksConnection.class, + label = "i18n::DatabricksVfsConnection.Connection.Label", + toolTip = "i18n::DatabricksVfsConnection.Connection.Tooltip") + @HopMetadataProperty + private String databricksConnectionName; + + /** + * Optional absolute workspace root for this scheme (for example {@code + * /Volumes/catalog/schema/volume}). When set, short URIs are relative to it: {@code + * name:///input} → {@code /Volumes/…/volume/input}. When blank, URIs must use full {@code + * /Volumes/…} or {@code /Workspace/…} paths. + */ + @GuiWidgetElement( + id = "10020-default-base-path", + parentId = PARENT, + type = GuiElementType.TEXT, + label = "i18n::DatabricksVfsConnection.DefaultBasePath.Label", + toolTip = "i18n::DatabricksVfsConnection.DefaultBasePath.Tooltip") + @HopMetadataProperty + private String defaultBasePath; + + public DatabricksVfsConnection() { + // defaults + } +} diff --git a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnectionEditor.java b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnectionEditor.java new file mode 100644 index 00000000000..faa824caa62 --- /dev/null +++ b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnectionEditor.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks.metadata; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.Const; +import org.apache.hop.core.gui.plugin.GuiPlugin; +import org.apache.hop.databricks.client.DatabricksJobsClient; +import org.apache.hop.databricks.client.RestDatabricksJobsClient; +import org.apache.hop.databricks.metadata.DatabricksConnection; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.dialog.MessageBox; +import org.apache.hop.ui.core.gui.GuiCompositeWidgets; +import org.apache.hop.ui.core.gui.GuiCompositeWidgetsAdapter; +import org.apache.hop.ui.core.metadata.MetadataEditor; +import org.apache.hop.ui.core.metadata.MetadataManager; +import org.apache.hop.ui.hopgui.HopGui; +import org.eclipse.swt.SWT; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Text; + +@GuiPlugin(description = "Editor for Databricks Volumes / Workspace VFS metadata") +public class DatabricksVfsConnectionEditor extends MetadataEditor { + + private static final Class PKG = DatabricksVfsConnection.class; + + public static final String GUI_WIDGETS_PARENT_ID = + "DatabricksVfsConnectionEditor-GuiWidgetsParent"; + + private Text wName; + private Composite wWidgetsComposite; + private GuiCompositeWidgets guiCompositeWidgets; + + public DatabricksVfsConnectionEditor( + HopGui hopGui, + MetadataManager manager, + DatabricksVfsConnection metadata) { + super(hopGui, manager, metadata); + } + + @Override + public void createControl(Composite parent) { + PropsUi props = PropsUi.getInstance(); + int middle = props.getMiddlePct(); + int margin = PropsUi.getMargin() + 2; + + Label wIcon = new Label(parent, SWT.RIGHT); + wIcon.setImage(getImage()); + FormData fdlIcon = new FormData(); + fdlIcon.top = new FormAttachment(0, 0); + fdlIcon.right = new FormAttachment(100, 0); + wIcon.setLayoutData(fdlIcon); + PropsUi.setLook(wIcon); + + Label wlName = new Label(parent, SWT.RIGHT); + PropsUi.setLook(wlName); + wlName.setText(BaseMessages.getString(PKG, "DatabricksVfsConnection.Name.Label")); + FormData fdlName = new FormData(); + fdlName.top = new FormAttachment(0, margin); + fdlName.left = new FormAttachment(0, 0); + fdlName.right = new FormAttachment(middle, -margin); + wlName.setLayoutData(fdlName); + wName = new Text(parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wName); + FormData fdName = new FormData(); + fdName.top = new FormAttachment(wlName, 0, SWT.CENTER); + fdName.left = new FormAttachment(middle, 0); + fdName.right = new FormAttachment(wIcon, -margin); + wName.setLayoutData(fdName); + Control lastControl = wName; + + wWidgetsComposite = new Composite(parent, SWT.NONE); + PropsUi.setLook(wWidgetsComposite); + wWidgetsComposite.setLayout(new FormLayout()); + FormData fdWidgetsComposite = new FormData(); + fdWidgetsComposite.top = new FormAttachment(lastControl, margin); + fdWidgetsComposite.left = new FormAttachment(0, 0); + fdWidgetsComposite.right = new FormAttachment(100, 0); + fdWidgetsComposite.bottom = new FormAttachment(100, -margin * 4); + wWidgetsComposite.setLayoutData(fdWidgetsComposite); + + guiCompositeWidgets = new GuiCompositeWidgets(manager.getVariables()); + guiCompositeWidgets.createCompositeWidgets( + metadata, null, wWidgetsComposite, GUI_WIDGETS_PARENT_ID, lastControl); + guiCompositeWidgets.setWidgetsListener( + new GuiCompositeWidgetsAdapter() { + @Override + public void widgetModified( + GuiCompositeWidgets compositeWidgets, Control changedWidget, String widgetId) { + setChanged(); + } + }); + + Button wTest = new Button(parent, SWT.PUSH); + PropsUi.setLook(wTest); + wTest.setText(BaseMessages.getString(PKG, "DatabricksVfsConnection.Test.Label")); + FormData fdTest = new FormData(); + fdTest.bottom = new FormAttachment(100, 0); + fdTest.left = new FormAttachment(middle, 0); + wTest.setLayoutData(fdTest); + wTest.addListener(SWT.Selection, e -> testConnection()); + + setWidgetsContent(); + resetChanged(); + wName.addModifyListener(e -> setChanged()); + } + + private void testConnection() { + try { + getWidgetsContent(getMetadata()); + DatabricksVfsConnection vfs = getMetadata(); + if (StringUtils.isBlank(vfs.getDatabricksConnectionName())) { + throw new IllegalStateException( + BaseMessages.getString(PKG, "DatabricksVfsConnection.Test.MissingConnection")); + } + IHopMetadataProvider provider = manager.getMetadataProvider(); + DatabricksConnection conn = + provider + .getSerializer(DatabricksConnection.class) + .load(vfs.getDatabricksConnectionName()); + if (conn == null) { + throw new IllegalStateException( + BaseMessages.getString( + PKG, + "DatabricksVfsConnection.Test.ConnectionNotFound", + vfs.getDatabricksConnectionName())); + } + try (DatabricksJobsClient client = + RestDatabricksJobsClient.create(conn, manager.getVariables())) { + String who = client.testConnection(); + MessageBox box = + new MessageBox(HopGui.getInstance().getShell(), SWT.OK | SWT.ICON_INFORMATION); + box.setText(BaseMessages.getString(PKG, "DatabricksVfsConnection.Test.Success.Title")); + box.setMessage( + BaseMessages.getString(PKG, "DatabricksVfsConnection.Test.Success.Message", who)); + box.open(); + } + } catch (Exception e) { + new ErrorDialog( + HopGui.getInstance().getShell(), + BaseMessages.getString(PKG, "DatabricksVfsConnection.Test.Error.Title"), + BaseMessages.getString(PKG, "DatabricksVfsConnection.Test.Error.Message"), + e); + } + } + + @Override + public void setWidgetsContent() { + DatabricksVfsConnection meta = this.getMetadata(); + wName.setText(Const.NVL(meta.getName(), "")); + guiCompositeWidgets.setWidgetsContents(metadata, wWidgetsComposite, GUI_WIDGETS_PARENT_ID); + } + + @Override + public void getWidgetsContent(DatabricksVfsConnection meta) { + meta.setName(wName.getText()); + guiCompositeWidgets.getWidgetsContents(metadata, GUI_WIDGETS_PARENT_ID); + } + + @Override + public boolean setFocus() { + if (wName == null || wName.isDisposed()) { + return false; + } + return wName.setFocus(); + } +} diff --git a/plugins/tech/databricks/src/main/resources/databricks-connection.svg b/plugins/tech/databricks/src/main/resources/databricks-connection.svg index e9c86c7a1dc..198e679b1e9 100644 --- a/plugins/tech/databricks/src/main/resources/databricks-connection.svg +++ b/plugins/tech/databricks/src/main/resources/databricks-connection.svg @@ -16,8 +16,8 @@ limitations under the License. --> - - - - + diff --git a/plugins/tech/databricks/src/main/resources/databricks-job-run.svg b/plugins/tech/databricks/src/main/resources/databricks-job-run.svg deleted file mode 100644 index 5b74b5c3f70..00000000000 --- a/plugins/tech/databricks/src/main/resources/databricks-job-run.svg +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - diff --git a/plugins/tech/databricks/src/main/resources/databricks-job-wait.svg b/plugins/tech/databricks/src/main/resources/databricks-job-wait.svg deleted file mode 100644 index fcf3d78b8c3..00000000000 --- a/plugins/tech/databricks/src/main/resources/databricks-job-wait.svg +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - diff --git a/plugins/tech/databricks/src/main/resources/databricks-logo.svg b/plugins/tech/databricks/src/main/resources/databricks-logo.svg deleted file mode 100644 index 4928c857888..00000000000 --- a/plugins/tech/databricks/src/main/resources/databricks-logo.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - diff --git a/plugins/tech/databricks/src/main/resources/org/apache/hop/vfs/databricks/metadata/messages/messages_en_US.properties b/plugins/tech/databricks/src/main/resources/org/apache/hop/vfs/databricks/metadata/messages/messages_en_US.properties new file mode 100644 index 00000000000..83bc0802405 --- /dev/null +++ b/plugins/tech/databricks/src/main/resources/org/apache/hop/vfs/databricks/metadata/messages/messages_en_US.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +DatabricksVfsConnection.Name=Databricks VFS Connection +DatabricksVfsConnection.Description=Named VFS scheme for Unity Catalog Volumes and Workspace files (Files API) +DatabricksVfsConnection.Name.Label=Name (URI scheme) +DatabricksVfsConnection.Description.Label=Description +DatabricksVfsConnection.Description.Tooltip=Optional free-text description +DatabricksVfsConnection.Connection.Label=Databricks Connection +DatabricksVfsConnection.Connection.Tooltip=Jobs/workspace connection that provides host and personal access token +DatabricksVfsConnection.DefaultBasePath.Label=Root path +DatabricksVfsConnection.DefaultBasePath.Tooltip=Absolute UC Volume or Workspace path that is the root of this scheme. Shortens URIs: name:///input maps to /input. Leave empty to require full /Volumes/… or /Workspace/… paths in every URI. +DatabricksVfsConnection.Test.Label=Test connection +DatabricksVfsConnection.Test.Success.Title=Connection OK +DatabricksVfsConnection.Test.Success.Message=Referenced Databricks connection works as: {0} +DatabricksVfsConnection.Test.Error.Title=Connection failed +DatabricksVfsConnection.Test.Error.Message=Unable to validate the referenced Databricks connection +DatabricksVfsConnection.Test.MissingConnection=Select a Databricks Connection first +DatabricksVfsConnection.Test.ConnectionNotFound=Databricks Connection not found: {0} diff --git a/plugins/tech/databricks/src/test/java/org/apache/hop/databricks/client/RestDatabricksFilesClientTest.java b/plugins/tech/databricks/src/test/java/org/apache/hop/databricks/client/RestDatabricksFilesClientTest.java new file mode 100644 index 00000000000..d676d1d7879 --- /dev/null +++ b/plugins/tech/databricks/src/test/java/org/apache/hop/databricks/client/RestDatabricksFilesClientTest.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.databricks.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import java.io.InputStream; +import java.net.http.HttpClient; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.apache.hop.core.exception.HopException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class RestDatabricksFilesClientTest { + + private WireMockServer server; + private RestDatabricksFilesClient client; + + @BeforeEach + void setUp() { + server = new WireMockServer(WireMockConfiguration.wireMockConfig().dynamicPort()); + server.start(); + WireMock.configureFor("localhost", server.port()); + client = + RestDatabricksFilesClient.createForTest( + "http://localhost:" + server.port(), "test-token", HttpClient.newHttpClient()); + } + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(); + } + } + + @Test + void isFilesApiPathDetectsVolumesAndWorkspace() { + assertTrue(RestDatabricksFilesClient.isFilesApiPath("/Volumes/c/s/v/file.jar")); + assertTrue(RestDatabricksFilesClient.isFilesApiPath("/Workspace/Users/a@b.com/x")); + assertFalse(RestDatabricksFilesClient.isFilesApiPath("/FileStore/hop/x.jar")); + } + + @Test + void requireFilesApiPathRejectsClassicDbfs() { + HopException ex = + assertThrows( + HopException.class, + () -> RestDatabricksFilesClient.requireFilesApiPath("/FileStore/hop/x.jar")); + assertTrue(ex.getMessage().contains("Files API")); + } + + @Test + void listDirectoryPaginates() throws Exception { + server.stubFor( + WireMock.get(WireMock.urlEqualTo("/api/2.0/fs/directories/Volumes/apache-hop/default/jars")) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + """ + {"contents":[ + {"name":"a.jar","path":"/Volumes/apache-hop/default/jars/a.jar","is_directory":false,"file_size":10,"last_modified":1000} + ],"next_page_token":"page2"} + """))); + server.stubFor( + WireMock.get( + WireMock.urlEqualTo( + "/api/2.0/fs/directories/Volumes/apache-hop/default/jars?page_token=page2")) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + """ + {"contents":[ + {"name":"sub","path":"/Volumes/apache-hop/default/jars/sub","is_directory":true,"last_modified":2000} + ]} + """))); + + List entries = client.listDirectory("/Volumes/apache-hop/default/jars"); + assertEquals(2, entries.size()); + assertEquals("a.jar", entries.get(0).name()); + assertFalse(entries.get(0).directory()); + assertEquals(10L, entries.get(0).sizeBytes()); + assertEquals("sub", entries.get(1).name()); + assertTrue(entries.get(1).directory()); + } + + @Test + void createDirectoryPutsWithTrailingSlash() throws Exception { + server.stubFor( + WireMock.put( + WireMock.urlEqualTo( + "/api/2.0/fs/directories/Volumes/apache-hop/default/jars/newdir/")) + .withHeader("Authorization", WireMock.equalTo("Bearer test-token")) + .willReturn(WireMock.aResponse().withStatus(204))); + + client.createDirectory("/Volumes/apache-hop/default/jars/newdir"); + + server.verify( + WireMock.putRequestedFor( + WireMock.urlEqualTo( + "/api/2.0/fs/directories/Volumes/apache-hop/default/jars/newdir/"))); + } + + @Test + void deleteFileUsesFilesApi() throws Exception { + server.stubFor( + WireMock.delete( + WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/apache-hop/default/jars/old.jar")) + .willReturn(WireMock.aResponse().withStatus(204))); + + client.deleteFile("/Volumes/apache-hop/default/jars/old.jar"); + + server.verify( + WireMock.deleteRequestedFor( + WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/apache-hop/default/jars/old.jar"))); + } + + @Test + void deleteDirectoryDoesNotSendRecursiveQueryParam() throws Exception { + // Empty directory: no list children needed beyond empty contents + server.stubFor( + WireMock.get( + WireMock.urlEqualTo("/api/2.0/fs/directories/Volumes/apache-hop/default/jars/tmp")) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody("{\"contents\":[]}"))); + server.stubFor( + WireMock.delete( + WireMock.urlEqualTo("/api/2.0/fs/directories/Volumes/apache-hop/default/jars/tmp/")) + .willReturn(WireMock.aResponse().withStatus(204))); + + client.deleteDirectory("/Volumes/apache-hop/default/jars/tmp", true); + + server.verify( + WireMock.deleteRequestedFor( + WireMock.urlEqualTo("/api/2.0/fs/directories/Volumes/apache-hop/default/jars/tmp/"))); + // API rejects ?recursive=… + server.verify( + 0, WireMock.deleteRequestedFor(WireMock.urlMatching(".*/fs/directories/.*recursive.*"))); + } + + @Test + void deleteDirectoryRecursiveDeletesChildrenThenDir() throws Exception { + server.stubFor( + WireMock.get( + WireMock.urlEqualTo("/api/2.0/fs/directories/Volumes/apache-hop/default/jars/tmp")) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + """ + {"contents":[ + {"name":"f.txt","path":"/Volumes/apache-hop/default/jars/tmp/f.txt","is_directory":false,"file_size":1} + ]} + """))); + server.stubFor( + WireMock.delete( + WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/apache-hop/default/jars/tmp/f.txt")) + .willReturn(WireMock.aResponse().withStatus(204))); + server.stubFor( + WireMock.delete( + WireMock.urlEqualTo("/api/2.0/fs/directories/Volumes/apache-hop/default/jars/tmp/")) + .willReturn(WireMock.aResponse().withStatus(204))); + + client.deleteDirectory("/Volumes/apache-hop/default/jars/tmp", true); + + server.verify( + WireMock.deleteRequestedFor( + WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/apache-hop/default/jars/tmp/f.txt"))); + server.verify( + WireMock.deleteRequestedFor( + WireMock.urlEqualTo("/api/2.0/fs/directories/Volumes/apache-hop/default/jars/tmp/"))); + } + + @Test + void openInputStreamDownloadsBytes() throws Exception { + byte[] payload = "hello-volume".getBytes(StandardCharsets.UTF_8); + server.stubFor( + WireMock.get( + WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/apache-hop/default/jars/hello.txt")) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/octet-stream") + .withBody(payload))); + + try (InputStream in = client.openInputStream("/Volumes/apache-hop/default/jars/hello.txt")) { + assertEquals("hello-volume", new String(in.readAllBytes(), StandardCharsets.UTF_8)); + } + } + + @Test + void uploadUsesPutOverwrite() throws Exception { + Path temp = Files.createTempFile("hop-dbx-files-", ".bin"); + try { + byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + Files.write(temp, payload); + server.stubFor( + WireMock.put( + WireMock.urlEqualTo( + "/api/2.0/fs/files/Volumes/apache-hop/default/jars/x.bin?overwrite=true")) + .withRequestBody(WireMock.binaryEqualTo(payload)) + .willReturn(WireMock.aResponse().withStatus(204))); + + client.upload(temp, "/Volumes/apache-hop/default/jars/x.bin"); + + server.verify( + WireMock.putRequestedFor( + WireMock.urlEqualTo( + "/api/2.0/fs/files/Volumes/apache-hop/default/jars/x.bin?overwrite=true"))); + } finally { + Files.deleteIfExists(temp); + } + } +} diff --git a/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksFileObjectTest.java b/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksFileObjectTest.java new file mode 100644 index 00000000000..1078e584eea --- /dev/null +++ b/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksFileObjectTest.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import java.io.OutputStream; +import java.net.http.HttpClient; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; +import org.apache.commons.vfs2.Capability; +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystem; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemOptions; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.impl.DefaultFileSystemManager; +import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider; +import org.apache.hop.databricks.client.RestDatabricksFilesClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class DatabricksFileObjectTest { + + private WireMockServer server; + private RestDatabricksFilesClient client; + private DefaultFileSystemManager fsm; + private DefaultFileSystemManager rootedFsm; + + @BeforeEach + void setUp() throws Exception { + server = new WireMockServer(WireMockConfiguration.wireMockConfig().dynamicPort()); + server.start(); + WireMock.configureFor("localhost", server.port()); + client = + RestDatabricksFilesClient.createForTest( + "http://localhost:" + server.port(), "test-token", HttpClient.newHttpClient()); + + fsm = new DefaultFileSystemManager(); + fsm.addProvider("testdbx", new TestProvider(client, "")); + fsm.init(); + + rootedFsm = new DefaultFileSystemManager(); + rootedFsm.addProvider("dbvol", new TestProvider(client, "/Volumes/apache-hop/default/testing")); + rootedFsm.init(); + } + + @AfterEach + void tearDown() { + if (fsm != null) { + fsm.close(); + } + if (rootedFsm != null) { + rootedFsm.close(); + } + if (server != null) { + server.stop(); + } + } + + private FileObject resolve(String path) throws Exception { + return fsm.resolveFile("testdbx://" + path); + } + + private FileObject resolveRooted(String path) throws Exception { + return rootedFsm.resolveFile("dbvol://" + path); + } + + @Test + void listDirectoryReturnsChildren() throws Exception { + server.stubFor( + WireMock.head(WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/c/s/v")) + .willReturn(WireMock.aResponse().withStatus(404))); + server.stubFor( + WireMock.get(WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/c/s/v")) + .willReturn(WireMock.aResponse().withStatus(404))); + server.stubFor( + WireMock.get(WireMock.urlEqualTo("/api/2.0/fs/directories/Volumes/c/s/v")) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + """ + {"contents":[ + {"name":"data.csv","path":"/Volumes/c/s/v/data.csv","is_directory":false,"file_size":42,"last_modified":1}, + {"name":"subdir","path":"/Volumes/c/s/v/subdir","is_directory":true,"last_modified":2} + ]} + """))); + + FileObject dir = resolve("/Volumes/c/s/v"); + assertEquals(FileType.FOLDER, dir.getType()); + String[] names = + Arrays.stream(dir.getChildren()) + .map(fo -> fo.getName().getBaseName()) + .sorted() + .toArray(String[]::new); + assertArrayEquals(new String[] {"data.csv", "subdir"}, names); + } + + @Test + void listCacheProvidesLastModifiedOnChildAttach() throws Exception { + long mtime = 1_700_000_000_000L; + server.stubFor( + WireMock.head(WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/c/s/v")) + .willReturn(WireMock.aResponse().withStatus(404))); + server.stubFor( + WireMock.get(WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/c/s/v")) + .willReturn(WireMock.aResponse().withStatus(404))); + server.stubFor( + WireMock.get(WireMock.urlEqualTo("/api/2.0/fs/directories/Volumes/c/s/v")) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + """ + {"contents":[ + {"name":"run.json","path":"/Volumes/c/s/v/run.json","is_directory":false,"file_size":99,"last_modified":%d} + ]} + """ + .formatted(mtime)))); + + FileObject dir = resolve("/Volumes/c/s/v"); + FileObject[] children = dir.getChildren(); + assertEquals(1, children.length); + assertEquals(FileType.FILE, children[0].getType()); + assertEquals(99L, children[0].getContent().getSize()); + assertEquals(mtime, children[0].getContent().getLastModifiedTime()); + } + + @Test + void writeThenReadRoundTrip() throws Exception { + byte[] payload = "vfs-round-trip".getBytes(StandardCharsets.UTF_8); + server.stubFor( + WireMock.head(WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/c/s/v/out.txt")) + .willReturn(WireMock.aResponse().withStatus(404))); + server.stubFor( + WireMock.get(WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/c/s/v/out.txt")) + .willReturn(WireMock.aResponse().withStatus(404))); + server.stubFor( + WireMock.get(WireMock.urlEqualTo("/api/2.0/fs/directories/Volumes/c/s/v/out.txt")) + .willReturn(WireMock.aResponse().withStatus(404))); + server.stubFor( + WireMock.put(WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/c/s/v/out.txt?overwrite=true")) + .withRequestBody(WireMock.binaryEqualTo(payload)) + .willReturn(WireMock.aResponse().withStatus(204))); + + FileObject file = resolve("/Volumes/c/s/v/out.txt"); + try (OutputStream out = file.getContent().getOutputStream()) { + out.write(payload); + } + + server.stubFor( + WireMock.head(WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/c/s/v/out.txt")) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(payload.length)))); + server.stubFor( + WireMock.get(WireMock.urlEqualTo("/api/2.0/fs/files/Volumes/c/s/v/out.txt")) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/octet-stream") + .withBody(payload))); + + fsm.getFilesCache().clear(file.getFileSystem()); + FileObject written = resolve("/Volumes/c/s/v/out.txt"); + assertEquals(FileType.FILE, written.getType()); + try (var in = written.getContent().getInputStream()) { + assertEquals("vfs-round-trip", new String(in.readAllBytes(), StandardCharsets.UTF_8)); + } + } + + @Test + void createFolderUnderRootPathUsesJoinedWorkspacePath() throws Exception { + server.stubFor( + WireMock.get( + WireMock.urlEqualTo("/api/2.0/fs/directories/Volumes/apache-hop/default/testing")) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody("{\"contents\":[]}"))); + server.stubFor( + WireMock.put( + WireMock.urlEqualTo( + "/api/2.0/fs/directories/Volumes/apache-hop/default/testing/input/")) + .withHeader("Authorization", WireMock.equalTo("Bearer test-token")) + .willReturn(WireMock.aResponse().withStatus(204))); + + FileObject folder = resolveRooted("/input"); + folder.createFolder(); + + server.verify( + WireMock.putRequestedFor( + WireMock.urlEqualTo( + "/api/2.0/fs/directories/Volumes/apache-hop/default/testing/input/"))); + assertTrue(folder.exists()); + } + + @Test + void renameFileCopiesThenDeletesViaFilesApi() throws Exception { + byte[] payload = "rename-me".getBytes(StandardCharsets.UTF_8); + String src = "/Volumes/apache-hop/default/testing/input/a.txt"; + String dst = "/Volumes/apache-hop/default/testing/processing/a.txt"; + + server.stubFor( + WireMock.head(WireMock.urlEqualTo("/api/2.0/fs/files" + src)) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Length", String.valueOf(payload.length)))); + server.stubFor( + WireMock.get(WireMock.urlEqualTo("/api/2.0/fs/files" + src)) + .willReturn( + WireMock.aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/octet-stream") + .withBody(payload))); + server.stubFor( + WireMock.put(WireMock.urlEqualTo("/api/2.0/fs/files" + dst + "?overwrite=true")) + .withRequestBody(WireMock.binaryEqualTo(payload)) + .willReturn(WireMock.aResponse().withStatus(204))); + server.stubFor( + WireMock.delete(WireMock.urlEqualTo("/api/2.0/fs/files" + src)) + .willReturn(WireMock.aResponse().withStatus(204))); + + FileObject source = resolveRooted("/input/a.txt"); + FileObject target = resolveRooted("/processing/a.txt"); + source.moveTo(target); + + server.verify( + WireMock.putRequestedFor( + WireMock.urlEqualTo("/api/2.0/fs/files" + dst + "?overwrite=true"))); + server.verify(WireMock.deleteRequestedFor(WireMock.urlEqualTo("/api/2.0/fs/files" + src))); + } + + /** Provider that injects a pre-built Files client and optional scheme root. */ + private static final class TestProvider extends AbstractOriginatingFileProvider { + private final RestDatabricksFilesClient client; + private final String rootPath; + + TestProvider(RestDatabricksFilesClient client, String rootPath) { + this.client = client; + this.rootPath = rootPath == null ? "" : rootPath; + setFileNameParser(DatabricksFileNameParser.getInstance()); + } + + @Override + public Collection getCapabilities() { + return DatabricksFileProvider.CAPABILITIES; + } + + @Override + protected FileSystem doCreateFileSystem(FileName rootName, FileSystemOptions fileSystemOptions) + throws FileSystemException { + FileSystemOptions opts = + fileSystemOptions != null ? fileSystemOptions : new FileSystemOptions(); + return new DatabricksFileSystem(rootName, client, opts, rootPath); + } + } +} diff --git a/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksListCacheTest.java b/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksListCacheTest.java new file mode 100644 index 00000000000..b1319a83662 --- /dev/null +++ b/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksListCacheTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.Map; +import org.apache.commons.vfs2.FileType; +import org.junit.jupiter.api.Test; + +class DatabricksListCacheTest { + + @Test + void putAndGetByChildPath() { + DatabricksListCache cache = new DatabricksListCache(60_000L); + cache.put( + "/Volumes/c/s/v", + Map.of( + "/Volumes/c/s/v/a.json", new DatabricksListCache.ChildInfo(FileType.FILE, 10L, 1234L))); + + DatabricksListCache.ChildInfo info = cache.get("/Volumes/c/s/v/a.json"); + assertNotNull(info); + assertEquals(FileType.FILE, info.type); + assertEquals(10L, info.size); + assertEquals(1234L, info.lastModifiedEpochMs); + } + + @Test + void expiredEntriesAreDropped() throws Exception { + DatabricksListCache cache = new DatabricksListCache(1L); + cache.put( + "/Volumes/c/s/v", + Map.of( + "/Volumes/c/s/v/a.json", new DatabricksListCache.ChildInfo(FileType.FILE, 10L, 1234L))); + Thread.sleep(5L); + assertNull(cache.get("/Volumes/c/s/v/a.json")); + } + + @Test + void parentPathHelpers() { + assertEquals("/Volumes/c/s/v", DatabricksListCache.parentPath("/Volumes/c/s/v/a.json")); + assertEquals("/", DatabricksListCache.parentPath("/Volumes")); + assertEquals("/Volumes/c/s/v", DatabricksListCache.normalizePath("/Volumes/c/s/v/")); + } +} diff --git a/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksPathResolverTest.java b/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksPathResolverTest.java new file mode 100644 index 00000000000..a7981435fc3 --- /dev/null +++ b/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksPathResolverTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hop.core.exception.HopException; +import org.junit.jupiter.api.Test; + +class DatabricksPathResolverTest { + + @Test + void normalizeRootPathStripsSlashAndValidates() throws Exception { + assertEquals( + "/Volumes/apache-hop/default/testing", + DatabricksPathResolver.normalizeRootPath("/Volumes/apache-hop/default/testing/")); + assertEquals("/Volumes/c/s/v", DatabricksPathResolver.normalizeRootPath("dbfs:/Volumes/c/s/v")); + } + + @Test + void normalizeRootPathRejectsNonFilesApi() { + HopException ex = + assertThrows( + HopException.class, () -> DatabricksPathResolver.normalizeRootPath("/FileStore/x")); + assertTrue(ex.getMessage().contains("/Volumes")); + } + + @Test + void toWorkspacePathWithoutRootKeepsLogical() { + assertEquals("/input", DatabricksPathResolver.toWorkspacePath("/input", "")); + assertEquals( + "/Volumes/c/s/v/x", DatabricksPathResolver.toWorkspacePath("/Volumes/c/s/v/x", null)); + } + + @Test + void toWorkspacePathWithRootJoinsRelative() { + String root = "/Volumes/apache-hop/default/testing"; + assertEquals(root, DatabricksPathResolver.toWorkspacePath("/", root)); + assertEquals(root, DatabricksPathResolver.toWorkspacePath("", root)); + assertEquals(root + "/input", DatabricksPathResolver.toWorkspacePath("/input", root)); + assertEquals( + root + "/input/customers-1M.txt", + DatabricksPathResolver.toWorkspacePath("/input/customers-1M.txt", root)); + } + + @Test + void toWorkspacePathWithRootAllowsAbsoluteEscape() { + String root = "/Volumes/apache-hop/default/testing"; + assertEquals( + "/Volumes/other/default/vol/file", + DatabricksPathResolver.toWorkspacePath("/Volumes/other/default/vol/file", root)); + } +} diff --git a/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksVfsPluginTest.java b/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksVfsPluginTest.java new file mode 100644 index 00000000000..120d4192c31 --- /dev/null +++ b/plugins/tech/databricks/src/test/java/org/apache/hop/vfs/databricks/DatabricksVfsPluginTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.databricks; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; +import org.apache.commons.vfs2.provider.FileProvider; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.vfs.databricks.metadata.DatabricksVfsConnection; +import org.junit.jupiter.api.Test; + +class DatabricksVfsPluginTest { + + @Test + void urlSchemesEmptyLikeMinio() { + DatabricksVfsPlugin plugin = new DatabricksVfsPlugin(); + assertEquals(0, plugin.getUrlSchemes().length); + } + + @Test + void getProviderReturnsDatabricksProvider() { + DatabricksVfsPlugin plugin = new DatabricksVfsPlugin(); + FileProvider provider = plugin.getProvider(); + assertNotNull(provider); + assertTrue(provider instanceof DatabricksFileProvider); + } + + @Test + void getProvidersSurvivesMissingMetadata() { + DatabricksVfsPlugin plugin = new DatabricksVfsPlugin(); + Map providers = plugin.getProviders(new Variables()); + assertNotNull(providers); + } + + @Test + void fileProviderAcceptsVfsConnection() { + DatabricksVfsConnection vfs = new DatabricksVfsConnection(); + vfs.setName("dbx-jars"); + vfs.setDatabricksConnectionName("prod-workspace"); + DatabricksFileProvider provider = new DatabricksFileProvider(new Variables(), vfs); + assertNotNull(provider); + assertTrue( + provider.getCapabilities().contains(org.apache.commons.vfs2.Capability.READ_CONTENT)); + } +} diff --git a/ui/src/main/java/org/apache/hop/ui/core/vfs/HopVfsFileDialog.java b/ui/src/main/java/org/apache/hop/ui/core/vfs/HopVfsFileDialog.java index 31646d696ff..76cd073df0e 100644 --- a/ui/src/main/java/org/apache/hop/ui/core/vfs/HopVfsFileDialog.java +++ b/ui/src/main/java/org/apache/hop/ui/core/vfs/HopVfsFileDialog.java @@ -985,40 +985,30 @@ private String getTreeItemPath(TreeItem item) { private void populateFolder(FileObject folder, TreeItem folderItem) throws FileSystemException { FileObject[] children = folder.getChildren(); + if (children == null || children.length == 0) { + return; + } + // Comparator must define a total order (TimSort). Never treat unequal FileObjects as equal + // when metadata fails — that used to return 0 from catch and caused: + // IllegalArgumentException: Comparison method violates its general contract! Arrays.sort( children, (child1, child2) -> { - try { - int cmp; - switch (sortIndex) { - case 0: - String name1 = child1.getName().getBaseName(); - String name2 = child2.getName().getBaseName(); - cmp = name1.compareToIgnoreCase(name2); - break; - case 1: - long time1 = child1.getContent().getLastModifiedTime(); - long time2 = child2.getContent().getLastModifiedTime(); - cmp = Long.compare(time1, time2); - break; - case 2: - long size1 = child1.getContent().getSize(); - long size2 = child2.getContent().getSize(); - cmp = Long.compare(size1, size2); - break; - - default: - cmp = 0; - } - if (ascending) { - return -cmp; - } else { - return cmp; - } - } catch (Exception e) { - return 0; + int cmp = + switch (sortIndex) { + case 1 -> Long.compare(safeLastModified(child1), safeLastModified(child2)); + case 2 -> Long.compare(safeSize(child1), safeSize(child2)); + default -> compareBaseName(child1, child2); + }; + if (cmp == 0) { + cmp = compareBaseName(child1, child2); } + if (cmp == 0) { + cmp = compareUri(child1, child2); + } + // Preserve existing UI: ascending flag is inverted relative to natural order + return ascending ? -cmp : cmp; }); // First the child folders @@ -1088,6 +1078,43 @@ private void populateFolder(FileObject folder, TreeItem folderItem) throws FileS } } + private static int compareBaseName(FileObject child1, FileObject child2) { + try { + String name1 = child1.getName().getBaseName(); + String name2 = child2.getName().getBaseName(); + return name1.compareToIgnoreCase(name2); + } catch (Exception e) { + return Integer.compare(System.identityHashCode(child1), System.identityHashCode(child2)); + } + } + + private static int compareUri(FileObject child1, FileObject child2) { + try { + return child1.getName().getURI().compareTo(child2.getName().getURI()); + } catch (Exception e) { + return Integer.compare(System.identityHashCode(child1), System.identityHashCode(child2)); + } + } + + private static long safeLastModified(FileObject child) { + try { + return child.getContent().getLastModifiedTime(); + } catch (Exception e) { + return Long.MIN_VALUE; + } + } + + private static long safeSize(FileObject child) { + try { + if (child.isFolder()) { + return -1L; + } + return child.getContent().getSize(); + } catch (Exception e) { + return Long.MIN_VALUE; + } + } + /** Case-insensitive filename filter for the current folder listing. */ private boolean shouldFilterOut(String name) { return !Utils.isEmpty(filterText) && !filterMatcher.matches(name); diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/base/BaseExplorerFileTypeHandler.java b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/base/BaseExplorerFileTypeHandler.java index bff3fe94fdf..7ac2f0a5761 100644 --- a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/base/BaseExplorerFileTypeHandler.java +++ b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/base/BaseExplorerFileTypeHandler.java @@ -112,9 +112,9 @@ public String getFilename() { @Override public void setFilename(String filename) { explorerFile.setFilename(filename); - // Update name based on filename + // Update name based on filename (use variables so named VFS schemes resolve) try { - String name = HopVfs.getFileObject(filename).getName().getBaseName(); + String name = HopVfs.getFileObject(filename, getVariables()).getName().getBaseName(); explorerFile.setName(name); } catch (HopFileException e) { // Ignore diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/raw/RawExplorerFileTypeHandler.java b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/raw/RawExplorerFileTypeHandler.java index 3745eb873b2..b850eb5433c 100644 --- a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/raw/RawExplorerFileTypeHandler.java +++ b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/raw/RawExplorerFileTypeHandler.java @@ -113,8 +113,10 @@ public void save() throws HopException { } try { String filename = explorerFile.getFilename(); - boolean fileExist = HopVfs.fileExists(filename); - try (java.io.OutputStream outputStream = HopVfs.getOutputStream(filename, false)) { + // Named VFS schemes need variables so providers load from project metadata + boolean fileExist = HopVfs.fileExists(filename, getVariables()); + try (java.io.OutputStream outputStream = + HopVfs.getOutputStream(filename, false, getVariables())) { outputStream.write(editorWidget.getText().getBytes(StandardCharsets.UTF_8)); outputStream.flush(); } @@ -134,8 +136,13 @@ public void saveAs(String filename) throws HopException { throw new HopException("Binary file cannot be saved as text."); } try { - filename = HopVfs.normalize(filename); - FileObject fileObject = HopVfs.getFileObject(filename); + FileObject fileObject = HopVfs.getFileObject(filename, getVariables()); + if (!HopVfs.startsWithScheme(filename, getVariables()) && !filename.contains("://")) { + filename = HopVfs.normalize(filename); + fileObject = HopVfs.getFileObject(filename, getVariables()); + } else { + filename = fileObject.getName().getURI(); + } if (fileObject.exists()) { MessageBox box = new MessageBox(hopGui.getActiveShell(), SWT.YES | SWT.NO | SWT.ICON_QUESTION); diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/text/BaseTextExplorerFileTypeHandler.java b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/text/BaseTextExplorerFileTypeHandler.java index a6f72ace63e..e206454e079 100644 --- a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/text/BaseTextExplorerFileTypeHandler.java +++ b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/text/BaseTextExplorerFileTypeHandler.java @@ -82,14 +82,15 @@ public void save() throws HopException { try { // Save the current explorer file .... - // + // Named VFS schemes (MinIO, Databricks, WebDAV, …) only resolve when variables are passed so + // HopVfs can load connection providers from project metadata. String filename = explorerFile.getFilename(); - boolean fileExist = HopVfs.fileExists(filename); + boolean fileExist = HopVfs.fileExists(filename, getVariables()); // Save the file... // - try (OutputStream outputStream = HopVfs.getOutputStream(filename, false)) { + try (OutputStream outputStream = HopVfs.getOutputStream(filename, false, getVariables())) { outputStream.write(editorWidget.getText().getBytes(StandardCharsets.UTF_8)); outputStream.flush(); } @@ -117,10 +118,15 @@ public void saveAs(String filename) throws HopException { filename = filename + this.getFileType().getDefaultFileExtension(); } - // Normalize file name - filename = HopVfs.normalize(filename); + // Resolve via VFS with variables (named schemes). Only normalize local/relative paths. + FileObject fileObject = HopVfs.getFileObject(filename, getVariables()); + if (!HopVfs.startsWithScheme(filename, getVariables()) && !filename.contains("://")) { + filename = HopVfs.normalize(filename); + fileObject = HopVfs.getFileObject(filename, getVariables()); + } else { + filename = fileObject.getName().getURI(); + } - FileObject fileObject = HopVfs.getFileObject(filename); if (fileObject.exists()) { MessageBox box = new MessageBox(hopGui.getActiveShell(), SWT.YES | SWT.NO | SWT.ICON_QUESTION);