diff --git a/.gitignore b/.gitignore index 6091d76..78ccd78 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,6 @@ out/ .DS_Store # GraalVM -.graalvm \ No newline at end of file +.graalvm + +grimoires/ \ No newline at end of file diff --git a/build.gradle b/build.gradle index 9792b28..43b0575 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,13 @@ +buildscript { + repositories { + gradlePluginPortal() + mavenCentral() + } + dependencies { + classpath "org.graalvm.buildtools.native:org.graalvm.buildtools.native.gradle.plugin:0.11.2" + } +} + plugins { id "scala" id "maven-publish" @@ -11,6 +21,8 @@ subprojects { apply plugin: 'maven-publish' apply plugin: 'scala' + apply plugin: 'org.graalvm.buildtools.native' + group = 'org.mule.weave.native' version = nativeVersion diff --git a/gradle.properties b/gradle.properties index 01b5261..7b51347 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,10 +1,10 @@ -weaveVersion=2.11.0-20251023 -weaveTestSuiteVersion=2.11.0-20251023 +weaveVersion=2.12.0-SNAPSHOT +weaveTestSuiteVersion=2.12.0-SNAPSHOT nativeVersion=100.100.100 scalaVersion=2.12.18 -ioVersion=2.11.0-SNAPSHOT +ioVersion=2.12.0-SNAPSHOT graalvmVersion=24.0.2 -weaveSuiteVersion=2.11.0-20251023 +weaveSuiteVersion=2.12.0-SNAPSHOT #Libaries scalaTestVersion=3.2.15 scalaTestPluginVersion=0.33 diff --git a/native-cli/build.gradle b/native-cli/build.gradle index b567687..5ce3b5e 100644 --- a/native-cli/build.gradle +++ b/native-cli/build.gradle @@ -2,8 +2,6 @@ plugins { id "com.github.maiflai.scalatest" version "${scalaTestPluginVersion}" id 'application' - // Apply GraalVM Native Image plugin - id 'org.graalvm.buildtools.native' version '0.11.2' } sourceSets { diff --git a/native-lib/.gitignore b/native-lib/.gitignore new file mode 100644 index 0000000..4e19c5e --- /dev/null +++ b/native-lib/.gitignore @@ -0,0 +1,2 @@ +python/src/dataweave/native/ +python/dist/ \ No newline at end of file diff --git a/native-lib/README.md b/native-lib/README.md new file mode 100644 index 0000000..aa993d9 --- /dev/null +++ b/native-lib/README.md @@ -0,0 +1,222 @@ +# native-lib + +## Overview + +`native-lib` builds a **GraalVM native shared library** that embeds the MuleSoft **DataWeave runtime** and exposes a small C-compatible API. + +The main purpose is to allow non-JVM consumers (most notably the Python package in `native-lib/python`) to execute DataWeave scripts **without running a JVM**, while still using the official DataWeave runtime. + +## Architecture (GraalVM + FFI) + +``` +┌─────────────────────────────────────────────┐ +│ Python Process │ +│ │ +│ ┌────────────────────────────────────────┐ │ +│ │ Application Script │ │ +│ │ - Python: ctypes │ │ +│ └──────────────┬─────────────────────────┘ │ +│ │ │ +│ │ FFI Call │ +│ ▼ │ +│ ┌────────────────────────────────────────┐ │ +│ │ Native Shared Library (dwlib) │ │ +│ │ ┌──────────────────────────────────┐ │ │ +│ │ │ GraalVM Isolate │ │ │ +│ │ │ - NativeLib.run_script() │ │ │ +│ │ │ - DataWeave script execution │ │ │ +│ │ └──────────────────────────────────┘ │ │ +│ └────────────────────────────────────────┘ │ +└─────────────────────────────────────────────┘ +``` + +## Building with Gradle + +### Prerequisites + +- A GraalVM distribution installed that includes `native-image`. +- Enough memory for native-image (this build config uses `-J-Xmx6G`). + +### Build the shared library + +From the repository root: + +```bash +./gradlew :native-lib:nativeCompile +``` + +The shared library is produced under: + +- `native-lib/build/native/nativeCompile/` + +and is named: + +- macOS: `dwlib.dylib` +- Linux: `dwlib.so` +- Windows: `dwlib.dll` + +### Stage the library into the Python package (dev workflow) + +```bash +./gradlew :native-lib:stagePythonNativeLib +``` + +This copies `dwlib.*` into: + +- `native-lib/python/src/dataweave/native/` + +### Build a Python wheel (bundles the native library) + +```bash +./gradlew :native-lib:buildPythonWheel +``` + +The wheel will be created in: + +- `native-lib/python/dist/` + +## Installing for use in a Python project + +### Option A: Install the produced wheel (recommended) + +After `:native-lib:buildPythonWheel`: + +```bash +python3 -m pip install native-lib/python/dist/dataweave_native-0.0.1-*.whl +``` + +This wheel includes the `dwlib.*` shared library inside the Python package. + +### Option B: Editable install for development + +1. Stage the native library: + +```bash +./gradlew :native-lib:stagePythonNativeLib +``` + +2. Install the Python package in editable mode: + +```bash +python3 -m pip install -e native-lib/python +``` + +### Option C: Use an externally-built library via an environment variable + +If you want to point Python at a specific built artifact, set: + +- `DATAWEAVE_NATIVE_LIB=/absolute/path/to/dwlib.(dylib|so|dll)` + +The Python module will also try a few fallbacks (including the wheel-bundled location). + +## Using the library (Python examples) + +All examples below assume: + +```python +import dataweave +``` + +### 1) Simple script + +```python +result = dataweave.run_script("2 + 2") +assert result.success is True +print(result.get_string()) # "4" +``` + +### 2) Script with inputs (no explicit `mimeType`) + +Inputs can be plain Python values. The wrapper auto-encodes them as JSON or text. + +```python +result = dataweave.run_script( + "num1 + num2", + {"num1": 25, "num2": 17}, +) +print(result.get_string()) # "42" +``` + +### 3) Script with inputs (explicit `mimeType`, `charset`, `properties`) + +Use an explicit input dict when you need full control over how DataWeave interprets bytes. + +```python +script = "payload.person" +xml_bytes = b"Billy31".decode("utf-8").encode("utf-16") + +result = dataweave.run_script( + script, + { + "payload": { + "content": xml_bytes, + "mimeType": "application/xml", + "charset": "UTF-16", + "properties": { + "nullValueOn": "empty", + "maxAttributeSize": 256 + }, + } + }, +) + +if result.success: + print(result.get_string()) +else: + print(result.error) +``` + +You can also use `InputValue` for the same purpose: + +```python +input_value = dataweave.InputValue( + content="1234567", + mimeType="application/csv", + properties={"header": False, "separator": "4"}, +) + +result = dataweave.run_script("in0.column_1[0]", {"in0": input_value}) +print(result.get_string()) # '"567"' +``` + +### 4) Reusing a DataWeave context to run multiple scripts quicker + +Creating an isolate/runtime has overhead. For repeated executions, reuse a single `DataWeave` instance: + +```python +with dataweave.DataWeave() as dw: + r1 = dw.run("2 + 2") + r2 = dw.run("x + y", {"x": 10, "y": 32}) + + print(r1.get_string()) # "4" + print(r2.get_string()) # "42" +``` + +### 5) Error handling + +There are two common classes of errors: + +- The native library cannot be located/loaded. +- Script compilation/execution fails (reported as an unsuccessful `ExecutionResult`). + +```python +try: + result = dataweave.run_script("invalid syntax here") + + if not result.success: + raise dataweave.DataWeaveError(result.error or "Unknown DataWeave error") + + print(result.get_string()) + +except dataweave.DataWeaveLibraryNotFoundError as e: + # Build it (and/or install a wheel) first. + # Example build command (from repo root): ./gradlew :native-lib:nativeCompile + raise + +except dataweave.DataWeaveError: + raise + +finally: + # Optional: if you used the global API and want to force cleanup + dataweave.cleanup() +``` diff --git a/native-lib/build.gradle b/native-lib/build.gradle new file mode 100644 index 0000000..6b3d122 --- /dev/null +++ b/native-lib/build.gradle @@ -0,0 +1,97 @@ +dependencies { + api group: 'org.mule.weave', name: 'runtime', version: weaveVersion + api group: 'org.mule.weave', name: 'core-modules', version: weaveVersion + + implementation group: 'org.mule.weave', name: 'parser', version: weaveVersion + implementation group: 'org.mule.weave', name: 'wlang', version: weaveVersion + compileOnly group: 'org.graalvm.sdk', name: 'graal-sdk', version: graalvmVersion + compileOnly group: 'org.graalvm.nativeimage', name: 'svm', version: graalvmVersion + + implementation "org.scala-lang:scala-library:${scalaVersion}" + + testImplementation platform('org.junit:junit-bom:5.10.0') + testImplementation 'org.junit.jupiter:junit-jupiter' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' +} + +test { + useJUnitPlatform() +} + +tasks.matching { it.name == 'nativeCompileClasspathJar' }.configureEach { t -> + t.exclude('META-INF/services/org.mule.weave.v2.module.DataFormat') + t.from("${projectDir}/src/main/resources/META-INF/services/org.mule.weave.v2.module.DataFormat") { + into('META-INF/services') + rename { 'org.mule.weave.v2.module.DataFormat' } + } +} + +// Configure GraalVM native-image to build a shared library +graalvmNative { +// toolchainDetection = true + binaries { + main { + sharedLibrary = true + debug = true + verbose = true + fallback = false + //agent = false + useFatJar = true + //buildArgs.add('-Ob') // quick build mode to speed up builds during development + buildArgs.add('--no-fallback') + buildArgs.add('-H:Name=dwlib') + buildArgs.add('--verbose') + buildArgs.add("--report-unsupported-elements-at-runtime") + buildArgs.add("-J-Xmx6G") + + buildArgs.add("-H:+ReportExceptionStackTraces") + buildArgs.add("-H:+UnlockExperimentalVMOptions") + buildArgs.add("--initialize-at-build-time=sun.instrument.InstrumentationImpl") + buildArgs.add("-H:DeadlockWatchdogInterval=1000") + buildArgs.add("-H:CompilationExpirationPeriod=0") + buildArgs.add("-H:+AddAllCharsets") + buildArgs.add("-H:+IncludeAllLocales") + // Pass project directory as system property for header path resolution + buildArgs.add("-Dproject.root=${projectDir}") + } + } +} + +def pythonExe = (project.findProperty('pythonExe') ?: 'python3') as String + +tasks.register('stagePythonNativeLib', Copy) { + dependsOn tasks.named('nativeCompile') + from("${buildDir}/native/nativeCompile") { + include('dwlib.*') + } + into("${projectDir}/python/src/dataweave/native") +} + +tasks.register('buildPythonWheel', Exec) { + dependsOn tasks.named('stagePythonNativeLib') + workingDir("${projectDir}/python") + outputs.dir("${projectDir}/python/dist") + doFirst { + file("${projectDir}/python/dist").mkdirs() + } + commandLine(pythonExe, '-m', 'pip', 'wheel', '--no-deps', '-w', 'dist', '.') +} + +tasks.register('pythonTest', Exec) { + if (project.findProperty('skipPythonTests')?.toString()?.toBoolean() == true) { + enabled = false + } + + dependsOn tasks.named('stagePythonNativeLib') + workingDir("${projectDir}/python") + commandLine(pythonExe, 'tests/test_dataweave_module.py') +} + +tasks.named('test') { + dependsOn tasks.named('pythonTest') +} + +tasks.named('clean') { + delete("${projectDir}/python/dist") + delete("${projectDir}/python/src/dataweave/native") +} diff --git a/native-lib/example_dataweave_module.py b/native-lib/example_dataweave_module.py new file mode 100755 index 0000000..d1740a2 --- /dev/null +++ b/native-lib/example_dataweave_module.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +""" +Example demonstrating the simplified DataWeave Python module. + +This shows how easy it is to use DataWeave without dealing with +any GraalVM or native library complexity. +""" + +import sys +from pathlib import Path + +_PYTHON_SRC_DIR = Path(__file__).resolve().parent / "python" / "src" +sys.path.insert(0, str(_PYTHON_SRC_DIR)) + +import dataweave + +def example_simple_functions(): + """Example using simple function API""" + print("="*70) + print("Example 1: Simple Function API") + print("="*70) + + ok = True + + # Simple script execution + print("\n[*] Simple arithmetic:") + script = "2 + 2" + result = dataweave.run_script(script) + ok = assert_result(script, result, "4") and ok + + print("\n[*] Square root:") + script = "sqrt(144)" + result = dataweave.run_script(script) + ok = assert_result(script, result, "12") and ok + + print("\n[*] Array operations:") + script = "[1, 2, 3] map $ * 2" + result = dataweave.run_script(script) + ok = assert_result(script, result, "[\n 2, \n 4, \n 6\n]") and ok + + print("\n[*] String operations:") + script = "upper('hello world')" + result = dataweave.run_script(script) + ok = assert_result(script, result, '"HELLO WORLD"') and ok + + # Script with inputs (simple values - auto-converted) + print("\n[*] Script with inputs (auto-converted):") + script = "num1 + num2" + result = dataweave.run_script(script, {"num1": 25, "num2": 17}) + ok = assert_result(script, result, "42") and ok + + # Script with complex inputs + print("\n[*] Script with complex object:") + script = "payload.name" + result = dataweave.run_script(script, {"payload": {"content": '{"name": "John", "age": 30}', "mimeType": "application/json"}}) + ok = assert_result(script, result, '"John"') and ok + + # Script with mixed input types + print("\n[*] Script with mixed input types:") + script = "greeting ++ ' ' ++ payload.name" + result = dataweave.run_script(script, {"greeting": "Hello", "payload": {"content": '{"name": "Alice", "role": "Developer"}', "mimeType": "application/json"}}) + ok = assert_result(script, result, '"Hello Alice"') and ok + + # Binary output + print("\n[*] Binary output:") + script = "output application/octet-stream\n---\ndw::core::Binaries::fromBase64(\"holamund\")" + result = dataweave.run_script(script) + ok = assert_result(script, result, "holamund") and ok + + # Script with InputValue + print("\n[*] Inputs:") + input_value = dataweave.InputValue( + content="1234567", + mimeType="application/csv", + properties={"header": False, "separator": "4"} + ) + script = "in0.column_1[0]" + result = dataweave.run_script(script, {"in0": input_value}) + ok = assert_result(script, result, '"567"') and ok + + # Cleanup when done + dataweave.cleanup() + print("\n[OK] Cleanup completed") + + return ok + + +def assert_result(script, result, expected): + print(f" {script} = {result}") + ok = result.get_string() == expected + if ok: + status = "[OK]" + else: + status = f"[FAIL] (expected: {expected})" + print(f" result as string = {result.get_string()} {status}") + print(f" result as bytes = {result.get_bytes()}") + return ok + + +def example_context_manager(): + """Example using context manager (recommended)""" + print("\n" + "="*70) + print("Example 2: Context Manager API (Recommended)") + print("="*70) + + ok = True + + with dataweave.DataWeave() as dw: + print("\n[*] Multiple operations with same runtime:") + + script = "2 + 2" + result = dw.run(script) + ok = assert_result(script, result, "4") and ok + + script = "x + y + z" + result = dw.run(script, {"x": 1, "y": 2, "z": 3}) + ok = assert_result(script, result, "6") and ok + + script = "numbers map $ * multiplier" + result = dw.run(script, {"numbers": [1, 2, 3, 4, 5], "multiplier": 10}) + ok = assert_result(script, result, "[\n 10, \n 20, \n 30, \n 40, \n 50\n]") and ok + + print("\n[OK] Context manager automatically cleaned up resources") + + return ok + + +def example_explicit_format(): + """Example using explicit content/mimeType format""" + print("\n" + "="*70) + print("Example 3: Explicit Format (Advanced)") + print("="*70) + + print("\n[*] Using explicit content and mimeType:") + + ok = True + + script = "payload.message" + result = dataweave.run_script(script, {"payload": {"content": '{"message": "Hello from JSON!", "value": 42}', "mimeType": "application/json"}}) + ok = assert_result(script, result, '"Hello from JSON!"') and ok + + script = "payload.value + offset" + result = dataweave.run_script(script, {"payload": {"content": '{"value": 100}', "mimeType": "application/json"}, "offset": 50}) + ok = assert_result(script, result, "150") and ok + + return ok + + +def example_error_handling(): + """Example with error handling""" + print("\n" + "="*70) + print("Example 4: Error Handling") + print("="*70) + + try: + print("\n[*] Invalid script (will show error):") + result = dataweave.run_script("invalid syntax here", {}) + print(f" Result: {result} {'[OK]' if result.success == False else '[FAIL]'}") + + except dataweave.DataWeaveLibraryNotFoundError as e: + print(f"[ERROR] Library not found: {e}") + print(" Please build the library first: ./gradlew nativeCompile") + except dataweave.DataWeaveError as e: + print(f"[ERROR] DataWeave error: {e}") + + +def main(): + """Run all examples""" + print("\n" + "="*70) + print("DataWeave Python Module - Examples") + print("="*70) + print("\nThis module abstracts all GraalVM/native complexity!") + print("Just import and use - no ctypes, no manual memory management.\n") + + try: + all_ok = True + all_ok = example_simple_functions() and all_ok + all_ok = example_context_manager() and all_ok + all_ok = example_explicit_format() and all_ok + example_error_handling() + + print("\n" + "="*70) + if all_ok: + print("[OK] All examples completed successfully!") + else: + print("[FAIL] One or more examples failed") + print("="*70) + + except dataweave.DataWeaveLibraryNotFoundError as e: + print(f"\n[ERROR] {e}") + print("\nPlease build the native library first:") + print(" ./gradlew nativeCompile") + except Exception as e: + print(f"\n[ERROR] Unexpected error: {e}") + import traceback + traceback.print_exc() + + +if __name__ == "__main__": + main() diff --git a/native-lib/python/pyproject.toml b/native-lib/python/pyproject.toml new file mode 100644 index 0000000..642ab3c --- /dev/null +++ b/native-lib/python/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/native-lib/python/setup.cfg b/native-lib/python/setup.cfg new file mode 100644 index 0000000..a1635ae --- /dev/null +++ b/native-lib/python/setup.cfg @@ -0,0 +1,18 @@ +[metadata] +name = dataweave-native +version = 0.0.1 +description = Python bindings for the DataWeave native library + +[options] +package_dir = + = src +packages = find: +include_package_data = True +python_requires = >=3.9 + +[options.packages.find] +where = src + +[options.package_data] +dataweave = + native/* diff --git a/native-lib/python/src/dataweave/__init__.py b/native-lib/python/src/dataweave/__init__.py new file mode 100644 index 0000000..9357515 --- /dev/null +++ b/native-lib/python/src/dataweave/__init__.py @@ -0,0 +1,364 @@ +""" +DataWeave Python Module + +A simple Python wrapper for executing DataWeave scripts via the native library. +This module abstracts all GraalVM and native library complexity, providing a +clean Python API for executing DataWeave scripts with or without inputs. + +Basic Usage: + import dataweave + + result = dataweave.run_script("2 + 2") + print(result.get_string()) +""" + +import base64 +import ctypes +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Optional, Union + + +class DataWeaveError(Exception): + pass + + +class DataWeaveLibraryNotFoundError(Exception): + pass + + +_ENV_NATIVE_LIB = "DATAWEAVE_NATIVE_LIB" + + +@dataclass +class InputValue: + content: Union[str, bytes] + mimeType: Optional[str] = None + charset: Optional[str] = None + properties: Optional[Dict[str, Union[str, int, bool]]] = None + + def encode_content(self) -> str: + if isinstance(self.content, bytes): + raw = self.content + else: + raw = self.content.encode(self.charset or "utf-8") + return base64.b64encode(raw).decode("ascii") + + +@dataclass +class ExecutionResult: + success: bool + result: Optional[str] + error: Optional[str] + binary: bool + mimeType: Optional[str] + charset: Optional[str] + + def get_bytes(self) -> Optional[bytes]: + if not self.success or self.result is None: + return None + return base64.b64decode(self.result) + + def get_string(self) -> Optional[str]: + if not self.success or self.result is None: + return None + if self.binary: + return self.result + return self.get_bytes().decode(self.charset or "utf-8") + + +def _parse_native_encoded_response(raw: str) -> ExecutionResult: + if raw is None: + return ExecutionResult(False, None, "Native returned null", False, None, None) + + if raw == "": + return ExecutionResult(False, None, "Native returned empty response", False, None, None) + + try: + parsed = json.loads(raw) + except Exception as e: + return ExecutionResult(False, None, f"Failed to parse native JSON response: {e}", False, None, None) + + if not isinstance(parsed, dict): + return ExecutionResult(False, None, "Native response JSON is not an object", False, None, None) + + success = bool(parsed.get("success", False)) + if not success: + return ExecutionResult(False, None, parsed.get("error"), False, None, None) + + return ExecutionResult( + success=True, + result=parsed.get("result"), + error=None, + binary=bool(parsed.get("binary", False)), + mimeType=parsed.get("mimeType"), + charset=parsed.get("charset"), + ) + + +def _candidate_library_paths() -> list[Path]: + paths: list[Path] = [] + + env_value = (__import__("os").environ.get(_ENV_NATIVE_LIB) or "").strip() + if env_value: + paths.append(Path(env_value)) + + pkg_dir = Path(__file__).resolve().parent + native_dir = pkg_dir / "native" + paths.append(native_dir / "dwlib.dylib") + paths.append(native_dir / "dwlib.so") + paths.append(native_dir / "dwlib.dll") + + # Dev fallback: if this package is being used from the data-weave-cli repo + # tree, locate native-lib/build/native/nativeCompile. + for parent in pkg_dir.parents: + build_dir = parent / "build" / "native" / "nativeCompile" + if build_dir.name == "nativeCompile" and build_dir.parent.name == "native" and build_dir.parent.parent.name == "build": + paths.append(build_dir / "dwlib.dylib") + paths.append(build_dir / "dwlib.so") + paths.append(build_dir / "dwlib.dll") + break + + # CWD fallback + paths.append(Path("dwlib.dylib")) + paths.append(Path("dwlib.so")) + paths.append(Path("dwlib.dll")) + + return paths + + +def _find_library() -> str: + for p in _candidate_library_paths(): + if p.exists() and p.is_file(): + return str(p) + + raise DataWeaveLibraryNotFoundError( + "Could not find DataWeave native library (dwlib). " + f"Set {_ENV_NATIVE_LIB} to an absolute path or install a wheel that bundles the native library." + ) + + +def _normalize_input_value(value: Any, mime_type: Optional[str] = None) -> Dict[str, Any]: + if isinstance(value, dict): + allowed_keys = {"content", "mimeType", "charset", "properties"} + extra_keys = set(value.keys()) - allowed_keys + if extra_keys: + raise DataWeaveError( + "Explicit input dict contains unsupported keys: " + ", ".join(sorted(extra_keys)) + ) + + if "content" in value or "mimeType" in value: + if "content" not in value or "mimeType" not in value: + raise DataWeaveError( + "Explicit input dict must include both 'content' and 'mimeType'" + ) + + raw_content = value.get("content") + charset = value.get("charset") or "utf-8" + if isinstance(raw_content, bytes): + encoded_content = base64.b64encode(raw_content).decode("ascii") + else: + encoded_content = base64.b64encode(str(raw_content).encode(charset)).decode("ascii") + + normalized: Dict[str, Any] = { + "content": encoded_content, + "mimeType": value.get("mimeType"), + } + if "charset" in value: + normalized["charset"] = value.get("charset") + if "properties" in value: + normalized["properties"] = value.get("properties") + return normalized + + if isinstance(value, InputValue): + out: Dict[str, Any] = { + "content": value.encode_content(), + "mimeType": value.mimeType or mime_type, + } + if value.charset is not None: + out["charset"] = value.charset + if value.properties is not None: + out["properties"] = value.properties + return out + + if isinstance(value, str): + content = value + default_mime = "text/plain" + elif isinstance(value, (int, float, bool)): + content = json.dumps(value) + default_mime = "application/json" + elif value is None: + content = "null" + default_mime = "application/json" + else: + try: + content = json.dumps(value) + default_mime = "application/json" + except (TypeError, ValueError): + content = str(value) + default_mime = "text/plain" + + charset = "utf-8" + encoded_content = base64.b64encode(content.encode(charset)).decode("ascii") + + return { + "content": encoded_content, + "mimeType": mime_type or default_mime, + "charset": charset, + "properties": None, + } + + +class DataWeave: + def __init__(self, lib_path: Optional[str] = None): + self._lib_path = lib_path or _find_library() + self._lib = None + self._isolate = None + self._thread = None + self._initialized = False + + def _load_library(self): + try: + self._lib = ctypes.CDLL(self._lib_path) + except OSError as e: + raise DataWeaveError(f"Failed to load library from {self._lib_path}: {e}") + + def _setup_graal_structures(self): + class graal_isolate_t(ctypes.Structure): + pass + + class graal_isolatethread_t(ctypes.Structure): + pass + + self._graal_isolate_t_ptr = ctypes.POINTER(graal_isolate_t) + self._graal_isolatethread_t_ptr = ctypes.POINTER(graal_isolatethread_t) + + def _create_isolate(self): + self._lib.graal_create_isolate.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(self._graal_isolate_t_ptr), + ctypes.POINTER(self._graal_isolatethread_t_ptr), + ] + self._lib.graal_create_isolate.restype = ctypes.c_int + + self._isolate = self._graal_isolate_t_ptr() + self._thread = self._graal_isolatethread_t_ptr() + + result = self._lib.graal_create_isolate(None, ctypes.byref(self._isolate), ctypes.byref(self._thread)) + if result != 0: + raise DataWeaveError(f"Failed to create GraalVM isolate. Error code: {result}") + + def _setup_functions(self): + if not hasattr(self._lib, "run_script"): + raise DataWeaveError("Native library does not export run_script") + + self._lib.run_script.argtypes = [ + self._graal_isolatethread_t_ptr, + ctypes.c_char_p, + ctypes.c_char_p, + ] + self._lib.run_script.restype = ctypes.c_void_p + + if hasattr(self._lib, "free_cstring"): + self._lib.free_cstring.argtypes = [self._graal_isolatethread_t_ptr, ctypes.c_void_p] + self._lib.free_cstring.restype = None + + def _decode_and_free(self, ptr: Optional[int]) -> str: + if not ptr: + return "" + + try: + result_bytes = ctypes.string_at(ptr) + return result_bytes.decode("utf-8") + finally: + if self._lib is not None and hasattr(self._lib, "free_cstring"): + self._lib.free_cstring(self._thread, ptr) + + def initialize(self): + if self._initialized: + return + + self._load_library() + self._setup_graal_structures() + self._create_isolate() + self._setup_functions() + self._initialized = True + + def cleanup(self): + if not self._initialized: + return + + if hasattr(self._lib, "graal_detach_thread") and self._thread: + try: + self._lib.graal_detach_thread.argtypes = [self._graal_isolatethread_t_ptr] + self._lib.graal_detach_thread.restype = ctypes.c_int + self._lib.graal_detach_thread(self._thread) + except Exception: + pass + + self._initialized = False + self._thread = None + self._isolate = None + self._lib = None + + def run(self, script: str, inputs: Optional[Dict[str, Any]] = None) -> ExecutionResult: + if not self._initialized: + raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") + + if inputs is None: + inputs = {} + + normalized_inputs = {key: _normalize_input_value(val) for key, val in inputs.items()} + inputs_json = json.dumps(normalized_inputs) + + try: + result_ptr = self._lib.run_script( + self._thread, + script.encode("utf-8"), + inputs_json.encode("utf-8"), + ) + raw = self._decode_and_free(result_ptr) + return _parse_native_encoded_response(raw) + except Exception as e: + raise DataWeaveError(f"Failed to execute script: {e}") + + def __enter__(self): + self.initialize() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.cleanup() + return False + + +_global_instance: Optional[DataWeave] = None + + +def _get_global_instance() -> DataWeave: + global _global_instance + if _global_instance is None: + _global_instance = DataWeave() + _global_instance.initialize() + return _global_instance + + +def run_script(script: str, inputs: Optional[Dict[str, Any]] = None) -> ExecutionResult: + return _get_global_instance().run(script, inputs) + + +def cleanup(): + global _global_instance + if _global_instance is not None: + _global_instance.cleanup() + _global_instance = None + + +__all__ = [ + "DataWeaveError", + "DataWeaveLibraryNotFoundError", + "ExecutionResult", + "InputValue", + "run_script", + "cleanup", +] diff --git a/native-lib/python/tests/person.xml b/native-lib/python/tests/person.xml new file mode 100644 index 0000000..376a6b7 Binary files /dev/null and b/native-lib/python/tests/person.xml differ diff --git a/native-lib/python/tests/test_dataweave_module.py b/native-lib/python/tests/test_dataweave_module.py new file mode 100755 index 0000000..c508378 --- /dev/null +++ b/native-lib/python/tests/test_dataweave_module.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +""" +Quick test script for the DataWeave Python module. +""" + +import sys +from pathlib import Path + +_PYTHON_SRC_DIR = Path(__file__).resolve().parents[1] / "src" +sys.path.insert(0, str(_PYTHON_SRC_DIR)) + +import dataweave + +def test_basic(): + """Test basic functionality""" + print("Testing basic script execution...") + try: + result = dataweave.run_script("2 + 2", {}) + assert result.get_string() == "4", f"Expected '4', got '{result.get_string()}'" + print("[OK] Basic script execution works") + return True + except Exception as e: + print(f"[FAIL] Basic script execution failed: {e}") + return False + +def test_with_inputs(): + """Test script with inputs""" + print("\nTesting script with inputs...") + try: + result = dataweave.run_script("num1 + num2", {"num1": 25, "num2": 17}) + assert result.get_string() == "42", f"Expected '42', got '{result.get_string()}'" + print("[OK] Script with inputs works") + return True + except Exception as e: + print(f"[FAIL] Script with inputs failed: {e}") + return False + +def test_context_manager(): + """Test context manager""" + print("\nTesting with context manager...") + try: + with dataweave.DataWeave() as dw: + + result = dataweave.run_script("sqrt(144)") + assert result.get_string() == "12", f"Expected '12', got '{result.get_string()}'" + result = dataweave.run_script("sqrt(10000)") + assert result.get_string() == "100", f"Expected '100', got '{result.get_string()}'" + print("[OK] Script execution witch context manager works") + return True + except Exception as e: + print(f"[FAIL] Script execution witch context manager failed: {e}") + return False + +def test_encoding(): + """Test reading UTF-16 XML input and producing CSV output""" + print("\nTesting encoding (UTF-16 XML -> CSV)...") + try: + xml_path = ( + Path(__file__).resolve().parent / "person.xml" + ) + xml_bytes = xml_path.read_bytes() + + script = """output application/csv header=true +--- +[payload.person] +""" + + result = dataweave.run_script( + script, + { + "payload": { + "content": xml_bytes, + "mimeType": "application/xml", + "charset": "UTF-16", + } + }, + ) + + out = result.get_string() or "" + print(f"out: \n{out}") + assert result.success is True, f"Expected success=true, got: {result}" + assert "name" in out and "age" in out, f"CSV header missing, got: {out!r}" + assert "Billy" in out, f"Expected name 'Billy' in CSV, got: {out!r}" + assert "31" in out, f"Expected age '31' in CSV, got: {out!r}" + + print("[OK] Encoding conversion works") + return True + except Exception as e: + print(f"[FAIL] Encoding conversion failed: {e}") + return False + +def test_auto_conversion(): + """Test auto-conversion of different types""" + print("\nTesting auto-conversion...") + try: + + # Test array + result = dataweave.run_script( + "numbers[0]", + {"numbers": [1, 2, 3]} + ) + assert result.get_string() == "1", f"Expected '1', got '{result.get_string()}'" + + print("[OK] Auto-conversion works") + return True + except Exception as e: + print(f"[FAIL] Auto-conversion failed: {e}") + return False + +def main(): + """Run all tests""" + print("="*70) + print("DataWeave Python Module - Test Suite") + print("="*70) + + try: + results = [] + results.append(test_basic()) + results.append(test_with_inputs()) + results.append(test_context_manager()) + results.append(test_encoding()) + results.append(test_auto_conversion()) + + # Cleanup + dataweave.cleanup() + + print("\n" + "="*70) + passed = sum(results) + total = len(results) + print(f"Results: {passed}/{total} tests passed") + print("="*70) + + if passed == total: + print("\n[OK] All tests passed!") + sys.exit(0) + else: + print(f"\n[FAIL] {total - passed} test(s) failed") + sys.exit(1) + + except dataweave.DataWeaveLibraryNotFoundError as e: + print(f"\n[ERROR] {e}") + print("\nPlease build the native library first:") + print(" ./gradlew nativeCompile") + sys.exit(2) + except Exception as e: + print(f"\n[ERROR] Unexpected error: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/native-lib/src/main/java/org/mule/weave/lib/NativeLib.java b/native-lib/src/main/java/org/mule/weave/lib/NativeLib.java new file mode 100644 index 0000000..f22fe57 --- /dev/null +++ b/native-lib/src/main/java/org/mule/weave/lib/NativeLib.java @@ -0,0 +1,62 @@ +package org.mule.weave.lib; + +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.UnmanagedMemory; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.nativeimage.c.type.CTypeConversion; + +import java.nio.charset.StandardCharsets; + +/** + * GraalVM native entry points exposed for FFI consumers. + * + *

This class provides C-callable functions to execute DataWeave scripts and to free the returned + * unmanaged strings.

+ */ +public class NativeLib { + + /** + * Native method that executes a DataWeave script with inputs and returns the result. + * Can be called from Python via FFI. + * + * @param thread the isolate thread (automatically provided by GraalVM) + * @param script the DataWeave script to execute (C string pointer) + * @param inputsJson JSON string containing the inputs map with content (base64 encoded), mimeType, properties and charset for each binding + * @return the script execution result (C string pointer) + */ + @CEntryPoint(name = "run_script") + public static CCharPointer runDwScriptEncoded(IsolateThread thread, CCharPointer script, CCharPointer inputsJson) { + String dwScript = CTypeConversion.toJavaString(script); + String inputs = CTypeConversion.toJavaString(inputsJson); + + ScriptRuntime runtime = ScriptRuntime.getInstance(); + String result = runtime.run(dwScript, inputs); + return toUnmanagedCString(result); + } + + /** + * Frees a C string previously returned by {@link #runDwScriptEncoded(IsolateThread, CCharPointer, CCharPointer)}. + * + * @param thread the isolate thread (automatically provided by GraalVM) + * @param pointer the pointer to the unmanaged C string to free; if null, this is a no-op + */ + @CEntryPoint(name = "free_cstring") + public static void freeCString(IsolateThread thread, CCharPointer pointer) { + if (pointer.isNull()) { + return; + } + UnmanagedMemory.free(pointer); + } + + private static CCharPointer toUnmanagedCString(String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + CCharPointer ptr = UnmanagedMemory.malloc(bytes.length + 1); + for (int i = 0; i < bytes.length; i++) { + ptr.write(i, bytes[i]); + } + ptr.write(bytes.length, (byte) 0); + return ptr; + } + +} diff --git a/native-lib/src/main/java/org/mule/weave/lib/ScriptRuntime.java b/native-lib/src/main/java/org/mule/weave/lib/ScriptRuntime.java new file mode 100644 index 0000000..bb96893 --- /dev/null +++ b/native-lib/src/main/java/org/mule/weave/lib/ScriptRuntime.java @@ -0,0 +1,417 @@ +package org.mule.weave.lib; + +import org.mule.weave.v2.runtime.BindingValue; +import org.mule.weave.v2.runtime.DataWeaveResult; +import org.mule.weave.v2.runtime.ScriptingBindings; +import org.mule.weave.v2.runtime.api.DWResult; +import org.mule.weave.v2.runtime.api.DWScript; +import org.mule.weave.v2.runtime.api.DWScriptingEngine; +import scala.Option; +import scala.Tuple2; +import scala.collection.immutable.Map; +import scala.collection.immutable.Map$; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.Base64; + +/** + * Singleton wrapper around a {@link DWScriptingEngine} used to compile and execute DataWeave scripts. + * + *

Execution results are returned as a JSON string containing a base64-encoded payload plus metadata + * (mime type, charset, and whether the result is binary). Errors are returned as a JSON string with + * {@code success=false} and an escaped error message.

+ */ +public class ScriptRuntime { + + private static final ScriptRuntime INSTANCE = new ScriptRuntime(); + + /** + * Returns the singleton instance. + * + * @return the shared {@link ScriptRuntime} + */ + public static ScriptRuntime getInstance() { + return INSTANCE; + } + + private DWScriptingEngine engine; + + private ScriptRuntime() { + engine = DWScriptingEngine.builder().build(); + } + + /** + * Executes a DataWeave script with no input bindings. + * + * @param script the DataWeave script source + * @return a JSON string describing either the successful result or an error + */ + public String run(String script) { + return run(script, null); + } + + /** + * Executes a DataWeave script with optional input bindings encoded as JSON. + * + *

The expected JSON structure maps binding names to an object containing {@code content} + * (base64), {@code mimeType}, optional {@code charset}, and optional {@code properties}.

+ * + * @param script the DataWeave script source + * @param inputsJson JSON string encoding the input bindings map, or {@code null} + * @return a JSON string describing either the successful result or an error + */ + public String run(String script, String inputsJson) { + ScriptingBindings bindings = parseJsonInputsToBindings(inputsJson); + String[] inputs = bindings.bindingNames(); + + try { + DWScript compiled = engine.compileDWScript(script, inputs); + DWResult dwResult = compiled.writeDWResult(bindings); + + String encodedResult; + if (dwResult.getContent() instanceof InputStream) { + try { + byte[] ba = ((InputStream) dwResult.getContent()).readAllBytes(); + encodedResult = Base64.getEncoder().encodeToString(ba); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + throw new RuntimeException("Result is not an InputStream: " + dwResult.getContent().getClass().getName()); + } + + return "{" + + "\"success\":true," + + "\"result\":\"" + encodedResult + "\"," + + "\"mimeType\":\"" + dwResult.getMimeType() + "\"," + + "\"charset\":\"" + dwResult.getCharset() + "\"," + + "\"binary\":" + ((DataWeaveResult) dwResult).isBinary() + + "}"; + } catch (Exception e) { + String message = e.getMessage(); + if (message == null || message.trim().isEmpty()) { + message = e.toString(); + } + + return "{" + + "\"success\":false," + + "\"error\":\"" + escapeJsonString(message) + "\"" + + "}"; + } + } + + private ScriptingBindings parseJsonInputsToBindings(String inputsJson) { + ScriptingBindings bindings = new ScriptingBindings(); + + if (inputsJson == null || inputsJson.trim().isEmpty()) { + return bindings; + } + + try { + String json = inputsJson.trim(); + + // Parse top-level entries: "name": { ... } + int pos = 1; // Skip opening brace + + while (pos < json.length()) { + // Skip whitespace, commas + while (pos < json.length() && (Character.isWhitespace(json.charAt(pos)) || json.charAt(pos) == ',')) { + pos++; + } + + if (pos >= json.length() || json.charAt(pos) == '}') break; + + // Expect a quoted string (binding name) + if (json.charAt(pos) != '"') break; + + int nameEnd = findClosingQuote(json, pos + 1); + if (nameEnd == -1) break; + + String name = json.substring(pos + 1, nameEnd); + pos = nameEnd + 1; // Move past the closing quote + + // Skip whitespace and colon + while (pos < json.length() && (Character.isWhitespace(json.charAt(pos)) || json.charAt(pos) == ':')) { + pos++; + } + + // Expect opening brace for nested object + if (pos >= json.length() || json.charAt(pos) != '{') break; + + int objEnd = findClosingBrace(json, pos + 1); + if (objEnd == -1) break; + + String nestedContent = json.substring(pos + 1, objEnd); + pos = objEnd + 1; + + String contentRaw = extractStringValue(nestedContent, "content"); + if (contentRaw != null) { + String mimeTypeRaw = extractStringValue(nestedContent, "mimeType"); + String propertiesRaw = null; + if (nestedContent.indexOf("\"properties\": {") != -1) { + propertiesRaw = nestedContent.substring(nestedContent.indexOf("\"properties\": {") + 14, nestedContent.lastIndexOf("}") + 1); + } + String charsetRaw = extractStringValue(nestedContent, "charset"); + + Map properties = Map$.MODULE$.empty(); + if (propertiesRaw != null) { + properties = parseJsonProperties(propertiesRaw); + } + Charset charset = Charset.forName(charsetRaw != null ? charsetRaw : "UTF-8"); + Option mimeType = Option.apply(mimeTypeRaw); + + byte[] content = Base64.getDecoder().decode(contentRaw); + BindingValue bindingValue = new BindingValue(content, mimeType, properties, charset); + bindings.addBinding(name, bindingValue); + + } + } + } catch (Exception e) { + System.err.println("Error parsing JSON inputs: " + e.getMessage()); + e.printStackTrace(); + } + + return bindings; + } + + private Map parseJsonProperties(String jsonProperties) { + if (jsonProperties == null || jsonProperties.trim().isEmpty()) { + return Map$.MODULE$.empty(); + } + + String json = jsonProperties.trim(); + if (json.charAt(0) != '{') { + throw new IllegalArgumentException("properties must be a JSON object (must start with '{'): " + jsonProperties); + } + + int end = findClosingBrace(json, 1); + if (end == -1) { + throw new IllegalArgumentException("properties must be a valid JSON object (missing closing '}'): " + jsonProperties); + } + + // Disallow trailing non-whitespace after the object + for (int i = end + 1; i < json.length(); i++) { + if (!Character.isWhitespace(json.charAt(i))) { + throw new IllegalArgumentException("properties must contain a single JSON object (unexpected trailing content): " + jsonProperties); + } + } + + Map result = Map$.MODULE$.empty(); + int pos = 1; // skip '{' + + while (pos < end) { + while (pos < end && (Character.isWhitespace(json.charAt(pos)) || json.charAt(pos) == ',')) { + pos++; + } + + if (pos >= end) { + break; + } + + if (json.charAt(pos) != '"') { + throw new IllegalArgumentException("properties keys must be quoted strings at position " + pos + ": " + jsonProperties); + } + + int keyEnd = findClosingQuote(json, pos + 1); + if (keyEnd == -1 || keyEnd > end) { + throw new IllegalArgumentException("properties has an unterminated key string: " + jsonProperties); + } + + String key = unescapeJsonString(json.substring(pos + 1, keyEnd)); + pos = keyEnd + 1; + + while (pos < end && Character.isWhitespace(json.charAt(pos))) { + pos++; + } + if (pos >= end || json.charAt(pos) != ':') { + throw new IllegalArgumentException("properties expected ':' after key '" + key + "': " + jsonProperties); + } + pos++; + + while (pos < end && Character.isWhitespace(json.charAt(pos))) { + pos++; + } + if (pos >= end) { + throw new IllegalArgumentException("properties missing value for key '" + key + "': " + jsonProperties); + } + + Object value; + char c = json.charAt(pos); + if (c == '"') { + int valueEnd = findClosingQuote(json, pos + 1); + if (valueEnd == -1 || valueEnd > end) { + throw new IllegalArgumentException("properties has an unterminated string value for key '" + key + "': " + jsonProperties); + } + value = unescapeJsonString(json.substring(pos + 1, valueEnd)); + pos = valueEnd + 1; + } else if (c == 't' || c == 'f') { + if (json.startsWith("true", pos)) { + value = Boolean.TRUE; + pos += 4; + } else if (json.startsWith("false", pos)) { + value = Boolean.FALSE; + pos += 5; + } else { + throw new IllegalArgumentException("properties invalid boolean value for key '" + key + "' at position " + pos + ": " + jsonProperties); + } + } else if (c == 'n') { + throw new IllegalArgumentException("properties values cannot be null (key '" + key + "'): " + jsonProperties); + } else if (c == '{' || c == '[') { + throw new IllegalArgumentException("properties values must be primitive (string/number/boolean) (key '" + key + "'): " + jsonProperties); + } else { + int numEnd = pos; + while (numEnd < end) { + char nc = json.charAt(numEnd); + if (nc == ',' || nc == '}' || Character.isWhitespace(nc)) { + break; + } + numEnd++; + } + String numStr = json.substring(pos, numEnd); + if (numStr.isEmpty()) { + throw new IllegalArgumentException("properties invalid number value for key '" + key + "' at position " + pos + ": " + jsonProperties); + } + try { + if (numStr.indexOf('.') >= 0 || numStr.indexOf('e') >= 0 || numStr.indexOf('E') >= 0) { + value = Double.parseDouble(numStr); + } else { + value = Long.parseLong(numStr); + } + } catch (NumberFormatException nfe) { + throw new IllegalArgumentException("properties invalid number value for key '" + key + "': " + numStr, nfe); + } + pos = numEnd; + } + + result = (Map) result.$plus(new Tuple2<>(key, value)); + } + + return result; + } + + /** + * Parse a JSON string value starting at position (which should be at the opening quote). + * Returns the unescaped string content (without quotes). + */ + private String parseString(String json, int startPos) { + if (json.charAt(startPos) != '"') return null; + + int endPos = findClosingQuote(json, startPos + 1); + if (endPos == -1) return null; + + String escaped = json.substring(startPos + 1, endPos); + return unescapeJsonString(escaped); + } + + /** + * Extract a string value by key from a JSON object content. + * Simplified version assuming all values are strings. + */ + private String extractStringValue(String json, String key) { + String searchKey = "\"" + key + "\""; + int keyPos = json.indexOf(searchKey); + if (keyPos == -1) return null; + + // Find the colon after the key + int colonPos = keyPos + searchKey.length(); + while (colonPos < json.length() && json.charAt(colonPos) != ':') { + colonPos++; + } + if (colonPos >= json.length()) return null; + + // Skip whitespace after colon + int valueStart = colonPos + 1; + while (valueStart < json.length() && Character.isWhitespace(json.charAt(valueStart))) { + valueStart++; + } + + if (valueStart >= json.length() || json.charAt(valueStart) != '"') return null; + + return parseString(json, valueStart); + } + + /** + * Find the closing quote, skipping escaped quotes. + * Properly handles escaped backslashes. + */ + private int findClosingQuote(String str, int startPos) { + for (int i = startPos; i < str.length(); i++) { + char c = str.charAt(i); + + if (c == '\\' && i + 1 < str.length()) { + // Skip the escaped character + i++; + } else if (c == '"') { + // Found unescaped quote + return i; + } + } + return -1; + } + + /** + * Find the closing brace, properly handling nested braces and strings. + */ + private int findClosingBrace(String str, int startPos) { + int depth = 1; + boolean inString = false; + + for (int i = startPos; i < str.length(); i++) { + char c = str.charAt(i); + + if (inString) { + if (c == '\\' && i + 1 < str.length()) { + i++; // Skip escaped character + } else if (c == '"') { + inString = false; + } + } else { + if (c == '"') { + inString = true; + } else if (c == '{') { + depth++; + } else if (c == '}') { + depth--; + if (depth == 0) { + return i; + } + } + } + } + return -1; + } + + /** + * Unescapes JSON string escape sequences. + * Order matters: handle \\\\ first to avoid conflicts with other escape sequences. + */ + private String unescapeJsonString(String input) { + if (input == null) { + return null; + } + // Use a placeholder for escaped backslashes to avoid conflicts + String placeholder = "\u0000"; // null character as temporary placeholder + return input + .replace("\\\\", placeholder) + .replace("\\n", "\n") + .replace("\\r", "\r") + .replace("\\t", "\t") + .replace("\\\"", "\"") + .replace(placeholder, "\\"); + } + + private String escapeJsonString(String input) { + if (input == null) { + return ""; + } + + return input + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } +} diff --git a/native-lib/src/main/resources/META-INF/services/org.mule.weave.v2.module.DataFormat b/native-lib/src/main/resources/META-INF/services/org.mule.weave.v2.module.DataFormat new file mode 100644 index 0000000..22c55c7 --- /dev/null +++ b/native-lib/src/main/resources/META-INF/services/org.mule.weave.v2.module.DataFormat @@ -0,0 +1,9 @@ +org.mule.weave.v2.interpreted.module.WeaveDataFormat +org.mule.weave.v2.module.core.json.JsonDataFormat +org.mule.weave.v2.module.core.xml.XmlDataFormat +org.mule.weave.v2.module.core.csv.CSVDataFormat +org.mule.weave.v2.module.core.octetstream.OctetStreamDataFormat +org.mule.weave.v2.module.core.textplain.TextPlainDataFormat +org.mule.weave.v2.module.core.urlencoded.UrlEncodedDataFormat +org.mule.weave.v2.module.core.multipart.MultiPartDataFormat +org.mule.weave.v2.module.core.properties.PropertiesDataFormat diff --git a/native-lib/src/test/java/org/mule/weave/lib/ScriptRuntimeTest.java b/native-lib/src/test/java/org/mule/weave/lib/ScriptRuntimeTest.java new file mode 100644 index 0000000..2b67a50 --- /dev/null +++ b/native-lib/src/test/java/org/mule/weave/lib/ScriptRuntimeTest.java @@ -0,0 +1,280 @@ +package org.mule.weave.lib; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.Charset; +import java.util.Base64; + +class ScriptRuntimeTest { + + @Test + void runSimpleScript() { + ScriptRuntime runtime = ScriptRuntime.getInstance(); + + System.out.println("Running sqrt(144) 10 times with timing:"); + System.out.println("=".repeat(50)); + + for (int i = 1; i <= 20; i++) { + long startTime = System.nanoTime(); + String result = runtime.run("sqrt(144)"); + long endTime = System.nanoTime(); + double executionTimeMs = (endTime - startTime) / 1_000_000.0; + + assertEquals("12", Result.parse(result).result); + System.out.printf("Run %2d: %.3f ms - Result: %s%n", i, executionTimeMs, result); + } + + System.out.println("=".repeat(50)); + } + + @Test + void runParseError() { + ScriptRuntime runtime = ScriptRuntime.getInstance(); + + System.out.println("Running sqrt(144) 10 times with timing:"); + System.out.println("=".repeat(50)); + + String result = runtime.run("invalid syntax here"); + + String error = Result.parse(result).error; + assertTrue(error.contains("Unable to resolve reference")); + System.out.printf("Error: %s%n", result); + + System.out.println("=".repeat(50)); + } + + @Test + void runWithInputs() { + ScriptRuntime runtime = ScriptRuntime.getInstance(); + + System.out.println("Testing runWithInputs with two integer numbers:"); + System.out.println("=".repeat(50)); + + // Test 1: Sum 25 + 17 + int num1 = 25; + int num2 = 17; + int expected = num1 + num2; + + // Create inputs JSON with content and mimeType for each binding + String inputsJson = String.format( + "{\"num1\": {\"content\": \"%s\", \"mimeType\": \"application/json\"}, " + + "\"num2\": {\"content\": \"%s\", \"mimeType\": \"application/json\"}}", + encode(num1), encode(num2) + ); + + String script = "num1 + num2"; + + System.out.printf("Test 1: %d + %d%n", num1, num2); + System.out.printf("Script: %s%n", script); + System.out.printf("Inputs: %s%n", inputsJson); + + long startTime = System.nanoTime(); + String result = Result.parse(runtime.run(script, inputsJson)).result; + long endTime = System.nanoTime(); + double executionTimeMs = (endTime - startTime) / 1_000_000.0; + + System.out.printf("Result: %s%n", result); + System.out.printf("Expected: %d%n", expected); + System.out.printf("Execution time: %.3f ms%n", executionTimeMs); + + assertEquals(String.valueOf(expected), result); + System.out.println("✓ Test 1 passed!"); + + System.out.println("-".repeat(50)); + + // Test 2: Sum 100 + 250 + num1 = 100; + num2 = 250; + expected = num1 + num2; + + inputsJson = String.format( + "{\"num1\": {\"content\": \"%s\", \"mimeType\": \"application/json\"}, " + + "\"num2\": {\"content\": \"%s\", \"mimeType\": \"application/json\"}}", + encode(num1), encode(num2) + ); + + System.out.printf("Test 2: %d + %d%n", num1, num2); + System.out.printf("Script: %s%n", script); + + startTime = System.nanoTime(); + result = Result.parse(runtime.run(script, inputsJson)).result; + endTime = System.nanoTime(); + executionTimeMs = (endTime - startTime) / 1_000_000.0; + + System.out.printf("Result: %s%n", result); + System.out.printf("Expected: %d%n", expected); + System.out.printf("Execution time: %.3f ms%n", executionTimeMs); + + assertEquals(String.valueOf(expected), result); + System.out.println("✓ Test 2 passed!"); + + System.out.println("=".repeat(50)); + } + + private String encode(Object value) { + byte[] bytes = value instanceof byte[] ? (byte[]) value : String.valueOf(value).getBytes(); + return Base64.getEncoder().encodeToString(bytes); + + } + + @Test + void runWithXmlInput() { + ScriptRuntime runtime = ScriptRuntime.getInstance(); + + System.out.println("Testing runWithInputs with XML input to calculate average age:"); + System.out.println("=".repeat(50)); + + // XML input with two people + String xmlInput = """ + + + 19 + john + + + 25 + jane + + + """; + + String inputsJson = String.format( + "{\"people\": {\"content\": \"%s\", \"mimeType\": \"application/xml\"}}", + encode(xmlInput) + ); + + // DataWeave script to calculate average age + String script = """ + output application/json + --- + avg(people.people.*person.age) + """; + + System.out.printf("XML Input:%n%s%n", xmlInput); + System.out.printf("Script:%n%s%n", script); + + long startTime = System.nanoTime(); + String result = runtime.run(script, inputsJson); + long endTime = System.nanoTime(); + double executionTimeMs = (endTime - startTime) / 1_000_000.0; + + System.out.printf("Result: %s%n", result); + System.out.printf("Expected: 22 (average of 19 and 25)%n"); + System.out.printf("Execution time: %.3f ms%n", executionTimeMs); + + // The average of 19 and 25 is 22 + assertEquals("22", Result.parse(result).result); + System.out.println("✓ Test passed!"); + + System.out.println("=".repeat(50)); + } + + @Test + void runWithJsonObjectInput() { + ScriptRuntime runtime = ScriptRuntime.getInstance(); + + System.out.println("Testing runWithInputs with JSON object input:"); + System.out.println("=".repeat(50)); + + String jsonInput = "{\"name\": \"John\", \"age\": 30}"; + + String inputsJson = String.format( + "{\"payload\": {\"content\": \"%s\", \"mimeType\": \"application/json\"}}", + encode(jsonInput) + ); + + // DataWeave script to extract name + String script = "output application/json\n---\npayload.name"; + + System.out.printf("JSON Input: %s%n", jsonInput); + System.out.printf("Script: %s%n", script); + + long startTime = System.nanoTime(); + String result = Result.parse(runtime.run(script, inputsJson)).result; + long endTime = System.nanoTime(); + double executionTimeMs = (endTime - startTime) / 1_000_000.0; + + System.out.printf("Result: %s%n", result); + System.out.printf("Expected: \"John\"%n"); + System.out.printf("Execution time: %.3f ms%n", executionTimeMs); + + assertEquals("\"John\"", result); + System.out.println("✓ Test passed!"); + + System.out.println("=".repeat(50)); + } + + @Test + void runWithBinaryResult() { + ScriptRuntime runtime = ScriptRuntime.getInstance(); + + System.out.println("Running fromBase64 10 times with timing:"); + System.out.println("=".repeat(50)); + + for (int i = 1; i <= 1; i++) { + long startTime = System.nanoTime(); + Result result = Result.parse(runtime.run("import fromBase64 from dw::core::Binaries\n" + + "output application/octet-stream\n" + + "---\n" + + "fromBase64(\"12345678\")", "")); + long endTime = System.nanoTime(); + double executionTimeMs = (endTime - startTime) / 1_000_000.0; + + assertEquals("12345678", result.result); + System.out.printf("Run %2d: %.3f ms - Result: %s%n", i, executionTimeMs, result.result); + } + + System.out.println("=".repeat(50)); + } + + @Test + void runWithInputProperties() { + ScriptRuntime runtime = ScriptRuntime.getInstance(); + String encodedIn0 = Base64.getEncoder().encodeToString("1234567".getBytes()); + Result result = Result.parse(runtime.run("in0.column_1[0] as Number", + "{\"in0\": " + + "{\"content\": \"" + encodedIn0 + "\", " + + "\"mimeType\": \"application/csv\", " + + "\"properties\": {\"header\": false, \"separator\": \"4\"}}}")); + assertEquals("567", result.result); + + } + + static class Result { + boolean success; + String result; + String error; + boolean binary; + String mimeType; + String charset; + + static Result parse(String json) { + Result result = new Result(); + + String successString = json.substring(json.indexOf(":") + 1, json.indexOf(",")); + result.success = Boolean.parseBoolean(successString); + if (result.success) { + String binaryString = json.substring(json.indexOf(",\"binary\":") + 10, json.indexOf("}")); + result.binary = Boolean.parseBoolean(binaryString); + String resultString = json.substring(json.indexOf(",\"result\":") + 11, json.indexOf(",\"mimeType\":")-1); + String mimeTypeString = json.substring(json.indexOf(",\"mimeType\":") + 13, json.indexOf(",\"charset\":")-1); + result.mimeType = mimeTypeString; + String charsetString = json.substring(json.indexOf(",\"charset\":") + 12, json.indexOf(",\"binary\":")-1); + result.charset = charsetString; + if (result.binary) { + result.result = resultString; + } else { + result.result = new String(Base64.getDecoder().decode(resultString), Charset.forName(result.charset)); + } + + } else { + result.error = json.substring(json.indexOf(",\"error\":") + 10, json.length()-2); + } + return result; + } + } + +} diff --git a/settings.gradle b/settings.gradle index a47c02c..befbb96 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,3 +1,3 @@ include 'native-cli' include 'native-cli-integration-tests' - +include 'native-lib'