From aab7fa4dba5e2450c7f42f6f7906fbef7c0445cc Mon Sep 17 00:00:00 2001 From: ionmincu Date: Wed, 14 Jan 2026 12:34:38 +0200 Subject: [PATCH 1/4] tests: add profiler testcase --- .github/workflows/integration_tests.yml | 8 +++++ testcases/performance-testcase/main.py | 29 +++++++++++++++++ testcases/performance-testcase/pyproject.toml | 12 +++++++ testcases/performance-testcase/run.sh | 28 ++++++++++++++++ testcases/performance-testcase/src/assert.py | 32 +++++++++++++++++++ testcases/performance-testcase/uipath.json | 5 +++ 6 files changed, 114 insertions(+) create mode 100644 testcases/performance-testcase/main.py create mode 100644 testcases/performance-testcase/pyproject.toml create mode 100644 testcases/performance-testcase/run.sh create mode 100644 testcases/performance-testcase/src/assert.py create mode 100644 testcases/performance-testcase/uipath.json diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 448435f91..3b05c5fc2 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -80,6 +80,14 @@ jobs: bash run.sh bash ../common/validate_output.sh + - name: Upload testcase artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: artifacts-${{ matrix.testcase }}-${{ matrix.environment }} + path: testcases/${{ matrix.testcase }}/artifacts/ + if-no-files-found: ignore + summarize-results: needs: [integration-tests] runs-on: ubuntu-latest diff --git a/testcases/performance-testcase/main.py b/testcases/performance-testcase/main.py new file mode 100644 index 000000000..49fa0e055 --- /dev/null +++ b/testcases/performance-testcase/main.py @@ -0,0 +1,29 @@ +import logging +from dataclasses import dataclass + + +logger = logging.getLogger(__name__) + + +@dataclass +class EchoIn: + message: str + repeat: int | None = 1 + prefix: str | None = None + + +@dataclass +class EchoOut: + message: str + + +def main(input: EchoIn) -> EchoOut: + result = [] + + for _ in range(input.repeat): + line = input.message + if input.prefix: + line = f"{input.prefix}: {line}" + result.append(line) + + return EchoOut(message="\n".join(result)) diff --git a/testcases/performance-testcase/pyproject.toml b/testcases/performance-testcase/pyproject.toml new file mode 100644 index 000000000..cc8781624 --- /dev/null +++ b/testcases/performance-testcase/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "agent" +version = "0.0.1" +description = "agent" +authors = [{ name = "John Doe", email = "john.doe@myemail.com" }] +dependencies = [ + "uipath", +] +requires-python = ">=3.11" + +[tool.uv.sources] +uipath = { path = "../../", editable = true } \ No newline at end of file diff --git a/testcases/performance-testcase/run.sh b/testcases/performance-testcase/run.sh new file mode 100644 index 000000000..c682e90ca --- /dev/null +++ b/testcases/performance-testcase/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +echo "Syncing dependencies..." +uv sync + +uv add py-spy memray + +echo "Authenticating with UiPath..." +uv run uipath auth --client-id="$CLIENT_ID" --client-secret="$CLIENT_SECRET" --base-url="$BASE_URL" + +echo "Run init..." +uv run uipath init + +echo "Packing agent..." +uv run uipath pack + +echo "Creating artifacts directory..." +mkdir -p artifacts + +echo "Run agent with py-spy profiling (raw text format for LLM analysis)" +uv run py-spy record --subprocesses -f raw -o artifacts/cpu_profile.txt -- uv run uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' + +echo "Run agent with py-spy profiling (flamegraph SVG for visualization)" +uv run py-spy record --subprocesses -f flamegraph -o artifacts/cpu_profile.svg -- uv run uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' + +echo "Run agent with py-spy profiling (speedscope JSON for interactive viewing)" +uv run py-spy record --subprocesses -f speedscope -o artifacts/cpu_profile_speedscope.json -- uv run uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' diff --git a/testcases/performance-testcase/src/assert.py b/testcases/performance-testcase/src/assert.py new file mode 100644 index 000000000..025ddf212 --- /dev/null +++ b/testcases/performance-testcase/src/assert.py @@ -0,0 +1,32 @@ +import json +import os + +# Check NuGet package +uipath_dir = ".uipath" +assert os.path.exists(uipath_dir), "NuGet package directory (.uipath) not found" + +nupkg_files = [f for f in os.listdir(uipath_dir) if f.endswith(".nupkg")] +assert nupkg_files, "NuGet package file (.nupkg) not found in .uipath directory" + +print(f"NuGet package found: {nupkg_files[0]}") + +# Check agent output file +output_file = "__uipath/output.json" +assert os.path.isfile(output_file), "Agent output file not found" + +print("Agent output file found") + +# Check status and required fields +with open(output_file, "r", encoding="utf-8") as f: + output_data = json.load(f) + +# Check status +status = output_data.get("status") +assert status == "successful", f"Agent execution failed with status: {status}" + +print("Agent execution status: successful") + +# Check required fields for ticket classification agent +assert "output" in output_data, "Missing 'output' field in agent response" + +print("Required fields validation passed") diff --git a/testcases/performance-testcase/uipath.json b/testcases/performance-testcase/uipath.json new file mode 100644 index 000000000..ee698d9df --- /dev/null +++ b/testcases/performance-testcase/uipath.json @@ -0,0 +1,5 @@ +{ + "functions": { + "main": "main.py:main" + } +} \ No newline at end of file From 472cb91063d4c7eacc969c3eeefd3696ee69af6d Mon Sep 17 00:00:00 2001 From: ionmincu Date: Thu, 15 Jan 2026 17:56:45 +0200 Subject: [PATCH 2/4] xyz --- .github/workflows/integration_tests.yml | 3 + testcases/performance-testcase/README.md | 424 ++++++++++++++++++ .../performance-testcase/collect_metrics.py | 339 ++++++++++++++ .../performance-testcase/profile_memory.py | 96 ++++ testcases/performance-testcase/run.sh | 15 +- tmpclaude-46e7-cwd | 1 + tmpclaude-6847-cwd | 1 + 7 files changed, 872 insertions(+), 7 deletions(-) create mode 100644 testcases/performance-testcase/README.md create mode 100644 testcases/performance-testcase/collect_metrics.py create mode 100644 testcases/performance-testcase/profile_memory.py create mode 100644 tmpclaude-46e7-cwd create mode 100644 tmpclaude-6847-cwd diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 3b05c5fc2..e328d6dad 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -67,6 +67,9 @@ jobs: APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING }} APP_INSIGHTS_APP_ID: ${{ secrets.APP_INSIGHTS_APP_ID }} APP_INSIGHTS_API_KEY: ${{ secrets.APP_INSIGHTS_API_KEY }} + + # Azure Blob Storage for performance metrics + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} working-directory: testcases/${{ matrix.testcase }} run: | # If any errors occur execution will stop with exit code diff --git a/testcases/performance-testcase/README.md b/testcases/performance-testcase/README.md new file mode 100644 index 000000000..1b5529093 --- /dev/null +++ b/testcases/performance-testcase/README.md @@ -0,0 +1,424 @@ +# Performance Testcase + +This testcase measures the **framework overhead** of `uipath run` by profiling a minimal function that does almost no work. This helps identify performance bottlenecks in the CLI, runtime, and platform infrastructure. + +## What It Does + +1. **Runs a minimal function** ([main.py](main.py)) that only performs basic string operations +2. **Profiles with py-spy** (speedscope format) to capture execution timing +3. **Profiles memory usage** with Python's tracemalloc +4. **Collects performance metrics** including: + - Total execution time + - Time spent in user function vs framework overhead + - Time spent in imports/module loading + - Memory usage (peak and current) + - File sizes of profiling artifacts +5. **Uploads metrics to Azure Blob Storage** for historical tracking and analysis + +## Test Function + +```python +def main(input: EchoIn) -> EchoOut: + result = [] + for _ in range(input.repeat): + line = input.message + if input.prefix: + line = f"{input.prefix}: {line}" + result.append(line) + return EchoOut(message="\n".join(result)) +``` + +This function is deliberately minimal to isolate framework overhead. + +## Metrics Collected + +The testcase generates a `metrics.json` file with: + +```json +{ + "timestamp": "2026-01-15T10:30:45.123456+00:00", + "framework": "uipath", + "testcase": "performance-testcase", + "function": "main (echo function - minimal work)", + "timing": { + "total_time_seconds": 2.456, + "total_time_ms": 2456.78, + "user_function": { + "time_ms": 12.34, + "time_seconds": 0.012, + "percentage": 0.50 + }, + "framework_overhead": { + "time_ms": 412.89, + "time_seconds": 0.413, + "percentage": 16.81 + }, + "import_time": { + "time_ms": 2031.55, + "time_seconds": 2.032, + "percentage": 82.69 + }, + "sample_count": 24567, + "unique_frames": 245 + }, + "memory": { + "current_bytes": 45678912, + "peak_bytes": 52341256, + "current_mb": 43.56, + "peak_mb": 49.91 + }, + "execution_time_seconds": 2.458, + "file_sizes": { + "profile_json": 123456, + "memory_profile_json": 5678 + }, + "environment": { + "python_version": "3.11.0", + "platform": "linux", + "ci": "true", + "runner": "Linux", + "github_run_id": "12345", + "github_sha": "abc123", + "branch": "main" + } +} +``` + +### Key Metrics Explained + +- **framework**: Framework discriminator (`uipath`, `uipath-langgraph`, or `uipath-llamaindex`) +- **timing.total_time_seconds**: Total execution time from start to finish +- **timing.user_function**: Time spent executing the user's `main()` function +- **timing.framework_overhead**: Time spent in framework code (excluding imports) +- **timing.import_time**: Time spent loading Python modules +- **memory.peak_mb**: Peak memory usage during execution +- **memory.current_mb**: Memory usage at the end of execution +- **execution_time_seconds**: Wall-clock time measured by tracemalloc + +## Artifacts Generated + +| File | Format | Purpose | +|------|--------|---------| +| `profile.json` | Speedscope JSON | CPU profiling data with timing information - view at [speedscope.app](https://speedscope.app) | +| `memory_profile.json` | JSON | Memory profiling data with peak/current usage and top allocations | +| `metrics.json` | JSON | Combined metrics (timing + memory) for Azure Data Explorer ingestion | + +## Azure Blob Storage Setup + +### 1. Create Storage Account + +```bash +# Using Azure CLI +az storage account create \ + --name uipathperfmetrics \ + --resource-group uipath-performance \ + --location eastus \ + --sku Standard_LRS + +# Create container +az storage container create \ + --name performance-metrics \ + --account-name uipathperfmetrics +``` + +### 2. Get Connection String + +```bash +az storage account show-connection-string \ + --name uipathperfmetrics \ + --resource-group uipath-performance \ + --output tsv +``` + +### 3. Configure GitHub Secret + +1. Go to repository Settings → Secrets and variables → Actions +2. Add new secret: `AZURE_STORAGE_CONNECTION_STRING` +3. Paste the connection string from step 2 + +### Blob Naming Convention + +Metrics are uploaded with hierarchical names including the framework discriminator: + +``` +{framework}/{branch}/{github_run_id}/{timestamp}_metrics.json +``` + +Example: +``` +uipath/main/12345678/20260115_103045_metrics.json +uipath-langgraph/main/12345679/20260115_110230_metrics.json +uipath-llamaindex/feature/optimize-imports/12345680/20260115_112015_metrics.json +``` + +This allows: +- **Comparing frameworks** (uipath vs uipath-langgraph vs uipath-llamaindex) +- Tracking metrics over time +- Comparing branches within the same framework +- Correlating with CI runs + +## Running Locally + +### Prerequisites + +```bash +# Install dependencies +uv add py-spy azure-storage-blob + +# Set environment variables +export CLIENT_ID="your-client-id" +export CLIENT_SECRET="your-client-secret" +export BASE_URL="https://cloud.uipath.com/your-org" +export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=https;..." + +# Optional: Set framework discriminator (defaults to "uipath") +# export FRAMEWORK="uipath" # or "uipath-langgraph" or "uipath-llamaindex" +``` + +### Run Test + +```bash +cd testcases/performance-testcase +bash run.sh +``` + +**Note**: The `FRAMEWORK` environment variable defaults to `uipath`. For testing other frameworks: +```bash +export FRAMEWORK="uipath-langgraph" +bash run.sh +``` + +### View Results + +```bash +# View combined metrics +cat artifacts/metrics.json | jq + +# View timing breakdown +cat artifacts/metrics.json | jq '.timing' + +# View memory usage +cat artifacts/memory_profile.json | jq '.memory' + +# Upload to speedscope.app for interactive timeline view +# Go to https://speedscope.app +# Load artifacts/profile.json +``` + +## CI/CD Integration + +The testcase runs automatically in GitHub Actions across three environments: +- Alpha +- Staging +- Cloud (production) + +Each run: +1. Profiles the agent execution +2. Collects metrics +3. Uploads to Azure Blob Storage +4. Uploads artifacts to GitHub Actions + +### Workflow File + +See [`.github/workflows/integration_tests.yml`](../../.github/workflows/integration_tests.yml) + +## Azure Data Explorer (ADX) Ingestion + +### Create ADX Table + +```kql +.create table PerformanceMetrics ( + Timestamp: datetime, + Testcase: string, + Function: string, + Framework: string, + TotalTimeSeconds: real, + UserFunctionTimeSeconds: real, + UserFunctionPercentage: real, + FrameworkOverheadSeconds: real, + FrameworkOverheadPercentage: real, + ImportTimeSeconds: real, + ImportPercentage: real, + SampleCount: int, + UniqueFrames: int, + PeakMemoryMB: real, + CurrentMemoryMB: real, + ExecutionTimeSeconds: real, + ProfileSizeBytes: long, + MemoryProfileSizeBytes: long, + PythonVersion: string, + Platform: string, + CI: bool, + RunnerOS: string, + GitHubRunId: string, + GitHubSHA: string, + Branch: string +) +``` + +### Create Data Connection + +```kql +.create table PerformanceMetrics ingestion json mapping 'PerformanceMetricsMapping' +``` +```json +[ + {"column": "Timestamp", "path": "$.timestamp", "datatype": "datetime"}, + {"column": "Testcase", "path": "$.testcase", "datatype": "string"}, + {"column": "Function", "path": "$.function", "datatype": "string"}, + {"column": "Framework", "path": "$.framework", "datatype": "string"}, + {"column": "TotalTimeSeconds", "path": "$.timing.total_time_seconds", "datatype": "real"}, + {"column": "UserFunctionTimeSeconds", "path": "$.timing.user_function.time_seconds", "datatype": "real"}, + {"column": "UserFunctionPercentage", "path": "$.timing.user_function.percentage", "datatype": "real"}, + {"column": "FrameworkOverheadSeconds", "path": "$.timing.framework_overhead.time_seconds", "datatype": "real"}, + {"column": "FrameworkOverheadPercentage", "path": "$.timing.framework_overhead.percentage", "datatype": "real"}, + {"column": "ImportTimeSeconds", "path": "$.timing.import_time.time_seconds", "datatype": "real"}, + {"column": "ImportPercentage", "path": "$.timing.import_time.percentage", "datatype": "real"}, + {"column": "SampleCount", "path": "$.timing.sample_count", "datatype": "int"}, + {"column": "UniqueFrames", "path": "$.timing.unique_frames", "datatype": "int"}, + {"column": "PeakMemoryMB", "path": "$.memory.peak_mb", "datatype": "real"}, + {"column": "CurrentMemoryMB", "path": "$.memory.current_mb", "datatype": "real"}, + {"column": "ExecutionTimeSeconds", "path": "$.execution_time_seconds", "datatype": "real"}, + {"column": "ProfileSizeBytes", "path": "$.file_sizes.profile_json", "datatype": "long"}, + {"column": "MemoryProfileSizeBytes", "path": "$.file_sizes.memory_profile_json", "datatype": "long"}, + {"column": "PythonVersion", "path": "$.environment.python_version", "datatype": "string"}, + {"column": "Platform", "path": "$.environment.platform", "datatype": "string"}, + {"column": "CI", "path": "$.environment.ci", "datatype": "bool"}, + {"column": "RunnerOS", "path": "$.environment.runner", "datatype": "string"}, + {"column": "GitHubRunId", "path": "$.environment.github_run_id", "datatype": "string"}, + {"column": "GitHubSHA", "path": "$.environment.github_sha", "datatype": "string"}, + {"column": "Branch", "path": "$.environment.branch", "datatype": "string"} +] +``` + +### Setup Event Grid Ingestion + +```bash +# Create data connection from Blob Storage to ADX +az kusto data-connection event-grid create \ + --cluster-name uipath-performance-cluster \ + --database-name PerformanceDB \ + --data-connection-name blob-ingestion \ + --resource-group uipath-performance \ + --storage-account-resource-id "/subscriptions/.../uipathperfmetrics" \ + --event-hub-resource-id "/subscriptions/.../eventhub" \ + --consumer-group '$Default' \ + --table-name PerformanceMetrics \ + --mapping-rule-name PerformanceMetricsMapping \ + --data-format json \ + --blob-storage-event-type Microsoft.Storage.BlobCreated +``` + +### Query Metrics in ADX + +```kql +// View recent metrics for a specific framework +PerformanceMetrics +| where Timestamp > ago(7d) and Framework == "uipath" +| project Timestamp, Framework, Branch, TotalTimeSeconds, ImportPercentage, PeakMemoryMB, UserFunctionPercentage +| order by Timestamp desc + +// Compare frameworks - execution time breakdown +PerformanceMetrics +| where Timestamp > ago(30d) and Branch == "main" +| summarize + AvgTotalTime = avg(TotalTimeSeconds), + AvgUserFunctionTime = avg(UserFunctionTimeSeconds), + AvgFrameworkOverhead = avg(FrameworkOverheadSeconds), + AvgImportTime = avg(ImportTimeSeconds), + AvgPeakMemoryMB = avg(PeakMemoryMB) + by Framework +| order by AvgTotalTime desc + +// Compare branches within a framework +PerformanceMetrics +| where Timestamp > ago(30d) and Framework == "uipath" +| summarize + AvgTotalTime = avg(TotalTimeSeconds), + AvgImportPct = avg(ImportPercentage), + AvgFrameworkPct = avg(FrameworkOverheadPercentage) + by Branch +| order by AvgTotalTime desc + +// Trend over time for a framework - import percentage +PerformanceMetrics +| where Branch == "main" and Framework == "uipath" +| summarize ImportPct = avg(ImportPercentage) by bin(Timestamp, 1d) +| render timechart + +// Compare all three frameworks side by side +PerformanceMetrics +| where Timestamp > ago(7d) and Branch == "main" +| summarize + AvgTotalTime = avg(TotalTimeSeconds), + AvgUserFunctionPct = avg(UserFunctionPercentage), + AvgFrameworkPct = avg(FrameworkOverheadPercentage), + AvgImportPct = avg(ImportPercentage), + AvgPeakMemoryMB = avg(PeakMemoryMB) + by Framework +| order by AvgTotalTime desc + +// Memory usage trend +PerformanceMetrics +| where Branch == "main" and Framework == "uipath" +| summarize AvgMemory = avg(PeakMemoryMB) by bin(Timestamp, 1d) +| render timechart +``` + +## Troubleshooting + +### Metrics not uploading + +1. Check `AZURE_STORAGE_CONNECTION_STRING` is set: + ```bash + echo $AZURE_STORAGE_CONNECTION_STRING + ``` + +2. Verify storage account exists and is accessible: + ```bash + az storage container list \ + --connection-string "$AZURE_STORAGE_CONNECTION_STRING" + ``` + +3. Check script output for error messages + +### Profile files empty or missing + +1. Ensure py-spy is installed: `pip show py-spy` +2. Check process permissions (py-spy needs ptrace access on Linux) +3. Verify `uv run uipath run` executes successfully + +### Timing metrics seem off + +Timing metrics are extracted from speedscope profile data: +- **User function time**: Samples where stack contains `main` from testcases directory +- **Import time**: Samples where stack contains `_find_and_load` or `_load_unlocked` +- **Framework overhead**: All other samples (excluding imports and user function) + +The percentages should add up to 100%. If they don't, check that the speedscope JSON is valid. + +### Memory profiling failed + +Memory profiling uses Python's `tracemalloc` which may fail if: +1. The subprocess can't be executed +2. Insufficient permissions +3. Python crashes during execution + +Check the error output from `profile_memory.py` for details. + +## Related Files + +- [collect_metrics.py](collect_metrics.py) - Metrics collection script (parses speedscope and memory data) +- [profile_memory.py](profile_memory.py) - Memory profiling script using tracemalloc +- [main.py](main.py) - Minimal test function +- [run.sh](run.sh) - Test runner with profiling commands +- [../../.github/workflows/integration_tests.yml](../../.github/workflows/integration_tests.yml) - CI workflow + +## Future Enhancements + +- [ ] Add more granular timing breakdowns (e.g., HTTP requests, database queries) +- [ ] Track process startup time separately +- [ ] Measure network latency to UiPath services +- [ ] Compare performance across Python versions +- [ ] Add alerting for performance regressions (e.g., >10% slowdown) +- [ ] Generate performance regression reports in PRs diff --git a/testcases/performance-testcase/collect_metrics.py b/testcases/performance-testcase/collect_metrics.py new file mode 100644 index 000000000..aa93d2b19 --- /dev/null +++ b/testcases/performance-testcase/collect_metrics.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""Collect performance metrics and upload to Azure Blob Storage.""" + +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def parse_speedscope_profile(profile_path: str) -> dict: + """Extract timing metrics from py-spy speedscope output. + + Speedscope format contains: + - profiles: Array of profile data with weights (time samples) + - shared: Frame information with function names + - samples: Timeline of stack samples + + We extract: + - Total execution time + - Time spent in user function (main) + - Time spent in framework overhead + """ + if not Path(profile_path).exists(): + return {"error": f"Profile file not found: {profile_path}"} + + try: + with open(profile_path, "r", encoding="utf-8") as f: + data = json.load(f) + except json.JSONDecodeError as e: + return {"error": f"Invalid JSON: {e}"} + + # Extract timing information + profiles = data.get("profiles", []) + if not profiles: + return {"error": "No profile data found"} + + profile = profiles[0] # Use first profile (main process) + samples = profile.get("samples", []) + weights = profile.get("weights", []) + shared = data.get("shared", {}) + frames = shared.get("frames", []) + + # Calculate total execution time (sum of all weights, in microseconds) + total_time_us = sum(weights) if weights else 0 + total_time_ms = total_time_us / 1000 + total_time_s = total_time_us / 1_000_000 + + # Find user function time + # Look for frames containing "main" from our testcase + user_function_time_us = 0 + framework_time_us = 0 + import_time_us = 0 + + for i, sample_frames in enumerate(samples): + weight = weights[i] if i < len(weights) else 0 + + # Get frame information + frame_names = [] + for frame_idx in sample_frames: + if frame_idx < len(frames): + frame = frames[frame_idx] + frame_name = frame.get("name", "") + frame_names.append(frame_name) + + # Check if this sample is in user function + is_user_function = any("main" in name and "testcases" in str(frame.get("file", "")) + for frame_idx in sample_frames + for name in [frames[frame_idx].get("name", "")] + if frame_idx < len(frames)) + + # Check if this sample is in import machinery + is_import = any("_find_and_load" in name or "_load_unlocked" in name + for frame_idx in sample_frames + for name in [frames[frame_idx].get("name", "")] + if frame_idx < len(frames)) + + if is_import: + import_time_us += weight + elif is_user_function: + user_function_time_us += weight + else: + framework_time_us += weight + + # Calculate percentages + user_percentage = (user_function_time_us / total_time_us * 100) if total_time_us > 0 else 0 + framework_percentage = (framework_time_us / total_time_us * 100) if total_time_us > 0 else 0 + import_percentage = (import_time_us / total_time_us * 100) if total_time_us > 0 else 0 + + return { + "total_time_seconds": round(total_time_s, 3), + "total_time_ms": round(total_time_ms, 2), + "user_function": { + "time_ms": round(user_function_time_us / 1000, 2), + "time_seconds": round(user_function_time_us / 1_000_000, 3), + "percentage": round(user_percentage, 2) + }, + "framework_overhead": { + "time_ms": round(framework_time_us / 1000, 2), + "time_seconds": round(framework_time_us / 1_000_000, 3), + "percentage": round(framework_percentage, 2) + }, + "import_time": { + "time_ms": round(import_time_us / 1000, 2), + "time_seconds": round(import_time_us / 1_000_000, 3), + "percentage": round(import_percentage, 2) + }, + "sample_count": len(samples), + "unique_frames": len(frames) + } + + +def get_file_size(file_path: str) -> int: + """Get file size in bytes.""" + if not Path(file_path).exists(): + return 0 + return Path(file_path).stat().st_size + + +def load_memory_metrics(artifacts_dir: str = "artifacts") -> dict: + """Load memory profiling metrics if available.""" + memory_profile_path = Path(artifacts_dir) / "memory_profile.json" + if not memory_profile_path.exists(): + return {} + + try: + with open(memory_profile_path, "r", encoding="utf-8") as f: + return json.load(f) + except json.JSONDecodeError: + return {} + + +def collect_metrics( + artifacts_dir: str = "artifacts", + framework: str | None = None, + testcase: str | None = None, +) -> dict: + """Collect all performance metrics from artifacts directory. + + Args: + artifacts_dir: Directory containing profile artifacts + framework: Framework discriminator (uipath, uipath-langgraph, uipath-llamaindex) + If None, auto-detects from FRAMEWORK env var or defaults to 'uipath' + testcase: Testcase name (defaults to TESTCASE env var or current directory name) + """ + artifacts_path = Path(artifacts_dir) + + # Auto-detect framework if not specified + if framework is None: + framework = os.getenv("FRAMEWORK", "uipath") + + # Auto-detect testcase from current directory if not specified + if testcase is None: + testcase = os.getenv("TESTCASE", Path.cwd().name) + + # Parse speedscope profile for timing metrics + profile_path = artifacts_path / "profile.json" + timing_metrics = parse_speedscope_profile(str(profile_path)) + + # Load memory metrics + memory_metrics = load_memory_metrics(artifacts_dir) + + # Get artifact file sizes + file_sizes = { + "profile_json": get_file_size(str(profile_path)), + "memory_profile_json": get_file_size(str(artifacts_path / "memory_profile.json")), + } + + # Build complete metrics object + metrics = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "framework": framework, # Discriminator: uipath, uipath-langgraph, uipath-llamaindex + "testcase": testcase, + "function": "main (echo function - minimal work)", + "timing": timing_metrics, + "memory": memory_metrics.get("memory", {}), + "execution_time_seconds": memory_metrics.get("execution_time_seconds", 0), + "file_sizes": file_sizes, + "environment": { + "python_version": sys.version.split()[0], + "platform": sys.platform, + "ci": os.getenv("CI", "false"), + "runner": os.getenv("RUNNER_OS", "unknown"), + "github_run_id": os.getenv("GITHUB_RUN_ID", "local"), + "github_sha": os.getenv("GITHUB_SHA", "unknown"), + "branch": os.getenv("GITHUB_REF_NAME", "unknown"), + }, + } + + return metrics + + +def save_metrics_json(metrics: dict, output_path: str = "artifacts/metrics.json"): + """Save metrics to JSON file.""" + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(metrics, f, indent=2) + print(f"✓ Metrics saved to {output_path}") + + +def upload_to_blob_storage( + file_path: str, + connection_string: str | None = None, + container_name: str = "performance-metrics", + blob_name: str | None = None, +) -> bool: + """Upload file to Azure Blob Storage. + + Args: + file_path: Path to file to upload + connection_string: Azure Storage connection string (or uses AZURE_STORAGE_CONNECTION_STRING env var) + container_name: Blob container name + blob_name: Name for blob (defaults to filename with timestamp) + + Returns: + True if upload succeeded, False otherwise + """ + try: + from azure.storage.blob import BlobServiceClient + except ImportError: + print("⚠️ azure-storage-blob not installed. Run: pip install azure-storage-blob") + return False + + connection_string = connection_string or os.getenv("AZURE_STORAGE_CONNECTION_STRING") + if not connection_string: + print("⚠️ AZURE_STORAGE_CONNECTION_STRING not set") + return False + + if not Path(file_path).exists(): + print(f"⚠️ File not found: {file_path}") + return False + + # Generate blob name with timestamp if not provided + if blob_name is None: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + filename = Path(file_path).name + blob_name = f"{timestamp}_{filename}" + + try: + # Create BlobServiceClient + blob_service_client = BlobServiceClient.from_connection_string(connection_string) + + # Create container if it doesn't exist + try: + container_client = blob_service_client.get_container_client(container_name) + if not container_client.exists(): + blob_service_client.create_container(container_name) + print(f"✓ Created container: {container_name}") + except Exception as e: + print(f"⚠️ Container check/creation warning: {e}") + + # Upload file + blob_client = blob_service_client.get_blob_client( + container=container_name, blob=blob_name + ) + + with open(file_path, "rb") as data: + blob_client.upload_blob(data, overwrite=True) + + blob_url = blob_client.url + print(f"✓ Uploaded to Azure Blob Storage: {blob_url}") + return True + + except Exception as e: + print(f"⚠️ Upload failed: {e}") + return False + + +def main(): + """Main entry point.""" + print("=" * 60) + print("Performance Metrics Collection") + print("=" * 60) + + # Collect metrics + print("\n📊 Collecting metrics...") + metrics = collect_metrics() + + # Print summary + print("\n📈 Metrics Summary:") + print(f" Framework: {metrics['framework']}") + print(f" Testcase: {metrics['testcase']}") + + # Timing metrics + timing = metrics.get('timing', {}) + if timing and 'error' not in timing: + print(f"\n⏱️ Timing Metrics:") + print(f" Total execution time: {timing.get('total_time_seconds', 0)}s ({timing.get('total_time_ms', 0)}ms)") + + user_func = timing.get('user_function', {}) + print(f" User function time: {user_func.get('time_seconds', 0)}s ({user_func.get('percentage', 0)}%)") + + framework = timing.get('framework_overhead', {}) + print(f" Framework overhead: {framework.get('time_seconds', 0)}s ({framework.get('percentage', 0)}%)") + + import_time = timing.get('import_time', {}) + print(f" Import time: {import_time.get('time_seconds', 0)}s ({import_time.get('percentage', 0)}%)") + + # Memory metrics + memory = metrics.get('memory', {}) + if memory: + print(f"\n💾 Memory Metrics:") + print(f" Peak memory: {memory.get('peak_mb', 0)} MB") + print(f" Current memory: {memory.get('current_mb', 0)} MB") + + # Save metrics JSON + print("\n💾 Saving metrics...") + metrics_path = "artifacts/metrics.json" + save_metrics_json(metrics, metrics_path) + + # Upload to Azure Blob Storage if connection string is available + connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING") + if connection_string: + print("\n☁️ Uploading to Azure Blob Storage...") + + # Generate blob name with metadata including framework discriminator + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + framework = metrics["framework"] + branch = metrics["environment"]["branch"].replace("/", "_") + run_id = metrics["environment"]["github_run_id"] + blob_name = f"{framework}/{branch}/{run_id}/{timestamp}_metrics.json" + + upload_to_blob_storage( + metrics_path, + connection_string=connection_string, + container_name="performance-metrics", + blob_name=blob_name, + ) + else: + print("\n⚠️ AZURE_STORAGE_CONNECTION_STRING not set - skipping upload") + print(" To enable upload, set environment variable:") + print(" export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=...'") + + print("\n✅ Metrics collection complete!") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/testcases/performance-testcase/profile_memory.py b/testcases/performance-testcase/profile_memory.py new file mode 100644 index 000000000..8d0c2ed24 --- /dev/null +++ b/testcases/performance-testcase/profile_memory.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Profile memory usage during agent execution.""" + +import json +import subprocess +import sys +import time +import tracemalloc +from pathlib import Path + + +def profile_agent_execution(): + """Run the agent with memory profiling enabled.""" + print("Starting memory profiling...") + + # Start tracking memory allocations + tracemalloc.start() + start_time = time.perf_counter() + + # Record initial memory snapshot + snapshot_start = tracemalloc.take_snapshot() + + # Run the agent + try: + result = subprocess.run( + [ + "uv", "run", "uipath", "run", "main", + '{"message": "abc", "repeat": 2, "prefix": "xyz"}' + ], + capture_output=True, + text=True, + check=True + ) + success = True + error = None + except subprocess.CalledProcessError as e: + success = False + error = str(e) + result = e + + # Record final memory snapshot + end_time = time.perf_counter() + snapshot_end = tracemalloc.take_snapshot() + + # Get peak memory usage + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + # Calculate memory difference + top_stats = snapshot_end.compare_to(snapshot_start, 'lineno') + + # Extract top memory allocations + top_allocations = [] + for stat in top_stats[:10]: + top_allocations.append({ + "file": str(stat.traceback.format()[0]) if stat.traceback else "unknown", + "size_bytes": stat.size, + "size_diff_bytes": stat.size_diff, + "count": stat.count, + "count_diff": stat.count_diff + }) + + # Build metrics + metrics = { + "execution_time_seconds": round(end_time - start_time, 3), + "memory": { + "current_bytes": current, + "peak_bytes": peak, + "current_mb": round(current / 1024 / 1024, 2), + "peak_mb": round(peak / 1024 / 1024, 2), + }, + "top_allocations": top_allocations, + "success": success, + "error": error + } + + # Save metrics + output_path = Path("artifacts/memory_profile.json") + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(metrics, f, indent=2) + + print(f"✓ Memory profile saved to {output_path}") + print(f" Execution time: {metrics['execution_time_seconds']}s") + print(f" Peak memory: {metrics['memory']['peak_mb']} MB") + print(f" Current memory: {metrics['memory']['current_mb']} MB") + + return metrics + + +if __name__ == "__main__": + try: + profile_agent_execution() + except Exception as e: + print(f"⚠️ Memory profiling failed: {e}", file=sys.stderr) + sys.exit(1) diff --git a/testcases/performance-testcase/run.sh b/testcases/performance-testcase/run.sh index c682e90ca..247d5e88c 100644 --- a/testcases/performance-testcase/run.sh +++ b/testcases/performance-testcase/run.sh @@ -4,7 +4,8 @@ set -e echo "Syncing dependencies..." uv sync -uv add py-spy memray +echo "Installing profiling tools and Azure SDK..." +uv add py-spy azure-storage-blob echo "Authenticating with UiPath..." uv run uipath auth --client-id="$CLIENT_ID" --client-secret="$CLIENT_SECRET" --base-url="$BASE_URL" @@ -18,11 +19,11 @@ uv run uipath pack echo "Creating artifacts directory..." mkdir -p artifacts -echo "Run agent with py-spy profiling (raw text format for LLM analysis)" -uv run py-spy record --subprocesses -f raw -o artifacts/cpu_profile.txt -- uv run uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' +echo "Run agent with py-spy profiling (speedscope JSON with timing data)" +uv run py-spy record --subprocesses -f speedscope -o artifacts/profile.json -- uv run uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' -echo "Run agent with py-spy profiling (flamegraph SVG for visualization)" -uv run py-spy record --subprocesses -f flamegraph -o artifacts/cpu_profile.svg -- uv run uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' +echo "Run agent with memory profiling using tracemalloc" +uv run python profile_memory.py -echo "Run agent with py-spy profiling (speedscope JSON for interactive viewing)" -uv run py-spy record --subprocesses -f speedscope -o artifacts/cpu_profile_speedscope.json -- uv run uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' +echo "Collecting performance metrics and uploading to Azure..." +uv run python collect_metrics.py diff --git a/tmpclaude-46e7-cwd b/tmpclaude-46e7-cwd new file mode 100644 index 000000000..9c7a044e4 --- /dev/null +++ b/tmpclaude-46e7-cwd @@ -0,0 +1 @@ +/c/Work/uipath-python2 diff --git a/tmpclaude-6847-cwd b/tmpclaude-6847-cwd new file mode 100644 index 000000000..9c7a044e4 --- /dev/null +++ b/tmpclaude-6847-cwd @@ -0,0 +1 @@ +/c/Work/uipath-python2 From ea6708c3598e73d91e8bef86865b067509c492e3 Mon Sep 17 00:00:00 2001 From: ionmincu Date: Thu, 15 Jan 2026 18:18:34 +0200 Subject: [PATCH 3/4] xyz --- .../performance-testcase/collect_metrics.py | 183 +++++++----------- .../extract_memory_stats.py | 94 +++++++++ testcases/performance-testcase/main.py | 19 ++ .../performance-testcase/profile_memory.py | 62 ++---- testcases/performance-testcase/run.sh | 29 ++- 5 files changed, 224 insertions(+), 163 deletions(-) create mode 100644 testcases/performance-testcase/extract_memory_stats.py diff --git a/testcases/performance-testcase/collect_metrics.py b/testcases/performance-testcase/collect_metrics.py index aa93d2b19..dd5e4d135 100644 --- a/testcases/performance-testcase/collect_metrics.py +++ b/testcases/performance-testcase/collect_metrics.py @@ -8,128 +8,82 @@ from pathlib import Path -def parse_speedscope_profile(profile_path: str) -> dict: - """Extract timing metrics from py-spy speedscope output. - - Speedscope format contains: - - profiles: Array of profile data with weights (time samples) - - shared: Frame information with function names - - samples: Timeline of stack samples - - We extract: - - Total execution time - - Time spent in user function (main) - - Time spent in framework overhead +def load_timing_metrics(artifacts_dir: str = "artifacts") -> dict: + """Load timing metrics from instrumentation files. + + Returns timing breakdown: + - User code time (from main.py instrumentation) + - Total execution time (from run.sh measurement) + - Framework overhead (calculated as total - user code) """ - if not Path(profile_path).exists(): - return {"error": f"Profile file not found: {profile_path}"} + artifacts_path = Path(artifacts_dir) - try: - with open(profile_path, "r", encoding="utf-8") as f: - data = json.load(f) - except json.JSONDecodeError as e: - return {"error": f"Invalid JSON: {e}"} - - # Extract timing information - profiles = data.get("profiles", []) - if not profiles: - return {"error": "No profile data found"} - - profile = profiles[0] # Use first profile (main process) - samples = profile.get("samples", []) - weights = profile.get("weights", []) - shared = data.get("shared", {}) - frames = shared.get("frames", []) - - # Calculate total execution time (sum of all weights, in microseconds) - total_time_us = sum(weights) if weights else 0 - total_time_ms = total_time_us / 1000 - total_time_s = total_time_us / 1_000_000 - - # Find user function time - # Look for frames containing "main" from our testcase - user_function_time_us = 0 - framework_time_us = 0 - import_time_us = 0 - - for i, sample_frames in enumerate(samples): - weight = weights[i] if i < len(weights) else 0 - - # Get frame information - frame_names = [] - for frame_idx in sample_frames: - if frame_idx < len(frames): - frame = frames[frame_idx] - frame_name = frame.get("name", "") - frame_names.append(frame_name) - - # Check if this sample is in user function - is_user_function = any("main" in name and "testcases" in str(frame.get("file", "")) - for frame_idx in sample_frames - for name in [frames[frame_idx].get("name", "")] - if frame_idx < len(frames)) - - # Check if this sample is in import machinery - is_import = any("_find_and_load" in name or "_load_unlocked" in name - for frame_idx in sample_frames - for name in [frames[frame_idx].get("name", "")] - if frame_idx < len(frames)) - - if is_import: - import_time_us += weight - elif is_user_function: - user_function_time_us += weight - else: - framework_time_us += weight + # Load user code timing (instrumented in main.py) + user_timing_path = artifacts_path / "user_code_timing.json" + user_code_time = 0 + + if user_timing_path.exists(): + try: + with open(user_timing_path, "r", encoding="utf-8") as f: + user_timing = json.load(f) + user_code_time = user_timing.get("user_code_time_seconds", 0) + except json.JSONDecodeError: + pass + + # Load total execution timing (from run.sh memray measurement) + total_timing_path = artifacts_path / "total_execution.json" + total_time = 0 + if total_timing_path.exists(): + try: + with open(total_timing_path, "r", encoding="utf-8") as f: + total_timing = json.load(f) + total_time = total_timing.get("total_execution_time_seconds", 0) + except json.JSONDecodeError: + pass + + # Calculate framework overhead + framework_overhead = total_time - user_code_time # Calculate percentages - user_percentage = (user_function_time_us / total_time_us * 100) if total_time_us > 0 else 0 - framework_percentage = (framework_time_us / total_time_us * 100) if total_time_us > 0 else 0 - import_percentage = (import_time_us / total_time_us * 100) if total_time_us > 0 else 0 + user_percentage = (user_code_time / total_time * 100) if total_time > 0 else 0 + framework_percentage = (framework_overhead / total_time * 100) if total_time > 0 else 0 return { - "total_time_seconds": round(total_time_s, 3), - "total_time_ms": round(total_time_ms, 2), - "user_function": { - "time_ms": round(user_function_time_us / 1000, 2), - "time_seconds": round(user_function_time_us / 1_000_000, 3), + "total_time_seconds": round(total_time, 3), + "total_time_ms": round(total_time * 1000, 2), + "user_code_time": { + "time_ms": round(user_code_time * 1000, 2), + "time_seconds": round(user_code_time, 6), "percentage": round(user_percentage, 2) }, "framework_overhead": { - "time_ms": round(framework_time_us / 1000, 2), - "time_seconds": round(framework_time_us / 1_000_000, 3), + "time_ms": round(framework_overhead * 1000, 2), + "time_seconds": round(framework_overhead, 3), "percentage": round(framework_percentage, 2) - }, - "import_time": { - "time_ms": round(import_time_us / 1000, 2), - "time_seconds": round(import_time_us / 1_000_000, 3), - "percentage": round(import_percentage, 2) - }, - "sample_count": len(samples), - "unique_frames": len(frames) + } } -def get_file_size(file_path: str) -> int: - """Get file size in bytes.""" - if not Path(file_path).exists(): - return 0 - return Path(file_path).stat().st_size - - def load_memory_metrics(artifacts_dir: str = "artifacts") -> dict: - """Load memory profiling metrics if available.""" - memory_profile_path = Path(artifacts_dir) / "memory_profile.json" - if not memory_profile_path.exists(): + """Load memory metrics from memray stats output.""" + memory_stats_path = Path(artifacts_dir) / "memory_stats.json" + if not memory_stats_path.exists(): return {} try: - with open(memory_profile_path, "r", encoding="utf-8") as f: + with open(memory_stats_path, "r", encoding="utf-8") as f: return json.load(f) except json.JSONDecodeError: return {} +def get_file_size(file_path: str) -> int: + """Get file size in bytes.""" + if not Path(file_path).exists(): + return 0 + return Path(file_path).stat().st_size + + def collect_metrics( artifacts_dir: str = "artifacts", framework: str | None = None, @@ -153,17 +107,19 @@ def collect_metrics( if testcase is None: testcase = os.getenv("TESTCASE", Path.cwd().name) - # Parse speedscope profile for timing metrics - profile_path = artifacts_path / "profile.json" - timing_metrics = parse_speedscope_profile(str(profile_path)) + # Load timing metrics (user code + total execution) + timing_metrics = load_timing_metrics(artifacts_dir) - # Load memory metrics + # Load memory metrics (from memray) memory_metrics = load_memory_metrics(artifacts_dir) # Get artifact file sizes file_sizes = { - "profile_json": get_file_size(str(profile_path)), - "memory_profile_json": get_file_size(str(artifacts_path / "memory_profile.json")), + "profile_json": get_file_size(str(artifacts_path / "profile.json")), + "user_code_timing_json": get_file_size(str(artifacts_path / "user_code_timing.json")), + "total_execution_json": get_file_size(str(artifacts_path / "total_execution.json")), + "memory_bin": get_file_size(str(artifacts_path / "memory.bin")), + "memory_stats_json": get_file_size(str(artifacts_path / "memory_stats.json")), } # Build complete metrics object @@ -173,8 +129,7 @@ def collect_metrics( "testcase": testcase, "function": "main (echo function - minimal work)", "timing": timing_metrics, - "memory": memory_metrics.get("memory", {}), - "execution_time_seconds": memory_metrics.get("execution_time_seconds", 0), + "memory": memory_metrics, "file_sizes": file_sizes, "environment": { "python_version": sys.version.split()[0], @@ -283,25 +238,23 @@ def main(): # Timing metrics timing = metrics.get('timing', {}) - if timing and 'error' not in timing: + if timing: print(f"\n⏱️ Timing Metrics:") print(f" Total execution time: {timing.get('total_time_seconds', 0)}s ({timing.get('total_time_ms', 0)}ms)") - user_func = timing.get('user_function', {}) - print(f" User function time: {user_func.get('time_seconds', 0)}s ({user_func.get('percentage', 0)}%)") + user_code = timing.get('user_code_time', {}) + print(f" User code time: {user_code.get('time_seconds', 0)}s ({user_code.get('percentage', 0)}%)") framework = timing.get('framework_overhead', {}) print(f" Framework overhead: {framework.get('time_seconds', 0)}s ({framework.get('percentage', 0)}%)") - import_time = timing.get('import_time', {}) - print(f" Import time: {import_time.get('time_seconds', 0)}s ({import_time.get('percentage', 0)}%)") - # Memory metrics memory = metrics.get('memory', {}) if memory: print(f"\n💾 Memory Metrics:") print(f" Peak memory: {memory.get('peak_mb', 0)} MB") - print(f" Current memory: {memory.get('current_mb', 0)} MB") + if 'total_allocations' in memory: + print(f" Total allocations: {memory.get('total_allocations', 0):,}") # Save metrics JSON print("\n💾 Saving metrics...") diff --git a/testcases/performance-testcase/extract_memory_stats.py b/testcases/performance-testcase/extract_memory_stats.py new file mode 100644 index 000000000..b27011753 --- /dev/null +++ b/testcases/performance-testcase/extract_memory_stats.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Extract memory statistics from memray binary output.""" + +import json +import subprocess +import sys +from pathlib import Path + + +def extract_memory_stats(): + """Extract memory stats from memray binary file using memray stats command.""" + print("Extracting memory stats from memray output...") + + memray_file = Path("artifacts/memory.bin") + if not memray_file.exists(): + print(f"⚠️ Memray file not found: {memray_file}") + return False + + try: + # Run memray stats to get peak memory and total allocations + result = subprocess.run( + ["memray", "stats", str(memray_file)], + capture_output=True, + text=True, + check=True + ) + + # Parse the stats output + # Format: + # Total memory allocated: X.XX GB + # Total allocations: X + # Histogram of allocation size: ... + # ... + # High watermark: X.XX GB + output_lines = result.stdout.split("\n") + + peak_memory_mb = 0 + total_allocations = 0 + + for line in output_lines: + if "High watermark" in line or "peak memory" in line.lower(): + # Extract memory value (could be in KB, MB, or GB) + parts = line.split(":") + if len(parts) >= 2: + value_str = parts[1].strip().split()[0] + try: + value = float(value_str) + # Check units + if "GB" in line: + peak_memory_mb = value * 1024 + elif "MB" in line: + peak_memory_mb = value + elif "KB" in line: + peak_memory_mb = value / 1024 + except ValueError: + pass + + if "Total allocations" in line: + parts = line.split(":") + if len(parts) >= 2: + try: + total_allocations = int(parts[1].strip()) + except ValueError: + pass + + # Save memory metrics + memory_metrics = { + "peak_mb": round(peak_memory_mb, 2), + "total_allocations": total_allocations + } + + output_path = Path("artifacts/memory_stats.json") + with open(output_path, "w") as f: + json.dump(memory_metrics, f, indent=2) + + print(f"✓ Memory stats saved to {output_path}") + print(f" Peak memory: {memory_metrics['peak_mb']} MB") + print(f" Total allocations: {memory_metrics['total_allocations']}") + + return True + + except subprocess.CalledProcessError as e: + print(f"⚠️ Failed to extract memory stats: {e}") + print(f" stdout: {e.stdout}") + print(f" stderr: {e.stderr}") + return False + except Exception as e: + print(f"⚠️ Error extracting memory stats: {e}") + return False + + +if __name__ == "__main__": + success = extract_memory_stats() + sys.exit(0 if success else 1) diff --git a/testcases/performance-testcase/main.py b/testcases/performance-testcase/main.py index 49fa0e055..2fc7c1343 100644 --- a/testcases/performance-testcase/main.py +++ b/testcases/performance-testcase/main.py @@ -1,5 +1,8 @@ +import json import logging +import time from dataclasses import dataclass +from pathlib import Path logger = logging.getLogger(__name__) @@ -18,6 +21,9 @@ class EchoOut: def main(input: EchoIn) -> EchoOut: + # Record start time + start_time = time.perf_counter() + result = [] for _ in range(input.repeat): @@ -26,4 +32,17 @@ def main(input: EchoIn) -> EchoOut: line = f"{input.prefix}: {line}" result.append(line) + # Record end time + end_time = time.perf_counter() + user_code_time = end_time - start_time + + # Write timing to file for later collection + timing_file = Path("artifacts/user_code_timing.json") + timing_file.parent.mkdir(parents=True, exist_ok=True) + timing_file.write_text(json.dumps({ + "user_code_time_seconds": user_code_time, + "start_time": start_time, + "end_time": end_time + })) + return EchoOut(message="\n".join(result)) diff --git a/testcases/performance-testcase/profile_memory.py b/testcases/performance-testcase/profile_memory.py index 8d0c2ed24..ca7a48226 100644 --- a/testcases/performance-testcase/profile_memory.py +++ b/testcases/performance-testcase/profile_memory.py @@ -1,26 +1,25 @@ #!/usr/bin/env python3 -"""Profile memory usage during agent execution.""" +"""Profile memory usage and total execution time during agent execution.""" import json import subprocess import sys import time -import tracemalloc from pathlib import Path def profile_agent_execution(): - """Run the agent with memory profiling enabled.""" - print("Starting memory profiling...") + """Run the agent and measure total execution time. - # Start tracking memory allocations - tracemalloc.start() - start_time = time.perf_counter() + Note: Memory profiling of subprocesses requires psutil which adds overhead. + For now, we focus on accurate timing measurement. + """ + print("Starting performance profiling...") - # Record initial memory snapshot - snapshot_start = tracemalloc.take_snapshot() + # Record total execution start time + total_start_time = time.perf_counter() - # Run the agent + # Run the agent and measure total time try: result = subprocess.run( [ @@ -38,52 +37,25 @@ def profile_agent_execution(): error = str(e) result = e - # Record final memory snapshot - end_time = time.perf_counter() - snapshot_end = tracemalloc.take_snapshot() - - # Get peak memory usage - current, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() - - # Calculate memory difference - top_stats = snapshot_end.compare_to(snapshot_start, 'lineno') - - # Extract top memory allocations - top_allocations = [] - for stat in top_stats[:10]: - top_allocations.append({ - "file": str(stat.traceback.format()[0]) if stat.traceback else "unknown", - "size_bytes": stat.size, - "size_diff_bytes": stat.size_diff, - "count": stat.count, - "count_diff": stat.count_diff - }) + # Record total execution end time + total_end_time = time.perf_counter() + total_execution_time = total_end_time - total_start_time # Build metrics metrics = { - "execution_time_seconds": round(end_time - start_time, 3), - "memory": { - "current_bytes": current, - "peak_bytes": peak, - "current_mb": round(current / 1024 / 1024, 2), - "peak_mb": round(peak / 1024 / 1024, 2), - }, - "top_allocations": top_allocations, + "total_execution_time_seconds": round(total_execution_time, 3), "success": success, "error": error } # Save metrics - output_path = Path("artifacts/memory_profile.json") + output_path = Path("artifacts/total_execution.json") output_path.parent.mkdir(parents=True, exist_ok=True) with open(output_path, "w") as f: json.dump(metrics, f, indent=2) - print(f"✓ Memory profile saved to {output_path}") - print(f" Execution time: {metrics['execution_time_seconds']}s") - print(f" Peak memory: {metrics['memory']['peak_mb']} MB") - print(f" Current memory: {metrics['memory']['current_mb']} MB") + print(f"✓ Performance profile saved to {output_path}") + print(f" Total execution time: {metrics['total_execution_time_seconds']}s") return metrics @@ -92,5 +64,5 @@ def profile_agent_execution(): try: profile_agent_execution() except Exception as e: - print(f"⚠️ Memory profiling failed: {e}", file=sys.stderr) + print(f"⚠️ Performance profiling failed: {e}", file=sys.stderr) sys.exit(1) diff --git a/testcases/performance-testcase/run.sh b/testcases/performance-testcase/run.sh index 247d5e88c..079a5a9de 100644 --- a/testcases/performance-testcase/run.sh +++ b/testcases/performance-testcase/run.sh @@ -5,7 +5,7 @@ echo "Syncing dependencies..." uv sync echo "Installing profiling tools and Azure SDK..." -uv add py-spy azure-storage-blob +uv add py-spy memray azure-storage-blob echo "Authenticating with UiPath..." uv run uipath auth --client-id="$CLIENT_ID" --client-secret="$CLIENT_SECRET" --base-url="$BASE_URL" @@ -22,8 +22,31 @@ mkdir -p artifacts echo "Run agent with py-spy profiling (speedscope JSON with timing data)" uv run py-spy record --subprocesses -f speedscope -o artifacts/profile.json -- uv run uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' -echo "Run agent with memory profiling using tracemalloc" -uv run python profile_memory.py +echo "Run agent with memray memory profiling and timing measurement" +uv run python -c " +import subprocess +import time +import json +from pathlib import Path + +# Measure total execution time with memray +start_time = time.perf_counter() +result = subprocess.run( + ['uv', 'run', 'memray', 'run', '--output', 'artifacts/memory.bin', '-m', 'uipath', 'run', 'main', '{\"message\": \"abc\", \"repeat\": 2, \"prefix\": \"xyz\"}'], + capture_output=True +) +end_time = time.perf_counter() + +# Save total execution time +Path('artifacts/total_execution.json').write_text(json.dumps({ + 'total_execution_time_seconds': round(end_time - start_time, 3), + 'success': result.returncode == 0, + 'error': result.stderr.decode() if result.returncode != 0 else None +})) +" + +echo "Extract memory stats from memray output" +uv run python extract_memory_stats.py echo "Collecting performance metrics and uploading to Azure..." uv run python collect_metrics.py From 743edff769e79a58fc70b6c0cf6028fbc79e1942 Mon Sep 17 00:00:00 2001 From: ionmincu Date: Fri, 16 Jan 2026 15:27:10 +0200 Subject: [PATCH 4/4] xyz --- .../performance-testcase/collect_metrics.py | 23 +++++++++++------- testcases/performance-testcase/run.sh | 24 ++----------------- 2 files changed, 16 insertions(+), 31 deletions(-) diff --git a/testcases/performance-testcase/collect_metrics.py b/testcases/performance-testcase/collect_metrics.py index dd5e4d135..e3cd66278 100644 --- a/testcases/performance-testcase/collect_metrics.py +++ b/testcases/performance-testcase/collect_metrics.py @@ -13,7 +13,7 @@ def load_timing_metrics(artifacts_dir: str = "artifacts") -> dict: Returns timing breakdown: - User code time (from main.py instrumentation) - - Total execution time (from run.sh measurement) + - Total execution time (from py-spy speedscope profile) - Framework overhead (calculated as total - user code) """ artifacts_path = Path(artifacts_dir) @@ -30,15 +30,21 @@ def load_timing_metrics(artifacts_dir: str = "artifacts") -> dict: except json.JSONDecodeError: pass - # Load total execution timing (from run.sh memray measurement) - total_timing_path = artifacts_path / "total_execution.json" + # Extract total execution time from py-spy speedscope profile + profile_path = artifacts_path / "profile.json" total_time = 0 - if total_timing_path.exists(): + if profile_path.exists(): try: - with open(total_timing_path, "r", encoding="utf-8") as f: - total_timing = json.load(f) - total_time = total_timing.get("total_execution_time_seconds", 0) - except json.JSONDecodeError: + with open(profile_path, "r", encoding="utf-8") as f: + speedscope_data = json.load(f) + # Get weights from first profile (main process) + profiles = speedscope_data.get("profiles", []) + if profiles: + weights = profiles[0].get("weights", []) + # Sum all weights (in microseconds) to get total time + total_time_us = sum(weights) + total_time = total_time_us / 1_000_000 # Convert to seconds + except (json.JSONDecodeError, KeyError, IndexError): pass # Calculate framework overhead @@ -117,7 +123,6 @@ def collect_metrics( file_sizes = { "profile_json": get_file_size(str(artifacts_path / "profile.json")), "user_code_timing_json": get_file_size(str(artifacts_path / "user_code_timing.json")), - "total_execution_json": get_file_size(str(artifacts_path / "total_execution.json")), "memory_bin": get_file_size(str(artifacts_path / "memory.bin")), "memory_stats_json": get_file_size(str(artifacts_path / "memory_stats.json")), } diff --git a/testcases/performance-testcase/run.sh b/testcases/performance-testcase/run.sh index 079a5a9de..14eb82fa9 100644 --- a/testcases/performance-testcase/run.sh +++ b/testcases/performance-testcase/run.sh @@ -22,28 +22,8 @@ mkdir -p artifacts echo "Run agent with py-spy profiling (speedscope JSON with timing data)" uv run py-spy record --subprocesses -f speedscope -o artifacts/profile.json -- uv run uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' -echo "Run agent with memray memory profiling and timing measurement" -uv run python -c " -import subprocess -import time -import json -from pathlib import Path - -# Measure total execution time with memray -start_time = time.perf_counter() -result = subprocess.run( - ['uv', 'run', 'memray', 'run', '--output', 'artifacts/memory.bin', '-m', 'uipath', 'run', 'main', '{\"message\": \"abc\", \"repeat\": 2, \"prefix\": \"xyz\"}'], - capture_output=True -) -end_time = time.perf_counter() - -# Save total execution time -Path('artifacts/total_execution.json').write_text(json.dumps({ - 'total_execution_time_seconds': round(end_time - start_time, 3), - 'success': result.returncode == 0, - 'error': result.stderr.decode() if result.returncode != 0 else None -})) -" +echo "Run agent with memray memory profiling" +uv run memray run --output artifacts/memory.bin -m uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' 2>&1 | tee artifacts/memray_output.txt echo "Extract memory stats from memray output" uv run python extract_memory_stats.py