From 5d470f037c6690c49391d0cd20bb222543e5ebe4 Mon Sep 17 00:00:00 2001 From: arthursita-plank Date: Tue, 20 Jan 2026 15:09:07 -0300 Subject: [PATCH 01/11] feat: lead-scraper template --- .../python/lead-scraper/.env.example | 2 + pkg/templates/python/lead-scraper/README.md | 113 ++++++++++ pkg/templates/python/lead-scraper/_gitignore | 79 +++++++ .../python/lead-scraper/formaters.py | 208 ++++++++++++++++++ pkg/templates/python/lead-scraper/main.py | 171 ++++++++++++++ pkg/templates/python/lead-scraper/models.py | 65 ++++++ .../python/lead-scraper/pyproject.toml | 11 + 7 files changed, 649 insertions(+) create mode 100644 pkg/templates/python/lead-scraper/.env.example create mode 100644 pkg/templates/python/lead-scraper/README.md create mode 100644 pkg/templates/python/lead-scraper/_gitignore create mode 100644 pkg/templates/python/lead-scraper/formaters.py create mode 100644 pkg/templates/python/lead-scraper/main.py create mode 100644 pkg/templates/python/lead-scraper/models.py create mode 100644 pkg/templates/python/lead-scraper/pyproject.toml diff --git a/pkg/templates/python/lead-scraper/.env.example b/pkg/templates/python/lead-scraper/.env.example new file mode 100644 index 0000000..b74e0a2 --- /dev/null +++ b/pkg/templates/python/lead-scraper/.env.example @@ -0,0 +1,2 @@ +# Copy this file to .env and fill in your API key +OPENAI_API_KEY=your_openai_api_key_here diff --git a/pkg/templates/python/lead-scraper/README.md b/pkg/templates/python/lead-scraper/README.md new file mode 100644 index 0000000..e9e98b0 --- /dev/null +++ b/pkg/templates/python/lead-scraper/README.md @@ -0,0 +1,113 @@ +# Kernel Lead Scraper Template - Google Maps + +A ready-to-use lead scraper that extracts local business data from Google Maps using [browser-use](https://github.com/browser-use/browser-use) and the Kernel platform. + +## What It Does + +This template creates an AI-powered web scraper that: +1. Navigates to Google Maps +2. Searches for businesses by type and location +3. Scrolls through results to load more listings +4. Extracts structured lead data (name, phone, address, website, rating, reviews) +5. Returns clean JSON ready for your CRM or outreach tools + +## Quick Start + +### 1. Install Dependencies + +```bash +uv sync +``` + +### 2. Set Up Environment + +```bash +cp .env.example .env +# Edit .env and add your OpenAI API key +``` + +### 3. Deploy to Kernel + +```bash +kernel deploy main.py -e OPENAI_API_KEY=$OPENAI_API_KEY +``` + +### 4. Run the Scraper + +```bash +kernel run lead-scraper scrape-leads \ + --data '{"query": "restaurants", "location": "Austin, TX", "max_results": 10}' +``` + +## API Reference + +### Action: `scrape-leads` + +**Input Parameters:** + +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| `query` | string | ✅ | - | Business type to search (e.g., "plumbers", "gyms") | +| `location` | string | ✅ | - | Geographic location (e.g., "Miami, FL") | +| `max_results` | integer | ❌ | 20 | Maximum leads to scrape (1-50) | + +**Example Output:** + +```json +{ + "leads": [ + { + "name": "Joe's Pizza", + "phone": "(512) 555-0123", + "address": "123 Main St, Austin, TX 78701", + "website": "https://joespizza.com", + "rating": 4.5, + "review_count": 234, + "category": "Pizza restaurant" + } + ], + "total_found": 1, + "query": "pizza restaurants", + "location": "Austin, TX" +} +``` + +## Use Cases + +- **Sales Teams**: Build targeted prospect lists for cold outreach +- **Marketing Agencies**: Find local businesses needing marketing services +- **Service Providers**: Identify potential B2B clients in your area +- **Market Research**: Analyze competitor density and ratings by location + +## Customization + +### Modify the Search Prompt + +Edit the `SCRAPER_PROMPT` in `main.py` to customize what data the AI extracts: + +```python +SCRAPER_PROMPT = """ +Navigate to Google Maps and search for {query} in {location}. +# Add your custom extraction instructions here +""" +``` + +### Add New Fields + +1. Update `BusinessLead` model in `models.py` +2. Modify the prompt to extract the new fields +3. Redeploy with `kernel deploy main.py` + +## Troubleshooting + +| Issue | Solution | +|-------|----------| +| No results found | Try a broader search query or different location | +| Timeout errors | Reduce `max_results` or check your network | +| Rate limiting | Add delays between requests in production | + +## Resources + +- [Kernel Documentation](https://www.kernel.sh/docs) +- [Browser Use Docs](https://docs.browser-use.com) +- [Pydantic Models](https://docs.pydantic.dev) diff --git a/pkg/templates/python/lead-scraper/_gitignore b/pkg/templates/python/lead-scraper/_gitignore new file mode 100644 index 0000000..75475bc --- /dev/null +++ b/pkg/templates/python/lead-scraper/_gitignore @@ -0,0 +1,79 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual Environment +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +.project +.pydevproject +.settings/ + +# Testing +.coverage +htmlcov/ +.pytest_cache/ +.tox/ +.nox/ +coverage.xml +*.cover +.hypothesis/ + +# Logs +*.log +logs/ + +# OS +.DS_Store +Thumbs.db + +# Browser Use specific +.playwright-screenshots/ +.playwright-videos/ +.playwright-report/ +test-results/ +blob-report/ +playwright/.cache/ +playwright/.local-browsers/ + +# Lead Scraper specific +leads_output/ +*.csv +*.json + +# Misc +.cache/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.temp/ +.tmp/ diff --git a/pkg/templates/python/lead-scraper/formaters.py b/pkg/templates/python/lead-scraper/formaters.py new file mode 100644 index 0000000..60256c2 --- /dev/null +++ b/pkg/templates/python/lead-scraper/formaters.py @@ -0,0 +1,208 @@ +import json +import re +from typing import Any, Iterable +from models import BusinessLead + +_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.IGNORECASE | re.DOTALL) +_TRAILING_COMMA_RE = re.compile(r",\s*([\]}])") +_SMART_QUOTES = { + "\u201c": '"', "\u201d": '"', # “ ” + "\u2018": "'", "\u2019": "'", # ‘ ’ +} + + +def parse_leads_from_result(result_text: str) -> list[BusinessLead]: + """ + Robustly extract a JSON array of leads from an LLM/browser agent output and + convert it into BusinessLead objects. + + Strategy: + 1) Prefer JSON inside ```json ... ``` fenced blocks + 2) Else try to decode from the first '[' onwards using JSONDecoder.raw_decode + 3) Normalize a few common LLM issues (smart quotes, trailing commas, "null" strings) + """ + if not result_text or not result_text.strip(): + return [] + + candidates = _extract_json_candidates(result_text) + + for candidate in candidates: + parsed = _try_parse_json_list(candidate) + if parsed is None: + continue + + leads: list[BusinessLead] = [] + for raw in parsed: + lead = _to_business_lead(raw) + if lead is not None: + leads.append(lead) + + if leads: + return leads # first successful parse wins + + # Fallback: try to parse markdown format (when agent returns numbered lists) + leads = _parse_markdown_leads(result_text) + if leads: + return leads + + return [] + + +def _parse_markdown_leads(text: str) -> list[BusinessLead]: + """ + Parse markdown-formatted lead data when JSON parsing fails. + Handles format like: + 1. **Business Name** + - Address: 123 Main St + - Rating: 4.5 + - Phone: +1 555-1234 + """ + leads = [] + + # Pattern to match numbered entries with bold names + entry_pattern = re.compile( + r'\d+\.\s*\*\*(.+?)\*\*\s*\n((?:\s*-\s*.+\n?)+)', + re.MULTILINE + ) + + for match in entry_pattern.finditer(text): + name = match.group(1).strip() + details = match.group(2) + + # Extract fields from the dash-prefixed lines + def extract_field(pattern: str, txt: str) -> str | None: + m = re.search(pattern, txt, re.IGNORECASE) + return m.group(1).strip() if m else None + + address = extract_field(r'-\s*Address:\s*(.+?)(?:\n|$)', details) + rating_str = extract_field(r'-\s*Rating:\s*([\d.]+)', details) + review_str = extract_field(r'-\s*Review\s*Count:\s*([\d,]+)', details) + category = extract_field(r'-\s*Category:\s*(.+?)(?:\n|$)', details) + phone = extract_field(r'-\s*Phone:\s*(.+?)(?:\n|$)', details) + website = extract_field(r'-\s*Website:\s*(.+?)(?:\n|$)', details) + + # Clean up "Not available" etc + if phone and phone.lower() in ('not available', 'n/a', 'none'): + phone = None + if website and website.lower() in ('not available', 'n/a', 'none'): + website = None + + try: + lead = BusinessLead( + name=name, + address=address, + rating=float(rating_str) if rating_str else None, + review_count=int(review_str.replace(',', '')) if review_str else None, + category=category, + phone=phone, + website=website, + ) + leads.append(lead) + except Exception: + continue + + return leads + + +def _extract_json_candidates(text: str) -> list[str]: + """ + Return possible JSON snippets, ordered from most to least likely. + """ + # 1) Fenced code blocks first + fenced = [m.group(1) for m in _JSON_FENCE_RE.finditer(text)] + if fenced: + return fenced + + # 2) Otherwise try from first '[' onward (common "Return ONLY a JSON array") + idx = text.find("[") + return [text[idx:]] if idx != -1 else [] + + +def _normalize_llm_json(s: str) -> str: + # Replace smart quotes + for k, v in _SMART_QUOTES.items(): + s = s.replace(k, v) + + # Some models do ``key``: ``value``. Convert double-backticks to quotes carefully. + # (Keep this minimal: it can still be wrong, but it helps common cases.) + s = s.replace("``", '"') + + # Convert string "null" to JSON null + s = s.replace('"null"', "null") + + # Remove trailing commas before ] or } + s = _TRAILING_COMMA_RE.sub(r"\1", s) + + return s.strip() + + +def _try_parse_json_list(candidate: str) -> list[dict[str, Any]] | None: + """ + Attempt to parse a JSON array from a candidate snippet. + Returns a list of dicts or None. + """ + candidate = _normalize_llm_json(candidate) + + # 1) Direct parse + try: + data = json.loads(candidate) + return data if isinstance(data, list) else None + except json.JSONDecodeError: + pass + + # 2) Decoder-based parse from first '[' (more robust than find/rfind slicing) + start = candidate.find("[") + if start == -1: + return None + + decoder = json.JSONDecoder() + try: + obj, _end = decoder.raw_decode(candidate[start:]) + return obj if isinstance(obj, list) else None + except json.JSONDecodeError: + return None + + +def _to_business_lead(raw: Any) -> BusinessLead | None: + """ + Convert one raw object into a BusinessLead, best-effort. + """ + if not isinstance(raw, dict): + return None + + try: + # Optionally coerce some common fields + rating = raw.get("rating") + if isinstance(rating, str): + rating = _safe_float(rating) + + review_count = raw.get("review_count") + if isinstance(review_count, str): + review_count = _safe_int(review_count) + + return BusinessLead( + name=(raw.get("name") or "Unknown").strip() if isinstance(raw.get("name"), str) else (raw.get("name") or "Unknown"), + phone=raw.get("phone"), + address=raw.get("address"), + website=raw.get("website"), + rating=rating, + review_count=review_count, + category=raw.get("category"), + ) + except Exception: + # Keep parsing the rest; caller decides how to log + return None + + +def _safe_float(x: str) -> float | None: + try: + return float(x.replace(",", "").strip()) + except Exception: + return None + + +def _safe_int(x: str) -> int | None: + try: + return int(x.replace(",", "").strip()) + except Exception: + return None diff --git a/pkg/templates/python/lead-scraper/main.py b/pkg/templates/python/lead-scraper/main.py new file mode 100644 index 0000000..12f3bb0 --- /dev/null +++ b/pkg/templates/python/lead-scraper/main.py @@ -0,0 +1,171 @@ +""" +Google Maps Lead Scraper - Kernel Template + +This template demonstrates how to build a lead scraper using browser-use +to extract local business data from Google Maps. + +Usage: + kernel deploy main.py -e OPENAI_API_KEY=$OPENAI_API_KEY + kernel invoke lead-scraper scrape-leads --data '{"query": "restaurants", "location": "Austin, TX"}' +""" + +import json + +import kernel +from browser_use import Agent, Browser +from browser_use.llm import ChatOpenAI +from kernel import Kernel +from formaters import parse_leads_from_result + +from models import BusinessLead, ScrapeInput, ScrapeOutput + +# Initialize Kernel client and app +client = Kernel() +app = kernel.App("lead-scraper") + +# LLM for the browser-use agent +# API key is set via: kernel deploy main.py -e OPENAI_API_KEY=XXX +llm = ChatOpenAI(model="gpt-4o") + +# ============================================================================ +# SCRAPER PROMPT +# Customize this prompt to change what data the agent extracts +# ============================================================================ +SCRAPER_PROMPT = """ +You are a lead generation assistant. Scrape business information from Google Maps. + +**Instructions:** +1. Navigate to https://www.google.com/maps +2. Search for: "{query} in {location}" +3. Wait for results to load +4. For each of the max {max_results} businesses in the list: + a. Click on the listing to open its detail view + b. SCROLL DOWN in the detail panel to see all info (phone/website are often below) + c. Extract: name, address, rating, review count, category, phone number, website + d. Click back or the X to close the detail view and return to the list +5. After collecting data for max {max_results} businesses, return the JSON + +**What to extract:** +- Business name (REQUIRED) +- Address (REQUIRED) +- Star rating (REQUIRED) +- Review count (optional) +- Category (optional) +- Phone number (scroll down in detail view to find it, null if not shown) +- Website URL (scroll down in detail view to find it, null if not shown) + +**Important:** +- SCROLL DOWN inside each business detail panel to find phone/website +- Use null for any field that isn't available +- Task is SUCCESSFUL when you return at least 1 complete business + +**CRITICAL - Output Format:** +You MUST return ONLY a valid JSON array. No markdown, no explanations, no numbered lists. +Return EXACTLY this format: +[ + {{"name": "Business Name", "address": "123 Main St", "rating": 4.5, "review_count": 100, "category": "Restaurant", "phone": "+1 555-1234", "website": "https://example.com"}} +] +""" + +@app.action("scrape-leads") +async def scrape_leads(ctx: kernel.KernelContext, input_data: dict) -> dict: + """ + Scrape local business leads from Google Maps. + + This action uses browser-use to navigate Google Maps, search for businesses, + and extract structured lead data. + + Args: + ctx: Kernel context containing invocation information + input_data: Dictionary with query, location, and max_results + + Returns: + ScrapeOutput containing list of leads and metadata + + Example: + kernel invoke lead-scraper scrape-leads \ + --data '{"query": "plumbers", "location": "Miami, FL", "max_results": 15}' + """ + # Validate input - default to empty dict if no payload provided + scrape_input = ScrapeInput(**(input_data or {})) + + # Use attribute access for Pydantic model (not dictionary subscript) + input_query = scrape_input.query + input_location = scrape_input.location + input_max_results = scrape_input.max_results + + # Format the prompt with user parameters + task_prompt = SCRAPER_PROMPT.format( + query=input_query, + location=input_location, + max_results=input_max_results, + ) + + print(f"Starting lead scrape: {input_query} in {input_location}") + print(f"Target: {input_max_results} leads") + + # Create Kernel browser session + kernel_browser = None + + try: + + kernel_browser = client.browsers.create( + invocation_id=ctx.invocation_id, + stealth=True, # Use stealth mode to avoid detection + ) + print(f"Browser live view: {kernel_browser.browser_live_view_url}") + + # Connect browser-use to the Kernel browser + browser = Browser( + cdp_url=kernel_browser.cdp_ws_url, + headless=False, + window_size={"width": 1920, "height": 1080}, + viewport={"width": 1920, "height": 1080}, + device_scale_factor=1.0, + ) + + # Create and run the browser-use agent + agent = Agent( + task=task_prompt, + llm=llm, + browser_session=browser, + ) + + print("Running browser-use agent...") + # Limit steps to prevent timeouts (this is a template demo) + result = await agent.run(max_steps=25) + + # Parse the result from final_result + leads = [] + final_text = result.final_result() + + if final_text: + print(f"Parsing final_result ({len(final_text)} chars)...") + leads = parse_leads_from_result(final_text) + else: + # If no final_result, check the last action for done text + print("No final_result, checking last action...") + action_results = result.action_results() + if action_results: + last_action = action_results[-1] + if hasattr(last_action, 'extracted_content') and last_action.extracted_content: + content = last_action.extracted_content + if '[' in content and '"name"' in content: + print(f"Found leads in last action ({len(content)} chars)...") + leads = parse_leads_from_result(content) + + print(f"Successfully extracted {len(leads)} leads") + + output = ScrapeOutput( + leads=leads, + total_found=len(leads), + query=input_query, + location=input_location, + ) + return output.model_dump() + + finally: + # Always clean up the browsers session + if kernel_browser is not None: + client.browsers.delete_by_id(kernel_browser.session_id) + print("Browser session cleaned up") diff --git a/pkg/templates/python/lead-scraper/models.py b/pkg/templates/python/lead-scraper/models.py new file mode 100644 index 0000000..2d3c6e4 --- /dev/null +++ b/pkg/templates/python/lead-scraper/models.py @@ -0,0 +1,65 @@ +from pydantic import BaseModel, Field +from typing import Optional + + +class ScrapeInput(BaseModel): + """Input parameters for the lead scraper. + + Attributes: + query: The type of business to search (e.g., "restaurants", "plumbers", "gyms") + location: The geographic location to search (e.g., "Austin, TX", "New York, NY") + max_results: Maximum number of leads to scrape (default: 2, max: 5) + """ + + query: str = Field( + default="restaurants", + description="Type of business to search for (e.g., 'restaurants', 'plumbers')" + ) + location: str = Field( + default="New York, NY", + description="Geographic location (e.g., 'Austin, TX', 'New York, NY')" + ) + max_results: int = Field( + default=1, + ge=1, + le=5, + description="Maximum number of leads to scrape (1-5)", + ) + + +class BusinessLead(BaseModel): + """Structured data for a business lead scraped from Google Maps. + + Attributes: + name: Business name + phone: Phone number (if available) + address: Full address + website: Website URL (if available) + rating: Star rating (1-5) + review_count: Number of reviews + category: Business category/type + """ + + name: str = Field(description="Business name") + phone: Optional[str] = Field(default=None, description="Phone number") + address: Optional[str] = Field(default=None, description="Full address") + website: Optional[str] = Field(default=None, description="Website URL") + rating: Optional[float] = Field(default=None, ge=1, le=5, description="Star rating") + review_count: Optional[int] = Field(default=None, ge=0, description="Number of reviews") + category: Optional[str] = Field(default=None, description="Business category") + + +class ScrapeOutput(BaseModel): + """Output from the lead scraper. + + Attributes: + leads: List of scraped business leads + total_found: Total number of leads found + query: The original search query + location: The original search location + """ + + leads: list[BusinessLead] = Field(default_factory=list, description="List of scraped leads") + total_found: int = Field(default=0, description="Total number of leads found") + query: str = Field(description="Original search query") + location: str = Field(description="Original search location") diff --git a/pkg/templates/python/lead-scraper/pyproject.toml b/pkg/templates/python/lead-scraper/pyproject.toml new file mode 100644 index 0000000..2c33639 --- /dev/null +++ b/pkg/templates/python/lead-scraper/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "lead-scraper" +version = "0.1.0" +description = "Google Maps Lead Scraper - A Kernel template for scraping local business leads" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "browser-use>=0.11.1", + "kernel>=0.23.0", + "pydantic>=2.12.5", +] From 8970c1190a7255b6f709d70378c538a4eaa04ecc Mon Sep 17 00:00:00 2001 From: arthursita-plank Date: Tue, 20 Jan 2026 15:12:06 -0300 Subject: [PATCH 02/11] chore: change the model version --- pkg/templates/python/lead-scraper/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/templates/python/lead-scraper/main.py b/pkg/templates/python/lead-scraper/main.py index 12f3bb0..8b57c7a 100644 --- a/pkg/templates/python/lead-scraper/main.py +++ b/pkg/templates/python/lead-scraper/main.py @@ -25,7 +25,7 @@ # LLM for the browser-use agent # API key is set via: kernel deploy main.py -e OPENAI_API_KEY=XXX -llm = ChatOpenAI(model="gpt-4o") +llm = ChatOpenAI(model="gpt-4.1") # ============================================================================ # SCRAPER PROMPT From 384aa3f8db50870f1cc3a61b6884f54f7737f8f1 Mon Sep 17 00:00:00 2001 From: arthursita-plank Date: Tue, 20 Jan 2026 15:41:02 -0300 Subject: [PATCH 03/11] chore: fix cursor bugbot comment --- pkg/templates/python/lead-scraper/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/templates/python/lead-scraper/main.py b/pkg/templates/python/lead-scraper/main.py index 8b57c7a..aa7a7eb 100644 --- a/pkg/templates/python/lead-scraper/main.py +++ b/pkg/templates/python/lead-scraper/main.py @@ -150,9 +150,8 @@ async def scrape_leads(ctx: kernel.KernelContext, input_data: dict) -> dict: last_action = action_results[-1] if hasattr(last_action, 'extracted_content') and last_action.extracted_content: content = last_action.extracted_content - if '[' in content and '"name"' in content: - print(f"Found leads in last action ({len(content)} chars)...") - leads = parse_leads_from_result(content) + print(f"Found content in last action ({len(content)} chars)...") + leads = parse_leads_from_result(content) print(f"Successfully extracted {len(leads)} leads") From 0cfb74d720efa098c2389702672034594d4c9556 Mon Sep 17 00:00:00 2001 From: arthursita-plank Date: Tue, 20 Jan 2026 15:41:55 -0300 Subject: [PATCH 04/11] feat: add template name to readme documentation --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index af92648..c40d2f0 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ Commands with JSON output support: - `captcha-solver` - Template demonstrating Kernel's auto-CAPTCHA solver - `stagehand` - Template with Stagehand SDK (TypeScript only) - `browser-use` - Template with Browser Use SDK (Python only) + - `lead-scraper` - Google Maps lead scraper using Browser Use (Python only) - `anthropic-computer-use` - Anthropic Computer Use prompt loop - `openai-computer-use` - OpenAI Computer Use Agent sample - `gemini-computer-use` - Implements a Gemini computer use agent (TypeScript only) @@ -449,6 +450,9 @@ kernel create --name my-cu-app --language py --template anthropic-computer-use # Create a Claude Agent SDK app (TypeScript or Python) kernel create --name my-claude-agent --language ts --template claude-agent-sdk + +# Create a Google Maps Lead Scraper (Python) +kernel create --name my-lead-scraper --language python --template lead-scraper ``` ### Deploy with environment variables From fdad1ca595928277e64a1664eda6503ebfa3ba40 Mon Sep 17 00:00:00 2001 From: arthursita-plank Date: Thu, 22 Jan 2026 10:19:57 -0300 Subject: [PATCH 05/11] feat: export csv from ehr system using CUA --- README.md | 4 + .../typescript/ehr-system/.env.example | 1 + .../typescript/ehr-system/.gitignore | 3 + pkg/templates/typescript/ehr-system/README.md | 38 + pkg/templates/typescript/ehr-system/index.ts | 106 +++ .../typescript/ehr-system/lib/agent.ts | 208 ++++++ .../typescript/ehr-system/lib/computers.ts | 28 + .../ehr-system/lib/playwright/base.ts | 242 +++++++ .../ehr-system/lib/playwright/kernel.ts | 43 ++ .../ehr-system/lib/playwright/local.ts | 43 ++ .../typescript/ehr-system/lib/toolset.ts | 40 ++ .../typescript/ehr-system/lib/utils.ts | 61 ++ .../typescript/ehr-system/package-lock.json | 650 ++++++++++++++++++ .../typescript/ehr-system/package.json | 20 + .../typescript/ehr-system/tsconfig.json | 9 + 15 files changed, 1496 insertions(+) create mode 100644 pkg/templates/typescript/ehr-system/.env.example create mode 100644 pkg/templates/typescript/ehr-system/.gitignore create mode 100644 pkg/templates/typescript/ehr-system/README.md create mode 100644 pkg/templates/typescript/ehr-system/index.ts create mode 100644 pkg/templates/typescript/ehr-system/lib/agent.ts create mode 100644 pkg/templates/typescript/ehr-system/lib/computers.ts create mode 100644 pkg/templates/typescript/ehr-system/lib/playwright/base.ts create mode 100644 pkg/templates/typescript/ehr-system/lib/playwright/kernel.ts create mode 100644 pkg/templates/typescript/ehr-system/lib/playwright/local.ts create mode 100644 pkg/templates/typescript/ehr-system/lib/toolset.ts create mode 100644 pkg/templates/typescript/ehr-system/lib/utils.ts create mode 100644 pkg/templates/typescript/ehr-system/package-lock.json create mode 100644 pkg/templates/typescript/ehr-system/package.json create mode 100644 pkg/templates/typescript/ehr-system/tsconfig.json diff --git a/README.md b/README.md index c40d2f0..7a721e9 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,7 @@ Commands with JSON output support: - `sample-app` - Basic template with Playwright integration - `captcha-solver` - Template demonstrating Kernel's auto-CAPTCHA solver - `stagehand` - Template with Stagehand SDK (TypeScript only) + - `ehr-system` - EHR system automation demo with Playwright/OpenAI (TypeScript only) - `browser-use` - Template with Browser Use SDK (Python only) - `lead-scraper` - Google Maps lead scraper using Browser Use (Python only) - `anthropic-computer-use` - Anthropic Computer Use prompt loop @@ -453,6 +454,9 @@ kernel create --name my-claude-agent --language ts --template claude-agent-sdk # Create a Google Maps Lead Scraper (Python) kernel create --name my-lead-scraper --language python --template lead-scraper + +# Create an EHR System Automation (TypeScript) +kernel create --name my-ehr-bot --language ts --template ehr-system ``` ### Deploy with environment variables diff --git a/pkg/templates/typescript/ehr-system/.env.example b/pkg/templates/typescript/ehr-system/.env.example new file mode 100644 index 0000000..9847a1d --- /dev/null +++ b/pkg/templates/typescript/ehr-system/.env.example @@ -0,0 +1 @@ +OPENAI_API_KEY= \ No newline at end of file diff --git a/pkg/templates/typescript/ehr-system/.gitignore b/pkg/templates/typescript/ehr-system/.gitignore new file mode 100644 index 0000000..d8f3372 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/.gitignore @@ -0,0 +1,3 @@ +node_modules +.DS_Store +.env diff --git a/pkg/templates/typescript/ehr-system/README.md b/pkg/templates/typescript/ehr-system/README.md new file mode 100644 index 0000000..0d4ddbe --- /dev/null +++ b/pkg/templates/typescript/ehr-system/README.md @@ -0,0 +1,38 @@ +# EHR System Automation Template + +This template demonstrates how to use **Playwright** with **OpenAI's Computer Use** capabilities on Kernel to automate an Electronic Health Records (EHR) system workflow. + +## Logic + +The automation performs the following steps: +1. Navigate to the EHR login page (`https://ehr-system-six.vercel.app/login`). +2. Authenticate using valid credentials (any email/password works for this demo). +3. Navigate to the **Reports** section in the dashboard. +4. Click the **Export CSV** button to download the patient report. + +This template uses an agentic loop where OpenAI Vision analyzes the page and directs Playwright to interact with elements. + +## Usage + +1. **Deploy the app:** + + ```bash + kernel deploy index.ts -e OPENAI_API_KEY=$OPENAI_API_KEY + ``` + +2. **Invoke the action:** + + ```bash + kernel invoke ehr-system export-report + ``` + +3. **View logs:** + + ```bash + kernel logs ehr-system --follow + ``` + +## Requirements + +- OPENAI_API_KEY environment variable set. +- Kernel CLI installed and authenticated. diff --git a/pkg/templates/typescript/ehr-system/index.ts b/pkg/templates/typescript/ehr-system/index.ts new file mode 100644 index 0000000..0d4351b --- /dev/null +++ b/pkg/templates/typescript/ehr-system/index.ts @@ -0,0 +1,106 @@ +import { Kernel, type KernelContext } from '@onkernel/sdk'; +import 'dotenv/config'; +import { Agent } from './lib/agent'; +import computers from './lib/computers'; + +interface Input { + task?: string; +} + +interface Output { + elapsed: number; + answer: string | null; + logs?: any[]; +} + +const kernel = new Kernel(); +const app = kernel.app('ehr-system'); + +if (!process.env.OPENAI_API_KEY) { + throw new Error('OPENAI_API_KEY is not set'); +} + +const DEFAULT_TASK = ` +Go to https://ehr-system-six.vercel.app/login +Login with any email and password (e.g. user@example.com / password). +Navigate to the "Reports" page. +Find the "Export CSV" button and click it to download the report. +Wait for the download to start. +CRITICAL: Do not ask for confirmation. Perform all steps immediately. +`; + +app.action( + 'export-report', + async (ctx: KernelContext, payload?: Input): Promise => { + const start = Date.now(); + const task = payload?.task || DEFAULT_TASK; + + const kb = await kernel.browsers.create({ + invocation_id: ctx.invocation_id, + stealth: true + }); + console.log('> Kernel browser live view url:', kb.browser_live_view_url); + + try { + const { computer } = await computers.create({ type: 'kernel', cdp_ws_url: kb.cdp_ws_url }); + + const agent = new Agent({ + model: 'computer-use-preview', // Using a capable model for computer use + computer, + tools: [], + acknowledge_safety_check_callback: (m: string): boolean => { + console.log(`> safety check: ${m}`); + return true; + }, + }); + + // run agent and get response + const logs = await agent.runFullTurn({ + messages: [ + { + role: 'system', + content: `You are an automated agent. Current date and time: ${new Date().toISOString()}. You must complete the task fully without asking for permission.`, + }, + { + type: 'message', + role: 'user', + content: [{ type: 'input_text', text: task }], + }, + ], + print_steps: true, + debug: true, + show_images: false, + }); + + const elapsed = parseFloat(((Date.now() - start) / 1000).toFixed(2)); + + // filter only LLM messages + + // filter only LLM messages + const messages = logs.filter( + (item: any) => + item.type === 'message' && + typeof item.role === 'string' && + Array.isArray(item.content), + ); + const assistant = messages.find((m: any) => m.role === 'assistant') as any; + const lastContentIndex = assistant?.content?.length ? assistant.content.length - 1 : -1; + const lastContent = lastContentIndex >= 0 ? assistant?.content?.[lastContentIndex] : null; + const answer = lastContent && 'text' in lastContent ? lastContent.text : null; + + return { + elapsed, + answer, + }; + } catch (error) { + const elapsed = parseFloat(((Date.now() - start) / 1000).toFixed(2)); + console.error('Error in export-report:', error); + return { + elapsed, + answer: null, + }; + } finally { + await kernel.browsers.deleteByID(kb.session_id); + } + }, +); diff --git a/pkg/templates/typescript/ehr-system/lib/agent.ts b/pkg/templates/typescript/ehr-system/lib/agent.ts new file mode 100644 index 0000000..28808d5 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/lib/agent.ts @@ -0,0 +1,208 @@ +import { + type ResponseItem, + type ResponseInputItem, + type ResponseOutputMessage, + type ResponseFunctionToolCallItem, + type ResponseFunctionToolCallOutputItem, + type ResponseComputerToolCall, + type ResponseComputerToolCallOutputItem, + type ComputerTool, +} from 'openai/resources/responses/responses'; + +import * as utils from './utils'; +import toolset from './toolset'; +import type { BasePlaywrightComputer } from './playwright/base'; +import type { LocalPlaywrightComputer } from './playwright/local'; +import type { KernelPlaywrightComputer } from './playwright/kernel'; + +export class Agent { + private model: string; + private computer: + | BasePlaywrightComputer + | LocalPlaywrightComputer + | KernelPlaywrightComputer + | undefined; + private tools: ComputerTool[]; + private print_steps = true; + private debug = false; + private show_images = false; + private ackCb: (msg: string) => boolean; + + constructor(opts: { + model?: string; + computer?: + | BasePlaywrightComputer + | LocalPlaywrightComputer + | KernelPlaywrightComputer + | undefined; + tools?: ComputerTool[]; + acknowledge_safety_check_callback?: (msg: string) => boolean; + }) { + this.model = opts.model ?? 'computer-use-preview'; + this.computer = opts.computer; + this.tools = [...toolset.shared, ...(opts.tools ?? [])] as ComputerTool[]; + this.ackCb = opts.acknowledge_safety_check_callback ?? ((): boolean => true); + + if (this.computer) { + const [w, h] = this.computer.getDimensions(); + this.tools.push({ + type: 'computer_use_preview', + display_width: w, + display_height: h, + environment: this.computer.getEnvironment(), + }); + } + } + + private debugPrint(...args: unknown[]): void { + if (this.debug) { + console.warn('--- debug:agent:debugPrint'); + try { + console.dir( + args.map((msg) => utils.sanitizeMessage(msg as ResponseItem)), + { depth: null }, + ); + } catch { + console.dir(args, { depth: null }); + } + } + } + + private async handleItem(item: ResponseItem): Promise { + if (item.type === 'message' && this.print_steps) { + const msg = item as ResponseOutputMessage; + const c = msg.content; + if (Array.isArray(c) && c[0] && 'text' in c[0] && typeof c[0].text === 'string') + console.log(c[0].text); + } + + if (item.type === 'function_call') { + const fc = item as ResponseFunctionToolCallItem; + const argsObj = JSON.parse(fc.arguments) as Record; + if (this.print_steps) console.log(`${fc.name}(${JSON.stringify(argsObj)})`); + if (this.computer) { + const fn = (this.computer as unknown as Record)[fc.name]; + if (typeof fn === 'function') + await (fn as (...a: unknown[]) => unknown)(...Object.values(argsObj)); + } + return [ + { + type: 'function_call_output', + call_id: fc.call_id, + output: 'success', + } as unknown as ResponseFunctionToolCallOutputItem, + ]; + } + + if (item.type === 'computer_call') { + const cc = item as ResponseComputerToolCall; + const { type: actionType, ...actionArgs } = cc.action; + if (this.print_steps) console.log(`${actionType}(${JSON.stringify(actionArgs)})`); + if (this.computer) { + const fn = (this.computer as unknown as Record)[actionType as string]; + if (typeof fn === 'function') { + await (fn as (...a: unknown[]) => unknown)(...Object.values(actionArgs)); + const screenshot = await this.computer.screenshot(); + const pending = cc.pending_safety_checks ?? []; + for (const { message } of pending) + if (message && !this.ackCb(message)) throw new Error(`Safety check failed: ${message}`); + const out: Omit = { + type: 'computer_call_output', + call_id: cc.call_id, + // id: "?", // <---- omitting to work - need to determine id source, != call_id + acknowledged_safety_checks: pending, + output: { + type: 'computer_screenshot', + image_url: `data:image/webp;base64,${screenshot}`, + }, + }; + if (this.computer.getEnvironment() === 'browser') + utils.checkBlocklistedUrl(this.computer.getCurrentUrl()); + return [out as ResponseItem]; + } + } + } + + return []; + } + + async runFullTurn(opts: { + messages: ResponseInputItem[]; + print_steps?: boolean; + debug?: boolean; + show_images?: boolean; + }): Promise { + this.print_steps = opts.print_steps ?? true; + this.debug = opts.debug ?? false; + this.show_images = opts.show_images ?? false; + const newItems: ResponseItem[] = []; + + while ( + newItems.length === 0 || + (newItems[newItems.length - 1] as ResponseItem & { role?: string }).role !== 'assistant' + ) { + // Add current URL to system message if in browser environment + const inputMessages = [...opts.messages]; + + if (this.computer?.getEnvironment() === 'browser') { + const current_url = this.computer.getCurrentUrl(); + // Find system message by checking if it has a role property with value 'system' + const sysIndex = inputMessages.findIndex((msg) => 'role' in msg && msg.role === 'system'); + + if (sysIndex >= 0) { + const msg = inputMessages[sysIndex]; + const urlInfo = `\n- Current URL: ${current_url}`; + + // Create a properly typed message based on the original + if (msg && 'content' in msg) { + if (typeof msg.content === 'string') { + // Create a new message with the updated content + const updatedMsg = { + ...msg, + content: msg.content + urlInfo, + }; + // Type assertion to ensure compatibility + inputMessages[sysIndex] = updatedMsg as typeof msg; + } else if (Array.isArray(msg.content) && msg.content.length > 0) { + // Handle array content case + const updatedContent = [...msg.content]; + + // Check if first item has text property + if (updatedContent[0] && 'text' in updatedContent[0]) { + updatedContent[0] = { + ...updatedContent[0], + text: updatedContent[0].text + urlInfo, + }; + } + + // Create updated message with new content + const updatedMsg = { + ...msg, + content: updatedContent, + }; + // Type assertion to ensure compatibility + inputMessages[sysIndex] = updatedMsg as typeof msg; + } + } + } + } + + this.debugPrint(...inputMessages, ...newItems); + const response = await utils.createResponse({ + model: this.model, + input: [...inputMessages, ...newItems], + tools: this.tools, + truncation: 'auto', + }); + if (!response.output) throw new Error('No output from model'); + for (const msg of response.output as ResponseItem[]) { + newItems.push(msg, ...(await this.handleItem(msg))); + } + } + + // Return sanitized messages if show_images is false + return !this.show_images + ? newItems.map((msg) => utils.sanitizeMessage(msg) as ResponseItem) + : newItems; + } +} diff --git a/pkg/templates/typescript/ehr-system/lib/computers.ts b/pkg/templates/typescript/ehr-system/lib/computers.ts new file mode 100644 index 0000000..5828fc8 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/lib/computers.ts @@ -0,0 +1,28 @@ +import { KernelPlaywrightComputer } from './playwright/kernel'; +import { LocalPlaywrightComputer } from './playwright/local'; + +interface KernelConfig { + type: 'kernel'; + cdp_ws_url: string; +} +interface LocalConfig { + type: 'local'; + headless?: boolean; +} +type ComputerConfig = KernelConfig | LocalConfig; + +export default { + async create( + cfg: ComputerConfig, + ): Promise<{ computer: KernelPlaywrightComputer | LocalPlaywrightComputer }> { + if (cfg.type === 'kernel') { + const computer = new KernelPlaywrightComputer(cfg.cdp_ws_url); + await computer.enter(); + return { computer }; + } else { + const computer = new LocalPlaywrightComputer(cfg.headless ?? false); + await computer.enter(); + return { computer }; + } + }, +}; diff --git a/pkg/templates/typescript/ehr-system/lib/playwright/base.ts b/pkg/templates/typescript/ehr-system/lib/playwright/base.ts new file mode 100644 index 0000000..b43a7d2 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/lib/playwright/base.ts @@ -0,0 +1,242 @@ +import type { Browser, Page, Request, Response, Route } from 'playwright-core'; +import sharp from 'sharp'; +import utils from '../utils'; + +// CUA key -> Playwright key mapping +const KEY_MAP: Record = { + '/': '/', + '\\': '\\', + alt: 'Alt', + arrowdown: 'ArrowDown', + arrowleft: 'ArrowLeft', + arrowright: 'ArrowRight', + arrowup: 'ArrowUp', + backspace: 'Backspace', + capslock: 'CapsLock', + cmd: 'Meta', + ctrl: 'Control', + delete: 'Delete', + end: 'End', + enter: 'Enter', + esc: 'Escape', + home: 'Home', + insert: 'Insert', + option: 'Alt', + pagedown: 'PageDown', + pageup: 'PageUp', + shift: 'Shift', + space: ' ', + super: 'Meta', + tab: 'Tab', + win: 'Meta', +}; + +interface Point { + x: number; + y: number; +} + +export class BasePlaywrightComputer { + protected _browser: Browser | null = null; + protected _page: Page | null = null; + + constructor() { + this._browser = null; + this._page = null; + } + + /** + * Type guard to assert that this._page is present and is a Playwright Page. + * Throws an error if not present. + */ + protected _assertPage(): asserts this is { _page: Page } { + if (!this._page) { + throw new Error('Playwright Page is not initialized. Did you forget to call enter()?'); + } + } + + protected _handleNewPage = (page: Page): void => { + /** Handle the creation of a new page. */ + console.log('New page created'); + this._page = page; + page.on('close', this._handlePageClose.bind(this)); + }; + + protected _handlePageClose = (page: Page): void => { + /** Handle the closure of a page. */ + console.log('Page closed'); + try { + this._assertPage(); + } catch { + return; + } + if (this._page !== page) return; + + const browser = this._browser; + if (!browser || typeof browser.contexts !== 'function') { + console.log('Warning: Browser or context not available.'); + this._page = undefined as unknown as Page; + return; + } + + const contexts = browser.contexts(); + if (!contexts.length) { + console.log('Warning: No browser contexts available.'); + this._page = undefined as unknown as Page; + return; + } + + const context = contexts[0]; + if (!context || typeof context.pages !== 'function') { + console.log('Warning: Context pages not available.'); + this._page = undefined as unknown as Page; + return; + } + + const pages = context.pages(); + if (pages.length) { + this._page = pages[pages.length - 1] as Page; + } else { + console.log('Warning: All pages have been closed.'); + this._page = undefined as unknown as Page; + } + }; + + // Subclass hook + protected _getBrowserAndPage = async (): Promise<[Browser, Page]> => { + // Subclasses must implement, returning [Browser, Page] + throw new Error('Subclasses must implement _getBrowserAndPage()'); + }; + + getEnvironment = (): 'windows' | 'mac' | 'linux' | 'ubuntu' | 'browser' => { + return 'browser'; + }; + + getDimensions = (): [number, number] => { + return [1024, 768]; + }; + + enter = async (): Promise => { + // Call the subclass hook for getting browser/page + [this._browser, this._page] = await this._getBrowserAndPage(); + + // Set up network interception to flag URLs matching domains in BLOCKED_DOMAINS + const handleRoute = (route: Route, request: Request): void => { + const url = request.url(); + if (utils.checkBlocklistedUrl(url)) { + console.log(`Flagging blocked domain: ${url}`); + route.abort(); + } else { + route.continue(); + } + }; + + this._assertPage(); + await this._page.route('**/*', handleRoute); + return this; + }; + + exit = async (): Promise => { + if (this._browser) await this._browser.close(); + }; + + getCurrentUrl = (): string => { + this._assertPage(); + return this._page.url(); + }; + + screenshot = async (): Promise => { + this._assertPage(); + const buf = await this._page.screenshot({ fullPage: false }); + const webp = await sharp(buf).webp().toBuffer(); + return webp.toString('base64'); + }; + + click = async ( + button: 'left' | 'right' | 'back' | 'forward' | 'wheel', + x: number, + y: number, + ): Promise => { + this._assertPage(); + switch (button) { + case 'back': + await this.back(); + return; + case 'forward': + await this.forward(); + return; + case 'wheel': + await this._page.mouse.wheel(x, y); + return; + default: { + const btn = button === 'right' ? 'right' : 'left'; + await this._page.mouse.click(x, y, { button: btn }); + return; + } + } + }; + + doubleClick = async (x: number, y: number): Promise => { + this._assertPage(); + await this._page.mouse.dblclick(x, y); + }; + + scroll = async (x: number, y: number, scrollX: number, scrollY: number): Promise => { + this._assertPage(); + await this._page.mouse.move(x, y); + await this._page.evaluate( + (params: { dx: number; dy: number }) => window.scrollBy(params.dx, params.dy), + { dx: scrollX, dy: scrollY }, + ); + }; + + type = async (text: string): Promise => { + this._assertPage(); + await this._page.keyboard.type(text); + }; + + keypress = async (keys: string[]): Promise => { + this._assertPage(); + const mapped = keys.map((k) => KEY_MAP[k.toLowerCase()] ?? k); + for (const k of mapped) await this._page.keyboard.down(k); + for (const k of [...mapped].reverse()) await this._page.keyboard.up(k); + }; + + wait = async (ms = 1000): Promise => { + await new Promise((resolve) => setTimeout(resolve, ms)); + }; + + move = async (x: number, y: number): Promise => { + this._assertPage(); + await this._page.mouse.move(x, y); + }; + + drag = async (path: Point[]): Promise => { + this._assertPage(); + const first = path[0]; + if (!first) return; + await this._page.mouse.move(first.x, first.y); + await this._page.mouse.down(); + for (const pt of path.slice(1)) await this._page.mouse.move(pt.x, pt.y); + await this._page.mouse.up(); + }; + + goto = async (url: string): Promise => { + this._assertPage(); + try { + return await this._page.goto(url); + } catch { + return null; + } + }; + + back = async (): Promise => { + this._assertPage(); + return (await this._page.goBack()) || null; + }; + + forward = async (): Promise => { + this._assertPage(); + return (await this._page.goForward()) || null; + }; +} diff --git a/pkg/templates/typescript/ehr-system/lib/playwright/kernel.ts b/pkg/templates/typescript/ehr-system/lib/playwright/kernel.ts new file mode 100644 index 0000000..4dd0c86 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/lib/playwright/kernel.ts @@ -0,0 +1,43 @@ +import { chromium, type Browser, type Page } from 'playwright-core'; +import { BasePlaywrightComputer } from './base'; + +/** + * KernelPlaywrightComputer connects to a remote browser instance via CDP WebSocket URL. + * Similar to LocalPlaywrightComputer but uses an existing browser instance instead of launching one. + */ +export class KernelPlaywrightComputer extends BasePlaywrightComputer { + private cdp_ws_url: string; + + constructor(cdp_ws_url: string) { + super(); + this.cdp_ws_url = cdp_ws_url; + } + + _getBrowserAndPage = async (): Promise<[Browser, Page]> => { + const [width, height] = this.getDimensions(); + + // Connect to existing browser instance via CDP + const browser = await chromium.connectOverCDP(this.cdp_ws_url); + + // Get existing context or create new one + let context = browser.contexts()[0]; + if (!context) { + context = await browser.newContext(); + } + + // Add event listeners for page creation and closure + context.on('page', this._handleNewPage.bind(this)); + + // Get existing page or create new one + let page = context.pages()[0]; + if (!page) { + page = await context.newPage(); + } + + // Set viewport size + await page.setViewportSize({ width, height }); + page.on('close', this._handlePageClose.bind(this)); + + return [browser, page]; + }; +} diff --git a/pkg/templates/typescript/ehr-system/lib/playwright/local.ts b/pkg/templates/typescript/ehr-system/lib/playwright/local.ts new file mode 100644 index 0000000..d043780 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/lib/playwright/local.ts @@ -0,0 +1,43 @@ +import { chromium, type Browser, type Page } from 'playwright-core'; +import { BasePlaywrightComputer } from './base'; + +/** + * Launches a local Chromium instance using Playwright. + */ +export class LocalPlaywrightComputer extends BasePlaywrightComputer { + private headless: boolean; + + constructor(headless = false) { + super(); + this.headless = headless; + } + + _getBrowserAndPage = async (): Promise<[Browser, Page]> => { + const [width, height] = this.getDimensions(); + const launchArgs = [ + `--window-size=${width},${height}`, + '--disable-extensions', + '--disable-file-system', + ]; + + const browser = await chromium.launch({ + headless: this.headless, + args: launchArgs, + env: { DISPLAY: ':0' }, + }); + + const context = await browser.newContext(); + + // Add event listeners for page creation and closure + context.on('page', this._handleNewPage.bind(this)); + + const page = await context.newPage(); + await page.setViewportSize({ width, height }); + page.on('close', this._handlePageClose.bind(this)); + + await page.goto('https://duckduckgo.com'); + + // console.dir({debug_getBrowserAndPage: [browser, page]}); + return [browser, page]; + }; +} diff --git a/pkg/templates/typescript/ehr-system/lib/toolset.ts b/pkg/templates/typescript/ehr-system/lib/toolset.ts new file mode 100644 index 0000000..2999d0b --- /dev/null +++ b/pkg/templates/typescript/ehr-system/lib/toolset.ts @@ -0,0 +1,40 @@ +const shared = [ + { + type: 'function', + name: 'goto', + description: 'Go to a specific URL.', + parameters: { + type: 'object', + properties: { + url: { + type: 'string', + description: 'Fully qualified URL to navigate to.', + }, + }, + additionalProperties: false, + required: ['url'], + }, + }, + { + type: 'function', + name: 'back', + description: 'Navigate back in the browser history.', + parameters: { + type: 'object', + properties: {}, + additionalProperties: false, + }, + }, + { + type: 'function', + name: 'forward', + description: 'Navigate forward in the browser history.', + parameters: { + type: 'object', + properties: {}, + additionalProperties: false, + }, + }, +]; + +export default { shared }; diff --git a/pkg/templates/typescript/ehr-system/lib/utils.ts b/pkg/templates/typescript/ehr-system/lib/utils.ts new file mode 100644 index 0000000..f2dc0fd --- /dev/null +++ b/pkg/templates/typescript/ehr-system/lib/utils.ts @@ -0,0 +1,61 @@ +import 'dotenv/config'; +import sharp from 'sharp'; +import OpenAI from 'openai'; +import { type ResponseItem } from 'openai/resources/responses/responses'; +const openai = new OpenAI(); + +const BLOCKED_DOMAINS: readonly string[] = [ + 'maliciousbook.com', + 'evilvideos.com', + 'darkwebforum.com', + 'shadytok.com', + 'suspiciouspins.com', + 'ilanbigio.com', +] as const; + +export async function calculateImageDimensions( + base64Image: string, +): Promise<{ width: number; height: number }> { + const buf = Buffer.from(base64Image, 'base64'); + const meta = await sharp(buf).metadata(); + return { width: meta.width ?? 0, height: meta.height ?? 0 }; +} +export function sanitizeMessage(msg: ResponseItem): ResponseItem { + const sanitizedMsg = { ...msg } as ResponseItem; + if ( + sanitizedMsg.type === 'computer_call_output' && + typeof sanitizedMsg.output === 'object' && + sanitizedMsg.output !== null + ) { + sanitizedMsg.output = { ...sanitizedMsg.output }; + const output = sanitizedMsg.output as { image_url?: string }; + if (output.image_url) { + output.image_url = '[omitted]'; + } + } + return sanitizedMsg; +} + +export async function createResponse( + params: OpenAI.Responses.ResponseCreateParams, +): Promise<{ output?: OpenAI.Responses.ResponseOutputItem[] }> { + try { + const response = await openai.responses.create(params); + return 'output' in response ? response : { output: undefined }; + } catch (err: unknown) { + console.error((err as Error).message); + throw err; + } +} + +export function checkBlocklistedUrl(url: string): boolean { + const host = new URL(url).hostname; + return BLOCKED_DOMAINS.some((d) => host === d || host.endsWith(`.${d}`)); +} + +export default { + calculateImageDimensions, + sanitizeMessage, + createResponse, + checkBlocklistedUrl, +}; diff --git a/pkg/templates/typescript/ehr-system/package-lock.json b/pkg/templates/typescript/ehr-system/package-lock.json new file mode 100644 index 0000000..358fa33 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/package-lock.json @@ -0,0 +1,650 @@ +{ + "name": "ehr-system", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "ehr-system", + "dependencies": { + "@onkernel/sdk": "^0.23.0", + "dotenv": "^17.2.3", + "openai": "^6.13.0", + "playwright-core": "^1.57.0", + "sharp": "^0.34.5" + }, + "devDependencies": { + "@types/node": "^22.15.17", + "typescript": "^5.9.3" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", + "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@img/colour": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", + "integrity": "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@onkernel/sdk": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@onkernel/sdk/-/sdk-0.23.0.tgz", + "integrity": "sha512-P/ez6HU8sO2QvqWATkvC+Wdv+fgto4KfBCHLl2T6EUpoU3LhgOZ/sJP2ZRf/vh5Vh7QR2Vf05RgMaFcIGBGD9Q==", + "license": "Apache-2.0" + }, + "node_modules/@types/node": { + "version": "22.19.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.7.tgz", + "integrity": "sha512-MciR4AKGHWl7xwxkBa6xUGxQJ4VBOmPTF7sL+iGzuahOFaO0jHCsuEfS80pan1ef4gWId1oWOweIhrDEYLuaOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/dotenv": { + "version": "17.2.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", + "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/openai": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.16.0.tgz", + "integrity": "sha512-fZ1uBqjFUjXzbGc35fFtYKEOxd20kd9fDpFeqWtsOZWiubY8CZ1NAlXHW3iathaFvqmNtCWMIsosCuyeI7Joxg==", + "license": "Apache-2.0", + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/playwright-core": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.57.0.tgz", + "integrity": "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/pkg/templates/typescript/ehr-system/package.json b/pkg/templates/typescript/ehr-system/package.json new file mode 100644 index 0000000..1fdc3b9 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/package.json @@ -0,0 +1,20 @@ +{ + "name": "ehr-system", + "module": "index.ts", + "type": "module", + "private": true, + "scripts": { + "build": "tsc" + }, + "dependencies": { + "@onkernel/sdk": "^0.23.0", + "dotenv": "^17.2.3", + "openai": "^6.13.0", + "playwright-core": "^1.57.0", + "sharp": "^0.34.5" + }, + "devDependencies": { + "@types/node": "^22.15.17", + "typescript": "^5.9.3" + } +} diff --git a/pkg/templates/typescript/ehr-system/tsconfig.json b/pkg/templates/typescript/ehr-system/tsconfig.json new file mode 100644 index 0000000..fa10973 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": ".", + "lib": ["ESNext", "DOM"] + }, + "include": ["."] +} From 750aeefc4d0a5a4bf4d9be405ba740c37111ce55 Mon Sep 17 00:00:00 2001 From: arthursita-plank Date: Fri, 23 Jan 2026 14:32:48 -0300 Subject: [PATCH 06/11] WIP: get all data from report and try to download --- pkg/templates/typescript/ehr-system/index.ts | 20 +++++++++++++++++++ .../ehr-system/lib/playwright/base.ts | 11 ++++++++++ 2 files changed, 31 insertions(+) diff --git a/pkg/templates/typescript/ehr-system/index.ts b/pkg/templates/typescript/ehr-system/index.ts index 0d4351b..e71b9a3 100644 --- a/pkg/templates/typescript/ehr-system/index.ts +++ b/pkg/templates/typescript/ehr-system/index.ts @@ -10,6 +10,7 @@ interface Input { interface Output { elapsed: number; answer: string | null; + download?: string | null; logs?: any[]; } @@ -54,6 +55,11 @@ app.action( }, }); + console.log('Starting download listener...'); + // Start listening for download before running the agent + // Set a long timeout (5 minutes) because the agent might take time to navigate + const downloadPromise = (computer as any).waitForDownload(300000); + // run agent and get response const logs = await agent.runFullTurn({ messages: [ @@ -72,6 +78,19 @@ app.action( show_images: false, }); + // Wait for download to resolve (it should have happened during the run) + // We use a small timeout race just in case it's still pending but not happening + const download = await Promise.race([ + downloadPromise, + new Promise(resolve => setTimeout(() => resolve(null), 5000)) + ]); + + if (download) { + console.log(`Download captured: ${download}`); + } else { + console.log('No download captured within timeout.'); + } + const elapsed = parseFloat(((Date.now() - start) / 1000).toFixed(2)); // filter only LLM messages @@ -91,6 +110,7 @@ app.action( return { elapsed, answer, + download }; } catch (error) { const elapsed = parseFloat(((Date.now() - start) / 1000).toFixed(2)); diff --git a/pkg/templates/typescript/ehr-system/lib/playwright/base.ts b/pkg/templates/typescript/ehr-system/lib/playwright/base.ts index b43a7d2..2f9cf35 100644 --- a/pkg/templates/typescript/ehr-system/lib/playwright/base.ts +++ b/pkg/templates/typescript/ehr-system/lib/playwright/base.ts @@ -239,4 +239,15 @@ export class BasePlaywrightComputer { this._assertPage(); return (await this._page.goForward()) || null; }; + waitForDownload = async (timeout = 30000): Promise => { + this._assertPage(); + try { + const downloadPromise = this._page.waitForEvent('download', { timeout }); + const download = await downloadPromise; + return await download.url(); + } catch (e) { + console.error('Download capture failed:', e); + return null; + } + }; } From 962c79f8438cf4f982c1a266d3b1a1223a890c36 Mon Sep 17 00:00:00 2001 From: arthursita-plank Date: Tue, 27 Jan 2026 20:20:05 -0300 Subject: [PATCH 07/11] refactor: Reorganize EHR system template, introducing new tool management, utilities, sampling loop, and session management. --- .../typescript/ehr-system/.env.example | 2 +- pkg/templates/typescript/ehr-system/index.ts | 131 ++-- .../typescript/ehr-system/lib/agent.ts | 208 ------ .../typescript/ehr-system/lib/computers.ts | 28 - .../ehr-system/lib/playwright/base.ts | 253 ------- .../ehr-system/lib/playwright/kernel.ts | 43 -- .../ehr-system/lib/playwright/local.ts | 43 -- .../typescript/ehr-system/lib/toolset.ts | 40 -- .../typescript/ehr-system/lib/utils.ts | 61 -- pkg/templates/typescript/ehr-system/loop.ts | 196 ++++++ .../typescript/ehr-system/package-lock.json | 635 ++---------------- .../typescript/ehr-system/package.json | 9 +- .../typescript/ehr-system/session.ts | 222 ++++++ .../typescript/ehr-system/tools/collection.ts | 61 ++ .../typescript/ehr-system/tools/computer.ts | 401 +++++++++++ .../ehr-system/tools/types/computer.ts | 64 ++ .../ehr-system/tools/utils/keyboard.ts | 88 +++ .../ehr-system/tools/utils/validator.ts | 67 ++ .../typescript/ehr-system/types/beta.ts | 58 ++ .../ehr-system/utils/message-processing.ts | 79 +++ .../ehr-system/utils/tool-results.ts | 49 ++ 21 files changed, 1395 insertions(+), 1343 deletions(-) delete mode 100644 pkg/templates/typescript/ehr-system/lib/agent.ts delete mode 100644 pkg/templates/typescript/ehr-system/lib/computers.ts delete mode 100644 pkg/templates/typescript/ehr-system/lib/playwright/base.ts delete mode 100644 pkg/templates/typescript/ehr-system/lib/playwright/kernel.ts delete mode 100644 pkg/templates/typescript/ehr-system/lib/playwright/local.ts delete mode 100644 pkg/templates/typescript/ehr-system/lib/toolset.ts delete mode 100644 pkg/templates/typescript/ehr-system/lib/utils.ts create mode 100644 pkg/templates/typescript/ehr-system/loop.ts create mode 100644 pkg/templates/typescript/ehr-system/session.ts create mode 100644 pkg/templates/typescript/ehr-system/tools/collection.ts create mode 100644 pkg/templates/typescript/ehr-system/tools/computer.ts create mode 100644 pkg/templates/typescript/ehr-system/tools/types/computer.ts create mode 100644 pkg/templates/typescript/ehr-system/tools/utils/keyboard.ts create mode 100644 pkg/templates/typescript/ehr-system/tools/utils/validator.ts create mode 100644 pkg/templates/typescript/ehr-system/types/beta.ts create mode 100644 pkg/templates/typescript/ehr-system/utils/message-processing.ts create mode 100644 pkg/templates/typescript/ehr-system/utils/tool-results.ts diff --git a/pkg/templates/typescript/ehr-system/.env.example b/pkg/templates/typescript/ehr-system/.env.example index 9847a1d..80a79e6 100644 --- a/pkg/templates/typescript/ehr-system/.env.example +++ b/pkg/templates/typescript/ehr-system/.env.example @@ -1 +1 @@ -OPENAI_API_KEY= \ No newline at end of file +ANTHROPIC_API_KEY= \ No newline at end of file diff --git a/pkg/templates/typescript/ehr-system/index.ts b/pkg/templates/typescript/ehr-system/index.ts index e71b9a3..b485dc9 100644 --- a/pkg/templates/typescript/ehr-system/index.ts +++ b/pkg/templates/typescript/ehr-system/index.ts @@ -1,33 +1,34 @@ import { Kernel, type KernelContext } from '@onkernel/sdk'; -import 'dotenv/config'; -import { Agent } from './lib/agent'; -import computers from './lib/computers'; +import { samplingLoop } from './loop'; +import { KernelBrowserSession } from './session'; interface Input { task?: string; + record_replay?: boolean; } interface Output { elapsed: number; - answer: string | null; - download?: string | null; - logs?: any[]; + result: string | null; + replay_url?: string | null; } const kernel = new Kernel(); const app = kernel.app('ehr-system'); -if (!process.env.OPENAI_API_KEY) { - throw new Error('OPENAI_API_KEY is not set'); +// LLM API Keys are set in the environment during `kernel deploy -e ANTHROPIC_API_KEY=XXX` +// See https://www.kernel.sh/docs/launch/deploy#environment-variables +const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY; + +if (!ANTHROPIC_API_KEY) { + throw new Error('ANTHROPIC_API_KEY is not set'); } const DEFAULT_TASK = ` -Go to https://ehr-system-six.vercel.app/login -Login with any email and password (e.g. user@example.com / password). -Navigate to the "Reports" page. -Find the "Export CSV" button and click it to download the report. -Wait for the download to start. -CRITICAL: Do not ask for confirmation. Perform all steps immediately. +Go to https://demo.openemr.io/openemr/portal/index.php +Login with username: Phil1 | password: phil | email: heya@invalid.email.com. +Navigate to the "Medical Reports" page. +Find the "Download Summary of Care" button and click it to download the report. `; app.action( @@ -36,91 +37,63 @@ app.action( const start = Date.now(); const task = payload?.task || DEFAULT_TASK; - const kb = await kernel.browsers.create({ - invocation_id: ctx.invocation_id, - stealth: true + // Create browser session with optional replay recording + const session = new KernelBrowserSession(kernel, { + stealth: true, + recordReplay: payload?.record_replay ?? false, }); - console.log('> Kernel browser live view url:', kb.browser_live_view_url); + + await session.start(); + console.log('> Kernel browser live view url:', session.liveViewUrl); try { - const { computer } = await computers.create({ type: 'kernel', cdp_ws_url: kb.cdp_ws_url }); - - const agent = new Agent({ - model: 'computer-use-preview', // Using a capable model for computer use - computer, - tools: [], - acknowledge_safety_check_callback: (m: string): boolean => { - console.log(`> safety check: ${m}`); - return true; - }, + // Run the sampling loop with Anthropic Computer Use + const finalMessages = await samplingLoop({ + model: 'claude-sonnet-4-5-20250929', + messages: [{ + role: 'user', + content: `You are an automated agent. Current date and time: ${new Date().toISOString()}. You must complete the task fully without asking for permission.\n\nTask: ${task}`, + }], + apiKey: ANTHROPIC_API_KEY, + thinkingBudget: 1024, + kernel, + sessionId: session.sessionId, }); - console.log('Starting download listener...'); - // Start listening for download before running the agent - // Set a long timeout (5 minutes) because the agent might take time to navigate - const downloadPromise = (computer as any).waitForDownload(300000); - - // run agent and get response - const logs = await agent.runFullTurn({ - messages: [ - { - role: 'system', - content: `You are an automated agent. Current date and time: ${new Date().toISOString()}. You must complete the task fully without asking for permission.`, - }, - { - type: 'message', - role: 'user', - content: [{ type: 'input_text', text: task }], - }, - ], - print_steps: true, - debug: true, - show_images: false, - }); + // Extract the final result from the messages + if (finalMessages.length === 0) { + throw new Error('No messages were generated during the sampling loop'); + } - // Wait for download to resolve (it should have happened during the run) - // We use a small timeout race just in case it's still pending but not happening - const download = await Promise.race([ - downloadPromise, - new Promise(resolve => setTimeout(() => resolve(null), 5000)) - ]); - - if (download) { - console.log(`Download captured: ${download}`); - } else { - console.log('No download captured within timeout.'); + const lastMessage = finalMessages[finalMessages.length - 1]; + if (!lastMessage) { + throw new Error('Failed to get the last message from the sampling loop'); } - const elapsed = parseFloat(((Date.now() - start) / 1000).toFixed(2)); + const result = typeof lastMessage.content === 'string' + ? lastMessage.content + : lastMessage.content.map(block => + block.type === 'text' ? block.text : '' + ).join(''); - // filter only LLM messages + const elapsed = parseFloat(((Date.now() - start) / 1000).toFixed(2)); - // filter only LLM messages - const messages = logs.filter( - (item: any) => - item.type === 'message' && - typeof item.role === 'string' && - Array.isArray(item.content), - ); - const assistant = messages.find((m: any) => m.role === 'assistant') as any; - const lastContentIndex = assistant?.content?.length ? assistant.content.length - 1 : -1; - const lastContent = lastContentIndex >= 0 ? assistant?.content?.[lastContentIndex] : null; - const answer = lastContent && 'text' in lastContent ? lastContent.text : null; + // Stop session and get replay URL if recording was enabled + const sessionInfo = await session.stop(); return { elapsed, - answer, - download + result, + replay_url: sessionInfo.replayViewUrl, }; } catch (error) { const elapsed = parseFloat(((Date.now() - start) / 1000).toFixed(2)); console.error('Error in export-report:', error); + await session.stop(); return { elapsed, - answer: null, + result: null, }; - } finally { - await kernel.browsers.deleteByID(kb.session_id); } }, ); diff --git a/pkg/templates/typescript/ehr-system/lib/agent.ts b/pkg/templates/typescript/ehr-system/lib/agent.ts deleted file mode 100644 index 28808d5..0000000 --- a/pkg/templates/typescript/ehr-system/lib/agent.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { - type ResponseItem, - type ResponseInputItem, - type ResponseOutputMessage, - type ResponseFunctionToolCallItem, - type ResponseFunctionToolCallOutputItem, - type ResponseComputerToolCall, - type ResponseComputerToolCallOutputItem, - type ComputerTool, -} from 'openai/resources/responses/responses'; - -import * as utils from './utils'; -import toolset from './toolset'; -import type { BasePlaywrightComputer } from './playwright/base'; -import type { LocalPlaywrightComputer } from './playwright/local'; -import type { KernelPlaywrightComputer } from './playwright/kernel'; - -export class Agent { - private model: string; - private computer: - | BasePlaywrightComputer - | LocalPlaywrightComputer - | KernelPlaywrightComputer - | undefined; - private tools: ComputerTool[]; - private print_steps = true; - private debug = false; - private show_images = false; - private ackCb: (msg: string) => boolean; - - constructor(opts: { - model?: string; - computer?: - | BasePlaywrightComputer - | LocalPlaywrightComputer - | KernelPlaywrightComputer - | undefined; - tools?: ComputerTool[]; - acknowledge_safety_check_callback?: (msg: string) => boolean; - }) { - this.model = opts.model ?? 'computer-use-preview'; - this.computer = opts.computer; - this.tools = [...toolset.shared, ...(opts.tools ?? [])] as ComputerTool[]; - this.ackCb = opts.acknowledge_safety_check_callback ?? ((): boolean => true); - - if (this.computer) { - const [w, h] = this.computer.getDimensions(); - this.tools.push({ - type: 'computer_use_preview', - display_width: w, - display_height: h, - environment: this.computer.getEnvironment(), - }); - } - } - - private debugPrint(...args: unknown[]): void { - if (this.debug) { - console.warn('--- debug:agent:debugPrint'); - try { - console.dir( - args.map((msg) => utils.sanitizeMessage(msg as ResponseItem)), - { depth: null }, - ); - } catch { - console.dir(args, { depth: null }); - } - } - } - - private async handleItem(item: ResponseItem): Promise { - if (item.type === 'message' && this.print_steps) { - const msg = item as ResponseOutputMessage; - const c = msg.content; - if (Array.isArray(c) && c[0] && 'text' in c[0] && typeof c[0].text === 'string') - console.log(c[0].text); - } - - if (item.type === 'function_call') { - const fc = item as ResponseFunctionToolCallItem; - const argsObj = JSON.parse(fc.arguments) as Record; - if (this.print_steps) console.log(`${fc.name}(${JSON.stringify(argsObj)})`); - if (this.computer) { - const fn = (this.computer as unknown as Record)[fc.name]; - if (typeof fn === 'function') - await (fn as (...a: unknown[]) => unknown)(...Object.values(argsObj)); - } - return [ - { - type: 'function_call_output', - call_id: fc.call_id, - output: 'success', - } as unknown as ResponseFunctionToolCallOutputItem, - ]; - } - - if (item.type === 'computer_call') { - const cc = item as ResponseComputerToolCall; - const { type: actionType, ...actionArgs } = cc.action; - if (this.print_steps) console.log(`${actionType}(${JSON.stringify(actionArgs)})`); - if (this.computer) { - const fn = (this.computer as unknown as Record)[actionType as string]; - if (typeof fn === 'function') { - await (fn as (...a: unknown[]) => unknown)(...Object.values(actionArgs)); - const screenshot = await this.computer.screenshot(); - const pending = cc.pending_safety_checks ?? []; - for (const { message } of pending) - if (message && !this.ackCb(message)) throw new Error(`Safety check failed: ${message}`); - const out: Omit = { - type: 'computer_call_output', - call_id: cc.call_id, - // id: "?", // <---- omitting to work - need to determine id source, != call_id - acknowledged_safety_checks: pending, - output: { - type: 'computer_screenshot', - image_url: `data:image/webp;base64,${screenshot}`, - }, - }; - if (this.computer.getEnvironment() === 'browser') - utils.checkBlocklistedUrl(this.computer.getCurrentUrl()); - return [out as ResponseItem]; - } - } - } - - return []; - } - - async runFullTurn(opts: { - messages: ResponseInputItem[]; - print_steps?: boolean; - debug?: boolean; - show_images?: boolean; - }): Promise { - this.print_steps = opts.print_steps ?? true; - this.debug = opts.debug ?? false; - this.show_images = opts.show_images ?? false; - const newItems: ResponseItem[] = []; - - while ( - newItems.length === 0 || - (newItems[newItems.length - 1] as ResponseItem & { role?: string }).role !== 'assistant' - ) { - // Add current URL to system message if in browser environment - const inputMessages = [...opts.messages]; - - if (this.computer?.getEnvironment() === 'browser') { - const current_url = this.computer.getCurrentUrl(); - // Find system message by checking if it has a role property with value 'system' - const sysIndex = inputMessages.findIndex((msg) => 'role' in msg && msg.role === 'system'); - - if (sysIndex >= 0) { - const msg = inputMessages[sysIndex]; - const urlInfo = `\n- Current URL: ${current_url}`; - - // Create a properly typed message based on the original - if (msg && 'content' in msg) { - if (typeof msg.content === 'string') { - // Create a new message with the updated content - const updatedMsg = { - ...msg, - content: msg.content + urlInfo, - }; - // Type assertion to ensure compatibility - inputMessages[sysIndex] = updatedMsg as typeof msg; - } else if (Array.isArray(msg.content) && msg.content.length > 0) { - // Handle array content case - const updatedContent = [...msg.content]; - - // Check if first item has text property - if (updatedContent[0] && 'text' in updatedContent[0]) { - updatedContent[0] = { - ...updatedContent[0], - text: updatedContent[0].text + urlInfo, - }; - } - - // Create updated message with new content - const updatedMsg = { - ...msg, - content: updatedContent, - }; - // Type assertion to ensure compatibility - inputMessages[sysIndex] = updatedMsg as typeof msg; - } - } - } - } - - this.debugPrint(...inputMessages, ...newItems); - const response = await utils.createResponse({ - model: this.model, - input: [...inputMessages, ...newItems], - tools: this.tools, - truncation: 'auto', - }); - if (!response.output) throw new Error('No output from model'); - for (const msg of response.output as ResponseItem[]) { - newItems.push(msg, ...(await this.handleItem(msg))); - } - } - - // Return sanitized messages if show_images is false - return !this.show_images - ? newItems.map((msg) => utils.sanitizeMessage(msg) as ResponseItem) - : newItems; - } -} diff --git a/pkg/templates/typescript/ehr-system/lib/computers.ts b/pkg/templates/typescript/ehr-system/lib/computers.ts deleted file mode 100644 index 5828fc8..0000000 --- a/pkg/templates/typescript/ehr-system/lib/computers.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { KernelPlaywrightComputer } from './playwright/kernel'; -import { LocalPlaywrightComputer } from './playwright/local'; - -interface KernelConfig { - type: 'kernel'; - cdp_ws_url: string; -} -interface LocalConfig { - type: 'local'; - headless?: boolean; -} -type ComputerConfig = KernelConfig | LocalConfig; - -export default { - async create( - cfg: ComputerConfig, - ): Promise<{ computer: KernelPlaywrightComputer | LocalPlaywrightComputer }> { - if (cfg.type === 'kernel') { - const computer = new KernelPlaywrightComputer(cfg.cdp_ws_url); - await computer.enter(); - return { computer }; - } else { - const computer = new LocalPlaywrightComputer(cfg.headless ?? false); - await computer.enter(); - return { computer }; - } - }, -}; diff --git a/pkg/templates/typescript/ehr-system/lib/playwright/base.ts b/pkg/templates/typescript/ehr-system/lib/playwright/base.ts deleted file mode 100644 index 2f9cf35..0000000 --- a/pkg/templates/typescript/ehr-system/lib/playwright/base.ts +++ /dev/null @@ -1,253 +0,0 @@ -import type { Browser, Page, Request, Response, Route } from 'playwright-core'; -import sharp from 'sharp'; -import utils from '../utils'; - -// CUA key -> Playwright key mapping -const KEY_MAP: Record = { - '/': '/', - '\\': '\\', - alt: 'Alt', - arrowdown: 'ArrowDown', - arrowleft: 'ArrowLeft', - arrowright: 'ArrowRight', - arrowup: 'ArrowUp', - backspace: 'Backspace', - capslock: 'CapsLock', - cmd: 'Meta', - ctrl: 'Control', - delete: 'Delete', - end: 'End', - enter: 'Enter', - esc: 'Escape', - home: 'Home', - insert: 'Insert', - option: 'Alt', - pagedown: 'PageDown', - pageup: 'PageUp', - shift: 'Shift', - space: ' ', - super: 'Meta', - tab: 'Tab', - win: 'Meta', -}; - -interface Point { - x: number; - y: number; -} - -export class BasePlaywrightComputer { - protected _browser: Browser | null = null; - protected _page: Page | null = null; - - constructor() { - this._browser = null; - this._page = null; - } - - /** - * Type guard to assert that this._page is present and is a Playwright Page. - * Throws an error if not present. - */ - protected _assertPage(): asserts this is { _page: Page } { - if (!this._page) { - throw new Error('Playwright Page is not initialized. Did you forget to call enter()?'); - } - } - - protected _handleNewPage = (page: Page): void => { - /** Handle the creation of a new page. */ - console.log('New page created'); - this._page = page; - page.on('close', this._handlePageClose.bind(this)); - }; - - protected _handlePageClose = (page: Page): void => { - /** Handle the closure of a page. */ - console.log('Page closed'); - try { - this._assertPage(); - } catch { - return; - } - if (this._page !== page) return; - - const browser = this._browser; - if (!browser || typeof browser.contexts !== 'function') { - console.log('Warning: Browser or context not available.'); - this._page = undefined as unknown as Page; - return; - } - - const contexts = browser.contexts(); - if (!contexts.length) { - console.log('Warning: No browser contexts available.'); - this._page = undefined as unknown as Page; - return; - } - - const context = contexts[0]; - if (!context || typeof context.pages !== 'function') { - console.log('Warning: Context pages not available.'); - this._page = undefined as unknown as Page; - return; - } - - const pages = context.pages(); - if (pages.length) { - this._page = pages[pages.length - 1] as Page; - } else { - console.log('Warning: All pages have been closed.'); - this._page = undefined as unknown as Page; - } - }; - - // Subclass hook - protected _getBrowserAndPage = async (): Promise<[Browser, Page]> => { - // Subclasses must implement, returning [Browser, Page] - throw new Error('Subclasses must implement _getBrowserAndPage()'); - }; - - getEnvironment = (): 'windows' | 'mac' | 'linux' | 'ubuntu' | 'browser' => { - return 'browser'; - }; - - getDimensions = (): [number, number] => { - return [1024, 768]; - }; - - enter = async (): Promise => { - // Call the subclass hook for getting browser/page - [this._browser, this._page] = await this._getBrowserAndPage(); - - // Set up network interception to flag URLs matching domains in BLOCKED_DOMAINS - const handleRoute = (route: Route, request: Request): void => { - const url = request.url(); - if (utils.checkBlocklistedUrl(url)) { - console.log(`Flagging blocked domain: ${url}`); - route.abort(); - } else { - route.continue(); - } - }; - - this._assertPage(); - await this._page.route('**/*', handleRoute); - return this; - }; - - exit = async (): Promise => { - if (this._browser) await this._browser.close(); - }; - - getCurrentUrl = (): string => { - this._assertPage(); - return this._page.url(); - }; - - screenshot = async (): Promise => { - this._assertPage(); - const buf = await this._page.screenshot({ fullPage: false }); - const webp = await sharp(buf).webp().toBuffer(); - return webp.toString('base64'); - }; - - click = async ( - button: 'left' | 'right' | 'back' | 'forward' | 'wheel', - x: number, - y: number, - ): Promise => { - this._assertPage(); - switch (button) { - case 'back': - await this.back(); - return; - case 'forward': - await this.forward(); - return; - case 'wheel': - await this._page.mouse.wheel(x, y); - return; - default: { - const btn = button === 'right' ? 'right' : 'left'; - await this._page.mouse.click(x, y, { button: btn }); - return; - } - } - }; - - doubleClick = async (x: number, y: number): Promise => { - this._assertPage(); - await this._page.mouse.dblclick(x, y); - }; - - scroll = async (x: number, y: number, scrollX: number, scrollY: number): Promise => { - this._assertPage(); - await this._page.mouse.move(x, y); - await this._page.evaluate( - (params: { dx: number; dy: number }) => window.scrollBy(params.dx, params.dy), - { dx: scrollX, dy: scrollY }, - ); - }; - - type = async (text: string): Promise => { - this._assertPage(); - await this._page.keyboard.type(text); - }; - - keypress = async (keys: string[]): Promise => { - this._assertPage(); - const mapped = keys.map((k) => KEY_MAP[k.toLowerCase()] ?? k); - for (const k of mapped) await this._page.keyboard.down(k); - for (const k of [...mapped].reverse()) await this._page.keyboard.up(k); - }; - - wait = async (ms = 1000): Promise => { - await new Promise((resolve) => setTimeout(resolve, ms)); - }; - - move = async (x: number, y: number): Promise => { - this._assertPage(); - await this._page.mouse.move(x, y); - }; - - drag = async (path: Point[]): Promise => { - this._assertPage(); - const first = path[0]; - if (!first) return; - await this._page.mouse.move(first.x, first.y); - await this._page.mouse.down(); - for (const pt of path.slice(1)) await this._page.mouse.move(pt.x, pt.y); - await this._page.mouse.up(); - }; - - goto = async (url: string): Promise => { - this._assertPage(); - try { - return await this._page.goto(url); - } catch { - return null; - } - }; - - back = async (): Promise => { - this._assertPage(); - return (await this._page.goBack()) || null; - }; - - forward = async (): Promise => { - this._assertPage(); - return (await this._page.goForward()) || null; - }; - waitForDownload = async (timeout = 30000): Promise => { - this._assertPage(); - try { - const downloadPromise = this._page.waitForEvent('download', { timeout }); - const download = await downloadPromise; - return await download.url(); - } catch (e) { - console.error('Download capture failed:', e); - return null; - } - }; -} diff --git a/pkg/templates/typescript/ehr-system/lib/playwright/kernel.ts b/pkg/templates/typescript/ehr-system/lib/playwright/kernel.ts deleted file mode 100644 index 4dd0c86..0000000 --- a/pkg/templates/typescript/ehr-system/lib/playwright/kernel.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { chromium, type Browser, type Page } from 'playwright-core'; -import { BasePlaywrightComputer } from './base'; - -/** - * KernelPlaywrightComputer connects to a remote browser instance via CDP WebSocket URL. - * Similar to LocalPlaywrightComputer but uses an existing browser instance instead of launching one. - */ -export class KernelPlaywrightComputer extends BasePlaywrightComputer { - private cdp_ws_url: string; - - constructor(cdp_ws_url: string) { - super(); - this.cdp_ws_url = cdp_ws_url; - } - - _getBrowserAndPage = async (): Promise<[Browser, Page]> => { - const [width, height] = this.getDimensions(); - - // Connect to existing browser instance via CDP - const browser = await chromium.connectOverCDP(this.cdp_ws_url); - - // Get existing context or create new one - let context = browser.contexts()[0]; - if (!context) { - context = await browser.newContext(); - } - - // Add event listeners for page creation and closure - context.on('page', this._handleNewPage.bind(this)); - - // Get existing page or create new one - let page = context.pages()[0]; - if (!page) { - page = await context.newPage(); - } - - // Set viewport size - await page.setViewportSize({ width, height }); - page.on('close', this._handlePageClose.bind(this)); - - return [browser, page]; - }; -} diff --git a/pkg/templates/typescript/ehr-system/lib/playwright/local.ts b/pkg/templates/typescript/ehr-system/lib/playwright/local.ts deleted file mode 100644 index d043780..0000000 --- a/pkg/templates/typescript/ehr-system/lib/playwright/local.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { chromium, type Browser, type Page } from 'playwright-core'; -import { BasePlaywrightComputer } from './base'; - -/** - * Launches a local Chromium instance using Playwright. - */ -export class LocalPlaywrightComputer extends BasePlaywrightComputer { - private headless: boolean; - - constructor(headless = false) { - super(); - this.headless = headless; - } - - _getBrowserAndPage = async (): Promise<[Browser, Page]> => { - const [width, height] = this.getDimensions(); - const launchArgs = [ - `--window-size=${width},${height}`, - '--disable-extensions', - '--disable-file-system', - ]; - - const browser = await chromium.launch({ - headless: this.headless, - args: launchArgs, - env: { DISPLAY: ':0' }, - }); - - const context = await browser.newContext(); - - // Add event listeners for page creation and closure - context.on('page', this._handleNewPage.bind(this)); - - const page = await context.newPage(); - await page.setViewportSize({ width, height }); - page.on('close', this._handlePageClose.bind(this)); - - await page.goto('https://duckduckgo.com'); - - // console.dir({debug_getBrowserAndPage: [browser, page]}); - return [browser, page]; - }; -} diff --git a/pkg/templates/typescript/ehr-system/lib/toolset.ts b/pkg/templates/typescript/ehr-system/lib/toolset.ts deleted file mode 100644 index 2999d0b..0000000 --- a/pkg/templates/typescript/ehr-system/lib/toolset.ts +++ /dev/null @@ -1,40 +0,0 @@ -const shared = [ - { - type: 'function', - name: 'goto', - description: 'Go to a specific URL.', - parameters: { - type: 'object', - properties: { - url: { - type: 'string', - description: 'Fully qualified URL to navigate to.', - }, - }, - additionalProperties: false, - required: ['url'], - }, - }, - { - type: 'function', - name: 'back', - description: 'Navigate back in the browser history.', - parameters: { - type: 'object', - properties: {}, - additionalProperties: false, - }, - }, - { - type: 'function', - name: 'forward', - description: 'Navigate forward in the browser history.', - parameters: { - type: 'object', - properties: {}, - additionalProperties: false, - }, - }, -]; - -export default { shared }; diff --git a/pkg/templates/typescript/ehr-system/lib/utils.ts b/pkg/templates/typescript/ehr-system/lib/utils.ts deleted file mode 100644 index f2dc0fd..0000000 --- a/pkg/templates/typescript/ehr-system/lib/utils.ts +++ /dev/null @@ -1,61 +0,0 @@ -import 'dotenv/config'; -import sharp from 'sharp'; -import OpenAI from 'openai'; -import { type ResponseItem } from 'openai/resources/responses/responses'; -const openai = new OpenAI(); - -const BLOCKED_DOMAINS: readonly string[] = [ - 'maliciousbook.com', - 'evilvideos.com', - 'darkwebforum.com', - 'shadytok.com', - 'suspiciouspins.com', - 'ilanbigio.com', -] as const; - -export async function calculateImageDimensions( - base64Image: string, -): Promise<{ width: number; height: number }> { - const buf = Buffer.from(base64Image, 'base64'); - const meta = await sharp(buf).metadata(); - return { width: meta.width ?? 0, height: meta.height ?? 0 }; -} -export function sanitizeMessage(msg: ResponseItem): ResponseItem { - const sanitizedMsg = { ...msg } as ResponseItem; - if ( - sanitizedMsg.type === 'computer_call_output' && - typeof sanitizedMsg.output === 'object' && - sanitizedMsg.output !== null - ) { - sanitizedMsg.output = { ...sanitizedMsg.output }; - const output = sanitizedMsg.output as { image_url?: string }; - if (output.image_url) { - output.image_url = '[omitted]'; - } - } - return sanitizedMsg; -} - -export async function createResponse( - params: OpenAI.Responses.ResponseCreateParams, -): Promise<{ output?: OpenAI.Responses.ResponseOutputItem[] }> { - try { - const response = await openai.responses.create(params); - return 'output' in response ? response : { output: undefined }; - } catch (err: unknown) { - console.error((err as Error).message); - throw err; - } -} - -export function checkBlocklistedUrl(url: string): boolean { - const host = new URL(url).hostname; - return BLOCKED_DOMAINS.some((d) => host === d || host.endsWith(`.${d}`)); -} - -export default { - calculateImageDimensions, - sanitizeMessage, - createResponse, - checkBlocklistedUrl, -}; diff --git a/pkg/templates/typescript/ehr-system/loop.ts b/pkg/templates/typescript/ehr-system/loop.ts new file mode 100644 index 0000000..06e22ca --- /dev/null +++ b/pkg/templates/typescript/ehr-system/loop.ts @@ -0,0 +1,196 @@ +import { Anthropic } from '@anthropic-ai/sdk'; +import { DateTime } from 'luxon'; +import type { Kernel } from '@onkernel/sdk'; +import { DEFAULT_TOOL_VERSION, TOOL_GROUPS_BY_VERSION, ToolCollection, type ToolVersion } from './tools/collection'; +import { ComputerTool20241022, ComputerTool20250124 } from './tools/computer'; +import type { ActionParams } from './tools/types/computer'; +import { Action } from './tools/types/computer'; +import type { BetaMessageParam, BetaTextBlock } from './types/beta'; +import { injectPromptCaching, maybeFilterToNMostRecentImages, PROMPT_CACHING_BETA_FLAG, responseToParams } from './utils/message-processing'; +import { makeApiToolResult } from './utils/tool-results'; + +// System prompt optimized for the environment +const SYSTEM_PROMPT = ` +* You are utilising an Ubuntu virtual machine using ${process.arch} architecture with internet access. +* When you connect to the display, CHROMIUM IS ALREADY OPEN. The url bar is not visible but it is there. +* If you need to navigate to a new page, use ctrl+l to focus the url bar and then enter the url. +* You won't be able to see the url bar from the screenshot but ctrl-l still works. +* As the initial step click on the search bar. +* When viewing a page it can be helpful to zoom out so that you can see everything on the page. +* Either that, or make sure you scroll down to see everything before deciding something isn't available. +* When using your computer function calls, they take a while to run and send back to you. +* Where possible/feasible, try to chain multiple of these calls all into one function calls request. +* The current date is ${DateTime.now().toFormat('EEEE, MMMM d, yyyy')}. +* After each step, take a screenshot and carefully evaluate if you have achieved the right outcome. +* Explicitly show your thinking: "I have evaluated step X..." If not correct, try again. +* Only when you confirm a step was executed correctly should you move on to the next one. + + + +* When using Chromium, if a startup wizard appears, IGNORE IT. Do not even click "skip this step". +* Instead, click on the search bar on the center of the screen where it says "Search or enter address", and enter the appropriate search term or URL there. +`; + +// Add new type definitions +interface ThinkingConfig { + type: 'enabled'; + budget_tokens: number; +} + +interface ExtraBodyConfig { + thinking?: ThinkingConfig; +} + +interface ToolUseInput extends Record { + action: Action; +} + +export async function samplingLoop({ + model, + systemPromptSuffix, + messages, + apiKey, + onlyNMostRecentImages, + maxTokens = 4096, + toolVersion, + thinkingBudget, + tokenEfficientToolsBeta = false, + kernel, + sessionId, +}: { + model: string; + systemPromptSuffix?: string; + messages: BetaMessageParam[]; + apiKey: string; + onlyNMostRecentImages?: number; + maxTokens?: number; + toolVersion?: ToolVersion; + thinkingBudget?: number; + tokenEfficientToolsBeta?: boolean; + kernel: Kernel; + sessionId: string; +}): Promise { + const selectedVersion = toolVersion || DEFAULT_TOOL_VERSION; + const toolGroup = TOOL_GROUPS_BY_VERSION[selectedVersion]; + const toolCollection = new ToolCollection(...toolGroup.tools.map((Tool: typeof ComputerTool20241022 | typeof ComputerTool20250124) => new Tool(kernel, sessionId))); + + const system: BetaTextBlock = { + type: 'text', + text: `${SYSTEM_PROMPT}${systemPromptSuffix ? ' ' + systemPromptSuffix : ''}`, + }; + + while (true) { + const betas: string[] = toolGroup.beta_flag ? [toolGroup.beta_flag] : []; + + if (tokenEfficientToolsBeta) { + betas.push('token-efficient-tools-2025-02-19'); + } + + let imageTruncationThreshold = onlyNMostRecentImages || 0; + + const client = new Anthropic({ apiKey, maxRetries: 4 }); + const enablePromptCaching = true; + + if (enablePromptCaching) { + betas.push(PROMPT_CACHING_BETA_FLAG); + injectPromptCaching(messages); + onlyNMostRecentImages = 0; + (system as BetaTextBlock).cache_control = { type: 'ephemeral' }; + } + + if (onlyNMostRecentImages) { + maybeFilterToNMostRecentImages( + messages, + onlyNMostRecentImages, + imageTruncationThreshold + ); + } + + const extraBody: ExtraBodyConfig = {}; + if (thinkingBudget) { + extraBody.thinking = { type: 'enabled', budget_tokens: thinkingBudget }; + } + + const toolParams = toolCollection.toParams(); + + const response = await client.beta.messages.create({ + max_tokens: maxTokens, + messages, + model, + system: [system], + tools: toolParams as any[], + betas, + ...extraBody, + }); + + const responseParams = responseToParams(response); + + const loggableContent = responseParams.map(block => { + if (block.type === 'tool_use') { + return { + type: 'tool_use', + name: block.name, + input: block.input + }; + } + return block; + }); + console.log('=== LLM RESPONSE ==='); + console.log('Stop reason:', response.stop_reason); + console.log(loggableContent); + console.log("===") + + messages.push({ + role: 'assistant', + content: responseParams, + }); + + if (response.stop_reason === 'end_turn') { + console.log('LLM has completed its task, ending loop'); + return messages; + } + + const toolResultContent = []; + let hasToolUse = false; + + for (const contentBlock of responseParams) { + if (contentBlock.type === 'tool_use' && contentBlock.name && contentBlock.input && typeof contentBlock.input === 'object') { + const input = contentBlock.input as ToolUseInput; + if ('action' in input && typeof input.action === 'string') { + hasToolUse = true; + const toolInput: ActionParams = { + action: input.action as Action, + ...Object.fromEntries( + Object.entries(input).filter(([key]) => key !== 'action') + ) + }; + + try { + const result = await toolCollection.run( + contentBlock.name, + toolInput + ); + + const toolResult = makeApiToolResult(result, contentBlock.id!); + toolResultContent.push(toolResult); + } catch (error) { + console.error(error); + throw error; + } + } + } + } + + if (toolResultContent.length === 0 && !hasToolUse && response.stop_reason !== 'tool_use') { + console.log('No tool use or results, and not waiting for tool use, ending loop'); + return messages; + } + + if (toolResultContent.length > 0) { + messages.push({ + role: 'user', + content: toolResultContent, + }); + } + } +} diff --git a/pkg/templates/typescript/ehr-system/package-lock.json b/pkg/templates/typescript/ehr-system/package-lock.json index 358fa33..e91f864 100644 --- a/pkg/templates/typescript/ehr-system/package-lock.json +++ b/pkg/templates/typescript/ehr-system/package-lock.json @@ -6,498 +6,58 @@ "": { "name": "ehr-system", "dependencies": { - "@onkernel/sdk": "^0.23.0", - "dotenv": "^17.2.3", - "openai": "^6.13.0", - "playwright-core": "^1.57.0", - "sharp": "^0.34.5" + "@anthropic-ai/sdk": "^0.71.2", + "@onkernel/sdk": "^0.24.0", + "luxon": "^3.7.2" }, "devDependencies": { + "@types/luxon": "^3.7.1", "@types/node": "^22.15.17", "typescript": "^5.9.3" } }, - "node_modules/@emnapi/runtime": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", - "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "node_modules/@anthropic-ai/sdk": { + "version": "0.71.2", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.71.2.tgz", + "integrity": "sha512-TGNDEUuEstk/DKu0/TflXAEt+p+p/WhTlFzEnoosvbaDU2LTjm42igSdlL0VijrKpWejtOKxX0b8A7uc+XiSAQ==", "license": "MIT", - "optional": true, "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@img/colour": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", - "integrity": "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", - "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", - "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", - "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", - "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", - "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", - "cpu": [ - "arm" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", - "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", - "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", - "cpu": [ - "ppc64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", - "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", - "cpu": [ - "riscv64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", - "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", - "cpu": [ - "s390x" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", - "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", - "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", - "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", - "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", - "cpu": [ - "arm" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", - "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-ppc64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", - "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", - "cpu": [ - "ppc64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-riscv64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", - "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", - "cpu": [ - "riscv64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", - "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", - "cpu": [ - "s390x" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", - "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", - "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", - "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" + "json-schema-to-ts": "^3.1.1" }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", - "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", - "cpu": [ - "wasm32" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", - "optional": true, - "dependencies": { - "@emnapi/runtime": "^1.7.0" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", - "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "bin": { + "anthropic-ai-sdk": "bin/cli" }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", - "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", - "cpu": [ - "ia32" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" }, - "funding": { - "url": "https://opencollective.com/libvips" + "peerDependenciesMeta": { + "zod": { + "optional": true + } } }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", - "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], + "node_modules/@babel/runtime": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz", + "integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==", + "license": "MIT", "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" + "node": ">=6.9.0" } }, "node_modules/@onkernel/sdk": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/@onkernel/sdk/-/sdk-0.23.0.tgz", - "integrity": "sha512-P/ez6HU8sO2QvqWATkvC+Wdv+fgto4KfBCHLl2T6EUpoU3LhgOZ/sJP2ZRf/vh5Vh7QR2Vf05RgMaFcIGBGD9Q==", + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@onkernel/sdk/-/sdk-0.24.0.tgz", + "integrity": "sha512-f0xZGSaC9Nlg7CwLw6agyw682sc9Q8rPRG6Zyk82JmCKETFBdMqfyXuxK5uESidk0pQp/GYGG8rHy+vGa5jgCQ==", "license": "Apache-2.0" }, + "node_modules/@types/luxon": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.1.tgz", + "integrity": "sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "22.19.7", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.7.tgz", @@ -508,122 +68,33 @@ "undici-types": "~6.21.0" } }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, - "node_modules/dotenv": { - "version": "17.2.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", - "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/openai": { - "version": "6.16.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-6.16.0.tgz", - "integrity": "sha512-fZ1uBqjFUjXzbGc35fFtYKEOxd20kd9fDpFeqWtsOZWiubY8CZ1NAlXHW3iathaFvqmNtCWMIsosCuyeI7Joxg==", - "license": "Apache-2.0", - "bin": { - "openai": "bin/cli" - }, - "peerDependencies": { - "ws": "^8.18.0", - "zod": "^3.25 || ^4.0" - }, - "peerDependenciesMeta": { - "ws": { - "optional": true - }, - "zod": { - "optional": true - } - } - }, - "node_modules/playwright-core": { - "version": "1.57.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.57.0.tgz", - "integrity": "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ==", - "license": "Apache-2.0", - "bin": { - "playwright-core": "cli.js" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" }, "engines": { - "node": ">=10" + "node": ">=16" } }, - "node_modules/sharp": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", - "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@img/colour": "^1.0.0", - "detect-libc": "^2.1.2", - "semver": "^7.7.3" - }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.34.5", - "@img/sharp-darwin-x64": "0.34.5", - "@img/sharp-libvips-darwin-arm64": "1.2.4", - "@img/sharp-libvips-darwin-x64": "1.2.4", - "@img/sharp-libvips-linux-arm": "1.2.4", - "@img/sharp-libvips-linux-arm64": "1.2.4", - "@img/sharp-libvips-linux-ppc64": "1.2.4", - "@img/sharp-libvips-linux-riscv64": "1.2.4", - "@img/sharp-libvips-linux-s390x": "1.2.4", - "@img/sharp-libvips-linux-x64": "1.2.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", - "@img/sharp-libvips-linuxmusl-x64": "1.2.4", - "@img/sharp-linux-arm": "0.34.5", - "@img/sharp-linux-arm64": "0.34.5", - "@img/sharp-linux-ppc64": "0.34.5", - "@img/sharp-linux-riscv64": "0.34.5", - "@img/sharp-linux-s390x": "0.34.5", - "@img/sharp-linux-x64": "0.34.5", - "@img/sharp-linuxmusl-arm64": "0.34.5", - "@img/sharp-linuxmusl-x64": "0.34.5", - "@img/sharp-wasm32": "0.34.5", - "@img/sharp-win32-arm64": "0.34.5", - "@img/sharp-win32-ia32": "0.34.5", - "@img/sharp-win32-x64": "0.34.5" + "node": ">=12" } }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD", - "optional": true + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" }, "node_modules/typescript": { "version": "5.9.3", diff --git a/pkg/templates/typescript/ehr-system/package.json b/pkg/templates/typescript/ehr-system/package.json index 1fdc3b9..7552171 100644 --- a/pkg/templates/typescript/ehr-system/package.json +++ b/pkg/templates/typescript/ehr-system/package.json @@ -7,13 +7,12 @@ "build": "tsc" }, "dependencies": { - "@onkernel/sdk": "^0.23.0", - "dotenv": "^17.2.3", - "openai": "^6.13.0", - "playwright-core": "^1.57.0", - "sharp": "^0.34.5" + "@anthropic-ai/sdk": "^0.71.2", + "@onkernel/sdk": "^0.24.0", + "luxon": "^3.7.2" }, "devDependencies": { + "@types/luxon": "^3.7.1", "@types/node": "^22.15.17", "typescript": "^5.9.3" } diff --git a/pkg/templates/typescript/ehr-system/session.ts b/pkg/templates/typescript/ehr-system/session.ts new file mode 100644 index 0000000..3aeb77c --- /dev/null +++ b/pkg/templates/typescript/ehr-system/session.ts @@ -0,0 +1,222 @@ +/** + * Kernel Browser Session Manager. + * + * Provides a class for managing Kernel browser lifecycle + * with optional video replay recording. + */ + +import type { Kernel } from '@onkernel/sdk'; + +export interface SessionOptions { + /** Enable stealth mode to avoid bot detection */ + stealth?: boolean; + /** Browser session timeout in seconds */ + timeoutSeconds?: number; + /** Enable replay recording (requires paid plan) */ + recordReplay?: boolean; + /** Grace period in seconds before stopping replay */ + replayGracePeriod?: number; +} + +export interface SessionInfo { + sessionId: string; + liveViewUrl: string; + replayId?: string; + replayViewUrl?: string; +} + +const DEFAULT_OPTIONS: Required = { + stealth: true, + timeoutSeconds: 300, + recordReplay: false, + replayGracePeriod: 5.0, +}; + +/** + * Manages Kernel browser lifecycle with optional replay recording. + * + * Usage: + * ```typescript + * const session = new KernelBrowserSession(kernel, options); + * await session.start(); + * try { + * // Use session.sessionId for computer controls + * } finally { + * await session.stop(); + * } + * ``` + */ +export class KernelBrowserSession { + private kernel: Kernel; + private options: Required; + + // Session state + private _sessionId: string | null = null; + private _liveViewUrl: string | null = null; + private _replayId: string | null = null; + private _replayViewUrl: string | null = null; + + constructor(kernel: Kernel, options: SessionOptions = {}) { + this.kernel = kernel; + this.options = { ...DEFAULT_OPTIONS, ...options }; + } + + get sessionId(): string { + if (!this._sessionId) { + throw new Error('Session not started. Call start() first.'); + } + return this._sessionId; + } + + get liveViewUrl(): string | null { + return this._liveViewUrl; + } + + get replayViewUrl(): string | null { + return this._replayViewUrl; + } + + get info(): SessionInfo { + return { + sessionId: this.sessionId, + liveViewUrl: this._liveViewUrl || '', + replayId: this._replayId || undefined, + replayViewUrl: this._replayViewUrl || undefined, + }; + } + + /** + * Create a Kernel browser session and optionally start recording. + */ + async start(): Promise { + // Create browser with specified settings + const browser = await this.kernel.browsers.create({ + stealth: this.options.stealth, + timeout_seconds: this.options.timeoutSeconds, + viewport: { + width: 1024, + height: 768, + refresh_rate: 60, + }, + }); + + this._sessionId = browser.session_id; + this._liveViewUrl = browser.browser_live_view_url ?? null; + + console.log(`Kernel browser created: ${this._sessionId}`); + console.log(`Live view URL: ${this._liveViewUrl}`); + + // Start replay recording if enabled + if (this.options.recordReplay) { + try { + await this.startReplay(); + } catch (error) { + console.warn(`Warning: Failed to start replay recording: ${error}`); + console.warn('Continuing without replay recording.'); + } + } + + return this.info; + } + + /** + * Start recording a replay of the browser session. + */ + private async startReplay(): Promise { + if (!this._sessionId) { + return; + } + + console.log('Starting replay recording...'); + const replay = await this.kernel.browsers.replays.start(this._sessionId); + this._replayId = replay.replay_id; + console.log(`Replay recording started: ${this._replayId}`); + } + + /** + * Stop recording and get the replay URL. + */ + private async stopReplay(): Promise { + if (!this._sessionId || !this._replayId) { + return; + } + + console.log('Stopping replay recording...'); + await this.kernel.browsers.replays.stop(this._replayId, { + id: this._sessionId, + }); + console.log('Replay recording stopped. Processing video...'); + + // Wait a moment for processing + await this.sleep(2000); + + // Poll for replay to be ready (with timeout) + const maxWait = 60000; // 60 seconds + const startTime = Date.now(); + let replayReady = false; + + while (Date.now() - startTime < maxWait) { + try { + const replays = await this.kernel.browsers.replays.list(this._sessionId); + for (const replay of replays) { + if (replay.replay_id === this._replayId) { + this._replayViewUrl = replay.replay_view_url ?? null; + replayReady = true; + break; + } + } + if (replayReady) { + break; + } + } catch { + // Ignore errors while polling + } + await this.sleep(1000); + } + + if (!replayReady) { + console.log('Warning: Replay may still be processing'); + } else if (this._replayViewUrl) { + console.log(`Replay view URL: ${this._replayViewUrl}`); + } + } + + /** + * Stop recording, and delete the browser session. + */ + async stop(): Promise { + const info = this.info; + + if (this._sessionId) { + try { + // Stop replay if recording was enabled + if (this.options.recordReplay && this._replayId) { + // Wait grace period before stopping to capture final state + if (this.options.replayGracePeriod > 0) { + console.log(`Waiting ${this.options.replayGracePeriod}s grace period...`); + await this.sleep(this.options.replayGracePeriod * 1000); + } + await this.stopReplay(); + info.replayViewUrl = this._replayViewUrl || undefined; + } + } finally { + // Always clean up the browser session, even if replay stopping fails + console.log(`Destroying browser session: ${this._sessionId}`); + await this.kernel.browsers.deleteByID(this._sessionId); + console.log('Browser session destroyed.'); + } + } + + // Reset state + this._sessionId = null; + this._liveViewUrl = null; + this._replayId = null; + this._replayViewUrl = null; + + return info; + } + + private sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } +} diff --git a/pkg/templates/typescript/ehr-system/tools/collection.ts b/pkg/templates/typescript/ehr-system/tools/collection.ts new file mode 100644 index 0000000..155352d --- /dev/null +++ b/pkg/templates/typescript/ehr-system/tools/collection.ts @@ -0,0 +1,61 @@ +import { ComputerTool20241022, ComputerTool20250124 } from './computer'; +import { Action } from './types/computer'; +import type { ActionParams, ToolResult } from './types/computer'; + +export type ToolVersion = 'computer_use_20250124' | 'computer_use_20241022' | 'computer_use_20250429'; + +export const DEFAULT_TOOL_VERSION: ToolVersion = 'computer_use_20250429'; + +interface ToolGroup { + readonly version: ToolVersion; + readonly tools: (typeof ComputerTool20241022 | typeof ComputerTool20250124)[]; + readonly beta_flag: string; +} + +export const TOOL_GROUPS: ToolGroup[] = [ + { + version: 'computer_use_20241022', + tools: [ComputerTool20241022], + beta_flag: 'computer-use-2024-10-22', + }, + { + version: 'computer_use_20250124', + tools: [ComputerTool20250124], + beta_flag: 'computer-use-2025-01-24', + }, + // 20250429 version inherits from 20250124 + { + version: 'computer_use_20250429', + tools: [ComputerTool20250124], + beta_flag: 'computer-use-2025-01-24', + }, +]; + +export const TOOL_GROUPS_BY_VERSION: Record = Object.fromEntries( + TOOL_GROUPS.map(group => [group.version, group]) +) as Record; + +export class ToolCollection { + private tools: Map; + + constructor(...tools: (ComputerTool20241022 | ComputerTool20250124)[]) { + this.tools = new Map(tools.map(tool => [tool.name, tool])); + } + + toParams(): unknown[] { + return Array.from(this.tools.values()).map(tool => tool.toParams()); + } + + async run(name: string, toolInput: ActionParams): Promise { + const tool = this.tools.get(name); + if (!tool) { + throw new Error(`Tool ${name} not found`); + } + + if (!Object.values(Action).includes(toolInput.action)) { + throw new Error(`Invalid action ${toolInput.action} for tool ${name}`); + } + + return await tool.call(toolInput); + } +} \ No newline at end of file diff --git a/pkg/templates/typescript/ehr-system/tools/computer.ts b/pkg/templates/typescript/ehr-system/tools/computer.ts new file mode 100644 index 0000000..dc0eb41 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/tools/computer.ts @@ -0,0 +1,401 @@ +import { Buffer } from 'buffer'; +import type { Kernel } from '@onkernel/sdk'; +import type { BaseAnthropicTool, ToolResult, ActionParams } from './types/computer'; +import { Action, ToolError } from './types/computer'; +import { ActionValidator } from './utils/validator'; + +const TYPING_DELAY_MS = 12; + +// Type for the tool parameters sent to Anthropic API +export interface ComputerToolParams { + name: 'computer'; + type: 'computer_20241022' | 'computer_20250124'; + display_width_px: number; + display_height_px: number; + display_number: null; +} + +export class ComputerTool implements BaseAnthropicTool { + name: 'computer' = 'computer'; + protected kernel: Kernel; + protected sessionId: string; + protected _screenshotDelay = 2.0; + protected version: '20241022' | '20250124'; + + private lastMousePosition: [number, number] = [0, 0]; + + private readonly mouseActions = new Set([ + Action.LEFT_CLICK, + Action.RIGHT_CLICK, + Action.MIDDLE_CLICK, + Action.DOUBLE_CLICK, + Action.TRIPLE_CLICK, + Action.MOUSE_MOVE, + Action.LEFT_MOUSE_DOWN, + Action.LEFT_MOUSE_UP, + ]); + + private readonly keyboardActions = new Set([ + Action.KEY, + Action.TYPE, + Action.HOLD_KEY, + ]); + + private readonly systemActions = new Set([ + Action.SCREENSHOT, + Action.CURSOR_POSITION, + Action.SCROLL, + Action.WAIT, + ]); + + constructor(kernel: Kernel, sessionId: string, version: '20241022' | '20250124' = '20250124') { + this.kernel = kernel; + this.sessionId = sessionId; + this.version = version; + } + + get apiType(): 'computer_20241022' | 'computer_20250124' { + return this.version === '20241022' ? 'computer_20241022' : 'computer_20250124'; + } + + toParams(): ComputerToolParams { + const params: ComputerToolParams = { + name: this.name, + type: this.apiType, + display_width_px: 1024, + display_height_px: 768, + display_number: null, + }; + return params; + } + + private getMouseButton(action: Action): 'left' | 'right' | 'middle' { + switch (action) { + case Action.LEFT_CLICK: + case Action.DOUBLE_CLICK: + case Action.TRIPLE_CLICK: + case Action.LEFT_CLICK_DRAG: + case Action.LEFT_MOUSE_DOWN: + case Action.LEFT_MOUSE_UP: + return 'left'; + case Action.RIGHT_CLICK: + return 'right'; + case Action.MIDDLE_CLICK: + return 'middle'; + default: + throw new ToolError(`Invalid mouse action: ${action}`); + } + } + + private async handleMouseAction(action: Action, coordinate: [number, number]): Promise { + const [x, y] = ActionValidator.validateAndGetCoordinates(coordinate); + + if (action === Action.MOUSE_MOVE) { + await this.kernel.browsers.computer.moveMouse(this.sessionId, { + x, + y, + }); + this.lastMousePosition = [x, y]; + } else if (action === Action.LEFT_MOUSE_DOWN) { + await this.kernel.browsers.computer.clickMouse(this.sessionId, { + x, + y, + button: 'left', + click_type: 'down', + }); + this.lastMousePosition = [x, y]; + } else if (action === Action.LEFT_MOUSE_UP) { + await this.kernel.browsers.computer.clickMouse(this.sessionId, { + x, + y, + button: 'left', + click_type: 'up', + }); + this.lastMousePosition = [x, y]; + } else { + const button = this.getMouseButton(action); + let numClicks = 1; + if (action === Action.DOUBLE_CLICK) { + numClicks = 2; + } else if (action === Action.TRIPLE_CLICK) { + numClicks = 3; + } + + await this.kernel.browsers.computer.clickMouse(this.sessionId, { + x, + y, + button, + click_type: 'click', + num_clicks: numClicks, + }); + this.lastMousePosition = [x, y]; + } + + await new Promise(resolve => setTimeout(resolve, 500)); + return await this.screenshot(); + } + + private async handleKeyboardAction(action: Action, text: string, duration?: number): Promise { + if (action === Action.HOLD_KEY) { + const key = this.convertToKernelKey(text); + await this.kernel.browsers.computer.pressKey(this.sessionId, { + keys: [key], + duration: duration ? duration * 1000 : undefined, + }); + } else if (action === Action.KEY) { + const key = this.convertKeyCombinationToKernel(text); + await this.kernel.browsers.computer.pressKey(this.sessionId, { + keys: [key], + }); + } else { + await this.kernel.browsers.computer.typeText(this.sessionId, { + text, + delay: TYPING_DELAY_MS, + }); + } + + await new Promise(resolve => setTimeout(resolve, 500)); + return await this.screenshot(); + } + + // Key mappings for Kernel Computer Controls API (xdotool format) + private static readonly KEY_MAP: Record = { + // Enter/Return + 'return': 'Return', + 'enter': 'Return', + 'Enter': 'Return', + // Arrow keys + 'left': 'Left', + 'right': 'Right', + 'up': 'Up', + 'down': 'Down', + 'ArrowLeft': 'Left', + 'ArrowRight': 'Right', + 'ArrowUp': 'Up', + 'ArrowDown': 'Down', + // Navigation + 'home': 'Home', + 'end': 'End', + 'pageup': 'Page_Up', + 'page_up': 'Page_Up', + 'PageUp': 'Page_Up', + 'pagedown': 'Page_Down', + 'page_down': 'Page_Down', + 'PageDown': 'Page_Down', + // Editing + 'delete': 'Delete', + 'backspace': 'BackSpace', + 'Backspace': 'BackSpace', + 'tab': 'Tab', + 'insert': 'Insert', + // Escape + 'esc': 'Escape', + 'escape': 'Escape', + // Function keys + 'f1': 'F1', + 'f2': 'F2', + 'f3': 'F3', + 'f4': 'F4', + 'f5': 'F5', + 'f6': 'F6', + 'f7': 'F7', + 'f8': 'F8', + 'f9': 'F9', + 'f10': 'F10', + 'f11': 'F11', + 'f12': 'F12', + // Misc + 'space': 'space', + 'minus': 'minus', + 'equal': 'equal', + 'plus': 'plus', + }; + + // Modifier key mappings (xdotool format) + private static readonly MODIFIER_MAP: Record = { + 'ctrl': 'ctrl', + 'control': 'ctrl', + 'Control': 'ctrl', + 'alt': 'alt', + 'Alt': 'alt', + 'shift': 'shift', + 'Shift': 'shift', + 'meta': 'super', + 'Meta': 'super', + 'cmd': 'super', + 'command': 'super', + 'win': 'super', + 'super': 'super', + }; + + private convertToKernelKey(key: string): string { + // Check modifier keys first + if (ComputerTool.MODIFIER_MAP[key]) { + return ComputerTool.MODIFIER_MAP[key]; + } + // Check special keys + if (ComputerTool.KEY_MAP[key]) { + return ComputerTool.KEY_MAP[key]; + } + // Return as-is if no mapping exists + return key; + } + + private convertKeyCombinationToKernel(combo: string): string { + // Handle key combinations (e.g., "ctrl+a", "Control+t") + if (combo.includes('+')) { + const parts = combo.split('+'); + const mappedParts = parts.map(part => this.convertToKernelKey(part.trim())); + return mappedParts.join('+'); + } + // Single key - just convert it + return this.convertToKernelKey(combo); + } + + async screenshot(): Promise { + try { + console.log('Starting screenshot...'); + await new Promise(resolve => setTimeout(resolve, this._screenshotDelay * 1000)); + const response = await this.kernel.browsers.computer.captureScreenshot(this.sessionId); + const blob = await response.blob(); + const arrayBuffer = await blob.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + console.log('Screenshot taken, size:', buffer.length, 'bytes'); + + return { + base64Image: buffer.toString('base64'), + }; + } catch (error) { + throw new ToolError(`Failed to take screenshot: ${error}`); + } + } + + async call(params: ActionParams): Promise { + const { + action, + text, + coordinate, + scrollDirection: scrollDirectionParam, + scroll_amount, + scrollAmount, + duration, + ...kwargs + } = params; + + ActionValidator.validateActionParams(params, this.mouseActions, this.keyboardActions); + + if (action === Action.SCREENSHOT) { + return await this.screenshot(); + } + + if (action === Action.CURSOR_POSITION) { + throw new ToolError('Cursor position is not available with Kernel Computer Controls API'); + } + + if (action === Action.SCROLL) { + if (this.version !== '20250124') { + throw new ToolError(`${action} is only available in version 20250124`); + } + + const scrollDirection = (scrollDirectionParam || kwargs.scroll_direction) as string | undefined; + const scrollAmountValue = scrollAmount || scroll_amount; + + if (!scrollDirection || !['up', 'down', 'left', 'right'].includes(String(scrollDirection))) { + throw new ToolError(`Scroll direction "${scrollDirection}" must be 'up', 'down', 'left', or 'right'`); + } + if (typeof scrollAmountValue !== 'number' || scrollAmountValue < 0) { + throw new ToolError(`Scroll amount "${scrollAmountValue}" must be a non-negative number`); + } + + const [x, y] = coordinate + ? ActionValidator.validateAndGetCoordinates(coordinate) + : this.lastMousePosition; + + let delta_x = 0; + let delta_y = 0; + // Each scroll_amount unit = 1 scroll wheel click ≈ 120 pixels (matches Anthropic's xdotool behavior) + const scrollDelta = (scrollAmountValue ?? 1) * 120; + + if (scrollDirection === 'down') { + delta_y = scrollDelta; + } else if (scrollDirection === 'up') { + delta_y = -scrollDelta; + } else if (scrollDirection === 'right') { + delta_x = scrollDelta; + } else if (scrollDirection === 'left') { + delta_x = -scrollDelta; + } + + await this.kernel.browsers.computer.scroll(this.sessionId, { + x, + y, + delta_x, + delta_y, + }); + + await new Promise(resolve => setTimeout(resolve, 500)); + return await this.screenshot(); + } + + if (action === Action.WAIT) { + if (this.version !== '20250124') { + throw new ToolError(`${action} is only available in version 20250124`); + } + await new Promise(resolve => setTimeout(resolve, duration! * 1000)); + return await this.screenshot(); + } + + if (action === Action.LEFT_CLICK_DRAG) { + if (!coordinate) { + throw new ToolError(`coordinate is required for ${action}`); + } + + const [endX, endY] = ActionValidator.validateAndGetCoordinates(coordinate); + const startCoordinate = kwargs.start_coordinate as [number, number] | undefined; + const [startX, startY] = startCoordinate + ? ActionValidator.validateAndGetCoordinates(startCoordinate) + : this.lastMousePosition; + + console.log(`Dragging from (${startX}, ${startY}) to (${endX}, ${endY})`); + + await this.kernel.browsers.computer.dragMouse(this.sessionId, { + path: [[startX, startY], [endX, endY]], + button: 'left', + }); + + this.lastMousePosition = [endX, endY]; + + await new Promise(resolve => setTimeout(resolve, 500)); + return await this.screenshot(); + } + + if (this.mouseActions.has(action)) { + if (!coordinate) { + throw new ToolError(`coordinate is required for ${action}`); + } + return await this.handleMouseAction(action, coordinate); + } + + if (this.keyboardActions.has(action)) { + if (!text) { + throw new ToolError(`text is required for ${action}`); + } + return await this.handleKeyboardAction(action, text, duration); + } + + throw new ToolError(`Invalid action: ${action}`); + } +} + +// For backward compatibility +export class ComputerTool20241022 extends ComputerTool { + constructor(kernel: Kernel, sessionId: string) { + super(kernel, sessionId, '20241022'); + } +} + +export class ComputerTool20250124 extends ComputerTool { + constructor(kernel: Kernel, sessionId: string) { + super(kernel, sessionId, '20250124'); + } +} diff --git a/pkg/templates/typescript/ehr-system/tools/types/computer.ts b/pkg/templates/typescript/ehr-system/tools/types/computer.ts new file mode 100644 index 0000000..d7ac72e --- /dev/null +++ b/pkg/templates/typescript/ehr-system/tools/types/computer.ts @@ -0,0 +1,64 @@ +export enum Action { + // Mouse actions + MOUSE_MOVE = 'mouse_move', + LEFT_CLICK = 'left_click', + RIGHT_CLICK = 'right_click', + MIDDLE_CLICK = 'middle_click', + DOUBLE_CLICK = 'double_click', + TRIPLE_CLICK = 'triple_click', + LEFT_CLICK_DRAG = 'left_click_drag', + LEFT_MOUSE_DOWN = 'left_mouse_down', + LEFT_MOUSE_UP = 'left_mouse_up', + + // Keyboard actions + KEY = 'key', + TYPE = 'type', + HOLD_KEY = 'hold_key', + + // System actions + SCREENSHOT = 'screenshot', + CURSOR_POSITION = 'cursor_position', + SCROLL = 'scroll', + WAIT = 'wait', +} + +// For backward compatibility +export type Action_20241022 = Action; +export type Action_20250124 = Action; + +export type MouseButton = 'left' | 'right' | 'middle'; +export type ScrollDirection = 'up' | 'down' | 'left' | 'right'; +export type Coordinate = [number, number]; +export type Duration = number; + +export interface ActionParams { + action: Action; + text?: string; + coordinate?: Coordinate; + scrollDirection?: ScrollDirection; + scroll_amount?: number; + scrollAmount?: number; + duration?: Duration; + key?: string; + [key: string]: Action | string | Coordinate | ScrollDirection | number | Duration | undefined; +} + +export interface ToolResult { + output?: string; + error?: string; + base64Image?: string; + system?: string; +} + +export interface BaseAnthropicTool { + name: string; + apiType: string; + toParams(): unknown; +} + +export class ToolError extends Error { + constructor(message: string) { + super(message); + this.name = 'ToolError'; + } +} \ No newline at end of file diff --git a/pkg/templates/typescript/ehr-system/tools/utils/keyboard.ts b/pkg/templates/typescript/ehr-system/tools/utils/keyboard.ts new file mode 100644 index 0000000..244cddf --- /dev/null +++ b/pkg/templates/typescript/ehr-system/tools/utils/keyboard.ts @@ -0,0 +1,88 @@ +export class KeyboardUtils { + // Only map alternative names to standard Playwright modifier keys + private static readonly modifierKeyMap: Record = { + 'ctrl': 'Control', + 'alt': 'Alt', + 'cmd': 'Meta', + 'command': 'Meta', + 'win': 'Meta', + }; + + // Essential key mappings for Playwright compatibility + private static readonly keyMap: Record = { + 'return': 'Enter', + 'space': ' ', + 'left': 'ArrowLeft', + 'right': 'ArrowRight', + 'up': 'ArrowUp', + 'down': 'ArrowDown', + 'home': 'Home', + 'end': 'End', + 'pageup': 'PageUp', + 'page_up': 'PageUp', + 'pagedown': 'PageDown', + 'page_down': 'PageDown', + 'delete': 'Delete', + 'backspace': 'Backspace', + 'tab': 'Tab', + 'esc': 'Escape', + 'escape': 'Escape', + 'insert': 'Insert', + 'super_l': 'Meta', + 'f1': 'F1', + 'f2': 'F2', + 'f3': 'F3', + 'f4': 'F4', + 'f5': 'F5', + 'f6': 'F6', + 'f7': 'F7', + 'f8': 'F8', + 'f9': 'F9', + 'f10': 'F10', + 'f11': 'F11', + 'f12': 'F12', + 'minus': '-', + 'equal': '=', + 'plus': '+', + }; + + static isModifierKey(key: string | undefined): boolean { + if (!key) return false; + const normalizedKey = this.modifierKeyMap[key.toLowerCase()] || key; + return ['Control', 'Alt', 'Shift', 'Meta'].includes(normalizedKey); + } + + static getPlaywrightKey(key: string | undefined): string { + if (!key) { + throw new Error('Key cannot be undefined'); + } + + const normalizedKey = key.toLowerCase(); + + // Handle special cases + if (normalizedKey in this.keyMap) { + return this.keyMap[normalizedKey] as string; + } + + // Normalize modifier keys + if (normalizedKey in this.modifierKeyMap) { + return this.modifierKeyMap[normalizedKey] as string; + } + + // Return the key as is - Playwright handles standard key names + return key; + } + + static parseKeyCombination(combo: string): string[] { + if (!combo) { + throw new Error('Key combination cannot be empty'); + } + return combo.toLowerCase().split('+').map(key => { + const trimmedKey = key.trim(); + if (!trimmedKey) { + throw new Error('Invalid key combination: empty key'); + } + return this.getPlaywrightKey(trimmedKey); + }); + } +} \ No newline at end of file diff --git a/pkg/templates/typescript/ehr-system/tools/utils/validator.ts b/pkg/templates/typescript/ehr-system/tools/utils/validator.ts new file mode 100644 index 0000000..b8522c8 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/tools/utils/validator.ts @@ -0,0 +1,67 @@ +import { Action, ToolError } from '../types/computer'; +import type { ActionParams, Coordinate, Duration } from '../types/computer'; + +export class ActionValidator { + static validateText(text: string | undefined, required: boolean, action: string): void { + if (required && text === undefined) { + throw new ToolError(`text is required for ${action}`); + } + if (text !== undefined && typeof text !== 'string') { + throw new ToolError(`${text} must be a string`); + } + } + + static validateCoordinate(coordinate: Coordinate | undefined, required: boolean, action: string): void { + if (required && !coordinate) { + throw new ToolError(`coordinate is required for ${action}`); + } + if (coordinate) { + this.validateAndGetCoordinates(coordinate); + } + } + + static validateDuration(duration: Duration | undefined): void { + if (duration === undefined || typeof duration !== 'number') { + throw new ToolError(`${duration} must be a number`); + } + if (duration < 0) { + throw new ToolError(`${duration} must be non-negative`); + } + if (duration > 100) { + throw new ToolError(`${duration} is too long`); + } + } + + static validateAndGetCoordinates(coordinate: Coordinate): Coordinate { + if (!Array.isArray(coordinate) || coordinate.length !== 2) { + throw new ToolError(`${coordinate} must be a tuple of length 2`); + } + if (!coordinate.every(i => typeof i === 'number' && i >= 0)) { + throw new ToolError(`${coordinate} must be a tuple of non-negative numbers`); + } + return coordinate; + } + + static validateActionParams(params: ActionParams, mouseActions: Set, keyboardActions: Set): void { + const { action, text, coordinate, duration } = params; + + // Validate text parameter + if (keyboardActions.has(action)) { + this.validateText(text, true, action); + } else { + this.validateText(text, false, action); + } + + // Validate coordinate parameter + if (mouseActions.has(action)) { + this.validateCoordinate(coordinate, true, action); + } else { + this.validateCoordinate(coordinate, false, action); + } + + // Validate duration parameter + if (action === Action.HOLD_KEY || action === Action.WAIT) { + this.validateDuration(duration); + } + } +} \ No newline at end of file diff --git a/pkg/templates/typescript/ehr-system/types/beta.ts b/pkg/templates/typescript/ehr-system/types/beta.ts new file mode 100644 index 0000000..35328d7 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/types/beta.ts @@ -0,0 +1,58 @@ +import type { BetaMessageParam as AnthropicMessageParam, BetaMessage as AnthropicMessage, BetaContentBlock as AnthropicContentBlock } from '@anthropic-ai/sdk/resources/beta/messages/messages'; +import type { ActionParams } from '../tools/types/computer'; + +// Re-export the SDK types +export type BetaMessageParam = AnthropicMessageParam; +export type BetaMessage = AnthropicMessage; +export type BetaContentBlock = AnthropicContentBlock; + +// Keep our local types for internal use +export interface BetaTextBlock { + type: 'text'; + text: string; + id?: string; + cache_control?: { type: 'ephemeral' }; +} + +export interface BetaImageBlock { + type: 'image'; + source: { + type: 'base64'; + media_type: 'image/png'; + data: string; + }; + id?: string; + cache_control?: { type: 'ephemeral' }; +} + +export interface BetaToolUseBlock { + type: 'tool_use'; + name: string; + input: ActionParams; + id?: string; + cache_control?: { type: 'ephemeral' }; +} + +export interface BetaThinkingBlock { + type: 'thinking'; + thinking: { + type: 'enabled'; + budget_tokens: number; + } | { + type: 'disabled'; + }; + signature?: string; + id?: string; + cache_control?: { type: 'ephemeral' }; +} + +export interface BetaToolResultBlock { + type: 'tool_result'; + content: (BetaTextBlock | BetaImageBlock)[] | string; + tool_use_id: string; + is_error: boolean; + id?: string; + cache_control?: { type: 'ephemeral' }; +} + +export type BetaLocalContentBlock = BetaTextBlock | BetaImageBlock | BetaToolUseBlock | BetaThinkingBlock | BetaToolResultBlock; \ No newline at end of file diff --git a/pkg/templates/typescript/ehr-system/utils/message-processing.ts b/pkg/templates/typescript/ehr-system/utils/message-processing.ts new file mode 100644 index 0000000..2595ec4 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/utils/message-processing.ts @@ -0,0 +1,79 @@ +import type { BetaMessage, BetaMessageParam, BetaToolResultBlock, BetaContentBlock, BetaLocalContentBlock } from '../types/beta'; + +export function responseToParams(response: BetaMessage): BetaContentBlock[] { + return response.content.map(block => { + if (block.type === 'text' && block.text) { + return { type: 'text', text: block.text } as BetaContentBlock; + } + if (block.type === 'thinking') { + const { thinking, signature, ...rest } = block as any; + return { ...rest, thinking, ...(signature && { signature }) } as BetaContentBlock; + } + return block as BetaContentBlock; + }); +} + +export function maybeFilterToNMostRecentImages( + messages: BetaMessageParam[], + imagesToKeep: number, + minRemovalThreshold: number +): void { + if (!imagesToKeep) return; + + const toolResultBlocks = messages + .flatMap(message => Array.isArray(message?.content) ? message.content : []) + .filter((item): item is BetaToolResultBlock => + typeof item === 'object' && item.type === 'tool_result' + ); + + const totalImages = toolResultBlocks.reduce((count, toolResult) => { + if (!Array.isArray(toolResult.content)) return count; + return count + toolResult.content.filter( + content => typeof content === 'object' && content.type === 'image' + ).length; + }, 0); + + let imagesToRemove = Math.floor((totalImages - imagesToKeep) / minRemovalThreshold) * minRemovalThreshold; + + for (const toolResult of toolResultBlocks) { + if (Array.isArray(toolResult.content)) { + toolResult.content = toolResult.content.filter(content => { + if (typeof content === 'object' && content.type === 'image') { + if (imagesToRemove > 0) { + imagesToRemove--; + return false; + } + } + return true; + }); + } + } +} + +const PROMPT_CACHING_BETA_FLAG = 'prompt-caching-2024-07-31'; + +export function injectPromptCaching(messages: BetaMessageParam[]): void { + let breakpointsRemaining = 3; + + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i]; + if (!message) continue; + if (message.role === 'user' && Array.isArray(message.content)) { + if (breakpointsRemaining > 0) { + breakpointsRemaining--; + const lastContent = message.content[message.content.length - 1]; + if (lastContent) { + (lastContent as BetaLocalContentBlock).cache_control = { type: 'ephemeral' }; + } + } else { + const lastContent = message.content[message.content.length - 1]; + if (lastContent) { + delete (lastContent as BetaLocalContentBlock).cache_control; + } + break; + } + } + } +} + +export { PROMPT_CACHING_BETA_FLAG }; \ No newline at end of file diff --git a/pkg/templates/typescript/ehr-system/utils/tool-results.ts b/pkg/templates/typescript/ehr-system/utils/tool-results.ts new file mode 100644 index 0000000..c18eab2 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/utils/tool-results.ts @@ -0,0 +1,49 @@ +import type { ToolResult } from '../tools/types/computer'; +import type { BetaToolResultBlock, BetaTextBlock, BetaImageBlock } from '../types/beta'; + +export function makeApiToolResult( + result: ToolResult, + toolUseId: string +): BetaToolResultBlock { + const toolResultContent: (BetaTextBlock | BetaImageBlock)[] = []; + let isError = false; + + if (result.error) { + isError = true; + toolResultContent.push({ + type: 'text', + text: maybePrependSystemToolResult(result, result.error), + }); + } else { + if (result.output) { + toolResultContent.push({ + type: 'text', + text: maybePrependSystemToolResult(result, result.output), + }); + } + if (result.base64Image) { + toolResultContent.push({ + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: result.base64Image, + }, + }); + } + } + + return { + type: 'tool_result', + content: toolResultContent, + tool_use_id: toolUseId, + is_error: isError, + }; +} + +export function maybePrependSystemToolResult(result: ToolResult, resultText: string): string { + if (result.system) { + return `${result.system}\n${resultText}`; + } + return resultText; +} \ No newline at end of file From ff2ce428d496cf66cdb86692eefc2356bbf2f4fd Mon Sep 17 00:00:00 2001 From: arthursita-plank Date: Fri, 30 Jan 2026 14:27:34 -0300 Subject: [PATCH 08/11] feat: create ehr system from scratch --- .../ehr-system/openEMR/dashboard.html | 72 +++++ .../typescript/ehr-system/openEMR/index.html | 40 +++ .../typescript/ehr-system/openEMR/report.csv | 4 + .../ehr-system/openEMR/reports.html | 66 +++++ .../typescript/ehr-system/openEMR/style.css | 257 ++++++++++++++++++ 5 files changed, 439 insertions(+) create mode 100644 pkg/templates/typescript/ehr-system/openEMR/dashboard.html create mode 100644 pkg/templates/typescript/ehr-system/openEMR/index.html create mode 100644 pkg/templates/typescript/ehr-system/openEMR/report.csv create mode 100644 pkg/templates/typescript/ehr-system/openEMR/reports.html create mode 100644 pkg/templates/typescript/ehr-system/openEMR/style.css diff --git a/pkg/templates/typescript/ehr-system/openEMR/dashboard.html b/pkg/templates/typescript/ehr-system/openEMR/dashboard.html new file mode 100644 index 0000000..af44135 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/openEMR/dashboard.html @@ -0,0 +1,72 @@ + + + + + + Dashboard - OpenEMR + + + + +
+ + + + +
+
+
+

Dashboard

+
+ +
+ +
+
+
+ 📄 + Clinical Documents +
+
+ 📅 + Appointments +
+
+ ✉️ + Secure Messaging +
+
+ 📋 + Health Snapshot +
+ + + + 📊 + Medical Reports + + +
+ 👤 + Profile +
+
+
+
+
+ + diff --git a/pkg/templates/typescript/ehr-system/openEMR/index.html b/pkg/templates/typescript/ehr-system/openEMR/index.html new file mode 100644 index 0000000..c0760e9 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/openEMR/index.html @@ -0,0 +1,40 @@ + + + + + + OpenEMR Portal Login + + + + + + + diff --git a/pkg/templates/typescript/ehr-system/openEMR/report.csv b/pkg/templates/typescript/ehr-system/openEMR/report.csv new file mode 100644 index 0000000..6dcf833 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/openEMR/report.csv @@ -0,0 +1,4 @@ +Date,Patient,Description,Code +2025-01-01,John Doe,Checkup,V70.0 +2025-01-15,Jane Smith,Flu Sypmtoms,487.1 +2025-02-10,John Doe,Follow-up,V67.0 diff --git a/pkg/templates/typescript/ehr-system/openEMR/reports.html b/pkg/templates/typescript/ehr-system/openEMR/reports.html new file mode 100644 index 0000000..37cf5ec --- /dev/null +++ b/pkg/templates/typescript/ehr-system/openEMR/reports.html @@ -0,0 +1,66 @@ + + + + + + Medical Reports - OpenEMR + + + + +
+ + + + +
+
+
+

Medical Reports

+
+ +
+ +
+ +
+

Patient Records

+ +
+ + + + ⬇️ + Download Summary of Care + + +
+ 📋 + Customized History +
+ +
+ 🤐 + Record Documents +
+
+
+
+
+
+ + diff --git a/pkg/templates/typescript/ehr-system/openEMR/style.css b/pkg/templates/typescript/ehr-system/openEMR/style.css new file mode 100644 index 0000000..93bca26 --- /dev/null +++ b/pkg/templates/typescript/ehr-system/openEMR/style.css @@ -0,0 +1,257 @@ +:root { + /* Dark Mode Palette - Inspired by reference */ + --bg-dark-main: #0B1015; /* Very dark background */ + --bg-dark-panel: #151A21; /* Slightly lighter for cards/sidebar */ + --bg-dark-element: #1E252E; /* Input/Button background */ + + --primary-accent: #3A86FF; /* Bright blue for primary actions */ + --secondary-accent: #00D2BA; /* Teal/Green for success/status */ + --highlight-pink: #FF006E; /* Pink for highlights (like teeth in ref) */ + + --text-main: #FFFFFF; + --text-muted: #8B949E; + + --border-subtle: #2D333B; + + --font-family: 'Inter', -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + --radius-lg: 16px; + --radius-md: 12px; + --radius-sm: 8px; +} + +body { + font-family: var(--font-family); + background-color: var(--bg-dark-main); + color: var(--text-main); + margin: 0; + height: 100vh; + display: flex; /* Default to flex for sidebar layout */ + overflow: hidden; /* App-like feel */ +} + +/* Reset & Utilities */ +* { box-sizing: border-box; } +a { text-decoration: none; color: inherit; } +ul { list-style: none; padding: 0; margin: 0; } + +.app-wrapper { + display: flex; + width: 100%; + height: 100%; +} + +/* Sidebar */ +.sidebar { + width: 80px; /* Collapsed state style similar to Ref */ + background-color: var(--bg-dark-main); + border-right: 1px solid var(--border-subtle); + display: flex; + flex-direction: column; + align-items: center; + padding: 20px 0; + justify-content: space-between; +} + +.sidebar-nav { + display: flex; + flex-direction: column; + gap: 20px; + width: 100%; + align-items: center; +} + +.nav-item { + width: 40px; + height: 40px; + border-radius: var(--radius-sm); + display: flex; + justify-content: center; + align-items: center; + color: var(--text-muted); + cursor: pointer; + transition: all 0.2s; +} + +.nav-item:hover, .nav-item.active { + background-color: var(--bg-dark-element); + color: var(--text-main); +} + +.sidebar-logo { + color: var(--secondary-accent); + font-size: 1.5rem; + margin-bottom: 30px; +} + +/* Main Content */ +.main-content { + flex: 1; + display: flex; + flex-direction: column; + background-color: var(--bg-dark-main); + overflow-y: auto; +} + +.top-bar { + padding: 20px 30px; + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid var(--border-subtle); +} + +.page-title h1 { + font-size: 1.25rem; + margin: 0; + font-weight: 600; +} + +.user-profile { + display: flex; + align-items: center; + gap: 10px; +} + +.user-avatar { + width: 36px; + height: 36px; + background: var(--bg-dark-element); + border-radius: 50%; +} + +/* Login Page specific overrides */ +body.login-body { + justify-content: center; + align-items: center; +} + +.login-card { + background: var(--bg-dark-panel); + padding: 50px; + border-radius: var(--radius-lg); + box-shadow: 0 10px 40px rgba(0,0,0,0.5); + max-width: 400px; + width: 90%; + text-align: center; + border: 1px solid var(--border-subtle); +} + +.login-logo { + font-size: 2rem; + margin-bottom: 40px; + display: flex; + justify-content: center; + align-items: center; + gap: 10px; +} +.login-logo span { color: var(--text-main); font-weight: 700; } +.login-logo .emr { color: var(--secondary-accent); } + +.form-group { + margin-bottom: 20px; + text-align: left; +} +.form-group label { + color: var(--text-muted); + font-size: 0.85rem; + margin-bottom: 8px; + display: block; +} + +.form-control { + background: var(--bg-dark-main); + border: 1px solid var(--border-subtle); + color: var(--text-main); + padding: 12px; + border-radius: var(--radius-sm); + width: 100%; +} +.form-control:focus { + outline: none; + border-color: var(--secondary-accent); +} + +.btn-primary { + background: var(--primary-accent); + color: white; + padding: 14px; + border-radius: var(--radius-sm); + width: 100%; + border: none; + font-weight: 600; + cursor: pointer; + margin-top: 10px; +} +.btn-primary:hover { + filter: brightness(1.1); +} + +/* Dashboard Grid */ +.dashboard-container { + padding: 30px; +} + +.grid-cards { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + gap: 20px; +} + +.card { + background: var(--bg-dark-panel); + border-radius: var(--radius-md); + padding: 25px; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + text-align: center; + aspect-ratio: 1/1; + transition: transform 0.2s, background 0.2s; + border: 1px solid transparent; +} +.card:hover { + transform: translateY(-5px); + background: #1A2028; + border-color: var(--border-subtle); +} + +.card-icon-lg { + font-size: 2.5rem; + margin-bottom: 15px; + display: inline-block; + padding: 15px; + background: var(--bg-dark-element); + border-radius: 50%; + color: var(--secondary-accent); +} + +.card-title-text { + font-weight: 500; + color: var(--text-main); + font-size: 0.95rem; +} + +/* Reports Specific */ +.report-action-bar { + display: flex; + justify-content: flex-end; + margin-bottom: 30px; +} + +.btn-download { + background: var(--bg-dark-element); + border: 1px solid var(--secondary-accent); + color: var(--secondary-accent); + padding: 10px 20px; + border-radius: var(--radius-sm); + display: inline-flex; + align-items: center; + gap: 8px; + cursor: pointer; + font-weight: 500; +} +.btn-download:hover { + background: var(--secondary-accent); + color: var(--bg-dark-main); +} From 0a5b9a3c7e57d701d1a7e95dd687ef257830795f Mon Sep 17 00:00:00 2001 From: arthursita-plank Date: Mon, 2 Feb 2026 15:35:56 -0300 Subject: [PATCH 09/11] feat (template): change default color schema for ehr demo app --- .../typescript/ehr-system/openEMR/style.css | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/templates/typescript/ehr-system/openEMR/style.css b/pkg/templates/typescript/ehr-system/openEMR/style.css index 93bca26..badfe39 100644 --- a/pkg/templates/typescript/ehr-system/openEMR/style.css +++ b/pkg/templates/typescript/ehr-system/openEMR/style.css @@ -1,17 +1,17 @@ :root { - /* Dark Mode Palette - Inspired by reference */ - --bg-dark-main: #0B1015; /* Very dark background */ - --bg-dark-panel: #151A21; /* Slightly lighter for cards/sidebar */ - --bg-dark-element: #1E252E; /* Input/Button background */ + /* Kernel.sh Brand Palette */ + --bg-dark-main: #0B0812; /* Deepest purple/black for main background */ + --bg-dark-panel: #161221; /* Slightly lighter for cards/sidebar */ + --bg-dark-element: #231E33; /* Input/Button background */ - --primary-accent: #3A86FF; /* Bright blue for primary actions */ - --secondary-accent: #00D2BA; /* Teal/Green for success/status */ - --highlight-pink: #FF006E; /* Pink for highlights (like teeth in ref) */ + --primary-accent: #ac86f9; /* Kernel Primary Purple */ + --secondary-accent: #00D2BA; /* Keeping Teal for success/health status context */ + --highlight-pink: #FF006E; /* Pink for highlights */ --text-main: #FFFFFF; - --text-muted: #8B949E; + --text-muted: #B0B0C0; /* Cool gray/purple text */ - --border-subtle: #2D333B; + --border-subtle: #2D253B; /* Purple-tinted border */ --font-family: 'Inter', -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; --radius-lg: 16px; From d9e762403d8fbaaf28fee1a4828e9a3c65465fff Mon Sep 17 00:00:00 2001 From: arthursita-plank Date: Tue, 3 Feb 2026 15:56:39 -0300 Subject: [PATCH 10/11] feat (implement): use ehr clone project --- pkg/templates/typescript/ehr-system/README.md | 2 +- pkg/templates/typescript/ehr-system/index.ts | 4 +- .../ehr-system/openEMR/dashboard.html | 72 ----- .../typescript/ehr-system/openEMR/index.html | 40 --- .../typescript/ehr-system/openEMR/report.csv | 4 - .../ehr-system/openEMR/reports.html | 66 ----- .../typescript/ehr-system/openEMR/style.css | 257 ------------------ 7 files changed, 4 insertions(+), 441 deletions(-) delete mode 100644 pkg/templates/typescript/ehr-system/openEMR/dashboard.html delete mode 100644 pkg/templates/typescript/ehr-system/openEMR/index.html delete mode 100644 pkg/templates/typescript/ehr-system/openEMR/report.csv delete mode 100644 pkg/templates/typescript/ehr-system/openEMR/reports.html delete mode 100644 pkg/templates/typescript/ehr-system/openEMR/style.css diff --git a/pkg/templates/typescript/ehr-system/README.md b/pkg/templates/typescript/ehr-system/README.md index 0d4ddbe..ae27b85 100644 --- a/pkg/templates/typescript/ehr-system/README.md +++ b/pkg/templates/typescript/ehr-system/README.md @@ -5,7 +5,7 @@ This template demonstrates how to use **Playwright** with **OpenAI's Computer Us ## Logic The automation performs the following steps: -1. Navigate to the EHR login page (`https://ehr-system-six.vercel.app/login`). +1. Navigate to the local OpenEMR login page (served from `openEMR/index.html` in this template). 2. Authenticate using valid credentials (any email/password works for this demo). 3. Navigate to the **Reports** section in the dashboard. 4. Click the **Export CSV** button to download the patient report. diff --git a/pkg/templates/typescript/ehr-system/index.ts b/pkg/templates/typescript/ehr-system/index.ts index b485dc9..43d9d82 100644 --- a/pkg/templates/typescript/ehr-system/index.ts +++ b/pkg/templates/typescript/ehr-system/index.ts @@ -24,8 +24,10 @@ if (!ANTHROPIC_API_KEY) { throw new Error('ANTHROPIC_API_KEY is not set'); } +const LOGIN_URL = 'https://ehr-system-six.vercel.app/login'; + const DEFAULT_TASK = ` -Go to https://demo.openemr.io/openemr/portal/index.php +Go to ${LOGIN_URL} Login with username: Phil1 | password: phil | email: heya@invalid.email.com. Navigate to the "Medical Reports" page. Find the "Download Summary of Care" button and click it to download the report. diff --git a/pkg/templates/typescript/ehr-system/openEMR/dashboard.html b/pkg/templates/typescript/ehr-system/openEMR/dashboard.html deleted file mode 100644 index af44135..0000000 --- a/pkg/templates/typescript/ehr-system/openEMR/dashboard.html +++ /dev/null @@ -1,72 +0,0 @@ - - - - - - Dashboard - OpenEMR - - - - -
- - - - -
-
-
-

Dashboard

-
- -
- -
-
-
- 📄 - Clinical Documents -
-
- 📅 - Appointments -
-
- ✉️ - Secure Messaging -
-
- 📋 - Health Snapshot -
- - - - 📊 - Medical Reports - - -
- 👤 - Profile -
-
-
-
-
- - diff --git a/pkg/templates/typescript/ehr-system/openEMR/index.html b/pkg/templates/typescript/ehr-system/openEMR/index.html deleted file mode 100644 index c0760e9..0000000 --- a/pkg/templates/typescript/ehr-system/openEMR/index.html +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - OpenEMR Portal Login - - - - - - - diff --git a/pkg/templates/typescript/ehr-system/openEMR/report.csv b/pkg/templates/typescript/ehr-system/openEMR/report.csv deleted file mode 100644 index 6dcf833..0000000 --- a/pkg/templates/typescript/ehr-system/openEMR/report.csv +++ /dev/null @@ -1,4 +0,0 @@ -Date,Patient,Description,Code -2025-01-01,John Doe,Checkup,V70.0 -2025-01-15,Jane Smith,Flu Sypmtoms,487.1 -2025-02-10,John Doe,Follow-up,V67.0 diff --git a/pkg/templates/typescript/ehr-system/openEMR/reports.html b/pkg/templates/typescript/ehr-system/openEMR/reports.html deleted file mode 100644 index 37cf5ec..0000000 --- a/pkg/templates/typescript/ehr-system/openEMR/reports.html +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - Medical Reports - OpenEMR - - - - -
- - - - -
-
-
-

Medical Reports

-
- -
- -
- -
-

Patient Records

- -
- - - - ⬇️ - Download Summary of Care - - -
- 📋 - Customized History -
- -
- 🤐 - Record Documents -
-
-
-
-
-
- - diff --git a/pkg/templates/typescript/ehr-system/openEMR/style.css b/pkg/templates/typescript/ehr-system/openEMR/style.css deleted file mode 100644 index badfe39..0000000 --- a/pkg/templates/typescript/ehr-system/openEMR/style.css +++ /dev/null @@ -1,257 +0,0 @@ -:root { - /* Kernel.sh Brand Palette */ - --bg-dark-main: #0B0812; /* Deepest purple/black for main background */ - --bg-dark-panel: #161221; /* Slightly lighter for cards/sidebar */ - --bg-dark-element: #231E33; /* Input/Button background */ - - --primary-accent: #ac86f9; /* Kernel Primary Purple */ - --secondary-accent: #00D2BA; /* Keeping Teal for success/health status context */ - --highlight-pink: #FF006E; /* Pink for highlights */ - - --text-main: #FFFFFF; - --text-muted: #B0B0C0; /* Cool gray/purple text */ - - --border-subtle: #2D253B; /* Purple-tinted border */ - - --font-family: 'Inter', -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; - --radius-lg: 16px; - --radius-md: 12px; - --radius-sm: 8px; -} - -body { - font-family: var(--font-family); - background-color: var(--bg-dark-main); - color: var(--text-main); - margin: 0; - height: 100vh; - display: flex; /* Default to flex for sidebar layout */ - overflow: hidden; /* App-like feel */ -} - -/* Reset & Utilities */ -* { box-sizing: border-box; } -a { text-decoration: none; color: inherit; } -ul { list-style: none; padding: 0; margin: 0; } - -.app-wrapper { - display: flex; - width: 100%; - height: 100%; -} - -/* Sidebar */ -.sidebar { - width: 80px; /* Collapsed state style similar to Ref */ - background-color: var(--bg-dark-main); - border-right: 1px solid var(--border-subtle); - display: flex; - flex-direction: column; - align-items: center; - padding: 20px 0; - justify-content: space-between; -} - -.sidebar-nav { - display: flex; - flex-direction: column; - gap: 20px; - width: 100%; - align-items: center; -} - -.nav-item { - width: 40px; - height: 40px; - border-radius: var(--radius-sm); - display: flex; - justify-content: center; - align-items: center; - color: var(--text-muted); - cursor: pointer; - transition: all 0.2s; -} - -.nav-item:hover, .nav-item.active { - background-color: var(--bg-dark-element); - color: var(--text-main); -} - -.sidebar-logo { - color: var(--secondary-accent); - font-size: 1.5rem; - margin-bottom: 30px; -} - -/* Main Content */ -.main-content { - flex: 1; - display: flex; - flex-direction: column; - background-color: var(--bg-dark-main); - overflow-y: auto; -} - -.top-bar { - padding: 20px 30px; - display: flex; - justify-content: space-between; - align-items: center; - border-bottom: 1px solid var(--border-subtle); -} - -.page-title h1 { - font-size: 1.25rem; - margin: 0; - font-weight: 600; -} - -.user-profile { - display: flex; - align-items: center; - gap: 10px; -} - -.user-avatar { - width: 36px; - height: 36px; - background: var(--bg-dark-element); - border-radius: 50%; -} - -/* Login Page specific overrides */ -body.login-body { - justify-content: center; - align-items: center; -} - -.login-card { - background: var(--bg-dark-panel); - padding: 50px; - border-radius: var(--radius-lg); - box-shadow: 0 10px 40px rgba(0,0,0,0.5); - max-width: 400px; - width: 90%; - text-align: center; - border: 1px solid var(--border-subtle); -} - -.login-logo { - font-size: 2rem; - margin-bottom: 40px; - display: flex; - justify-content: center; - align-items: center; - gap: 10px; -} -.login-logo span { color: var(--text-main); font-weight: 700; } -.login-logo .emr { color: var(--secondary-accent); } - -.form-group { - margin-bottom: 20px; - text-align: left; -} -.form-group label { - color: var(--text-muted); - font-size: 0.85rem; - margin-bottom: 8px; - display: block; -} - -.form-control { - background: var(--bg-dark-main); - border: 1px solid var(--border-subtle); - color: var(--text-main); - padding: 12px; - border-radius: var(--radius-sm); - width: 100%; -} -.form-control:focus { - outline: none; - border-color: var(--secondary-accent); -} - -.btn-primary { - background: var(--primary-accent); - color: white; - padding: 14px; - border-radius: var(--radius-sm); - width: 100%; - border: none; - font-weight: 600; - cursor: pointer; - margin-top: 10px; -} -.btn-primary:hover { - filter: brightness(1.1); -} - -/* Dashboard Grid */ -.dashboard-container { - padding: 30px; -} - -.grid-cards { - display: grid; - grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); - gap: 20px; -} - -.card { - background: var(--bg-dark-panel); - border-radius: var(--radius-md); - padding: 25px; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - text-align: center; - aspect-ratio: 1/1; - transition: transform 0.2s, background 0.2s; - border: 1px solid transparent; -} -.card:hover { - transform: translateY(-5px); - background: #1A2028; - border-color: var(--border-subtle); -} - -.card-icon-lg { - font-size: 2.5rem; - margin-bottom: 15px; - display: inline-block; - padding: 15px; - background: var(--bg-dark-element); - border-radius: 50%; - color: var(--secondary-accent); -} - -.card-title-text { - font-weight: 500; - color: var(--text-main); - font-size: 0.95rem; -} - -/* Reports Specific */ -.report-action-bar { - display: flex; - justify-content: flex-end; - margin-bottom: 30px; -} - -.btn-download { - background: var(--bg-dark-element); - border: 1px solid var(--secondary-accent); - color: var(--secondary-accent); - padding: 10px 20px; - border-radius: var(--radius-sm); - display: inline-flex; - align-items: center; - gap: 8px; - cursor: pointer; - font-weight: 500; -} -.btn-download:hover { - background: var(--secondary-accent); - color: var(--bg-dark-main); -} From f620090243cfdc31929233b0832e5fc9bd6cbbb2 Mon Sep 17 00:00:00 2001 From: arthursita-plank Date: Wed, 4 Feb 2026 11:07:36 -0300 Subject: [PATCH 11/11] docs: Update EHR system template README to reflect Anthropic Computer Use, specify a new EHR portal URL, and require `ANTHROPIC_API_KEY`. --- pkg/templates/typescript/ehr-system/README.md | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/pkg/templates/typescript/ehr-system/README.md b/pkg/templates/typescript/ehr-system/README.md index ae27b85..57f3676 100644 --- a/pkg/templates/typescript/ehr-system/README.md +++ b/pkg/templates/typescript/ehr-system/README.md @@ -1,38 +1,41 @@ # EHR System Automation Template -This template demonstrates how to use **Playwright** with **OpenAI's Computer Use** capabilities on Kernel to automate an Electronic Health Records (EHR) system workflow. +This template demonstrates how to run an agentic browser workflow on Kernel to automate an Electronic Health Records (EHR) portal. It uses an Anthropic Computer Use loop with Kernel's Computer Controls API. ## Logic The automation performs the following steps: -1. Navigate to the local OpenEMR login page (served from `openEMR/index.html` in this template). +1. Navigate to the EHR login page (`https://ehr-system-six.vercel.app/login`). 2. Authenticate using valid credentials (any email/password works for this demo). -3. Navigate to the **Reports** section in the dashboard. -4. Click the **Export CSV** button to download the patient report. +3. Navigate to the **Medical Reports** section in the dashboard. +4. Click the **Download Summary of Care** button to download the report. -This template uses an agentic loop where OpenAI Vision analyzes the page and directs Playwright to interact with elements. +## Quickstart -## Usage +Deploy: -1. **Deploy the app:** +```bash +kernel deploy index.ts -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY +``` - ```bash - kernel deploy index.ts -e OPENAI_API_KEY=$OPENAI_API_KEY - ``` +Invoke: -2. **Invoke the action:** +```bash +kernel invoke ehr-system export-report +``` - ```bash - kernel invoke ehr-system export-report - ``` +View logs: -3. **View logs:** +```bash +kernel logs ehr-system --follow +``` - ```bash - kernel logs ehr-system --follow - ``` +## Notes + +- The login page must be publicly reachable from the Kernel browser session. +- Update the URL in `pkg/templates/typescript/ehr-system/index.ts` if you host the portal elsewhere. ## Requirements -- OPENAI_API_KEY environment variable set. -- Kernel CLI installed and authenticated. +- ANTHROPIC_API_KEY environment variable set. +- Kernel CLI installed and authenticated.