diff --git a/py/packages/genkit-anthropic/src/genkit_anthropic/__init__.py b/py/packages/genkit-anthropic/src/genkit_anthropic/__init__.py index df2f8d0373..787d7590bc 100644 --- a/py/packages/genkit-anthropic/src/genkit_anthropic/__init__.py +++ b/py/packages/genkit-anthropic/src/genkit_anthropic/__init__.py @@ -20,165 +20,30 @@ Genkit framework. It registers Claude models as Genkit actions, enabling text generation operations. -Key Concepts (ELI5):: - - ┌─────────────────────┬────────────────────────────────────────────────────┐ - │ Concept │ ELI5 Explanation │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Claude │ Anthropic's AI assistant. Like a helpful friend │ - │ │ who's great at explaining things and writing. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Sonnet │ The "just right" Claude model. Good at most │ - │ │ tasks without being too slow or expensive. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Haiku │ The fast & cheap Claude model. Perfect for │ - │ │ quick tasks like classification or summaries. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Opus │ The most capable Claude. For complex tasks │ - │ │ like research, analysis, or creative writing. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ API Key │ Your password to use Claude. Keep it secret! │ - │ │ Set as ANTHROPIC_API_KEY environment variable. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ System Prompt │ Instructions that shape Claude's personality. │ - │ │ Like giving a new employee their job description. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Tool Calling │ Claude can use functions you define. Like │ - │ │ giving it a calculator or search engine to use. │ - └─────────────────────┴────────────────────────────────────────────────────┘ - -Data Flow:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ HOW CLAUDE PROCESSES YOUR REQUEST │ - │ │ - │ Your Code │ - │ ai.generate(prompt="Explain quantum computing") │ - │ │ │ - │ │ (1) Request goes to Anthropic plugin │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Anthropic │ Plugin adds API key to request │ - │ │ Plugin │ │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (2) Converts Genkit format → Claude Messages API │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ AnthropicModel │ Handles message roles, images, │ - │ │ │ tools, and streaming │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (3) HTTPS request to api.anthropic.com │ - │ ▼ │ - │ ════════════════════════════════════════════════════ │ - │ │ Internet │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Anthropic │ Claude thinks about your prompt │ - │ │ Claude API │ and generates a response │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (4) Response parsed back to Genkit format │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Your App │ response.text = "Quantum computing..." │ - │ └─────────────────┘ │ - └─────────────────────────────────────────────────────────────────────────┘ - -Architecture Overview:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Anthropic Plugin │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ Plugin Entry Point (__init__.py) │ - │ ├── Anthropic - Plugin class │ - │ └── anthropic_name() - Helper to create namespaced model names │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ plugin.py - Plugin Implementation │ - │ ├── Anthropic class (registers models) │ - │ └── Client initialization with Anthropic SDK │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ models.py - Model Implementation │ - │ ├── AnthropicModel (Messages API integration) │ - │ ├── Request/response conversion │ - │ └── Streaming support │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ model_info.py - Model Registry │ - │ ├── SUPPORTED_MODELS (claude-sonnet-4-5, opus, haiku, etc.) │ - │ └── Model capabilities and metadata │ - └─────────────────────────────────────────────────────────────────────────┘ - -Overview: - The Anthropic plugin adds support for Claude models to Genkit. It uses - the official Anthropic Python SDK and registers models that can be used - with ai.generate() and other Genkit generation methods. - -Supported Models: - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Model │ Description │ - ├───────────────────────────┼─────────────────────────────────────────────┤ - │ claude-sonnet-4-5 │ Balanced performance and capability │ - │ claude-haiku-4-5 │ Fast and cost-effective │ - │ claude-opus-4-5 │ Most capable, complex tasks │ - │ claude-sonnet-5 │ Latest Sonnet model │ - └───────────────────────────┴─────────────────────────────────────────────┘ - -Key Components: - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Component │ Purpose │ - ├─────────────────────┼───────────────────────────────────────────────────┤ - │ Anthropic │ Plugin class to register with Genkit │ - │ anthropic_name() │ Helper to create namespaced model names │ - └─────────────────────┴───────────────────────────────────────────────────┘ - Example: - Basic usage: - ```python from genkit import Genkit from genkit_anthropic import Anthropic, AnthropicConfig - # Uses ANTHROPIC_API_KEY env var or pass api_key explicitly - ai = Genkit( - plugins=[Anthropic()], - model='anthropic/claude-sonnet-4-5', - ) - - response = await ai.generate(prompt='Hello, Claude!') - print(response.text) - - # With custom configuration - response = await ai.generate( - model='anthropic/claude-haiku-4-5', - prompt='Write a haiku about AI', - config=AnthropicConfig(temperature=0.7, max_output_tokens=100), - ) - ``` + # 1. Initialize Genkit with the Anthropic plugin + ai = Genkit(plugins=[Anthropic()]) - With tools: - - ```python - @ai.tool() - def get_weather(city: str) -> str: - return f'Weather in {city}: Sunny, 72°F' - - - response = await ai.generate( + # 2. Generate content using Claude Sonnet 4.5 + res = await ai.generate( model='anthropic/claude-sonnet-4-5', - prompt='What is the weather in Paris?', - tools=['get_weather'], + prompt='Explain recursion in 10 words.', ) + + # 3. Inspect output shapes directly + print(res.text) + # => A function calling itself until reaching a base stopping condition. ``` -Caveats: - - Requires ANTHROPIC_API_KEY environment variable or api_key parameter - - Model names are prefixed with 'anthropic/' (e.g., 'anthropic/claude-sonnet-4-5') - - Anthropic models may have different tool calling behavior than Google models +Requirements: + - Requires the ``ANTHROPIC_API_KEY`` environment variable or explicit ``api_key``. See Also: - Anthropic documentation: https://docs.anthropic.com/ - - Genkit documentation: https://genkit.dev/ """ from genkit_anthropic.config import ( diff --git a/py/packages/genkit-django/src/genkit_django/__init__.py b/py/packages/genkit-django/src/genkit_django/__init__.py index 7154d79f05..97d04d4e6c 100644 --- a/py/packages/genkit-django/src/genkit_django/__init__.py +++ b/py/packages/genkit-django/src/genkit_django/__init__.py @@ -25,15 +25,25 @@ # myapp/views.py from genkit import Genkit from genkit_django import genkit_django_handler + from genkit_google_genai import GoogleAI - ai = Genkit(plugins=[...]) + # 1. Initialize Genkit + ai = Genkit(plugins=[GoogleAI()]) + # 2. Define flow and decorate as Django view @genkit_django_handler(ai) @ai.flow() async def chat(prompt: str) -> str: - response = await ai.generate(prompt=prompt) - return response.text + res = await ai.generate( + model='googleai/gemini-flash-latest', + prompt=f'Answer concisely: {prompt}', + ) + return res.text + + + # POST /chat/ {"data": "Hello!"} + # => {"result": "Hi there! How can I assist you today?"} ``` ```python diff --git a/py/packages/genkit-fastapi/src/genkit_fastapi/__init__.py b/py/packages/genkit-fastapi/src/genkit_fastapi/__init__.py index 9b9c874afb..2270ac24ea 100644 --- a/py/packages/genkit-fastapi/src/genkit_fastapi/__init__.py +++ b/py/packages/genkit-fastapi/src/genkit_fastapi/__init__.py @@ -29,16 +29,25 @@ from genkit_fastapi import genkit_fastapi_handler from genkit_google_genai import GoogleAI + # 1. Initialize Genkit and FastAPI app ai = Genkit(plugins=[GoogleAI()]) app = FastAPI() + # 2. Define flow and expose as FastAPI endpoint in one clean decorator stack @app.post('/chat', response_model=None) @genkit_fastapi_handler(ai) @ai.flow() async def chat_flow(prompt: str) -> str: - response = await ai.generate(prompt=prompt) - return response.text + res = await ai.generate( + model='googleai/gemini-flash-latest', + prompt=f'Answer concisely: {prompt}', + ) + return res.text + + + # POST /chat {"data": "Why is the sky blue?"} + # => {"result": "The sky appears blue due to Rayleigh scattering..."} ``` Running: diff --git a/py/packages/genkit-flask/src/genkit_flask/__init__.py b/py/packages/genkit-flask/src/genkit_flask/__init__.py index aac5389a2b..5f49e78a9c 100644 --- a/py/packages/genkit-flask/src/genkit_flask/__init__.py +++ b/py/packages/genkit-flask/src/genkit_flask/__init__.py @@ -20,118 +20,37 @@ This plugin provides Flask integration for Genkit, enabling you to expose Genkit flows as HTTP endpoints in a Flask application. -Key Concepts (ELI5):: - - ┌─────────────────────┬────────────────────────────────────────────────────┐ - │ Concept │ ELI5 Explanation │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Flask │ A simple Python web framework. Like a waiter │ - │ │ that takes HTTP requests and serves responses. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ HTTP Endpoint │ A URL that accepts requests. Like a phone number │ - │ │ your app answers when called. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Flow │ A Genkit function that does AI work. This plugin │ - │ │ lets you call flows via HTTP requests. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Route │ Maps a URL to a function. /api/chat → chat_flow │ - │ │ │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Request Handler │ Code that processes incoming requests. │ - │ │ genkit_flask_handler does this for you. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ POST │ HTTP method for sending data. Like mailing a │ - │ │ letter with your prompt inside. │ - └─────────────────────┴────────────────────────────────────────────────────┘ - -Data Flow:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ HOW FLASK SERVES YOUR GENKIT FLOWS │ - │ │ - │ Client (Browser, curl, etc.) │ - │ POST /api/chat {"prompt": "Hello!"} │ - │ │ │ - │ │ (1) HTTP request arrives │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Flask App │ Routes request to the right handler │ - │ │ @app.route() │ │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (2) Handler invoked │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ genkit_flask_ │ Parses JSON body, validates input │ - │ │ handler() │ │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (3) Calls your Genkit flow │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Your Flow │ Does AI magic (generate, tools, etc.) │ - │ │ async def ... │ │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (4) Response serialized to JSON │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Client │ {"result": "Hello! How can I help?"} │ - │ └─────────────────┘ │ - └─────────────────────────────────────────────────────────────────────────┘ - -Architecture Overview:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Flask Plugin │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ Plugin Entry Point (__init__.py) │ - │ └── genkit_flask_handler() - Create Flask route handler │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ handler.py - Request Handler │ - │ ├── genkit_flask_handler() - Factory for Flask handlers │ - │ ├── Request parsing and validation │ - │ └── Response serialization │ - └─────────────────────────────────────────────────────────────────────────┘ - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Request Flow │ - │ │ - │ HTTP Request ──► Flask Route ──► genkit_flask_handler ──► Genkit Flow │ - │ │ - │ HTTP Response ◄── Flask Route ◄── Handler ◄── Flow Result │ - └─────────────────────────────────────────────────────────────────────────┘ - Example: ```python from flask import Flask from genkit import Genkit from genkit_flask import genkit_flask_handler + from genkit_google_genai import GoogleAI + # 1. Initialize Flask app and Genkit with GoogleAI app = Flask(__name__) - ai = Genkit(...) + ai = Genkit(plugins=[GoogleAI()], model='googleai/gemini-flash-latest') + # 2. Stack Flask route + Genkit handler + flow on one function + @app.post('/api/greet') + @genkit_flask_handler(ai) @ai.flow() - async def my_flow(prompt: str) -> str: - response = await ai.generate(prompt=prompt) - return response.text + async def greet_user(name: str) -> str: + res = await ai.generate(prompt=f'Say hello to {name} in one sentence.') + return res.text - # Expose flow as HTTP endpoint - @app.route('/api/flow', methods=['POST']) - def handle_flow(): - return genkit_flask_handler(ai, my_flow) + # POST /api/greet {"data": "Alice"} + # => {"result": "Hello Alice! Welcome to our AI community."} ``` -Caveats: - - Requires Flask to be installed - - Async flows are run synchronously in Flask (use async frameworks for better performance) - - For production, consider using the async-native Genkit server +Requirements: + - Requires Flask 3.0+. + - Async flows are run via an asyncio event loop within the Flask request handler. See Also: - Flask documentation: https://flask.palletsprojects.com/ - - Genkit documentation: https://genkit.dev/ """ from .handler import genkit_flask_handler diff --git a/py/packages/genkit-google-cloud/src/genkit_google_cloud/__init__.py b/py/packages/genkit-google-cloud/src/genkit_google_cloud/__init__.py index bf13ca66e9..8ed4de1e11 100644 --- a/py/packages/genkit-google-cloud/src/genkit_google_cloud/__init__.py +++ b/py/packages/genkit-google-cloud/src/genkit_google_cloud/__init__.py @@ -18,126 +18,31 @@ """Google Cloud Plugin for Genkit. This plugin provides Google Cloud observability integration for Genkit, -enabling telemetry export to Cloud Trace and Cloud Monitoring. - -Key Concepts (ELI5):: - - ┌─────────────────────┬────────────────────────────────────────────────────┐ - │ Concept │ ELI5 Explanation │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Telemetry │ Data about how your app is running. Like a │ - │ │ fitness tracker for your code. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Cloud Trace │ Shows the path requests take through your app. │ - │ │ Like GPS tracking for your API calls. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Cloud Monitoring │ Graphs and alerts for your app's health. │ - │ │ Like a heart rate monitor dashboard. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Span │ One step in a request's journey. Like one │ - │ │ leg of a relay race. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Trace │ All spans for one request connected together. │ - │ │ The complete story of one API call. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Metrics │ Numbers that describe your app (requests/sec, │ - │ │ error rate, latency). Like a report card. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ PII Redaction │ Hiding sensitive data in traces. Like blurring │ - │ │ faces in photos before sharing. │ - └─────────────────────┴────────────────────────────────────────────────────┘ - -Data Flow:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ HOW TELEMETRY FLOWS TO GOOGLE CLOUD │ - │ │ - │ Your Genkit App │ - │ │ │ - │ │ (1) App runs flows, calls models, uses tools │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ OpenTelemetry │ Automatically creates spans for each │ - │ │ SDK │ operation (you don't write this code!) │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (2) Spans collected and processed │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ GCP Exporters │ • Redact PII (input/output) │ - │ │ │ • Add error markers │ - │ │ │ • Batch for efficiency │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (3) HTTPS to Google Cloud │ - │ ▼ │ - │ ════════════════════════════════════════════════════ │ - │ │ Internet │ - │ ▼ │ - │ ┌─────────────────────────────────────────────────────┐ │ - │ │ Google Cloud Console │ │ - │ │ ┌──────────────┐ ┌──────────────┐ │ │ - │ │ │ Cloud Trace │ │ Cloud │ │ │ - │ │ │ (waterfall │ │ Monitoring │ │ │ - │ │ │ diagrams) │ │ (dashboards) │ │ │ - │ │ └──────────────┘ └──────────────┘ │ │ - │ └─────────────────────────────────────────────────────┘ │ - └─────────────────────────────────────────────────────────────────────────┘ - -Architecture Overview:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Google Cloud Plugin │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ Plugin Entry Point (__init__.py) │ - │ └── enable_google_cloud_telemetry() - Enable Cloud Trace/Monitoring export │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ telemetry/__init__.py - Telemetry Module │ - │ └── Re-exports from submodules │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ telemetry/tracing.py - Distributed Tracing │ - │ ├── Cloud Trace exporter configuration │ - │ └── OpenTelemetry integration │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ telemetry/metrics.py - Metrics Collection │ - │ ├── Cloud Monitoring exporter │ - │ └── Custom Genkit metrics │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ telemetry/action.py - Action Instrumentation │ - │ └── Automatic span creation for Genkit actions │ - └─────────────────────────────────────────────────────────────────────────┘ - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Telemetry Data Flow │ - │ │ - │ ┌──────────────┐ ┌──────────────┐ ┌──────────────────────┐ │ - │ │ Genkit App │───►│ OpenTelemetry│───►│ Google Cloud │ │ - │ │ (actions, │ │ SDK │ │ (Trace, Monitoring) │ │ - │ │ flows) │ └──────────────┘ └──────────────────────┘ │ - │ └──────────────┘ │ - └─────────────────────────────────────────────────────────────────────────┘ +enabling telemetry export to Cloud Trace, Cloud Monitoring, and Cloud Logging. Example: ```python + from genkit import Genkit + from genkit_google_genai import GoogleAI from genkit_google_cloud import enable_google_cloud_telemetry - # Enable telemetry export to Google Cloud - enable_google_cloud_telemetry() - # Traces and metrics are now exported to: - # - Cloud Trace (distributed tracing) - # - Cloud Monitoring (metrics) + # 1. Enable Google Cloud Trace and Monitoring export + enable_google_cloud_telemetry(project_id='my-project') + + # 2. All subsequent Genkit actions automatically export telemetry + ai = Genkit(plugins=[GoogleAI()], model='googleai/gemini-flash-latest') + await ai.generate(prompt='Hello, world!') + # => Traces exported asynchronously to Cloud Trace (latency, tokens, status) ``` -Caveats: - - Requires Google Cloud credentials (ADC or explicit) - - Telemetry is disabled by default in development mode (GENKIT_ENV=dev) - - Requires opentelemetry and google-cloud-* packages +Requirements: + - Requires Google Cloud Application Default Credentials (ADC) or explicit credentials. + - Telemetry export is disabled by default in local dev environments unless explicitly configured. See Also: - Cloud Trace: https://cloud.google.com/trace - Cloud Monitoring: https://cloud.google.com/monitoring - - Genkit documentation: https://genkit.dev/ """ from .telemetry import add_gcp_telemetry, enable_google_cloud_telemetry diff --git a/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/__init__.py b/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/__init__.py index 12712a219f..d69cc7c3cc 100644 --- a/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/__init__.py +++ b/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/__init__.py @@ -20,53 +20,24 @@ enabling monitoring and debugging of Genkit applications through Cloud Trace, Cloud Monitoring, and Cloud Logging. -Module Structure: - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Module │ Purpose │ - ├─────────────────┼───────────────────────────────────────────────────────┤ - │ tracing.py │ Main entry point, exporters, configuration │ - │ feature.py │ Root span metrics (requests, latency) │ - │ path.py │ Error path tracking and failure metrics │ - │ generate.py │ Model/generate metrics (tokens, latency, media) │ - │ action.py │ Action I/O logging (tools, flows) │ - │ engagement.py │ User feedback and acceptance metrics │ - │ metrics.py │ Metric definitions and lazy initialization │ - │ utils.py │ Shared utilities (truncation, path parsing, logging) │ - └─────────────────┴───────────────────────────────────────────────────────┘ - -Quick Start: +Example: ```python + from genkit import Genkit + from genkit_google_genai import GoogleAI from genkit_google_cloud import enable_google_cloud_telemetry - # Enable telemetry with defaults (PII redaction enabled) - enable_google_cloud_telemetry() + # 1. Enable Google Cloud Trace and Monitoring export + enable_google_cloud_telemetry(project_id='my-project') - # Or with custom options - enable_google_cloud_telemetry( - project_id='my-project', - log_input_and_output=True, # Disable PII redaction (caution!) - ) + # 2. All subsequent Genkit actions automatically export telemetry + ai = Genkit(plugins=[GoogleAI()], model='googleai/gemini-flash-latest') + await ai.generate(prompt='Hello, world!') + # => Traces exported asynchronously to Cloud Trace (latency, tokens, status) ``` -Cross-Language Parity: - This implementation maintains feature parity with: - - JavaScript: js/plugins/google-cloud/src/gcpOpenTelemetry.ts - - Go: go/plugins/googlecloud/ and go/plugins/firebase/telemetry.go - See Also: - - tracing.py module docstring for detailed architecture documentation - -GCP Documentation: - Cloud Trace: - - Overview: https://cloud.google.com/trace/docs - - IAM Roles: https://cloud.google.com/trace/docs/iam - - Cloud Monitoring: - - Overview: https://cloud.google.com/monitoring/docs - - Quotas & Limits: https://cloud.google.com/monitoring/quotas - - OpenTelemetry GCP: - - Python Exporters: https://google-cloud-opentelemetry.readthedocs.io/ + - Cloud Trace: https://cloud.google.com/trace/docs + - Cloud Monitoring: https://cloud.google.com/monitoring/docs """ from .tracing import add_gcp_telemetry, enable_google_cloud_telemetry diff --git a/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/generate.py b/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/generate.py index f48f17342e..f4bc42a7b7 100644 --- a/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/generate.py +++ b/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/generate.py @@ -14,72 +14,25 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Generate action telemetry for GCP. +"""Generate action telemetry for Google Cloud. -This module tracks generate action metrics (tokens, latencies) and logs, -matching the JavaScript implementation in telemetry/generate.ts and Go -implementation in googlecloud/generate.go. +This module tracks generate action metrics (tokens, latencies) and structured logs, +maintaining cross-language parity with JavaScript and Go implementations. When It Fires: - The generate telemetry handler is called for spans where: - - genkit:type = "action" - - genkit:metadata:subtype = "model" - -Metrics Recorded: - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Metric Name │ Type │ Description │ - ├──────────────────────────────────────┼───────────┼──────────────────────┤ - │ genkit/ai/generate/requests │ Counter │ Model call count │ - │ genkit/ai/generate/latency │ Histogram │ Response time (ms) │ - │ genkit/ai/generate/input/tokens │ Counter │ Input token count │ - │ genkit/ai/generate/input/characters │ Counter │ Input char count │ - │ genkit/ai/generate/input/images │ Counter │ Input image count │ - │ genkit/ai/generate/input/videos │ Counter │ Input video count │ - │ genkit/ai/generate/input/audio │ Counter │ Input audio count │ - │ genkit/ai/generate/output/tokens │ Counter │ Output token count │ - │ genkit/ai/generate/output/characters │ Counter │ Output char count │ - │ genkit/ai/generate/output/images │ Counter │ Output image count │ - │ genkit/ai/generate/output/videos │ Counter │ Output video count │ - │ genkit/ai/generate/output/audio │ Counter │ Output audio count │ - │ genkit/ai/generate/thinking/tokens │ Counter │ Thinking token count │ - └──────────────────────────────────────┴───────────┴──────────────────────┘ + The generate telemetry handler executes for spans where: + - ``genkit:type`` = "action" + - ``genkit:metadata:subtype`` = "model" + +Recorded Metrics: + - ``genkit/ai/generate/requests`` (Counter): Model invocation count. + - ``genkit/ai/generate/latency`` (Histogram): Response latency in milliseconds. + - ``genkit/ai/generate/input/*`` and ``genkit/ai/generate/output/*`` (Counters): + Token, character, image, video, audio, and thinking token counts. Metric Dimensions: - All metrics include these dimensions: - - modelName: The model name (e.g., "gemini-2.0-flash") - - featureName: The outer flow/feature name - - path: The qualified Genkit path - - status: "success" or "failure" - - error: Error name (only on failure) - - source: "py" (language identifier) - - sourceVersion: Genkit version - -Logs Recorded: - 1. Config logs (always): Model configuration (maxOutputTokens, stopSequences) - 2. Input logs (when log_input_and_output=True): Per-message, per-part input - 3. Output logs (when log_input_and_output=True): Per-part output content - -Log Format: - - Config[path, model] - Model configuration - - Input[path, model] (part X of Y) - Input content with part indices - - Output[path, model] (part X of Y) - Output content with part indices - -Media Handling: - - Data URLs (base64) are hashed with SHA-256 to avoid logging large content - - Format: "data:image/png;base64," - -GCP Documentation: - Cloud Monitoring Metrics: - - Custom Metrics: https://cloud.google.com/monitoring/custom-metrics - - Quotas: https://cloud.google.com/monitoring/quotas - - Note: Rate limit is 1 point per 5 seconds per time series - - OpenTelemetry: - - Python Metrics SDK: https://opentelemetry-python.readthedocs.io/en/stable/sdk/metrics.html - -Cross-Language Parity: - - JavaScript: js/plugins/google-cloud/src/telemetry/generate.ts - - Go: go/plugins/googlecloud/generate.go + Includes ``modelName`` (e.g., "gemini-flash-latest"), ``featureName``, ``path``, + ``status``, ``source`` ("py"), and ``sourceVersion``. """ from __future__ import annotations @@ -553,6 +506,8 @@ def _to_part_log_content(self, part: dict[str, Any]) -> str: """Convert a part to log-safe content.""" if part.get('text'): return truncate(str(part['text'])) + if part.get('reasoning'): + return truncate(str(part['reasoning'])) if part.get('data'): return truncate(json.dumps(part['data'])) if part.get('media'): @@ -561,6 +516,8 @@ def _to_part_log_content(self, part: dict[str, Any]) -> str: return self._to_part_log_tool_request(part) if part.get('toolResponse'): return self._to_part_log_tool_response(part) + if part.get('resource'): + return truncate(json.dumps(part['resource'])) if part.get('custom'): return truncate(json.dumps(part['custom'])) return '' diff --git a/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/trace_exporter.py b/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/trace_exporter.py index dbeff19bc9..d5b459cbb1 100644 --- a/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/trace_exporter.py +++ b/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/trace_exporter.py @@ -140,11 +140,15 @@ class GcpAdjustingTraceExporter(AdjustingTraceExporter): Example: ```python + # 1. Wrap GCP trace exporter with PII redaction and metrics processing exporter = GcpAdjustingTraceExporter( exporter=GenkitGCPExporter(), log_input_and_output=False, project_id='my-project', ) + + # 2. Export spans processed through Genkit telemetry handlers + # => Automatically redacts inputs/outputs and records model metrics ``` """ diff --git a/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/tracing.py b/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/tracing.py index 93be4efc20..d6e897cf2a 100644 --- a/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/tracing.py +++ b/py/packages/genkit-google-cloud/src/genkit_google_cloud/telemetry/tracing.py @@ -17,195 +17,32 @@ """Telemetry and tracing functionality for the Genkit Google Cloud plugin. -This module provides functionality for collecting and exporting telemetry data -from Genkit operations to Google Cloud. It uses OpenTelemetry for tracing and -exports span data to Google Cloud Trace for monitoring and debugging purposes. - -Architecture Overview: - The telemetry system follows a pipeline architecture that processes spans - (traces) and metrics before exporting them to Google Cloud: - - ``` - ┌─────────────────────────────────────────────────────────────────────────┐ - │ TELEMETRY DATA FLOW │ - │ │ - │ Genkit Actions (flows, models, tools) │ - │ │ │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ OpenTelemetry │ Creates spans with genkit:* attributes │ - │ │ Tracer │ (type, name, input, output, state, path, etc.) │ - │ └────────┬────────┘ │ - │ │ │ - │ ▼ │ - │ ┌─────────────────────────────────────────────────────────────┐ │ - │ │ GcpAdjustingTraceExporter │ │ - │ │ ┌─────────────────────────────────────────────────────┐ │ │ - │ │ │ 1. _tick_telemetry() │ │ │ - │ │ │ - pathsTelemetry.tick() → Error metrics/logs │ │ │ - │ │ │ - featuresTelemetry.tick() → Feature metrics │ │ │ - │ │ │ - generateTelemetry.tick() → Model metrics │ │ │ - │ │ │ - actionTelemetry.tick() → Action I/O logs │ │ │ - │ │ │ - engagementTelemetry.tick() → Feedback metrics │ │ │ - │ │ │ - Sets genkit:rootState for root spans │ │ │ - │ │ └─────────────────────────────────────────────────────┘ │ │ - │ │ ┌─────────────────────────────────────────────────────┐ │ │ - │ │ │ 2. AdjustingTraceExporter._adjust() │ │ │ - │ │ │ - Redact genkit:input/output → "" │ │ │ - │ │ │ - Mark error spans with /http/status_code: 599 │ │ │ - │ │ │ - Mark failed spans with genkit:failedSpan │ │ │ - │ │ │ - Mark root spans with genkit:feature │ │ │ - │ │ │ - Mark model spans with genkit:model │ │ │ - │ │ │ - Normalize labels (: → /) for GCP compatibility │ │ │ - │ │ └─────────────────────────────────────────────────────┘ │ │ - │ └────────────────────────┬────────────────────────────────────┘ │ - │ │ │ - │ ┌───────────────┴───────────────┐ │ - │ ▼ ▼ │ - │ ┌─────────────────┐ ┌─────────────────┐ │ - │ │ GenkitGCPExporter│ │ Cloud Logging │ │ - │ │ (Cloud Trace) │ │ (via structlog) │ │ - │ └────────┬────────┘ └─────────────────┘ │ - │ │ │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Google Cloud │ │ - │ │ Trace API │ │ - │ └─────────────────┘ │ - │ │ - │ ─────────────────────── METRICS PIPELINE ──────────────────────── │ - │ │ - │ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ - │ │ OpenTelemetry │───▶│ GenkitMetric │───▶│ Cloud Monitoring│ │ - │ │ Meter │ │ Exporter │ │ API │ │ - │ │ (counters, │ │ (adjusts start │ │ │ │ - │ │ histograms) │ │ times for │ │ │ │ - │ └─────────────────┘ │ DELTA→CUMUL.) │ └─────────────────┘ │ - │ └─────────────────┘ │ - └─────────────────────────────────────────────────────────────────────────┘ - ``` - -Key Components: - 1. **GcpAdjustingTraceExporter**: Extends AdjustingTraceExporter to add - GCP-specific telemetry recording before spans are adjusted and exported. - - 2. **AdjustingTraceExporter** (from genkit._core._trace): Base class that - handles PII redaction, error marking, and label normalization. - - 3. **GenkitGCPExporter**: Extends CloudTraceSpanExporter with retry logic - for reliable delivery to Google Cloud Trace. - - 4. **GenkitMetricExporter**: Wraps CloudMonitoringMetricsExporter and - adjusts start times to prevent overlap when GCP converts DELTA to - CUMULATIVE aggregation. - - 5. **Telemetry Handlers** (in separate modules): - - feature.py: Tracks root span requests/latency - - path.py: Tracks error paths and failure metrics - - generate.py: Tracks model usage (tokens, latency, media) - - action.py: Logs tool and action I/O - - engagement.py: Tracks user feedback and acceptance - -Telemetry Types and When They Fire: - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Telemetry Type │ Condition │ What It Records │ - ├────────────────┼──────────────────────────────┼─────────────────────────┤ - │ paths │ Always (for all spans) │ Error paths, failures │ - │ features │ genkit:isRoot = true │ Request count, latency │ - │ generate │ type=action, subtype=model │ Tokens, latency, media │ - │ action │ type in (action,flow,...) │ Input/output logs │ - │ engagement │ type=userEngagement │ Feedback, acceptance │ - └────────────────┴──────────────────────────────┴─────────────────────────┘ - -Span Attributes Used: - The system reads these genkit:* attributes from spans: - - genkit:type - Span type (action, flow, flowStep, util, userEngagement) - - genkit:metadata:subtype - Subtype (model, tool, etc.) - - genkit:isRoot - Whether this is the root/entry span - - genkit:name - Action/flow name - - genkit:path - Hierarchical path like /{flow,t:flow}/{step,t:flowStep} - - genkit:input - JSON-encoded input data - - genkit:output - JSON-encoded output data - - genkit:state - Span state (success, error) - - genkit:isFailureSource - Whether this span is the source of a failure - -Configuration Options (matching JS/Go parity): - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Option │ Type │ Default │ Description │ - ├─────────────────────────────┼──────────┼────────────┼───────────────────┤ - │ project_id │ str │ Auto │ GCP project ID │ - │ credentials │ dict │ ADC │ Service account │ - │ log_input_and_output │ bool │ False │ Disable redaction │ - │ force_dev_export │ bool │ True │ Export in dev │ - │ disable_metrics │ bool │ False │ Skip metrics │ - │ disable_traces │ bool │ False │ Skip traces │ - │ metric_export_interval_ms │ int │ 60000 │ Export interval │ - │ metric_export_timeout_ms │ int │ None │ Export timeout │ - │ sampler │ Sampler │ AlwaysOn │ Trace sampler │ - └─────────────────────────────┴──────────┴────────────┴───────────────────┘ - -Project ID Resolution Order: - 1. Explicit project_id parameter - 2. FIREBASE_PROJECT_ID environment variable - 3. GOOGLE_CLOUD_PROJECT environment variable - 4. GCLOUD_PROJECT environment variable - 5. project_id from credentials dict +This module configures OpenTelemetry exporters to send distributed traces to +Google Cloud Trace and metrics to Google Cloud Monitoring. It includes automatic +PII redaction and error span adjustment. Usage: ```python + from genkit import Genkit + from genkit_google_genai import GoogleAI from genkit_google_cloud import enable_google_cloud_telemetry - # Enable telemetry with default settings (PII redaction enabled) - enable_google_cloud_telemetry() - - # Enable telemetry with input/output logging (disable PII redaction) - enable_google_cloud_telemetry(log_input_and_output=True) - - # Force export even in dev environment - enable_google_cloud_telemetry(force_dev_export=True) + # 1. Enable telemetry with default settings (PII redaction enabled) + enable_google_cloud_telemetry(project_id='my-project') - # Disable metrics but keep traces - enable_google_cloud_telemetry(disable_metrics=True) - - # Custom metric export interval (minimum 5000ms for GCP) - enable_google_cloud_telemetry(metric_export_interval_ms=30000) + # 2. All subsequent Genkit actions automatically export telemetry + ai = Genkit(plugins=[GoogleAI()], model='googleai/gemini-flash-latest') + await ai.generate(prompt='Hello, world!') + # => Traces exported asynchronously to Cloud Trace (latency, tokens, status) ``` -Caveats: - - By default, model inputs and outputs are redacted for privacy - - Set log_input_and_output=True only in trusted environments - - In dev environment, telemetry is skipped unless force_dev_export=True - - GCP requires minimum 5000ms metric export interval (see quotas link below) - -GCP Documentation References: - Cloud Trace: - - Overview: https://cloud.google.com/trace/docs - - IAM Roles: https://cloud.google.com/trace/docs/iam - - Required role: roles/cloudtrace.agent (Cloud Trace Agent) - - Cloud Monitoring: - - Overview: https://cloud.google.com/monitoring/docs - - Quotas & Limits: https://cloud.google.com/monitoring/quotas - - Required role: roles/monitoring.metricWriter (Monitoring Metric Writer) - or roles/telemetry.metricsWriter (Cloud Telemetry Metrics Writer) - - OpenTelemetry GCP Exporters: - - Documentation: https://google-cloud-opentelemetry.readthedocs.io/ - - Cloud Trace Exporter: https://google-cloud-opentelemetry.readthedocs.io/en/stable/cloud_trace/cloud_trace.html - - Cloud Monitoring Exporter: https://google-cloud-opentelemetry.readthedocs.io/en/stable/cloud_monitoring/cloud_monitoring.html - -Cross-Language Parity: - This implementation maintains parity with: - - JavaScript: js/plugins/google-cloud/src/gcpOpenTelemetry.ts - - Go: go/plugins/googlecloud/googlecloud.go - - Go: go/plugins/firebase/telemetry.go (FirebaseTelemetryOptions) +Requirements: + - Requires Google Cloud Application Default Credentials (ADC) or explicit credentials. + - Set ``log_input_and_output=True`` only in trusted environments where prompt/response logging is permitted. - Key parity points: - - Same configuration options with equivalent semantics - - Same telemetry dispatch logic (when each handler fires) - - Same metrics names and dimensions - - Same span adjustment pipeline (redaction, marking, normalization) - - Same project ID resolution order +See Also: + - Cloud Trace: https://cloud.google.com/trace/docs + - Cloud Monitoring: https://cloud.google.com/monitoring/docs """ import warnings diff --git a/py/packages/genkit-google-cloud/tests/gcp_telemetry_utils_test.py b/py/packages/genkit-google-cloud/tests/gcp_telemetry_utils_test.py index e753ed43ed..b6bbe15397 100644 --- a/py/packages/genkit-google-cloud/tests/gcp_telemetry_utils_test.py +++ b/py/packages/genkit-google-cloud/tests/gcp_telemetry_utils_test.py @@ -321,3 +321,22 @@ def test_plain_segments_not_matched(self) -> None: """Plain segments without type annotations are not extracted.""" # The regex only matches {name,t:type} patterns assert to_display_path('foo/bar') == '' + + +# --------------------------------------------------------------------------- +# _to_part_log_content() +# --------------------------------------------------------------------------- +class TestToPartLogContent: + """Tests for part content extraction in generate telemetry logs.""" + + def test_reasoning_part(self) -> None: + from genkit_google_cloud.telemetry.generate import generate_telemetry + + result = generate_telemetry._to_part_log_content({'reasoning': 'Thinking step 1...'}) + assert result == 'Thinking step 1...' + + def test_resource_part(self) -> None: + from genkit_google_cloud.telemetry.generate import generate_telemetry + + result = generate_telemetry._to_part_log_content({'resource': {'uri': 'gs://bucket/file'}}) + assert result == '{"uri": "gs://bucket/file"}' diff --git a/py/packages/genkit-google-genai/src/genkit_google_genai/__init__.py b/py/packages/genkit-google-genai/src/genkit_google_genai/__init__.py index 53d536b078..1a0b0ce7bd 100644 --- a/py/packages/genkit-google-genai/src/genkit_google_genai/__init__.py +++ b/py/packages/genkit-google-genai/src/genkit_google_genai/__init__.py @@ -17,137 +17,9 @@ """Google GenAI plugin for Genkit. -This plugin provides integration with Google's AI services, including -Google AI (Gemini API) and Vertex AI. It registers Gemini models and -embedders for use with the Genkit framework. - -Key Concepts (ELI5):: - - ┌─────────────────────┬────────────────────────────────────────────────────┐ - │ Concept │ ELI5 Explanation │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Gemini │ Google's AI model family. Like having a smart │ - │ │ assistant that can read, write, and understand. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ GoogleAI │ Access Gemini directly via API key. Like calling │ - │ │ a pizza place directly with your phone. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ VertexAI │ Access Gemini through Google Cloud. Like ordering │ - │ │ pizza through a delivery app with your account. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Embeddings │ Convert text to numbers that capture meaning. │ - │ │ Like a fingerprint for sentences. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Imagen │ Google's image generation model. Tell it what │ - │ │ you want and it draws a picture. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Context Caching │ Save conversation history to reuse later. │ - │ │ Like bookmarking your place in a conversation. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Multimodal │ Models that understand text AND images/audio. │ - │ │ Like a friend who can see photos you share. │ - └─────────────────────┴────────────────────────────────────────────────────┘ - -Data Flow:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ HOW YOUR PROMPT BECOMES A RESPONSE │ - │ │ - │ Your Code │ - │ ai.generate(prompt="Hello!") │ - │ │ │ - │ │ (1) Genkit receives your request │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ GoogleAI or │ Plugin handles auth & config │ - │ │ VertexAI │ (API key or GCP credentials) │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (2) Request converted to Gemini format │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ GeminiModel │ Handles message formatting, tools, │ - │ │ │ streaming, and response parsing │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (3) HTTP request to Google's servers │ - │ ▼ │ - │ ════════════════════════════════════════════════════ │ - │ │ Internet │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Google Gemini │ AI processes your prompt │ - │ │ API │ │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (4) Response streamed back │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Your App │ response.text contains the answer! │ - │ └─────────────────┘ │ - └─────────────────────────────────────────────────────────────────────────┘ - -Architecture Overview:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Google GenAI Plugin │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ Plugin Entry Point (__init__.py) │ - │ ├── GoogleAI - Plugin for Gemini API (api_key auth) │ - │ ├── VertexAI - Plugin for Vertex AI (GCP auth) │ - │ ├── Model version enums (GoogleAIGeminiVersion, VertexAIGeminiVersion) │ - │ └── Config schemas (GeminiConfigSchema, etc.) │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ google.py - Plugin Implementation │ - │ ├── GoogleAI class (Gemini API integration) │ - │ └── VertexAI class (Vertex AI integration) │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ models/gemini.py - Gemini Model Implementation │ - │ ├── GeminiModel (generation logic) │ - │ ├── Request/response conversion │ - │ └── Streaming and tool calling support │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ models/embedder.py - Embedding Implementation │ - │ ├── GeminiEmbedder (text embeddings) │ - │ └── EmbeddingTaskType enum │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ models/imagen.py - Image Generation │ - │ └── ImagenModel (Vertex AI only) │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ models/veo.py - Video Generation │ - │ └── VeoModel (Vertex AI only) │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ models/lyria.py - Audio Generation │ - │ └── LyriaModel (Vertex AI only) │ - └─────────────────────────────────────────────────────────────────────────┘ - -Overview: - The Google GenAI plugin supports two backends: - - **GoogleAI**: Direct Gemini API access (requires GEMINI_API_KEY) - - **VertexAI**: Google Cloud Vertex AI platform access - - Both plugins register Gemini models and text embedding models as - Genkit actions, enabling generation and embedding operations. - -Supported Models: - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Plugin │ Models │ - ├────────────┼─────────────────────────────────────────────────────────────┤ - │ GoogleAI │ gemini-flash-latest, gemini-pro-latest, etc. │ - │ VertexAI │ Same Gemini models + imagen-4.0-generate-001 │ - └────────────┴─────────────────────────────────────────────────────────────┘ - -Key Components: - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Component │ Purpose │ - ├───────────────────────┼─────────────────────────────────────────────────┤ - │ GoogleAI │ Plugin for Gemini API (api_key auth) │ - │ VertexAI │ Plugin for Vertex AI (GCP project auth) │ - │ GeminiConfigSchema │ Configuration schema for Gemini models │ - │ GeminiEmbeddingModels │ Enum of available GoogleAI embedding models │ - │ VertexEmbeddingModels │ Enum of available VertexAI embedding models │ - │ EmbeddingTaskType │ Task types for embeddings (CLUSTERING, etc.) │ - └───────────────────────┴─────────────────────────────────────────────────┘ +This plugin provides integration with Google's generative AI models through +either Google AI (Gemini API) or Google Cloud Vertex AI. It dynamically discovers +and registers available models and embedders at runtime. Example: Using GoogleAI (Gemini API): @@ -156,17 +28,19 @@ from genkit import Genkit from genkit_google_genai import GoogleAI - # Uses GEMINI_API_KEY env var or pass api_key explicitly - ai = Genkit(plugins=[GoogleAI()], model='googleai/gemini-flash-latest') + # 1. Initialize Genkit with the GoogleAI plugin + ai = Genkit(plugins=[GoogleAI()]) - response = await ai.generate(prompt='Hello, world!') - print(response.text) - - # Embeddings - embeddings = await ai.embed( - embedder='googleai/gemini-embedding-001', - content='Hello, world!', + # 2. Generate content using dynamic model discovery + res = await ai.generate( + model='googleai/gemini-flash-latest', + prompt='Suggest 2 catchy names for a space coffee shop.', ) + + # 3. Inspect output shapes directly + print(res.text) + # => 1. AstroBrew + # 2. Nebula Nectar ``` Using VertexAI (Google Cloud): @@ -175,31 +49,27 @@ from genkit import Genkit from genkit_google_genai import VertexAI - # Uses default GCP credentials; optionally pass project/location - ai = Genkit( - plugins=[VertexAI(project='my-project', location='us-central1')], - model='vertexai/gemini-flash-latest', - ) + # 1. Initialize with your GCP project and location + ai = Genkit(plugins=[VertexAI(project='my-project', location='us-central1')]) - response = await ai.generate(prompt='Hello, world!') - print(response.text) - - # Image generation with Imagen - response = await ai.generate( - model='vertexai/imagen-4.0-generate-001', - prompt='A beautiful sunset over mountains', + # 2. Generate content with Gemini Pro on Vertex AI + res = await ai.generate( + model='vertexai/gemini-pro-latest', + prompt='Explain quantum entanglement in one sentence.', ) + + # 3. Inspect output shapes directly + print(res.text) + # => "Quantum entanglement occurs when paired particles remain linked..." ``` -Caveats: - - GoogleAI requires GEMINI_API_KEY environment variable - - VertexAI uses Google Cloud credentials (ADC or explicit) - - Model names are prefixed with 'googleai/' or 'vertexai/' +Requirements: + - GoogleAI requires the ``GEMINI_API_KEY`` environment variable or explicit ``api_key``. + - VertexAI requires Google Cloud Application Default Credentials (ADC) or explicit credentials. See Also: - Gemini API: https://ai.google.dev/ - Vertex AI: https://cloud.google.com/vertex-ai - - Model catalog: https://genkit.dev/docs/models """ from genkit_google_genai.google import ( diff --git a/py/packages/genkit-google-genai/src/genkit_google_genai/evaluators/__init__.py b/py/packages/genkit-google-genai/src/genkit_google_genai/evaluators/__init__.py index 6c9f2e4001..88d30f1a69 100644 --- a/py/packages/genkit-google-genai/src/genkit_google_genai/evaluators/__init__.py +++ b/py/packages/genkit-google-genai/src/genkit_google_genai/evaluators/__init__.py @@ -20,119 +20,32 @@ These evaluators assess model outputs for quality metrics like BLEU, ROUGE, fluency, safety, groundedness, and summarization quality. -Key Concepts (ELI5):: - - ┌─────────────────────┬────────────────────────────────────────────────────┐ - │ Concept │ ELI5 Explanation │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Evaluator │ A "grader" that scores your AI's answers. │ - │ │ Like a teacher checking homework. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ BLEU Score │ Compares AI output to a "correct" answer. │ - │ │ Higher = closer to the reference text. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ ROUGE Score │ Measures how much key info is captured. │ - │ │ Good for checking if summaries hit key points. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Fluency │ How natural and readable the text is. │ - │ │ Does it sound like a human wrote it? │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Safety │ Is the content appropriate and safe? │ - │ │ No harmful, biased, or inappropriate content. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Groundedness │ Does the answer stick to the facts given? │ - │ │ No making things up (hallucinations). │ - └─────────────────────┴────────────────────────────────────────────────────┘ - -Data Flow:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ EVALUATION PIPELINE │ - │ │ - │ Test Dataset │ - │ [input, output, reference, context] │ - │ │ │ - │ ▼ │ - │ ┌─────────────────────────────────────────────────────────────────┐ │ - │ │ Vertex AI Evaluators │ │ - │ │ │ │ - │ │ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────────────────┐ │ │ - │ │ │ BLEU │ │ ROUGE │ │ Fluency │ │ Groundedness │ │ │ - │ │ │ (0.72) │ │ (0.68) │ │ (4/5) │ │ (5/5) │ │ │ - │ │ └─────────┘ └─────────┘ └─────────┘ └─────────────────────┘ │ │ - │ │ │ │ - │ │ ┌─────────┐ ┌──────────────┐ ┌───────────────────────────────┐│ │ - │ │ │ Safety │ │ Summarization│ │ Summarization Helpfulness ││ │ - │ │ │ (5/5) │ │ Quality (4/5)│ │ (4/5) ││ │ - │ │ └─────────┘ └──────────────┘ └───────────────────────────────┘│ │ - │ └─────────────────────────────────────────────────────────────────┘ │ - │ │ │ - │ ▼ │ - │ Evaluation Report │ - │ {"score": 0.85, "details": {"reasoning": "..."}} │ - └─────────────────────────────────────────────────────────────────────────┘ - -Overview: - Vertex AI offers built-in evaluation metrics that use machine learning - to score model outputs. These evaluators are useful for: - - - **Automated testing**: CI/CD quality gates for LLM outputs - - **Model comparison**: Compare different models or prompts - - **Quality assurance**: Catch regressions in output quality - - **Safety checks**: Ensure outputs meet safety standards - -Available Metrics: - +-----------------------------+-------------------------------------------+ - | Metric | Description | - +-----------------------------+-------------------------------------------+ - | BLEU | Compare output to reference (translation) | - | ROUGE | Compare output to reference (summarization)| - | FLUENCY | Assess language mastery and readability | - | SAFETY | Check for harmful/inappropriate content | - | GROUNDEDNESS | Verify output is grounded in context | - | SUMMARIZATION_QUALITY | Overall summarization ability | - | SUMMARIZATION_HELPFULNESS | Usefulness as a summary substitute | - | SUMMARIZATION_VERBOSITY | Conciseness of the summary | - +-----------------------------+-------------------------------------------+ - Example: - Running evaluations: - - >>> from genkit import Genkit - >>> from genkit_google_genai import VertexAI - >>> from genkit_google_genai.evaluators import VertexAIEvaluationMetricType - >>> - >>> ai = Genkit(plugins=[VertexAI(project='my-project')]) - >>> - >>> # Prepare test dataset - >>> dataset = [ - ... { - ... 'input': 'Summarize this article about AI...', - ... 'output': 'AI is transforming industries...', - ... 'reference': 'The article discusses how AI impacts...', - ... 'context': ['Article content here...'], - ... } - ... ] - >>> - >>> # Run fluency evaluation - >>> results = await ai.evaluate( - ... evaluator='vertexai/fluency', - ... dataset=dataset, - ... ) - >>> - >>> for result in results: - ... print(f'Score: {result.evaluation.score}') - ... print(f'Reasoning: {result.evaluation.details.get("reasoning")}') - -Caveats: - - Requires Google Cloud project with Vertex AI API enabled - - Evaluators are billed per API call - - Some metrics require specific fields (e.g., GROUNDEDNESS needs context) - - Scores are subjective assessments, not ground truth - -See Also: - - Vertex AI Evaluation API: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation - - Genkit evaluation docs: https://genkit.dev/docs/evaluation + ```python + from genkit import Genkit + from genkit_google_genai import VertexAI + + # 1. Initialize Genkit with VertexAI plugin + ai = Genkit(plugins=[VertexAI(project='my-project', location='us-central1')]) + + # 2. Prepare dataset with input and model output + dataset = [ + { + 'input': 'What is the capital of France?', + 'output': 'Paris is the capital of France.', + } + ] + + # 3. Evaluate output fluency using Vertex AI Evaluators + results = await ai.evaluate( + evaluator='vertexai/fluency', + dataset=dataset, + ) + + # 4. Inspect evaluation score directly + print(results[0].evaluation.score) + # => 5.0 + ``` """ from genkit_google_genai.evaluators.evaluation import ( diff --git a/py/packages/genkit-google-genai/src/genkit_google_genai/evaluators/evaluation.py b/py/packages/genkit-google-genai/src/genkit_google_genai/evaluators/evaluation.py index 1760856414..7c483586d2 100644 --- a/py/packages/genkit-google-genai/src/genkit_google_genai/evaluators/evaluation.py +++ b/py/packages/genkit-google-genai/src/genkit_google_genai/evaluators/evaluation.py @@ -17,40 +17,13 @@ """Vertex AI Evaluation implementation. This module implements the Vertex AI Evaluation API for evaluating model outputs -using built-in metrics like BLEU, ROUGE, fluency, safety, and more. - -Architecture:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Vertex AI Evaluators Module │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ Types & Configuration │ - │ ├── VertexAIEvaluationMetricType (enum) - Available metrics │ - │ └── VertexAIEvaluationMetricConfig - Per-metric configuration │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ EvaluatorFactory │ - │ ├── evaluate_instances() - Async API call to evaluateInstances │ - │ └── create_evaluator_fn() - Creates evaluator function for metric │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ Evaluator Configurations (per metric) │ - │ ├── BLEU - to_request(), response_handler() │ - │ ├── ROUGE - to_request(), response_handler() │ - │ ├── FLUENCY - to_request(), response_handler() │ - │ ├── SAFETY - to_request(), response_handler() │ - │ ├── GROUNDEDNESS - to_request(), response_handler() │ - │ ├── SUMMARIZATION_QUALITY - to_request(), response_handler() │ - │ ├── SUMMARIZATION_HELPFULNESS - to_request(), response_handler() │ - │ └── SUMMARIZATION_VERBOSITY - to_request(), response_handler() │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ Plugin Integration │ - │ └── create_vertex_evaluators() - Register evaluators with Genkit │ - └─────────────────────────────────────────────────────────────────────────┘ +using built-in metrics such as BLEU, ROUGE, fluency, safety, groundedness, and +summarization quality. Implementation Notes: - - Uses Google Cloud Application Default Credentials (ADC) for auth - - Calls the Vertex AI Platform evaluateInstances v1beta1 endpoint - - Each metric has a specific request format and response handler - - Supports custom metric_spec for fine-tuning metric behavior + - Uses Google Cloud Application Default Credentials (ADC) for authentication. + - Calls the Vertex AI Platform ``evaluateInstances`` v1beta1 endpoint. + - Supports custom metric specifications for fine-tuning evaluation behavior. """ from __future__ import annotations diff --git a/py/packages/genkit-google-genai/src/genkit_google_genai/google.py b/py/packages/genkit-google-genai/src/genkit_google_genai/google.py index 324ae204ab..7eb62cecab 100644 --- a/py/packages/genkit-google-genai/src/genkit_google_genai/google.py +++ b/py/packages/genkit-google-genai/src/genkit_google_genai/google.py @@ -18,78 +18,32 @@ """Google AI and Vertex AI plugin implementations for Genkit. This module provides the GoogleAI and VertexAI plugins that enable Genkit to use -Google's generative AI models. Both plugins use **dynamic model discovery** to -automatically detect and register available models from the Google GenAI SDK. +Google's generative AI models. Both plugins use dynamic model discovery via the +Google GenAI SDK to detect and register available models at runtime. -Architecture: - ``` - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Dynamic Model Discovery │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ │ - │ Plugin Init │ - │ ┌─────────┐ ┌──────────────┐ ┌─────────────────────────────┐ │ - │ │ GoogleAI│────►│client.models │────►│ Filter & Categorize │ │ - │ │ VertexAI│ │ .list() │ │ ┌─────────┬───────────────┐ │ │ - │ └─────────┘ └──────────────┘ │ │ Action │ Model Type │ │ │ - │ │ ├─────────┼───────────────┤ │ │ - │ │ │generate │ gemini, gemma │ │ │ - │ │ │Content │ │ │ │ - │ │ ├─────────┼───────────────┤ │ │ - │ │ │embed │ text-embedding│ │ │ - │ │ │Content │ │ │ │ - │ │ ├─────────┼───────────────┤ │ │ - │ │ │predict │ imagen │ │ │ - │ │ ├─────────┼───────────────┤ │ │ - │ │ │generate │ veo │ │ │ - │ │ │Videos │ │ │ │ - │ │ └─────────┴───────────────┘ │ │ - │ └─────────────────────────────┘ │ - │ │ - └─────────────────────────────────────────────────────────────────────────┘ - ``` - -Key Concepts: - +--------------------+-------------------------------------------------------+ - | Concept | Description | - +--------------------+-------------------------------------------------------+ - | Dynamic Discovery | Models are discovered at runtime via the API, not | - | | hardcoded. This ensures new models are automatically | - | | available without SDK updates. | - +--------------------+-------------------------------------------------------+ - | Background Models | Long-running operations (e.g., Veo video generation) | - | | use start/check pattern instead of blocking generate. | - +--------------------+-------------------------------------------------------+ - | Action Resolution | On-demand model instantiation when a model is first | - | | used, avoiding upfront initialization overhead. | - +--------------------+-------------------------------------------------------+ - | Namespacing | Models are prefixed with plugin name (e.g., | - | | 'googleai/gemini-flash-latest'). | - +--------------------+-------------------------------------------------------+ - -Supported Model Types: - - **Gemini/Gemma**: Text generation with generateContent action - - **Embedders**: Text embeddings with embedContent action - - **Imagen**: Image generation with predict action - - **Veo**: Video generation with generateVideos action +Supported capabilities include text generation (Gemini/Gemma), text embeddings, +image generation (Imagen), and video generation (Veo). Example: - >>> from genkit import Genkit - >>> from genkit_google_genai import GoogleAI - >>> - >>> # Models are discovered automatically - >>> ai = Genkit(plugins=[GoogleAI()]) - >>> - >>> # Use any available model - no pre-registration needed - >>> response = await ai.generate( - ... model='googleai/gemini-flash-latest', - ... prompt='Hello, world!', - ... ) - -See Also: - - https://ai.google.dev/gemini-api/docs - - https://cloud.google.com/vertex-ai/generative-ai/docs - - JS implementation: js/plugins/google-genai/src/ + ```python + from genkit import Genkit + from genkit_google_genai import GoogleAI + + # 1. Initialize Genkit with dynamic model discovery + ai = Genkit(plugins=[GoogleAI()]) + + # 2. Generate content using any discovered Gemini model + response = await ai.generate( + model='googleai/gemini-flash-latest', + prompt='Suggest 3 names for a space-themed coffee shop.', + ) + + # 3. Inspect output shapes directly + print(response.text) + # => 1. AstroBrew + # 2. Nebula Nectar + # 3. Cosmic Cup + ``` """ import os @@ -358,35 +312,31 @@ class GoogleAI(Plugin): initialization time, ensuring new models are available without SDK updates. Model Types: - +------------------+-------------------+--------------------------------+ - | Type | Action Kind | Example | - +------------------+-------------------+--------------------------------+ - | Gemini/Gemma | MODEL | googleai/gemini-flash-latest | - | Imagen | MODEL | googleai/imagen-3.0-generate | - | Embedders | EMBEDDER | googleai/gemini-embedding-001 | - | Veo (video) | BACKGROUND_MODEL | googleai/veo-2.0-generate-001 | - +------------------+-------------------+--------------------------------+ + | Type | Action Kind | Example | + |---|---|---| + | Gemini / Gemma | MODEL | ``googleai/gemini-flash-latest`` | + | Imagen | MODEL | ``googleai/imagen-3.0-generate-002`` | + | Embedders | EMBEDDER | ``googleai/text-embedding-004`` | + | Veo (Video) | BACKGROUND_MODEL | ``googleai/veo-2.0-generate-001`` | Example: - >>> from genkit import Genkit - >>> from genkit_google_genai import GoogleAI - >>> - >>> ai = Genkit(plugins=[GoogleAI()]) - >>> - >>> # Text generation - >>> response = await ai.generate( - ... model='googleai/gemini-flash-latest', - ... prompt='Explain quantum computing', - ... ) - >>> - >>> # Video generation (background model) - >>> op = await ai.generate( - ... model='googleai/veo-2.0-generate-001', - ... prompt='A sunset over mountains', - ... ) - >>> while not op.done: - ... await asyncio.sleep(5) - ... op = await ai.check_operation(op) + ```python + from genkit import Genkit + from genkit_google_genai import GoogleAI + + # 1. Initialize Genkit with dynamic model discovery + ai = Genkit(plugins=[GoogleAI()]) + + # 2. Generate text using Gemini Flash + res = await ai.generate( + model='googleai/gemini-flash-latest', + prompt='Explain quantum computing in one sentence.', + ) + + # 3. Inspect output text directly + print(res.text) + # => Quantum computing utilizes quantum bits to solve complex problems faster... + ``` Attributes: name: The plugin name ('googleai'). @@ -726,32 +676,31 @@ class VertexAI(Plugin): - Imagen image generation models Model Types: - +------------------+-------------------+--------------------------------+ - | Type | Action Kind | Example | - +------------------+-------------------+--------------------------------+ - | Gemini/Gemma | MODEL | vertexai/gemini-flash-latest | - | Imagen | MODEL | vertexai/imagen-3.0-generate | - | Veo (video) | MODEL | vertexai/veo-2.0-generate-001 | - | Embedders | EMBEDDER | vertexai/text-embedding-005 | - +------------------+-------------------+--------------------------------+ + | Type | Action Kind | Example | + |---|---|---| + | Gemini / Gemma | MODEL | ``vertexai/gemini-flash-latest`` | + | Imagen | MODEL | ``vertexai/imagen-3.0-generate-002`` | + | Veo (Video) | MODEL | ``vertexai/veo-2.0-generate-001`` | + | Embedders | EMBEDDER | ``vertexai/text-embedding-005`` | Example: - >>> from genkit import Genkit - >>> from genkit_google_genai import VertexAI - >>> - >>> ai = Genkit(plugins=[VertexAI(project='my-project')]) - >>> - >>> # Text generation - >>> response = await ai.generate( - ... model='vertexai/gemini-flash-latest', - ... prompt='Explain quantum computing', - ... ) - >>> - >>> # Image generation (Vertex AI only) - >>> response = await ai.generate( - ... model='vertexai/imagen-3.0-generate-002', - ... prompt='A serene mountain landscape', - ... ) + ```python + from genkit import Genkit + from genkit_google_genai import VertexAI + + # 1. Initialize Genkit with VertexAI plugin + ai = Genkit(plugins=[VertexAI(project='my-project', location='us-central1')]) + + # 2. Generate text using Gemini on Vertex AI + res = await ai.generate( + model='vertexai/gemini-flash-latest', + prompt='Explain quantum computing in one sentence.', + ) + + # 3. Inspect output text directly + print(res.text) + # => Quantum computing utilizes quantum bits to solve complex problems faster... + ``` Attributes: name: The plugin name ('vertexai'). diff --git a/py/packages/genkit-google-genai/src/genkit_google_genai/models/lyria.py b/py/packages/genkit-google-genai/src/genkit_google_genai/models/lyria.py index c93a75f2c2..e5823a1f53 100644 --- a/py/packages/genkit-google-genai/src/genkit_google_genai/models/lyria.py +++ b/py/packages/genkit-google-genai/src/genkit_google_genai/models/lyria.py @@ -14,52 +14,11 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Lyria audio generation model for Google Vertex AI plugin. - -Lyria is Google's music and audio generation model that creates audio from -text prompts. It's available through Vertex AI only (not Google AI). - -Architecture: - ``` - ┌──────────────────────────────────────────────────────────────────────┐ - │ Lyria Audio Generation Flow │ - ├──────────────────────────────────────────────────────────────────────┤ - │ │ - │ Input Model Output │ - │ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ - │ │ Text │ ─predict──► │ Lyria │ ──────────► │ Audio │ │ - │ │ Prompt │ │ Model │ │ (WAV) │ │ - │ └─────────┘ └─────────┘ └─────────┘ │ - │ │ - └──────────────────────────────────────────────────────────────────────┘ - ``` - -Supported Models: - +----------------------+--------------------------------------------------+ - | Model | Description | - +----------------------+--------------------------------------------------+ - | lyria-002 | Lyria 002 - Audio generation from text | - +----------------------+--------------------------------------------------+ - -Example: - >>> from genkit import Genkit - >>> from genkit_google_genai import VertexAI - >>> - >>> ai = Genkit(plugins=[VertexAI(project='my-project')]) - >>> - >>> # Generate audio - >>> response = await ai.generate( - ... model='vertexai/lyria-002', - ... prompt='A peaceful piano melody with gentle rain sounds', - ... ) - >>> - >>> # Response contains audio as base64-encoded WAV - >>> audio_content = response.message.content[0] - >>> print(audio_content.media.content_type) # 'audio/wav' - -See Also: - - Vertex AI Audio: https://cloud.google.com/vertex-ai/docs/generative-ai/audio - - JS implementation: js/plugins/google-genai/src/vertexai/lyria.ts +"""Lyria audio generation helpers for Google Vertex AI. + +Lyria is Google's music and audio generation model available through Vertex AI. +This module exposes config and request/response helpers for the predict-based +audio API. """ import sys diff --git a/py/packages/genkit-google-genai/src/genkit_google_genai/models/veo.py b/py/packages/genkit-google-genai/src/genkit_google_genai/models/veo.py index f66a127e10..660d8e3c1e 100644 --- a/py/packages/genkit-google-genai/src/genkit_google_genai/models/veo.py +++ b/py/packages/genkit-google-genai/src/genkit_google_genai/models/veo.py @@ -17,53 +17,6 @@ """Veo video generation model for Google GenAI plugin. Veo is Google's video generation model that creates videos from text prompts. -It uses the background model pattern because video generation is a long-running -operation that can take 30 seconds to several minutes. - -Architecture: - ``` - ┌──────────────────────────────────────────────────────────────────────┐ - │ Veo Video Generation Flow │ - ├──────────────────────────────────────────────────────────────────────┤ - │ │ - │ 1. START 2. POLL 3. COMPLETE │ - │ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ - │ │ Prompt │ ─predict──► │Operation│ ─getOp()──► │ Video │ │ - │ │ +cfg │ LongRun │ (name) │ ... │ (URI) │ │ - │ └─────────┘ └────┬────┘ └─────────┘ │ - │ │ - └──────────────────────────────────────────────────────────────────────┘ - ``` - -Note: - Veo models are discovered dynamically via the Google GenAI SDK's models.list() API. - Any model with 'generateVideos' in supported_actions or 'veo' in the name is treated - as a Veo model. - -Example: - >>> from genkit import Genkit - >>> from genkit_google_genai import GoogleAI - >>> - >>> ai = Genkit(plugins=[GoogleAI()]) - >>> - >>> # Start video generation - >>> response = await ai.generate( - ... model='googleai/veo-2.0-generate-001', - ... prompt='A cat playing piano in a jazz club', - ... ) - >>> - >>> # Poll until complete - >>> operation = response.operation - >>> while not operation.done: - ... await asyncio.sleep(5) - ... operation = await ai.check_operation(operation) - >>> - >>> # Get the video URL - >>> print(operation.output) - -See Also: - - https://ai.google.dev/gemini-api/docs/video - - JS implementation: js/plugins/google-genai/src/googleai/veo.ts """ import asyncio diff --git a/py/packages/genkit-middleware/src/genkit_middleware/__init__.py b/py/packages/genkit-middleware/src/genkit_middleware/__init__.py index 39a9ab3cb1..7efb44e0c4 100644 --- a/py/packages/genkit-middleware/src/genkit_middleware/__init__.py +++ b/py/packages/genkit-middleware/src/genkit_middleware/__init__.py @@ -77,14 +77,26 @@ class Middleware(MiddlewarePlugin): constructing an instance, for example ``Filesystem(root_dir='./workspace')``. - Usage: - from genkit_middleware import Middleware, Retry, Skills + Example: + ```python + from genkit import Genkit + from genkit_google_genai import GoogleAI + from genkit_middleware import Middleware, Retry + # 1. Register middleware plugin ai = Genkit(plugins=[GoogleAI(), Middleware()]) - await ai.generate( - prompt='Hello', - use=[Retry(max_retries=5), Skills(skill_paths=['skills'])], + + # 2. Generate with automatic retry resilience + res = await ai.generate( + model='googleai/gemini-flash-latest', + prompt='Summarize quantum computing.', + use=[Retry(max_retries=3)], ) + + # 3. Inspect output + print(res.text) + # => Quantum computing uses quantum mechanics for complex calculations... + ``` """ name = 'genkit-middleware' diff --git a/py/packages/genkit-ollama/src/genkit_ollama/__init__.py b/py/packages/genkit-ollama/src/genkit_ollama/__init__.py index 9d60ea1cde..aea40c40ad 100644 --- a/py/packages/genkit-ollama/src/genkit_ollama/__init__.py +++ b/py/packages/genkit-ollama/src/genkit_ollama/__init__.py @@ -16,143 +16,35 @@ """Ollama plugin for Genkit. -This plugin provides integration with Ollama for running local LLMs. Ollama -allows you to run models like Llama, Mistral, and others on your own hardware. - -Key Concepts (ELI5):: - - ┌─────────────────────┬────────────────────────────────────────────────────┐ - │ Concept │ ELI5 Explanation │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Ollama │ Software that runs AI models on YOUR computer. │ - │ │ Like having a mini ChatGPT at home. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Local LLM │ An AI that runs offline on your machine. │ - │ │ No internet needed, your data stays private. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Llama │ Meta's open-source AI model. Like a free │ - │ │ version of ChatGPT you can run yourself. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Model Pull │ Download a model to your computer. Like │ - │ │ installing an app before you can use it. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Server URL │ Where Ollama listens for requests. Default │ - │ │ is localhost:11434 (your own computer). │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ GGUF │ File format for AI models. Like .mp3 for │ - │ │ music, but for AI brains. │ - └─────────────────────┴────────────────────────────────────────────────────┘ - -Data Flow:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ HOW OLLAMA RUNS AI ON YOUR COMPUTER │ - │ │ - │ Your Code │ - │ ai.generate(prompt="Hello!") │ - │ │ │ - │ │ (1) Request goes to Ollama plugin │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Ollama Plugin │ Formats request for Ollama API │ - │ │ │ │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (2) HTTP to localhost:11434 (your computer!) │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Ollama Server │ Loads model into RAM/GPU │ - │ │ (on your PC) │ (first request may be slow) │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (3) Model processes on YOUR hardware │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Llama/Mistral │ CPU or GPU does the thinking │ - │ │ Model (local) │ No data leaves your machine! │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (4) Response streamed back │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Your App │ response.text = "Hello! How can I help?" │ - │ └─────────────────┘ │ - └─────────────────────────────────────────────────────────────────────────┘ - -Architecture Overview:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Ollama Plugin │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ Plugin Entry Point (__init__.py) │ - │ ├── Ollama - Plugin class │ - │ └── ollama_name() - Helper to create namespaced model names │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ plugin_api.py - Plugin Implementation │ - │ ├── Ollama class (registers models and embedders) │ - │ └── Configuration for server URL and models │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ models.py - Model Implementation │ - │ ├── OllamaModel (chat/generate API integration) │ - │ ├── Request/response conversion │ - │ └── Streaming support │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ embedders.py - Embedding Implementation │ - │ └── OllamaEmbedder (embedding API integration) │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ constants.py - Default Configuration │ - │ └── DEFAULT_OLLAMA_SERVER_URL │ - └─────────────────────────────────────────────────────────────────────────┘ - -Overview: - The Ollama plugin connects Genkit to locally running Ollama models. - This is ideal for development, privacy-sensitive applications, or - when you want to run models without cloud dependencies. +This plugin provides integration with Ollama for running local LLMs and text +embedders directly on your own infrastructure. Prerequisites: - Install Ollama: https://ollama.ai/ - - Pull a model: ``ollama pull llama3.2`` or ``ollama pull mistral`` - - Ollama server running (default: http://localhost:11434) - -Key Components: - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Component │ Purpose │ - ├───────────────────┼─────────────────────────────────────────────────────┤ - │ Ollama │ Plugin class to register with Genkit │ - │ ollama_name() │ Helper to create namespaced model names │ - └───────────────────┴─────────────────────────────────────────────────────┘ + - Ensure the Ollama server is running (default: ``http://localhost:11434``). + - Pull target models locally (e.g., ``ollama pull llama3.2``). Example: - Basic usage: - ```python from genkit import Genkit from genkit_ollama import Ollama - # Configure with model name and optional server URL - ai = Genkit( - plugins=[Ollama(models=['llama3.2', 'mistral'])], + # 1. Initialize Genkit with local Ollama plugin (models resolve on demand) + ai = Genkit(plugins=[Ollama()]) + + # 2. Generate content entirely on local hardware + res = await ai.generate( model='ollama/llama3.2', + prompt='Why run AI models locally in 10 words?', ) - response = await ai.generate(prompt='Hello, Llama!') - print(response.text) - - # Use a specific model - response = await ai.generate( - model='ollama/mistral', - prompt='Write a haiku about coding', - ) + # 3. Inspect output shapes directly + print(res.text) + # => Complete data privacy with zero cloud latency or API costs. ``` -Caveats: - - Requires Ollama installed and running locally - - Model names are prefixed with 'ollama/' (e.g., 'ollama/llama3.2') - - Performance depends on local hardware - See Also: - Ollama documentation: https://ollama.ai/ - - Genkit documentation: https://genkit.dev/ """ from genkit_ollama._errors import OllamaConnectionError diff --git a/py/packages/genkit-openai/src/genkit_openai/__init__.py b/py/packages/genkit-openai/src/genkit_openai/__init__.py index 30fd5f2a0d..4e0e421979 100644 --- a/py/packages/genkit-openai/src/genkit_openai/__init__.py +++ b/py/packages/genkit-openai/src/genkit_openai/__init__.py @@ -18,135 +18,34 @@ """OpenAI-compatible model provider for Genkit. This plugin provides integration with OpenAI and any OpenAI-compatible API -endpoints (like Azure OpenAI, Together AI, Anyscale, etc.) for the Genkit -framework. It uses the official OpenAI Python SDK. - -Key Concepts (ELI5):: - - ┌─────────────────────┬────────────────────────────────────────────────────┐ - │ Concept │ ELI5 Explanation │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ OpenAI │ The company that made ChatGPT. This plugin │ - │ │ talks to their API directly. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ OpenAI-compatible │ Many AI providers copy OpenAI's API format. │ - │ │ This plugin works with ALL of them! │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ GPT-4o │ OpenAI's latest flagship model. The "o" means │ - │ │ "omni" - it can see, hear, and chat. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ base_url │ Where to send requests. Change this to use │ - │ │ Together AI, Anyscale, or any compatible API. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Chat Completions │ The API endpoint for conversations. Send │ - │ │ messages, get responses - like texting. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Streaming │ Get the response word-by-word as it's generated. │ - │ │ Feels faster, like watching someone type. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Function Calling │ Let GPT use tools you define. Like giving it │ - │ │ a calculator or database access. │ - └─────────────────────┴────────────────────────────────────────────────────┘ - -Data Flow:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ HOW OPENAI-COMPATIBLE REQUESTS WORK │ - │ │ - │ Your Code │ - │ ai.generate(prompt="Write a poem") │ - │ │ │ - │ │ (1) Request goes to OpenAI plugin │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ OpenAI Plugin │ Adds API key, selects base_url │ - │ │ │ (openai.com, together.xyz, etc.) │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (2) Convert to Chat Completions format │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ OpenAIModel │ Standard OpenAI SDK format works │ - │ │ │ with any compatible provider │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (3) HTTPS to base_url/v1/chat/completions │ - │ ▼ │ - │ ════════════════════════════════════════════════════ │ - │ │ Internet │ - │ ▼ │ - │ ┌─────────────────────────────────────────────────────┐ │ - │ │ OpenAI / Together AI / Anyscale / etc. │ │ - │ │ (any OpenAI-compatible endpoint) │ │ - │ └─────────────────────────┬───────────────────────────┘ │ - │ │ │ - │ │ (4) Streaming response │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Your App │ response.text = "Roses are red..." │ - │ └─────────────────┘ │ - └─────────────────────────────────────────────────────────────────────────┘ - -Architecture Overview:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ OpenAI-Compatible Plugin │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ Plugin Entry Point (__init__.py) │ - │ ├── OpenAI - Plugin class │ - │ ├── openai_model() - Helper to create model references │ - │ └── OpenAIConfig - Configuration schema │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ typing.py - Type-Safe Configuration Classes │ - │ ├── OpenAIConfig (base configuration) │ - │ └── Model-specific parameters │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ openai_plugin.py - Plugin Implementation │ - │ ├── OpenAI class (registers models) │ - │ └── Client initialization with OpenAI SDK │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ models/model.py - Model Implementation │ - │ ├── OpenAIModel (chat completions API) │ - │ ├── Request/response conversion │ - │ └── Streaming support │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ models/handler.py - Request Handler │ - │ └── Message conversion and tool handling │ - └─────────────────────────────────────────────────────────────────────────┘ - -Supported Providers: - - OpenAI (api.openai.com) - - Azure OpenAI - - Together AI - - Anyscale - - Any OpenAI-compatible endpoint +endpoints (such as Azure OpenAI, Together AI, or Anyscale) using the official +OpenAI Python SDK. Example: ```python from genkit import Genkit from genkit_openai import OpenAI - # Uses OPENAI_API_KEY env var or pass api_key explicitly - ai = Genkit(plugins=[OpenAI()], model='openai/gpt-4o') + # 1. Initialize Genkit with OpenAI plugin + ai = Genkit(plugins=[OpenAI()]) - response = await ai.generate(prompt='Hello, GPT!') - print(response.text) - - # With custom endpoint (e.g., Together AI) - ai = Genkit( - plugins=[OpenAI(base_url='https://api.together.xyz/v1')], - model='openai/meta-llama/Llama-3-70b-chat-hf', + # 2. Generate content using GPT-4o + res = await ai.generate( + model='openai/gpt-4o', + prompt='Suggest 2 catchy names for an AI newsletter.', ) + + # 3. Inspect output shapes directly + print(res.text) + # => 1. Prompt Daily + # 2. Neural Notes ``` -Caveats: - - Requires OPENAI_API_KEY environment variable or api_key parameter - - Model names are prefixed with 'openai/' (e.g., 'openai/gpt-4o') - - Custom endpoints may have different model availability +Requirements: + - Requires the ``OPENAI_API_KEY`` environment variable or explicit ``api_key``. See Also: - OpenAI documentation: https://platform.openai.com/docs/ - - Genkit documentation: https://genkit.dev/ """ from .openai_plugin import OpenAI, openai_model diff --git a/py/packages/genkit-openai/src/genkit_openai/models/audio.py b/py/packages/genkit-openai/src/genkit_openai/models/audio.py index bef34227bf..7a597f636b 100644 --- a/py/packages/genkit-openai/src/genkit_openai/models/audio.py +++ b/py/packages/genkit-openai/src/genkit_openai/models/audio.py @@ -21,36 +21,6 @@ Supported TTS models: tts-1, tts-1-hd, gpt-4o-mini-tts Supported STT models: gpt-4o-transcribe, gpt-4o-mini-transcribe, whisper-1 - -Data Flow (TTS):: - - ┌─────────────────────────────────────────────────────────────────────┐ - │ ModelRequest (text input) │ - │ │ │ - │ ▼ │ - │ to_tts_params() ──► SpeechCreateParams │ - │ │ │ - │ ▼ │ - │ client.audio.speech.create() │ - │ │ │ - │ ▼ │ - │ to_tts_response() ──► ModelResponse (audio media part) │ - └─────────────────────────────────────────────────────────────────────┘ - -Data Flow (STT):: - - ┌─────────────────────────────────────────────────────────────────────┐ - │ ModelRequest (audio media input) │ - │ │ │ - │ ▼ │ - │ to_stt_params() ──► TranscriptionCreateParams │ - │ │ │ - │ ▼ │ - │ client.audio.transcriptions.create() │ - │ │ │ - │ ▼ │ - │ to_stt_response() ──► ModelResponse (text part) │ - └─────────────────────────────────────────────────────────────────────┘ """ from __future__ import annotations diff --git a/py/packages/genkit-openai/src/genkit_openai/models/image.py b/py/packages/genkit-openai/src/genkit_openai/models/image.py index 1e17e0d974..5b295c759c 100644 --- a/py/packages/genkit-openai/src/genkit_openai/models/image.py +++ b/py/packages/genkit-openai/src/genkit_openai/models/image.py @@ -18,21 +18,6 @@ Provides image generation capabilities via the OpenAI Images API, supporting models like DALL-E 3 and GPT-Image-1. - -Data Flow:: - - ┌─────────────────────────────────────────────────────────────────────┐ - │ ModelRequest (text prompt) │ - │ │ │ - │ ▼ │ - │ to_image_generate_params() ──► ImageGenerateParams │ - │ │ │ - │ ▼ │ - │ client.images.generate() │ - │ │ │ - │ ▼ │ - │ to_generate_response() ──► ModelResponse (media parts) │ - └─────────────────────────────────────────────────────────────────────┘ """ from __future__ import annotations diff --git a/py/packages/genkit-vertexai/src/genkit_vertexai/__init__.py b/py/packages/genkit-vertexai/src/genkit_vertexai/__init__.py index 6478275137..87cba565c0 100644 --- a/py/packages/genkit-vertexai/src/genkit_vertexai/__init__.py +++ b/py/packages/genkit-vertexai/src/genkit_vertexai/__init__.py @@ -20,116 +20,33 @@ including Model Garden for accessing third-party models and Vector Search for RAG applications. -Key Concepts (ELI5):: - - ┌─────────────────────┬────────────────────────────────────────────────────┐ - │ Concept │ ELI5 Explanation │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Vertex AI │ Google Cloud's AI platform. Like a shopping │ - │ │ mall where you can access many AI services. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Model Garden │ A catalog of AI models from different companies. │ - │ │ Like an app store but for AI models. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Vector Search │ Find similar items using math. Like asking │ - │ │ "show me documents similar to this one." │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ RAG │ Retrieval-Augmented Generation. Let AI search │ - │ │ your documents before answering. Like giving │ - │ │ AI a reference book to look things up. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ BigQuery │ Google's data warehouse. Store and search │ - │ │ huge amounts of data super fast. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Firestore │ Google's NoSQL database. Store documents │ - │ │ as flexible JSON-like data. │ - ├─────────────────────┼────────────────────────────────────────────────────┤ - │ Embeddings │ Turn text into numbers for comparison. │ - │ │ Like converting words to GPS coordinates. │ - └─────────────────────┴────────────────────────────────────────────────────┘ - -Data Flow (Vector Search):: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ HOW VECTOR SEARCH FINDS SIMILAR DOCUMENTS │ - │ │ - │ Your Query: "How do I reset my password?" │ - │ │ │ - │ │ (1) Query converted to embedding (numbers) │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Embedder │ Text → [0.12, -0.45, 0.78, ...] │ - │ │ (Gemini) │ (hundreds of numbers) │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (2) Search for similar embeddings │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Vector Index │ Find documents with similar │ - │ │ (BigQuery or │ number patterns │ - │ │ Firestore) │ │ - │ └────────┬────────┘ │ - │ │ │ - │ │ (3) Return matching documents │ - │ ▼ │ - │ ┌─────────────────┐ │ - │ │ Results │ "Password Reset Guide" (95% match) │ - │ │ │ "Account Recovery FAQ" (87% match) │ - │ └─────────────────┘ │ - └─────────────────────────────────────────────────────────────────────────┘ - -Architecture Overview:: - - ┌─────────────────────────────────────────────────────────────────────────┐ - │ Vertex AI Plugin │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ Plugin Entry Point (__init__.py) │ - │ ├── ModelGarden - Access third-party models via Model Garden │ - │ ├── Vector Search Retrievers (BigQuery, Firestore) │ - │ └── Helper functions for defining vector search │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ model_garden/modelgarden_plugin.py - Model Garden Integration │ - │ ├── ModelGarden class │ - │ └── Access to Anthropic, Llama, Mistral via Vertex AI │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ model_garden/client.py - API Client │ - │ └── Google Cloud client initialization │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ model_garden/anthropic.py - Anthropic Models │ - │ └── Claude models via Vertex AI Model Garden │ - ├─────────────────────────────────────────────────────────────────────────┤ - │ vector_search.py - Vector Search Integration │ - │ ├── BigQueryRetriever - Vector search with BigQuery backend │ - │ ├── FirestoreRetriever - Vector search with Firestore backend │ - │ └── RetrieverOptionsSchema - Configuration for retrievers │ - └─────────────────────────────────────────────────────────────────────────┘ - -Key Components: - - ModelGarden: Access third-party models (Anthropic, Meta, Mistral) - through Vertex AI Model Garden - - BigQueryRetriever: Vector similarity search using BigQuery - - FirestoreRetriever: Vector similarity search using Firestore - Example: ```python from genkit import Genkit from genkit_vertexai.model_garden import ModelGarden - # Model Garden for third-party models + # 1. Initialize Genkit with the Model Garden plugin ai = Genkit( plugins=[ModelGarden(project_id='my-project', location='us-central1')], ) + + # 2. Call models under the modelgarden/ namespace (not vertexai/) + res = await ai.generate( + model='modelgarden/anthropic/claude-3-5-sonnet-v2@20241022', + prompt='Explain recursion in 10 words.', + ) + + # 3. Inspect output shapes directly + print(res.text) + # => A function calling itself until reaching a base stopping condition. ``` -Caveats: - - Requires Google Cloud credentials (ADC or explicit) - - Model Garden requires models to be deployed in your project - - Vector Search requires appropriate index configuration +Requirements: + - Requires Google Cloud Application Default Credentials (ADC) or explicit credentials. See Also: - Vertex AI Model Garden: https://cloud.google.com/vertex-ai/docs/model-garden - Vertex AI Vector Search: https://cloud.google.com/vertex-ai/docs/vector-search - - Genkit documentation: https://genkit.dev/ """ from genkit_vertexai.model_garden import ModelGarden, ModelGardenPlugin