diff --git a/docs.json b/docs.json index f6821c8..8908b1e 100644 --- a/docs.json +++ b/docs.json @@ -170,6 +170,20 @@ "pages": [ "v3/expert-models/features/video/generation-async" ] + }, + { + "group": "Web Features", + "icon": "globe", + "expanded": false, + "pages": [ + "v3/expert-models/features/web/scraping", + "v3/expert-models/features/web/map", + "v3/expert-models/features/web/search", + "v3/expert-models/features/web/crawl-async", + "v3/expert-models/features/web/batch-scrape-async", + "v3/expert-models/features/web/structured-extraction-async", + "v3/expert-models/features/web/research-async" + ] } ] }, diff --git a/v3/expert-models/features/web/batch-scrape-async.mdx b/v3/expert-models/features/web/batch-scrape-async.mdx new file mode 100644 index 0000000..2fe3ce6 --- /dev/null +++ b/v3/expert-models/features/web/batch-scrape-async.mdx @@ -0,0 +1,101 @@ +--- +title: "Batch Web Scraping" +description: "Scrape many URLs in one request and return each page's content. Runs as an async job." +--- + +import { TechArticleSchema } from "/snippets/TechArticleSchema.mdx"; + + + +## Endpoint + +`POST /v3/universal-ai/async` (async) + +Model string pattern: `web/batch_scrape_async/{provider}[/{model}]` + +## Input + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| urls | array[string] | Yes | Explicit list of URLs to scrape (async job) | + + +## Output + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| **pages** | array[object] | No | | +|     url | string | Yes | | +|     content | string | No | | +|     title | string | No | | +|     description | string | No | | +|     status_code | int | No | | +|     links | array[string] | No | | +|     **images** | array[object] | No | | +|         url | string | Yes | | +|         description | string | No | | +|         width | int | No | | +|         height | int | No | | +|     raw_html | string | No | | +|     summary | string | No | | +|     published_date | string | No | | +|     author | string | No | | +|     favicon | string | No | | +| failed_results | array[dict] | No | | +| usage | dict | No | | + + +## Available Providers + +| Provider | Model String | Price | +|----------|-------------|-------| +| firecrawl | `web/batch_scrape_async/firecrawl` | $0.0038 per token | + + +## Quick Start + + +> This is an **async** feature. The initial response returns a job ID. Poll `GET /v3/universal-ai/async/{job_id}` until the job completes. + + +```python Python +import requests + +url = "https://api.edenai.run/v3/universal-ai/async" +headers = { + "Authorization": "Bearer YOUR_API_KEY", + "Content-Type": "application/json" +} + +payload = { + "model": "web/batch_scrape_async/firecrawl", + "input": { + "urls": ["value1", "value2"] + } +} + +response = requests.post(url, headers=headers, json=payload) +print(response.json()) +``` + +```bash cURL +curl -X POST https://api.edenai.run/v3/universal-ai/async \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "web/batch_scrape_async/firecrawl", + "input": {"urls": ["value1", "value2"]} + }' +``` + + diff --git a/v3/expert-models/features/web/crawl-async.mdx b/v3/expert-models/features/web/crawl-async.mdx new file mode 100644 index 0000000..3fd53da --- /dev/null +++ b/v3/expert-models/features/web/crawl-async.mdx @@ -0,0 +1,104 @@ +--- +title: "Web Crawl" +description: "Crawl a site from a starting URL and return the content of its pages. Runs as an async job." +--- + +import { TechArticleSchema } from "/snippets/TechArticleSchema.mdx"; + + + +## Endpoint + +`POST /v3/universal-ai/async` (async) + +Model string pattern: `web/crawl_async/{provider}[/{model}]` + +## Input + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| url | string | Yes | Root URL to crawl (async job) | + + +## Output + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| **pages** | array[object] | No | | +|     url | string | Yes | | +|     content | string | No | | +|     title | string | No | | +|     description | string | No | | +|     status_code | int | No | | +|     links | array[string] | No | | +|     **images** | array[object] | No | | +|         url | string | Yes | | +|         description | string | No | | +|         width | int | No | | +|         height | int | No | | +|     raw_html | string | No | | +|     summary | string | No | | +|     published_date | string | No | | +|     author | string | No | | +|     favicon | string | No | | +| status | string | No | | +| total | int | No | | +| completed | int | No | | +| base_url | string | No | | +| usage | dict | No | | + + +## Available Providers + +| Provider | Model String | Price | +|----------|-------------|-------| +| firecrawl | `web/crawl_async/firecrawl` | $0.0038 per token | + + +## Quick Start + + +> This is an **async** feature. The initial response returns a job ID. Poll `GET /v3/universal-ai/async/{job_id}` until the job completes. + + +```python Python +import requests + +url = "https://api.edenai.run/v3/universal-ai/async" +headers = { + "Authorization": "Bearer YOUR_API_KEY", + "Content-Type": "application/json" +} + +payload = { + "model": "web/crawl_async/firecrawl", + "input": { + "url": "example" + } +} + +response = requests.post(url, headers=headers, json=payload) +print(response.json()) +``` + +```bash cURL +curl -X POST https://api.edenai.run/v3/universal-ai/async \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "web/crawl_async/firecrawl", + "input": {"url": "example"} + }' +``` + + diff --git a/v3/expert-models/features/web/map.mdx b/v3/expert-models/features/web/map.mdx new file mode 100644 index 0000000..6725c11 --- /dev/null +++ b/v3/expert-models/features/web/map.mdx @@ -0,0 +1,87 @@ +--- +title: "Site Map" +description: "Discover and return the list of URLs on a site (a sitemap-style link map)." +--- + +import { TechArticleSchema } from "/snippets/TechArticleSchema.mdx"; + + + +## Endpoint + +`POST /v3/universal-ai` (sync) + +Model string pattern: `web/map/{provider}[/{model}]` + +## Input + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| url | string | Yes | Root URL to map (discover its links) | + + +## Output + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| url | string | Yes | | +| **links** | array[object] | No | | +|     url | string | Yes | | +|     title | string | No | | +|     description | string | No | | +| base_url | string | No | | +| usage | dict | No | | + + +## Available Providers + +| Provider | Model String | Price | +|----------|-------------|-------| +| firecrawl | `web/map/firecrawl` | $0.0038 per token | + + +## Quick Start + + + +```python Python +import requests + +url = "https://api.edenai.run/v3/universal-ai" +headers = { + "Authorization": "Bearer YOUR_API_KEY", + "Content-Type": "application/json" +} + +payload = { + "model": "web/map/firecrawl", + "input": { + "url": "example" + } +} + +response = requests.post(url, headers=headers, json=payload) +print(response.json()) +``` + +```bash cURL +curl -X POST https://api.edenai.run/v3/universal-ai \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "web/map/firecrawl", + "input": {"url": "example"} + }' +``` + + diff --git a/v3/expert-models/features/web/research-async.mdx b/v3/expert-models/features/web/research-async.mdx new file mode 100644 index 0000000..09ccc16 --- /dev/null +++ b/v3/expert-models/features/web/research-async.mdx @@ -0,0 +1,97 @@ +--- +title: "Deep Research" +description: "Run deep, multi-step web research on a query and return a synthesized answer with sources. Runs as an async job." +--- + +import { TechArticleSchema } from "/snippets/TechArticleSchema.mdx"; + + + +## Endpoint + +`POST /v3/universal-ai/async` (async) + +Model string pattern: `web/research_async/{provider}[/{model}]` + +## Input + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| prompt | string | Yes | Natural-language research task for the agent | +| schema | dict | No | JSON schema describing the structured output | + + +## Output + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| answer | string | No | | +| data | any | No | | +| **sources** | array[object] | No | | +|     title | string | No | | +|     url | string | Yes | | +|     snippet | string | No | | +|     favicon | string | No | | +|     published_date | string | No | | +|     author | string | No | | +|     id | string | No | | +| status | string | No | | +| request_id | string | No | | +| created_at | string | No | | +| usage | dict | No | | + + +## Available Providers + +| Provider | Model String | Price | +|----------|-------------|-------| +| firecrawl | `web/research_async/firecrawl` | $0.0038 per token | + + +## Quick Start + + +> This is an **async** feature. The initial response returns a job ID. Poll `GET /v3/universal-ai/async/{job_id}` until the job completes. + + +```python Python +import requests + +url = "https://api.edenai.run/v3/universal-ai/async" +headers = { + "Authorization": "Bearer YOUR_API_KEY", + "Content-Type": "application/json" +} + +payload = { + "model": "web/research_async/firecrawl", + "input": { + "prompt": "example" + } +} + +response = requests.post(url, headers=headers, json=payload) +print(response.json()) +``` + +```bash cURL +curl -X POST https://api.edenai.run/v3/universal-ai/async \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "web/research_async/firecrawl", + "input": {"prompt": "example"} + }' +``` + + diff --git a/v3/expert-models/features/web/scraping.mdx b/v3/expert-models/features/web/scraping.mdx new file mode 100644 index 0000000..fc8536a --- /dev/null +++ b/v3/expert-models/features/web/scraping.mdx @@ -0,0 +1,97 @@ +--- +title: "Web Scraping" +description: "Fetch a single URL and return its clean content as markdown, text, or HTML, with optional JavaScript rendering." +--- + +import { TechArticleSchema } from "/snippets/TechArticleSchema.mdx"; + + + +## Endpoint + +`POST /v3/universal-ai` (sync) + +Model string pattern: `web/scraping/{provider}[/{model}]` + +## Input + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| url | string | Yes | URL of the web page to scrape | + + +## Output + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| url | string | Yes | | +| content | string | No | | +| title | string | No | | +| description | string | No | | +| status_code | int | No | | +| links | array[string] | No | | +| **images** | array[object] | No | | +|     url | string | Yes | | +|     description | string | No | | +|     width | int | No | | +|     height | int | No | | +| raw_html | string | No | | +| summary | string | No | | +| published_date | string | No | | +| author | string | No | | +| favicon | string | No | | +| usage | dict | No | | + + +## Available Providers + +| Provider | Model String | Price | +|----------|-------------|-------| +| firecrawl | `web/scraping/firecrawl` | $0.0038 per token | + + +## Quick Start + + + +```python Python +import requests + +url = "https://api.edenai.run/v3/universal-ai" +headers = { + "Authorization": "Bearer YOUR_API_KEY", + "Content-Type": "application/json" +} + +payload = { + "model": "web/scraping/firecrawl", + "input": { + "url": "example" + } +} + +response = requests.post(url, headers=headers, json=payload) +print(response.json()) +``` + +```bash cURL +curl -X POST https://api.edenai.run/v3/universal-ai \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "web/scraping/firecrawl", + "input": {"url": "example"} + }' +``` + + diff --git a/v3/expert-models/features/web/search.mdx b/v3/expert-models/features/web/search.mdx new file mode 100644 index 0000000..ed7f829 --- /dev/null +++ b/v3/expert-models/features/web/search.mdx @@ -0,0 +1,108 @@ +--- +title: "Web Search" +description: "Search the web for a query and return ranked results (title, URL, and a snippet or the page content)." +--- + +import { TechArticleSchema } from "/snippets/TechArticleSchema.mdx"; + + + +## Endpoint + +`POST /v3/universal-ai` (sync) + +Model string pattern: `web/search/{provider}[/{model}]` + +## Input + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| query | string | Yes | The web search query | +| max_results | int | No | Max number of search results to return | + + +## Output + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| query | string | Yes | | +| **results** | array[object] | No | | +|     title | string | No | | +|     url | string | Yes | | +|     content | string | No | | +|     score | float | No | | +|     favicon | string | No | | +|     **images** | array[object] | No | | +|         url | string | Yes | | +|         description | string | No | | +|         width | int | No | | +|         height | int | No | | +| answer | string | No | | +| **sources** | array[object] | No | | +|     title | string | No | | +|     url | string | Yes | | +|     snippet | string | No | | +|     favicon | string | No | | +|     published_date | string | No | | +|     author | string | No | | +|     id | string | No | | +| **images** | array[object] | No | | +|     url | string | Yes | | +|     description | string | No | | +|     width | int | No | | +|     height | int | No | | +| usage | dict | No | | + + +## Available Providers + +| Provider | Model String | Price | +|----------|-------------|-------| +| firecrawl | `web/search/firecrawl` | $0.0038 per token | + + +## Quick Start + + + +```python Python +import requests + +url = "https://api.edenai.run/v3/universal-ai" +headers = { + "Authorization": "Bearer YOUR_API_KEY", + "Content-Type": "application/json" +} + +payload = { + "model": "web/search/firecrawl", + "input": { + "query": "example" + } +} + +response = requests.post(url, headers=headers, json=payload) +print(response.json()) +``` + +```bash cURL +curl -X POST https://api.edenai.run/v3/universal-ai \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "web/search/firecrawl", + "input": {"query": "example"} + }' +``` + + diff --git a/v3/expert-models/features/web/structured-extraction-async.mdx b/v3/expert-models/features/web/structured-extraction-async.mdx new file mode 100644 index 0000000..431f5a4 --- /dev/null +++ b/v3/expert-models/features/web/structured-extraction-async.mdx @@ -0,0 +1,96 @@ +--- +title: "Structured Extraction" +description: "Extract structured data from one or more web pages against a schema or prompt. Runs as an async job." +--- + +import { TechArticleSchema } from "/snippets/TechArticleSchema.mdx"; + + + +## Endpoint + +`POST /v3/universal-ai/async` (async) + +Model string pattern: `web/structured_extraction_async/{provider}[/{model}]` + +## Input + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| urls | array[string] | Yes | URLs to extract structured data from (async job) | +| prompt | string | No | Natural-language description of what to extract | +| schema | dict | No | JSON schema describing the structured output | + + +## Output + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| data | any | No | | +| **sources** | array[object] | No | | +|     title | string | No | | +|     url | string | Yes | | +|     snippet | string | No | | +|     favicon | string | No | | +|     published_date | string | No | | +|     author | string | No | | +|     id | string | No | | +| status | string | No | | +| failed_results | array[dict] | No | | +| usage | dict | No | | + + +## Available Providers + +| Provider | Model String | Price | +|----------|-------------|-------| +| firecrawl | `web/structured_extraction_async/firecrawl` | $0.000253333 per token | + + +## Quick Start + + +> This is an **async** feature. The initial response returns a job ID. Poll `GET /v3/universal-ai/async/{job_id}` until the job completes. + + +```python Python +import requests + +url = "https://api.edenai.run/v3/universal-ai/async" +headers = { + "Authorization": "Bearer YOUR_API_KEY", + "Content-Type": "application/json" +} + +payload = { + "model": "web/structured_extraction_async/firecrawl", + "input": { + "urls": ["value1", "value2"] + } +} + +response = requests.post(url, headers=headers, json=payload) +print(response.json()) +``` + +```bash cURL +curl -X POST https://api.edenai.run/v3/universal-ai/async \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "web/structured_extraction_async/firecrawl", + "input": {"urls": ["value1", "value2"]} + }' +``` + +