ingestlib/config.example.yaml at main · LangModule/ingestlib · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# ingestlib configuration

# Your AWS profile from ~/.aws/credentials, with Bedrock access.
aws:
  profile: your-aws-profile
  region: us-east-1
  account_id: "000000000000"

# pinecone | qdrant | sqlite | pgvector | mongodb | milvus |
# opensearch | weaviate — secrets in .env
vector_store: pinecone

# jina (needs JINA_API_KEY) | aws (same AWS credentials) | none
reranker: jina

# Who serves LLM and embedding calls: bedrock | openai (needs OPENAI_API_KEY).
# Switching embedding_provider changes the vector space — re-ingest after.
llm_provider: bedrock
embedding_provider: bedrock

# Where pipeline artifacts (parses, page images, chunks) live:
# s3 (durable, shareable) | local (a plain folder — zero cloud)
artifact_store: s3

# Bucket names are global across AWS. Created on first use.
# Default: ingestlib-{aws.account_id}
s3:
  bucket: your-unique-bucket-name

# local mode's folder; relative paths anchor to this file's dir
# (wizard-managed machines get ~/.ingestlib/artifacts)
artifacts:
  path: artifacts

bedrock:
  llm_model_id: us.amazon.nova-2-lite-v1:0
  embedding_model_id: amazon.nova-2-multimodal-embeddings-v1:0
  rerank_model_id: amazon.rerank-v1:0
  rerank_region: us-west-2

jina:
  base_url: https://api.jina.ai/v1
  rerank_model_id: jina-reranker-v3

# OpenAI backend (needs OPENAI_API_KEY): GPT-5 vision-capable LLM +
# text-embedding-3 (text-only embeddings; image embeddings stay on bedrock)
openai:
  llm_model_id: gpt-5-mini
  embedding_model_id: text-embedding-3-small

# OCR server for parse/ingest: mlx-vlm-server (Apple Silicon) | vllm-server (NVIDIA)
paddle_vl:
  backend: mlx-vlm-server
  server_url: http://localhost:8111/
  api_model_name: PaddlePaddle/PaddleOCR-VL-1.6

# Indexes, collections, and tables are created on first use.
pinecone:
  index_name: ingestlib
  sparse_index_name: ingestlib-sparse
  sparse_model_id: pinecone-sparse-english-v0
  cloud: aws
  region: us-east-1

qdrant:
  collection_name: ingestlib

sqlite:
  path: ingestlib.db    # relative paths anchor to this file's dir

pgvector:
  table_name: ingestlib

mongodb:
  database: ingestlib
  collection_name: ingestlib

milvus:
  collection_name: ingestlib

opensearch:
  index_name: ingestlib

weaviate:
  collection_name: Ingestlib    # Weaviate capitalizes collection names