pyproject.toml

[project]
name = "unstructured_ingest"
description = "Local ETL data pipeline to get data RAG ready"
requires-python = ">=3.11, <3.14"
authors = [{name = "Unstructured Technologies", email = "devops@unstructuredai.io"}]
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Education",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: Apache Software License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
]
readme = "README.md"
license = "Apache-2.0"
dynamic = ["version"]
dependencies = [
    "python-dateutil",
    # Pydantic generic Secret only introduced in 2.7
    "pydantic>=2.7",
    "tqdm",
    "click",
    "opentelemetry-sdk",
    "certifi>=2026.1.4",
]

[project.optional-dependencies]
# Connectors
airtable = ["pandas", "pyairtable"]
astradb = ["astrapy>2.0.0"]
azure-ai-search = ["azure-search-documents"]
azure = ["adlfs", "fsspec"]
biomed = ["beautifulsoup4", "requests"]
box = ["boxfs", "fsspec"]
chroma = ["chromadb"]
clarifai = ["clarifai"]
confluence = ["atlassian-python-api", "requests"]
couchbase = ["couchbase"]
databricks-delta-tables = ["pandas", "databricks-sql-connector"]
databricks-volumes = ["databricks-sdk>=0.85.0"]
delta-table = ["pandas", "deltalake", "boto3", "tenacity", "pyarrow"]
discord = ["discord.py"]
dropbox = ["dropboxdrivefs", "fsspec"]
duckdb = ["pandas", "duckdb"]
elasticsearch = ["elasticsearch[async]<9.0.0"]
gcs = ["gcsfs", "fsspec", "beautifulsoup4"]
github = ["pygithub>1.58.0", "requests"]
gitlab = ["python-gitlab"]
google-drive = ["google-api-python-client", "tenacity"]
hubspot = ["hubspot-api-client", "urllib3"]
ibm-watsonx-s3 = ["pyiceberg", "httpx", "pyarrow", "tenacity", "pandas"]
jira = ["atlassian-python-api"]
kafka = ["confluent-kafka"]
kdbai = ["pandas", "kdbai-client>=1.4.0"]
lancedb = ["lancedb"]
milvus = ["pymilvus"]
mongodb = ["pymongo"]
neo4j = ["neo4j-rust-ext", "cymple", "networkx"]
notion = ["notion-client", "htmlBuilder", "httpx", "backoff"]
onedrive = ["msal", "Office365-REST-Python-Client", "requests"]
opensearch = [
    # opensearch-py 3.x adds opensearch-protobufs dependency which pulls in grpcio/protobuf,
    # causing version conflicts with pymilvus and pyiceberg. Pin to 2.x to avoid this cascade.
    "opensearch-py>=2.4.0,<3.0.0",
    "boto3>=1.26.0",
    "botocore>=1.29.0",
]
outlook = ["msal", "Office365-REST-Python-Client"]
pinecone = ["pinecone"]
postgres = ["pandas", "psycopg2-binary"]
qdrant = ["qdrant-client"]
reddit = ["praw"]
redis = ["redis<=5.3.0"]
s3 = ["s3fs", "fsspec"]
salesforce = ["simple-salesforce"]
sftp = ["fsspec", "paramiko"]
sharepoint = ["msal", "Office365-REST-Python-Client", "requests"]
singlestore = ["pandas", "singlestoredb"]
slack = ["slack_sdk[optional]"]
snowflake = ["pandas", "snowflake-connector-python", "psycopg2-binary"]
teradata = ["pandas", "teradatasql"]
vastdb = ["pandas", "pyarrow", "vastdb", "ibis"]
vectara = ["requests", "aiofiles", "httpx"]
weaviate = ["weaviate-client"]
wikipedia = ["wikipedia"]
zendesk = ["httpx", "aiofiles", "beautifulsoup4"]

# Embedders
bedrock = ["boto3", "aioboto3", "aiobotocore[boto3]!=2.24.2"]
huggingface = ["sentence-transformers"]
mixedbreadai = ["mixedbread"]
octoai = ["openai", "tiktoken"]
openai = ["openai", "tiktoken"]
togetherai = ["together"]
vertexai = ["vertexai"]
voyageai = ["voyageai", "langchain-core>=1.2.22,<2.0.0"]

# Remote
remote = ["unstructured-client>=0.30.0"]

# Local partition
doc = ["unstructured[doc]"]
docx = ["unstructured[docx]"]
epub = ["unstructured[epub]"]
image = ["unstructured[image]"]
md = ["unstructured[md]"]
odt = ["unstructured[odt]"]
org = ["unstructured[org]"]
pdf = ["unstructured[pdf]"]
ppt = ["unstructured[ppt]"]
pptx = ["unstructured[pptx]"]
rst = ["unstructured[rst]"]
rtf = ["unstructured[rtf]"]
tsv = ["unstructured[tsv]"]
xlsx = ["unstructured[xlsx]"]

[tool.hatch.version]
path = "unstructured_ingest/__version__.py"

[dependency-groups]
release = [
    "twine",
    "wheel",
    "build",
]
lint = [
    "ruff",
]
test = [
    "pytest",
    "pytest-cov",
    "pytest-mock",
    "pytest-check",
    "pytest-asyncio",
    "pytest-tagging",
    "pytest-json-report",
    "pytest-timeout",
    "pytest-xdist",
    "faker",
    "docker",
    "universal_pathlib",
    "deepdiff",
    "beautifulsoup4",
    "pandas",
    # Connector specific deps
    "cryptography",
    "fsspec",
    "vertexai",
    "pyiceberg",
    "pyarrow",
    "networkx",
    "htmlBuilder",
    "office365-rest-python-client",
]

[project.scripts]
unstructured-ingest = "unstructured_ingest.main:main"

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.uv]
# clarifai 12.x pins ruff==0.11.4, psutil==7.0.0, and uv==0.7.12 as hard
# runtime deps; override to allow latest versions in our lockfile.
# uv < 0.10 has multiple CVEs — force latest.
# pykx 2.3.0 declares requires-python>=3.7 but only ships cp311 wheels;
# force >=2.4.0 which is the first version with cp312 wheels.
# aiobotocore 2.x requires wrapt<2.0 which conflicts with unstructured>=0.20
# (wrapt>=2.1.1); aiobotocore 3.x widens to wrapt<3.0. aioboto3 15.5.0
# exact-pins aiobotocore==2.25.1 but no newer aioboto3 exists yet.
override-dependencies = [
    "ruff>=0.15.0",
    "psutil>=7.0.0",
    "uv>=0.10.0",
    "pykx>=2.4.0",
    "aiobotocore>=3.0.0",
]

[tool.ruff]
line-length = 100

[tool.ruff.lint]
select = [
    # pycodestyle
    "E",
    # Pyflakes
    "F",
    # flake8-simplify
    "SIM",
    # isort
    "I",
]

[tool.pytest.ini_options]
asyncio_mode = "auto"
filterwarnings = [
    "ignore::DeprecationWarning:pyiceberg.*",
    "ignore::pydantic.warnings.PydanticDeprecatedSince212:pyiceberg.*",
    "ignore:Default value.*is not JSON serializable:pydantic.json_schema.PydanticJsonSchemaWarning",
]

[tool.coverage.report]
# TODO: Update as this improves
fail_under = 0

[tool.hatch.build.targets.wheel]
packages = ["/unstructured_ingest"]

[tool.hatch.build.targets.sdist]
packages = ["/unstructured_ingest"]