setup_data.py

from langchain_community.vectorstores import Chroma
from langchain_upstage import UpstageEmbeddings
from langchain.docstore.document import Document
import json, os
from dotenv import load_dotenv

# 환경변수 로드
load_dotenv()

os.environ["CHROMA_TELEMETRY"] = "FALSE"

def load_jsonl(file_path):
    documents = []
    with open(file_path, "r", encoding="utf-8") as f:
        for line in f:
            data = json.loads(line)
            content = data["content"]
            metadata = data.get("metadata", {})

            cleaned_metadata = {}
            for key, value in metadata.items():
                if isinstance(value, (str, int, float, bool)):
                    cleaned_metadata[key] = value
                else:
                    cleaned_metadata[key] = json.dumps(value, ensure_ascii=False)

            documents.append(Document(page_content=content, metadata=cleaned_metadata))
    return documents

persona = load_jsonl("data/yjs-persona.jsonl")
info = load_jsonl("data/yjs-info.jsonl")

print("유재석 페르소나 임베딩을 시작합니다.")

embedding = UpstageEmbeddings(model="solar-embedding-1-large")
vectorstore = Chroma.from_documents(
    documents=persona,
    embedding=embedding,
    persist_directory="./chroma_db",
    collection_name="yujaeseuk_persona"
)
print("유재석 페르소나 임베딩 완료!")

print("유재석 정보 임베딩을 시작합니다.")
vectorstore = Chroma.from_documents(
    documents=info,
    embedding=embedding,
    persist_directory="./chroma_db",
    collection_name="yujaeseuk_data"
)
print("유재석 정보 임베딩 완료!")