From 8046819257d178f64684ce0c03ae95cb43b10159 Mon Sep 17 00:00:00 2001 From: tillwf Date: Tue, 31 Mar 2026 14:26:22 +0200 Subject: [PATCH 1/2] chore(llmobs): move optimization prompt template from .md to Python module The markdown template file was not included in release wheels, causing runtime FileNotFoundError. Move the template into a Python module so it is always packaged with the library. Co-Authored-By: Claude Opus 4.6 (1M context) --- ddtrace/llmobs/_prompt_optimization.py | 11 ++----- ...tion.md => _prompt_optimization_prompt.py} | 29 +++++++++++++++---- 2 files changed, 26 insertions(+), 14 deletions(-) rename ddtrace/llmobs/{_prompt_optimization.md => _prompt_optimization_prompt.py} (77%) diff --git a/ddtrace/llmobs/_prompt_optimization.py b/ddtrace/llmobs/_prompt_optimization.py index b944bb8d1ae..d9d8d7af8f7 100644 --- a/ddtrace/llmobs/_prompt_optimization.py +++ b/ddtrace/llmobs/_prompt_optimization.py @@ -247,21 +247,16 @@ def run(self) -> str: def _load_system_prompt(self) -> str: """Load and prepare the optimization system prompt. - Loads the template from _prompt_optimization.md and replaces placeholders. + Loads the template from _prompt_optimization_prompt.py and replaces placeholders. Adds evaluation model information and random tip at the end. :return: System prompt string with output format injected. """ - import os import random - # Get the directory of this file - current_dir = os.path.dirname(os.path.abspath(__file__)) - template_path = os.path.join(current_dir, "_prompt_optimization.md") + from ddtrace.llmobs._prompt_optimization_prompt import OPTIMIZATION_SYSTEM_PROMPT_TEMPLATE - # Load template - with open(template_path, "r", encoding="utf-8") as f: - template = f.read() + template = OPTIMIZATION_SYSTEM_PROMPT_TEMPLATE output_format = self._config.get("evaluation_output_format") structure_placeholder = "" diff --git a/ddtrace/llmobs/_prompt_optimization.md b/ddtrace/llmobs/_prompt_optimization_prompt.py similarity index 77% rename from ddtrace/llmobs/_prompt_optimization.md rename to ddtrace/llmobs/_prompt_optimization_prompt.py index 2a8c842d343..adf69a8f023 100644 --- a/ddtrace/llmobs/_prompt_optimization.md +++ b/ddtrace/llmobs/_prompt_optimization_prompt.py @@ -1,4 +1,15 @@ -You are a systematic prompt engineering expert. Your task is to identify the root cause of false positives and create targeted fixes that preserve correct behavior. As a prompt engineering expert you know how to format a prompt with clear structure, examples, and guidance. You optimize prompts for any domain or evaluation task. +"""System prompt template for the prompt optimization framework. + +This module contains the system prompt used by OptimizationIteration to guide +the LLM that performs prompt optimization. It was extracted from a markdown file +to ensure it is included in release packages. +""" + +OPTIMIZATION_SYSTEM_PROMPT_TEMPLATE = """\ +You are a systematic prompt engineering expert. Your task is to identify the root cause of \ +false positives and create targeted fixes that preserve correct behavior. As a prompt engineering \ +expert you know how to format a prompt with clear structure, examples, and guidance. You optimize \ +prompts for any domain or evaluation task. **GIVE SYNTHETIC EXAMPLES OF INPUT AND EXPECTED OUTPUT** @@ -34,11 +45,14 @@ ### 0.5. Model-Aware Optimization **Consider the specific evaluation model characteristics:** -- **Model type and capabilities**: Is it a reasoning model (o3-mini), instruction-following model (GPT-4), or domain-specific model? +- **Model type and capabilities**: Is it a reasoning model (o3-mini), instruction-following model \ +(GPT-4), or domain-specific model? - **Model strengths**: What types of tasks does this model excel at? - **Model weaknesses**: What are known limitations or common failure modes? -- **Prompt format preferences**: Does this model respond better to specific prompt structures or instruction styles? -- **Context handling**: How well does this model handle long contexts, examples, or complex instructions? +- **Prompt format preferences**: Does this model respond better to specific prompt structures or \ +instruction styles? +- **Context handling**: How well does this model handle long contexts, examples, or complex \ +instructions? ### 1. Root Cause Analysis Compare the bad vs good examples and identify: @@ -61,7 +75,8 @@ - **Tighten decision criteria** for weak boundaries - **Enforce schema compliance** for format issues - **Add uncertainty expressions** for overconfidence -- **Model-specific optimization** - Consider the specific evaluation model's capabilities, training, and typical behavior patterns +- **Model-specific optimization** - Consider the specific evaluation model's capabilities, \ +training, and typical behavior patterns ### 4. Preservation Check **MANDATORY CHECKS:** @@ -70,4 +85,6 @@ - Confirm the domain and fundamental goal are identical to the original - Check that true positive behavior is maintained while reducing false positives -Make your fix as minimal and targeted as possible while maximizing false positive reduction WITHOUT altering the prompt's original purpose. +Make your fix as minimal and targeted as possible while maximizing false positive reduction \ +WITHOUT altering the prompt's original purpose. +""" From 0bcac0642049b010ac1dc44a102f653ec0d15b0b Mon Sep 17 00:00:00 2001 From: tillwf Date: Tue, 31 Mar 2026 17:45:21 +0200 Subject: [PATCH 2/2] chore(llmobs): add release note for prompt optimization template fix Co-Authored-By: Claude Opus 4.6 (1M context) --- ...s-fix-prompt-optimization-template-2c9645c3cf27d798.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 releasenotes/notes/llmobs-fix-prompt-optimization-template-2c9645c3cf27d798.yaml diff --git a/releasenotes/notes/llmobs-fix-prompt-optimization-template-2c9645c3cf27d798.yaml b/releasenotes/notes/llmobs-fix-prompt-optimization-template-2c9645c3cf27d798.yaml new file mode 100644 index 00000000000..1d9f5428768 --- /dev/null +++ b/releasenotes/notes/llmobs-fix-prompt-optimization-template-2c9645c3cf27d798.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + LLM Observability: Fixes a ``FileNotFoundError`` in prompt optimization where the system prompt + template was stored as a ``.md`` file that was excluded from release wheels. The template is now + embedded in a Python module to ensure it is always available at runtime.