[SC-15520] Expose qualitative text agent to vm library

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "validmind"
-version = "2.12.5"
+version = "2.13.0"
 description = "ValidMind Library"
 readme = "README.pypi.md"
 requires-python = ">=3.9,<3.15"

r/validmind/DESCRIPTION

@@ -1,7 +1,7 @@
 Package: validmind
 Type: Package
 Title: Interface to the 'ValidMind' Platform
-Version: 2.12.5
+Version: 2.13.0
 Authors@R: c(person("Andres", "Rodriguez", role = c("aut", "cre","cph"),
                       email = "andres@validmind.ai"))
 Maintainer: Andres Rodriguez <andres@validmind.ai>

tests/test_api_client.py

@@ -16,10 +16,11 @@
 import validmind.api_client as api_client
 from validmind.__version__ import __version__
 from validmind.errors import (
+    APIRequestError,
     MissingAPICredentialsError,
     MissingModelIdError,
-    APIRequestError,
 )
+from validmind.utils import md_to_html
 from validmind.vm_models.figure import Figure
 
 
@@ -225,6 +226,27 @@ def test_log_metadata(self, mock_post: MagicMock):
             ),
         )
 
+    @patch("aiohttp.ClientSession.post")
+    def test_log_metadata_with_section_id(self, mock_post: MagicMock):
+        mock_post.return_value = MockAsyncResponse(200, json={"cuid": "abc1234"})
+
+        self.run_async(
+            api_client.alog_metadata,
+            "1234",
+            text="Some Text",
+            section_id="intended_use",
+        )
+
+        mock_post.assert_called_with(
+            f"{os.environ['VM_API_HOST']}/log_metadata?section_id=intended_use",
+            data=json.dumps(
+                {
+                    "content_id": "1234",
+                    "text": "Some Text",
+                }
+            ),
+        )
+
     @patch("aiohttp.ClientSession.post")
     def test_log_test_result(self, mock_post):
         result = {
@@ -245,6 +267,128 @@ def test_log_test_result(self, mock_post):
 
         mock_post.assert_called_with(url, data=json.dumps(result))
 
+    @patch("requests.post")
+    @patch("aiohttp.ClientSession.post")
+    def test_log_text_generates_text_and_logs_metadata(
+        self, mock_aiohttp_post, mock_requests_post
+    ):
+        mock_requests_post.return_value = Mock(status_code=200)
+        mock_requests_post.return_value.json.return_value = {
+            "content": "## Generated Summary\nGenerated content."
+        }
+        mock_aiohttp_post.return_value = MockAsyncResponse(
+            200,
+            json={
+                "content_id": "dataset_summary_text",
+                "text": md_to_html("## Generated Summary\nGenerated content.", mathml=True),
+            },
+        )
+
+        api_client.log_text(
+            content_id="dataset_summary_text",
+            prompt="Summarize the dataset.",
+            context={"content_ids": ["train_dataset", "target_description_text"]},
+        )
+
+        mock_requests_post.assert_called_once_with(
+            url=f"{os.environ['VM_API_HOST']}/ai/generate/qualitative_text_generation",
+            headers={
+                "X-API-KEY": os.environ["VM_API_KEY"],
+                "X-API-SECRET": os.environ["VM_API_SECRET"],
+                "X-MODEL-CUID": os.environ["VM_API_MODEL"],
+                "X-MONITORING": "False",
+                "X-LIBRARY-VERSION": __version__,
+            },
+            json={
+                "content_id": "dataset_summary_text",
+                "generate": True,
+                "prompt": "Summarize the dataset.",
+                "context": {
+                    "content_ids": ["train_dataset", "target_description_text"]
+                },
+            },
+        )
+        mock_aiohttp_post.assert_called_once_with(
+            f"{os.environ['VM_API_HOST']}/log_metadata",
+            data=json.dumps(
+                {
+                    "content_id": "dataset_summary_text",
+                    "text": md_to_html(
+                        "## Generated Summary\nGenerated content.", mathml=True
+                    ),
+                }
+            ),
+        )
+
+    @patch("requests.post")
+    @patch("aiohttp.ClientSession.post")
+    def test_log_text_logs_metadata_with_section_id(
+        self, mock_aiohttp_post, mock_requests_post
+    ):
+        mock_requests_post.return_value = Mock(status_code=200)
+        mock_requests_post.return_value.json.return_value = {
+            "content": "Generated content."
+        }
+        mock_aiohttp_post.return_value = MockAsyncResponse(
+            200,
+            json={
+                "content_id": "dataset_summary_text",
+                "text": "Generated content.",
+            },
+        )
+
+        api_client.log_text(
+            content_id="dataset_summary_text",
+            prompt="Summarize the dataset.",
+            section_id="intended_use",
+        )
+
+        mock_requests_post.assert_called_once_with(
+            url=f"{os.environ['VM_API_HOST']}/ai/generate/qualitative_text_generation",
+            headers={
+                "X-API-KEY": os.environ["VM_API_KEY"],
+                "X-API-SECRET": os.environ["VM_API_SECRET"],
+                "X-MODEL-CUID": os.environ["VM_API_MODEL"],
+                "X-MONITORING": "False",
+                "X-LIBRARY-VERSION": __version__,
+            },
+            json={
+                "content_id": "dataset_summary_text",
+                "generate": True,
+                "prompt": "Summarize the dataset.",
+                "section_id": "intended_use",
+            },
+        )
+        mock_aiohttp_post.assert_called_once_with(
+            f"{os.environ['VM_API_HOST']}/log_metadata?section_id=intended_use",
+            data=json.dumps(
+                {
+                    "content_id": "dataset_summary_text",
+                    "text": md_to_html("Generated content.", mathml=True),
+                }
+            ),
+        )
+
+    def test_log_text_rejects_prompt_when_text_is_provided(self):
+        with self.assertRaisesRegex(
+            ValueError, "`prompt` is only supported when `text` is omitted"
+        ):
+            api_client.log_text(
+                content_id="dataset_summary_text",
+                text="Hello world",
+                prompt="Ignore the provided text.",
+            )
+
+    def test_log_text_rejects_invalid_context(self):
+        with self.assertRaisesRegex(
+            ValueError,
+            "`context\\['content_ids'\\]` must contain only non-empty strings",
+        ):
+            api_client.log_text(
+                content_id="dataset_summary_text",
+                context={"content_ids": ["valid", ""]},
+            )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_client.py

@@ -12,12 +12,16 @@
 
 import validmind
 from validmind import (
+    generate_documentation_text,
+    get_content_ids,
     init_dataset,
     init_model,
     get_test_suite,
+    run_text_generation,
     run_documentation_tests,
 )
 from validmind.errors import UnsupportedModelError
+from validmind.vm_models.result import TextGenerationResult
 
 
 @dataclass
@@ -115,7 +119,7 @@ def test_init_model_invalid_metadata_dict(self):
             "key": "value",
             "foo": "bar",
         }
-        with self.assertRaises(UnsupportedModelError) as context:
+        with self.assertRaises(UnsupportedModelError):
             init_model(attributes=metadata, __log=False)
 
     def test_init_model_metadata_dict(self):
@@ -163,6 +167,173 @@ def test_get_default_config(self):
             self.assertIn("params", config)
 
 
+class TestGetContentIds(TestCase):
+    @mock.patch(
+        "validmind.client_config.client_config.documentation_template",
+        MockedConfig.documentation_template,
+    )
+    def test_get_all_content_ids(self):
+        content_ids = get_content_ids()
+        self.assertEqual(
+            content_ids,
+            [
+                "validmind.data_validation.ClassImbalance",
+                "validmind.data_validation.DatasetSplit",
+            ],
+        )
+
+    @mock.patch(
+        "validmind.client_config.client_config.documentation_template",
+        MockedConfig.documentation_template,
+    )
+    def test_get_content_ids_for_single_section(self):
+        content_ids = get_content_ids("test_section_1")
+        self.assertEqual(content_ids, ["validmind.data_validation.ClassImbalance"])
+
+    @mock.patch(
+        "validmind.client_config.client_config.documentation_template",
+        MockedConfig.documentation_template,
+    )
+    def test_get_content_ids_for_multiple_sections(self):
+        content_ids = get_content_ids(["test_section_1", "test_section_2"])
+        self.assertEqual(
+            content_ids,
+            [
+                "validmind.data_validation.ClassImbalance",
+                "validmind.data_validation.DatasetSplit",
+            ],
+        )
+
+
+class TestRunTextGeneration(TestCase):
+    @mock.patch(
+        "validmind.client.api_client._generate_log_text",
+        return_value="<p>Generated text</p>",
+    )
+    def test_run_text_generation(self, mock_generate_text):
+        result = run_text_generation(
+            content_id="dataset_summary_text",
+            prompt="Summarize the dataset.",
+            section_id="data_description",
+            context={"content_ids": ["train_dataset"]},
+            show=False,
+        )
+
+        self.assertIsInstance(result, TextGenerationResult)
+        self.assertEqual(result.content_id, "dataset_summary_text")
+        self.assertEqual(result.prompt, "Summarize the dataset.")
+        self.assertEqual(result.section_id, "data_description")
+        self.assertEqual(result.context, {"content_ids": ["train_dataset"]})
+        self.assertEqual(result.description, "<p>Generated text</p>")
+        self.assertTrue(result._was_description_generated)
+        self.assertIn("validmind", result.metadata)
+        self.assertIn("timestamp", result.metadata)
+        self.assertIn("duration_seconds", result.metadata)
+        mock_generate_text.assert_called_once_with(
+            "dataset_summary_text",
+            "Summarize the dataset.",
+            {"content_ids": ["train_dataset"]},
+            section_id="data_description",
+        )
+
+
+class TestGenerateDocumentationText(TestCase):
+    @mock.patch(
+        "validmind.client_config.client_config.documentation_template",
+        MockedConfig.documentation_template,
+    )
+    @mock.patch("validmind.client.vm_display")
+    @mock.patch("validmind.client.DocumentationTextSummary")
+    @mock.patch("validmind.client.HTMLLabel")
+    @mock.patch("validmind.client.HTMLProgressBar")
+    @mock.patch("validmind.client.run_text_generation")
+    def test_generate_documentation_text(
+        self,
+        mock_run_text_generation,
+        mock_progress_bar,
+        mock_label,
+        mock_summary,
+        mock_display,
+    ):
+        first_result = mock.Mock()
+        second_result = mock.Mock()
+        mock_run_text_generation.side_effect = [first_result, second_result]
+
+        progress_instance = mock_progress_bar.return_value
+        progress_instance.value = 0
+        progress_instance.update.side_effect = (
+            lambda value, description=None: setattr(progress_instance, "value", value)
+        )
+
+        config = {
+            "monitoring_plan": {
+                "prompt": "Write the monitoring plan.",
+                "section_id": "monitoring",
+                "context": {"content_ids": ["model_summary"]},
+            },
+            "governance_plan": {
+                "prompt": "Write the governance plan.",
+            },
+        }
+
+        results = generate_documentation_text(config)
+
+        self.assertEqual(
+            results,
+            {
+                "monitoring_plan": first_result,
+                "governance_plan": second_result,
+            },
+        )
+        self.assertEqual(mock_run_text_generation.call_count, 2)
+        mock_run_text_generation.assert_has_calls(
+            [
+                mock.call(
+                    content_id="monitoring_plan",
+                    prompt="Write the monitoring plan.",
+                    section_id="monitoring",
+                    context={"content_ids": ["model_summary"]},
+                    show=False,
+                ),
+                mock.call(
+                    content_id="governance_plan",
+                    prompt="Write the governance plan.",
+                    section_id=None,
+                    context=None,
+                    show=False,
+                ),
+            ]
+        )
+        first_result.log.assert_called_once_with()
+        second_result.log.assert_called_once_with()
+        mock_label.assert_called_once_with(value="Generating documentation text...")
+        mock_progress_bar.assert_called_once_with(
+            max_value=4,
+            description="Generating documentation text...",
+        )
+        progress_instance.display.assert_called_once_with()
+        progress_instance.close.assert_called_once_with()
+        mock_label.return_value.update.assert_any_call(
+            "Sending result to ValidMind: monitoring_plan..."
+        )
+        mock_label.return_value.update.assert_any_call(
+            "Sending result to ValidMind: governance_plan..."
+        )
+        mock_label.return_value.update.assert_any_call(
+            "Documentation text generation complete!"
+        )
+        mock_summary.assert_called_once_with(
+            title="Binary classification",
+            description="Template for binary classification models.",
+            results={
+                "monitoring_plan": first_result,
+                "governance_plan": second_result,
+            },
+            template_sections=MockedConfig.documentation_template["sections"],
+        )
+        mock_display.assert_called_once_with(mock_summary.return_value)
+
+
 # TODO: Fix this test
 # class TestPreviewTemplate(TestCase):
 #     @mock.patch(