Add an end-to-end test for patch advisory creation.

Update get_or_create_advisory_references to store the reference type correctly.
Update get_or_create_advisory_package_commit_patches to correctly create or update the patch_text field.

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/pipes/advisory.py

@@ -19,7 +19,6 @@
 from django.db import transaction
 from django.db.models import Q
 from django.db.models.query import QuerySet
-from packageurl import PackageURL
 from packageurl.contrib.purl2url import purl2url
 from packageurl.contrib.url2purl import url2purl
 
@@ -46,7 +45,6 @@
 from vulnerabilities.models import VulnerabilitySeverity
 from vulnerabilities.models import Weakness
 from vulnerabilities.pipes.univers_utils import get_exact_purls_v2
-from vulnerabilities.utils import is_commit
 
 
 def get_or_create_aliases(aliases: List) -> QuerySet:
@@ -71,7 +69,9 @@ def get_or_create_advisory_references(references: List) -> List[AdvisoryReferenc
     existing_urls = {r.url for r in existing}
 
     to_create = [
-        AdvisoryReference(reference_id=ref.reference_id, url=ref.url)
+        AdvisoryReference(
+            reference_id=ref.reference_id, url=ref.url, reference_type=ref.reference_type
+        )
         for ref in references
         if ref.url not in existing_urls
     ]
@@ -110,11 +110,11 @@ def get_or_create_advisory_weaknesses(weaknesses: List[str]) -> List[AdvisoryWea
 def get_or_create_advisory_package_commit_patches(
     commit_patches_data: List,
 ) -> List["PackageCommitPatch"]:
-
     if not commit_patches_data:
         return []
 
-    pairs = [(c.commit_hash, c.vcs_url) for c in commit_patches_data]
+    data_map = {(c.commit_hash, c.vcs_url): c for c in commit_patches_data}
+    pairs = list(data_map.keys())
 
     query = Q()
     for commit_hash, vcs_url in pairs:
@@ -123,46 +123,63 @@ def get_or_create_advisory_package_commit_patches(
     existing_commits_qs = PackageCommitPatch.objects.filter(query)
     existing_pairs = set(existing_commits_qs.values_list("commit_hash", "vcs_url"))
 
+    to_update = []
+    for commit_obj in existing_commits_qs:
+        key = (commit_obj.commit_hash, commit_obj.vcs_url)
+        input_data = data_map[key]
+
+        if (
+            commit_obj.patch_checksum != input_data.patch_checksum
+            or commit_obj.patch_text != input_data.patch_text
+        ):
+            commit_obj.patch_checksum = input_data.patch_checksum
+            commit_obj.patch_text = input_data.patch_text
+            to_update.append(commit_obj)
+
+    if to_update:
+        PackageCommitPatch.objects.bulk_update(to_update, fields=["patch_checksum", "patch_text"])
+
     to_create = [
         PackageCommitPatch(
             commit_hash=c.commit_hash,
             vcs_url=c.vcs_url,
-            patch_text=getattr(c, "patch_text", None),
+            patch_checksum=c.patch_checksum,
+            patch_text=c.patch_text,
         )
         for c in commit_patches_data
         if (c.commit_hash, c.vcs_url) not in existing_pairs
     ]
 
     if to_create:
-        PackageCommitPatch.objects.bulk_create(to_create, ignore_conflicts=True)
+        PackageCommitPatch.objects.bulk_create(to_create)
 
     all_commits = PackageCommitPatch.objects.filter(query)
     return list(all_commits)
 
 
 def get_or_create_advisory_patches(
-    base_patches_data: List,
+    patches: List,
 ) -> List["Patch"]:
-    if not base_patches_data:
+    if not patches:
         return []
 
-    pairs = [(c.patch_url, c.patch_checksum) for c in base_patches_data]
+    pairs = [(c.patch_text, c.patch_url) for c in patches]
 
     query = Q()
-    for patch_checksum, patch_url in pairs:
-        query |= Q(patch_checksum=patch_checksum, patch_url=patch_url)
+    for patch_text, patch_url in pairs:
+        query |= Q(patch_text=patch_text, patch_url=patch_url)
 
     existing_commits_qs = Patch.objects.filter(query)
-    existing_pairs = set(existing_commits_qs.values_list("patch_url", "patch_checksum"))
+    existing_pairs = set(existing_commits_qs.values_list("patch_text", "patch_url"))
 
     to_create = [
         Patch(
             patch_url=getattr(c, "patch_url", None),
             patch_text=getattr(c, "patch_text", None),
             patch_checksum=getattr(c, "patch_checksum", None),
         )
-        for c in base_patches_data
-        if (c.patch_url, c.patch_checksum) not in existing_pairs
+        for c in patches
+        if (c.patch_text, c.patch_url) not in existing_pairs
     ]
 
     if to_create:
@@ -176,7 +193,13 @@ def get_or_create_advisory_patches(
 
 
 def classify_patch_source(url, commit_hash, patch_text):
-    """Classify a patch as a PackageCommitPatchData or PatchData using provided args."""
+    """
+    Classify the patch type based on the given URL, commit hash, and patch text.
+    Returns: a base_purl, patch_obj tuple where base_purl is a string PackageURL without version for supported VCS URLs, otherwise `None`.
+    patch_obj is one of: (PackageCommitPatchData for supported VCS URLs with a commit,
+    PatchData for raw patch text or non-VCS URLs, ReferenceV2 when unsupported VCS URL is paired with a commit hash)
+    Returns `None` only when both `url` and `patch_text` are missing.
+    """
     if not url:
         if not patch_text:
             return
@@ -254,7 +277,7 @@ def insert_advisory_v2(
     aliases = get_or_create_advisory_aliases(aliases=advisory.aliases)
     references = get_or_create_advisory_references(references=advisory.references_v2)
     severities = get_or_create_advisory_severities(severities=advisory.severities)
-    patches = get_or_create_advisory_patches(base_patches_data=advisory.patches)
+    patches = get_or_create_advisory_patches(patches=advisory.patches)
     weaknesses = get_or_create_advisory_weaknesses(weaknesses=advisory.weaknesses)
     content_id = compute_content_id(advisory_data=advisory)
 

vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2.py

@@ -20,11 +20,15 @@
 from vulnerabilities.importer import AffectedPackageV2
 from vulnerabilities.importer import PackageCommitPatchData
 from vulnerabilities.importer import PatchData
+from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.models import AdvisoryReference
 from vulnerabilities.models import AdvisoryV2
 from vulnerabilities.models import ImpactedPackage
 from vulnerabilities.models import PackageCommitPatch
 from vulnerabilities.models import PackageV2
+from vulnerabilities.models import Patch
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.pipes.advisory import classify_patch_source
 
 
 class DummyImporter(VulnerableCodeBaseImporterPipelineV2):
@@ -57,13 +61,13 @@ def dummy_advisory():
                 introduced_by_commit_patches=[
                     PackageCommitPatchData(
                         commit_hash="9ff29db8ec3adefefce0d37c3c9b5b2c22e59fac",
-                        vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+                        vcs_url="https://github.com/abc/def",
                     )
                 ],
                 fixed_by_commit_patches=[
                     PackageCommitPatchData(
                         commit_hash="ab99939678dc36b3bee0f366493df1aeef521df4",
-                        vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+                        vcs_url="https://github.com/abc/def",
                     )
                 ],
             ),
@@ -74,13 +78,13 @@ def dummy_advisory():
                 introduced_by_commit_patches=[
                     PackageCommitPatchData(
                         commit_hash="9ff29db8ec3adefefce0d37c3c9b5b2c22e59fac",
-                        vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+                        vcs_url="https://github.com/abc/def",
                     )
                 ],
                 fixed_by_commit_patches=[
                     PackageCommitPatchData(
                         commit_hash="ab99939678dc36b3bee0f366493df1aeef521df4",
-                        vcs_url="https://github.com/aboutcode-org/vulnerablecode",
+                        vcs_url="https://github.com/abc/def",
                     )
                 ],
             ),
@@ -127,3 +131,201 @@ def test_advisory_import_atomicity(dummy_importer):
     assert ImpactedPackage.objects.count() == 2
     assert PackageCommitPatch.objects.count() == 2
     assert PackageV2.objects.count() == 4
+
+
+@pytest.fixture
+def patch_source_samples():
+    return [
+        {"url": "https://github.com/abc/def", "commit_hash": None, "patch_text": None},  # PatchData
+        {
+            "url": "https://github.com/abc/def",
+            "commit_hash": None,
+            "patch_text": "+1-2",
+        },  # PatchData
+        {
+            "url": "https://github.com/abc/def",
+            "commit_hash": "be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "patch_text": None,
+        },  # PackageCommitPatchData
+        {
+            "url": "https://github.com/abc/def",
+            "commit_hash": "be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "patch_text": "+1-2",
+        },  # PackageCommitPatchData
+        {
+            "url": "https://github.com/abc/def/commit/be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "commit_hash": None,
+            "patch_text": None,
+        },  # PackageCommitPatchData
+        {
+            "url": "https://github.com/abc/def/commit/be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "commit_hash": None,
+            "patch_text": "+1-2",
+        },  # PackageCommitPatchData
+        {
+            "url": "https://github.com/abc/def/commit/a2a5b42fb829b4a873c832b805680fc19199a07e",
+            "commit_hash": "a2a5b42fb829b4a873c832b805680fc19199a07e",
+            "patch_text": None,
+        },  # PackageCommitPatchData
+        {
+            "url": "https://github.com/abc/def/commit/a2a5b42fb829b4a873c832b805680fc19199a07e",
+            "commit_hash": "a2a5b42fb829b4a873c832b805680fc19199a07e",
+            "patch_text": "+1-2",
+        },  # PackageCommitPatchData
+        {
+            "url": "https://unknown.com/abc/def",
+            "commit_hash": None,
+            "patch_text": None,
+        },  # PatchData
+        {
+            "url": "https://unknown.com/abc/def",
+            "commit_hash": None,
+            "patch_text": "+1-2",
+        },  # PatchData
+        {
+            "url": "https://unknown.com/abc/def",
+            "commit_hash": "8eb1b04ca4ae6fc0a0ef46f1b0c042f64db28ff9",
+            "patch_text": None,
+        },  # ReferenceV2
+        {
+            "url": "https://unknown.com/abc/def",
+            "commit_hash": "8eb1b04ca4ae6fc0a0ef46f1b0c042f64db28ff9",
+            "patch_text": "+1-2",
+        },  # ReferenceV2
+        {
+            "url": "https://unknown.com/abc/def/be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "commit_hash": None,
+            "patch_text": None,
+        },  # PatchData
+        {
+            "url": "https://unknown.com/abc/def/be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "commit_hash": None,
+            "patch_text": "+1-2",
+        },  # PatchData
+        {
+            "url": "https://unknown.com/abc/def/be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "commit_hash": "be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "patch_text": None,
+        },  # ReferenceV2
+        {
+            "url": "https://unknown.com/abc/def/be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "commit_hash": "be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "patch_text": "+1-2",
+        },  # ReferenceV2
+    ]
+
+
+@pytest.fixture
+def dumpy_patch_advisory(patch_source_samples):
+    references = []
+    patches = []
+    affected_packages = []
+    for entry in patch_source_samples:
+        url = entry["url"]
+        commit_hash = entry["commit_hash"]
+        patch_text = entry["patch_text"]
+
+        base_purl, patch_obj = classify_patch_source(
+            url=url, commit_hash=commit_hash, patch_text=patch_text
+        )
+
+        if isinstance(patch_obj, PackageCommitPatchData):
+            # For testing only: commit hashes starting with "a" are treated as introduced_by_commit_patches,
+            # all others are treated as fixed_by_commit_patches.
+            if patch_obj.commit_hash.startswith("a"):
+                affected_package = AffectedPackageV2(
+                    package=base_purl,
+                    introduced_by_commit_patches=[patch_obj],
+                )
+            else:
+                affected_package = AffectedPackageV2(
+                    package=base_purl,
+                    fixed_by_commit_patches=[patch_obj],
+                )
+            affected_packages.append(affected_package)
+        elif isinstance(patch_obj, PatchData):
+            patches.append(patch_obj)
+        elif isinstance(patch_obj, ReferenceV2):
+            references.append(patch_obj)
+
+    return AdvisoryData(
+        summary="Test patch advisory",
+        aliases=["CVE-2025-0001"],
+        affected_packages=affected_packages,
+        references_v2=references,
+        patches=patches,
+        advisory_id="ADV-1234",
+        date_published=datetime.now() - timedelta(days=10),
+        url="https://example.com/advisory/1",
+    )
+
+
+@pytest.mark.django_db
+def test_patch_advisory(dumpy_patch_advisory):
+    dumpy_patch_importer = DummyImporter()
+    dumpy_patch_importer._advisories = [dumpy_patch_advisory]
+    dumpy_patch_importer.collect_and_store_advisories()
+    assert AdvisoryV2.objects.count() == 1
+    adv = AdvisoryV2.objects.get(advisory_id="ADV-1234")
+
+    assert ImpactedPackage.objects.count() == 6
+    assert [
+        (
+            package_commit_patch.commit_hash,
+            package_commit_patch.vcs_url,
+            package_commit_patch.patch_text,
+            package_commit_patch.patch_checksum,
+        )
+        for package_commit_patch in PackageCommitPatch.objects.all()
+    ] == [
+        (
+            "be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "https://github.com/abc/def",
+            "+1-2",
+            "a5d6b89c35224d4ed69910a18fb544ca3fb26f62db53bc2769ce8a8d5cf8874c191186d170cb6e8896b0aaa8eaed891e7e819c4c0c7af499397c84761d6fb22d",
+        ),
+        (
+            "a2a5b42fb829b4a873c832b805680fc19199a07e",
+            "https://github.com/abc/def",
+            "+1-2",
+            "a5d6b89c35224d4ed69910a18fb544ca3fb26f62db53bc2769ce8a8d5cf8874c191186d170cb6e8896b0aaa8eaed891e7e819c4c0c7af499397c84761d6fb22d",
+        ),
+    ]
+    assert (
+        PackageCommitPatch.objects.count() == 2
+    )  # Only 2 are created because the 6 inputs include duplicates with the VCS URL and commit_hash
+    assert Patch.objects.count() == 6
+    assert [
+        (patch.patch_text, patch.patch_url, patch.patch_checksum) for patch in adv.patches.all()
+    ] == [
+        (None, "https://github.com/abc/def", None),
+        (
+            "+1-2",
+            "https://github.com/abc/def",
+            "a5d6b89c35224d4ed69910a18fb544ca3fb26f62db53bc2769ce8a8d5cf8874c191186d170cb6e8896b0aaa8eaed891e7e819c4c0c7af499397c84761d6fb22d",
+        ),
+        (None, "https://unknown.com/abc/def", None),
+        (
+            "+1-2",
+            "https://unknown.com/abc/def",
+            "a5d6b89c35224d4ed69910a18fb544ca3fb26f62db53bc2769ce8a8d5cf8874c191186d170cb6e8896b0aaa8eaed891e7e819c4c0c7af499397c84761d6fb22d",
+        ),
+        (None, "https://unknown.com/abc/def/be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d", None),
+        (
+            "+1-2",
+            "https://unknown.com/abc/def/be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "a5d6b89c35224d4ed69910a18fb544ca3fb26f62db53bc2769ce8a8d5cf8874c191186d170cb6e8896b0aaa8eaed891e7e819c4c0c7af499397c84761d6fb22d",
+        ),
+    ]
+
+    assert (
+        AdvisoryReference.objects.count() == 2
+    )  # Only 2 are created because the 6 inputs include duplicates with the same URL and reference ID.
+    assert [(ref.url, ref.reference_type, ref.reference_id) for ref in adv.references.all()] == [
+        ("https://unknown.com/abc/def", "commit", "8eb1b04ca4ae6fc0a0ef46f1b0c042f64db28ff9"),
+        (
+            "https://unknown.com/abc/def/be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+            "commit",
+            "be891173be2fbdc897116bf5aa4fc9fdc8dc4f3d",
+        ),
+    ]