Skip to content

Commit 60fee54

Browse files
ziadhanyziadhany
authored
fix: validate aliases for alpine linux, mattermost, fireeye, istio (#2274)
* fix: validate aliases for alpine linux, mattermost, fireeye, istio advisories Signed-off-by: ziad hany <ziadhany2016@gmail.com> * Add migration to drop malformed advisories Add a migration test Signed-off-by: ziad hany <ziadhany2016@gmail.com> * Fix a typo in parse_vuln_ids function docs & update the test Signed-off-by: ziad hany <ziadhany2016@gmail.com> * Add a filter to ensure that only malformed avid from alpine_linux, mattermost_importer_v2, fireeye_importer_v2, and istio_importer_v2 are deleted Signed-off-by: ziad hany <ziadhany2016@gmail.com> * Fix a bug in istio importer Signed-off-by: ziad hany <ziadhany2016@gmail.com> * Delete alias from target advisiories only Signed-off-by: ziad hany <ziadhany2016@gmail.com> * Resolve merge migration conflict Signed-off-by: ziad hany <ziadhany2016@gmail.com> --------- Signed-off-by: ziad hany <ziadhany2016@gmail.com>
1 parent 6ea0ced commit 60fee54

9 files changed

Lines changed: 405 additions & 14 deletions

‎vulnerabilities/migrations/0124_advisoryv2_remove_malformed_aliases_and_dvisory_id.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
from django.db import migrations
2+
from django.db.models import Q
3+
4+
5+
class Migration(migrations.Migration):
6+
dependencies = [
7+
("vulnerabilities", "0123_alter_packagev2_options_alter_packagev2_package_url_and_more"),
8+
]
9+
10+
def drop_malformed_advisory_v2(apps, _):
11+
AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2")
12+
AdvisoryAlias = apps.get_model("vulnerabilities", "AdvisoryAlias")
13+
14+
valid_alias_prefix = [
15+
"cve-", "osv-", "xsa-", "vsv", "zbx-", "zf2", "vu#", "gms-", "usn-",
16+
"sw-", "ss-", "ts-", "osvdb-", "ysa-", "se-core-", "pysec-", "alpine-",
17+
"dw2", "go-", "mal-", "zdi-can", "asa-", "ezsa-", "ghsl-", "ghsa-",
18+
"talos-", "srcclr-sid-", "bit-", "gnutls-", "rustsec-", "snyk-",
19+
"temp-", "TYPO3-", "wnpa-sec-", "sa-core-", "skcsirt-", "flow-", "gsd-"
20+
]
21+
22+
target_importers = ["alpine_linux_importer_v2",
23+
"fireeye_importer_v2",
24+
"istio_importer_v2",
25+
"mattermost_importer_v2"]
26+
query = Q()
27+
for alias_prefix in valid_alias_prefix:
28+
query |= Q(alias__istartswith=alias_prefix)
29+
30+
malformed_alias_ids = list(
31+
AdvisoryAlias.objects.filter(
32+
advisories__datasource_id__in=target_importers
33+
).exclude(query).values_list('id', flat=True).distinct()
34+
)
35+
36+
AdvisoryV2.objects.filter(
37+
datasource_id__in=target_importers,
38+
aliases__id__in=malformed_alias_ids
39+
).delete()
40+
41+
AdvisoryAlias.objects.filter(
42+
id__in=malformed_alias_ids,
43+
advisories__isnull=True
44+
).delete()
45+
46+
operations = [
47+
migrations.RunPython(drop_malformed_advisory_v2, reverse_code=migrations.RunPython.noop),
48+
]

‎vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
import json
1111
import logging
12+
import re
1213
from pathlib import Path
1314
from typing import Any
1415
from typing import Iterable
@@ -28,6 +29,7 @@
2829
from vulnerabilities.references import XsaReferenceV2
2930
from vulnerabilities.references import ZbxReferenceV2
3031
from vulnerabilities.utils import get_advisory_url
32+
from vulnerabilities.utils import is_cve
3133
from vulnerabilities.utils import load_json
3234

3335

@@ -164,8 +166,10 @@ def load_advisories(
164166
# fixed_vulns is a list of strings and each string is a space-separated
165167
# list of aliases and CVES
166168
for vuln_ids in fixed_vulns:
167-
aliases = vuln_ids.strip().split(" ")
168-
vuln_id = aliases[0]
169+
vuln_id, aliases = parse_vuln_ids(vuln_ids, logger=logger)
170+
171+
if not vuln_id:
172+
continue
169173

170174
references = []
171175
if vuln_id.startswith("XSA"):
@@ -249,3 +253,53 @@ def load_advisories(
249253
url=url,
250254
original_advisory_text=json.dumps(pkg_infos, indent=2, ensure_ascii=False),
251255
)
256+
257+
258+
PARENTHESES_RE = re.compile(r"\(.*?\)")
259+
260+
261+
def parse_vuln_ids(vuln_ids_string, logger=print):
262+
"""
263+
Parses a raw vulnerability ids, removes parentheses and returns the vuln_id and a list of all valid aliases.
264+
"""
265+
vuln_ids = PARENTHESES_RE.sub("", vuln_ids_string)
266+
if not vuln_ids_string:
267+
return None, []
268+
269+
cleaned_vuln_ids = []
270+
for alias in vuln_ids.split():
271+
clean_alias = alias.replace("_", "-").replace(".patch", "")
272+
cleaned_vuln_ids.append(clean_alias)
273+
274+
aliases = []
275+
valid_prefixes = (
276+
"XSA-",
277+
"GHSL-",
278+
"TALOS-",
279+
"RUSTSEC-",
280+
"GHSA-",
281+
"GNUTLS-",
282+
"VSV",
283+
"ZDI-CAN-",
284+
"DW",
285+
"YSA-",
286+
"ZBX-",
287+
"ALPINE-",
288+
"TS-",
289+
"OSEC-",
290+
"wnpa-sec-",
291+
)
292+
293+
for alias in cleaned_vuln_ids:
294+
if alias and (
295+
(alias.startswith("CVE-") and is_cve(alias)) or alias.startswith(valid_prefixes)
296+
):
297+
aliases.append(alias)
298+
else:
299+
logger(f"Malformed aliases found: {alias}")
300+
301+
if not aliases:
302+
return None, []
303+
304+
vuln_id = aliases[0]
305+
return vuln_id, aliases

‎vulnerabilities/pipelines/v2_importers/fireeye_importer_v2.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,15 +104,16 @@ def parse_advisory_data(raw_data, file_path, base_path) -> AdvisoryDataV2:
104104
cve_refs = md_dict.get("## CVE Reference") or []
105105
cve_ids = md_dict.get("## CVE ID") or []
106106
cleaned_cve_ids = []
107-
for line in cve_ids:
107+
for line in cve_ids + cve_refs:
108108
found_cves = find_all_cve(line)
109109
cleaned_cve_ids.extend(found_cves)
110110

111111
references = md_dict.get("## References") or []
112112
cwe_data = md_dict.get("## Common Weakness Enumeration") or []
113113

114114
advisory_id = file_path.stem
115-
aliases = dedupe([cve.strip() for cve in cleaned_cve_ids + cve_refs])
115+
aliases = dedupe([cve.strip() for cve in cleaned_cve_ids])
116+
116117
aliases = [aliase for aliase in aliases if aliase != advisory_id]
117118
advisory_url = get_advisory_url(
118119
file=file_path,

‎vulnerabilities/pipelines/v2_importers/istio_importer.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from vulnerabilities.importer import ReferenceV2
2929
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
3030
from vulnerabilities.utils import get_advisory_url
31+
from vulnerabilities.utils import is_cve
3132
from vulnerabilities.utils import split_markdown_front_matter
3233

3334
is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match
@@ -88,6 +89,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
8889
data.get("releases", []), GolangVersion
8990
)
9091
cves = data.get("cves", [])
92+
aliases = [cve for cve in cves if is_cve(cve)]
9193

9294
affected_packages = []
9395
if semver_constraints:
@@ -107,6 +109,10 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
107109
)
108110

109111
title = data.get("title") or ""
112+
if not title.startswith("ISTIO-SECURITY-"):
113+
self.log(f"Invalid advisory_id: {title}")
114+
continue
115+
110116
summary = data.get("description") or ""
111117
references = []
112118
if title:
@@ -119,7 +125,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
119125

120126
yield AdvisoryDataV2(
121127
advisory_id=title,
122-
aliases=cves,
128+
aliases=aliases,
123129
summary=summary,
124130
affected_packages=affected_packages,
125131
references=references,

‎vulnerabilities/pipelines/v2_importers/mattermost_importer.py‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from vulnerabilities.importer import VulnerabilitySeverity
2121
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
2222
from vulnerabilities.utils import fetch_response
23+
from vulnerabilities.utils import is_cve
2324

2425
MM_REPO = {
2526
"Mattermost Mobile Apps": "mattermost-mobile",
@@ -62,11 +63,17 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
6263
return
6364

6465
for advisory in data:
65-
vuln_id = advisory.get("issue_id")
66-
if not vuln_id or not vuln_id.startswith("MMSA-"):
67-
self.log(f"Skipping advisory with missing issue_id. {vuln_id}")
66+
issue_id = advisory.get("issue_id") or ""
67+
cve_id = advisory.get("cve_id") or ""
68+
69+
advisory_id, aliases = parse_vuln_ids(issue_id, cve_id)
70+
71+
if not advisory_id:
72+
self.log(
73+
f"Skipping advisory with missing advisory_id. issue_id:{issue_id} cve_id:{cve_id}"
74+
)
6875
continue
69-
cve_id = advisory.get("cve_id")
76+
7077
details = advisory.get("details")
7178

7279
platform = advisory.get("platform")
@@ -78,7 +85,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
7885
affected_packages = []
7986
severity = advisory.get("severity")
8087
if not package_name:
81-
self.log(f"Unknown platform '{platform}' in advisory '{vuln_id}'.")
88+
self.log(f"Unknown platform '{platform}' in advisory '{advisory_id}'.")
8289

8390
else:
8491
package = PackageURL(
@@ -105,7 +112,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
105112
)
106113
except Exception as e:
107114
self.log(
108-
f"Error processing fixed versions '{fixed_versions}' for advisory '{vuln_id}': {e}"
115+
f"Error processing fixed versions '{fixed_versions}' for advisory '{advisory_id}': {e}"
109116
)
110117

111118
severities = []
@@ -118,12 +125,36 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
118125
)
119126

120127
yield AdvisoryDataV2(
121-
advisory_id=vuln_id,
122-
aliases=[cve_id],
128+
advisory_id=advisory_id,
129+
aliases=aliases,
123130
summary=details,
124131
references=[reference],
125132
affected_packages=affected_packages,
126133
severities=severities,
127134
url=self.url,
128135
original_advisory_text=json.dumps(advisory, indent=2, ensure_ascii=False),
129136
)
137+
138+
139+
def parse_vuln_ids(issue_id, cve_id):
140+
"""
141+
Parses a raw issue_id, cve_id, validate and returns the advisory id and a list of all valid aliases.
142+
"""
143+
advisory_id = None
144+
aliases = []
145+
146+
cve_id = cve_id.strip()
147+
issue_id = issue_id.strip()
148+
149+
for vuln_id in issue_id.split(","):
150+
vuln_id = vuln_id.strip()
151+
if vuln_id.startswith("MMSA-") or vuln_id.startswith("CVE-"):
152+
aliases.append(vuln_id)
153+
154+
if cve_id and is_cve(cve_id):
155+
aliases.append(cve_id)
156+
157+
if aliases:
158+
advisory_id = aliases.pop(0)
159+
160+
return advisory_id, aliases

‎vulnerabilities/tests/pipelines/v2_importers/test_alpine_linux_importer_pipeline.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import pytest
1414

1515
from vulnerabilities.pipelines.v2_importers.alpine_linux_importer import load_advisories
16+
from vulnerabilities.pipelines.v2_importers.alpine_linux_importer import parse_vuln_ids
1617
from vulnerabilities.pipelines.v2_importers.alpine_linux_importer import process_record
1718
from vulnerabilities.tests import util_tests
1819
from vulnerabilities.tests.pipelines import TestLogger
@@ -95,3 +96,83 @@ def test_load_advisories_package_with_invalid_alpine_version(test_case):
9596
}
9697
result = list(load_advisories(package, "v3.11", "main", archs=[], url="", logger=logger.write))
9798
assert result != []
99+
100+
101+
@pytest.mark.parametrize(
102+
"raw_input, expected_vuln_id, expected_aliases",
103+
[
104+
("CVE-2022-42332 XSA-427", "CVE-2022-42332", ["CVE-2022-42332", "XSA-427"]),
105+
(
106+
"CVE-2022-42333 CVE-2022-43334 XSA-428",
107+
"CVE-2022-42333",
108+
["CVE-2022-42333", "CVE-2022-43334", "XSA-428"],
109+
),
110+
(
111+
"CVE-2020-11501 GNUTLS-SA-2020-03-31 CVE-2020-11501",
112+
"CVE-2020-11501",
113+
["CVE-2020-11501", "GNUTLS-SA-2020-03-31", "CVE-2020-11501"],
114+
),
115+
("CVE_2019-2426", "CVE-2019-2426", ["CVE-2019-2426"]),
116+
(
117+
"CVE-2024-22195 GHSA-h5c8-rqwp-cp95",
118+
"CVE-2024-22195",
119+
["CVE-2024-22195", "GHSA-h5c8-rqwp-cp95"],
120+
),
121+
("CVE-2023-44441 ZDI-CAN-22093", "CVE-2023-44441", ["CVE-2023-44441", "ZDI-CAN-22093"]),
122+
("CVE-2022-45059 VSV00010", "CVE-2022-45059", ["CVE-2022-45059", "VSV00010"]),
123+
("OSEC-2026-03", "OSEC-2026-03", ["OSEC-2026-03"]),
124+
("CVE-2021-35940.patch", "CVE-2021-35940", ["CVE-2021-35940"]),
125+
("XSA-207", "XSA-207", ["XSA-207"]),
126+
("ALPINE-13661", "ALPINE-13661", ["ALPINE-13661"]),
127+
("GHSA-vv2x-vrpj-qqpq", "GHSA-vv2x-vrpj-qqpq", ["GHSA-vv2x-vrpj-qqpq"]),
128+
("CVE N/A ZBX-11023", "ZBX-11023", ["ZBX-11023"]),
129+
("CVE-2017-2616 (+ regression fix)", "CVE-2017-2616", ["CVE-2017-2616"]),
130+
(
131+
"CVE-2020-14342 (Not affected, requires --with-systemd)",
132+
"CVE-2020-14342",
133+
["CVE-2020-14342"],
134+
),
135+
("CVE-2017-16808 (AoE)", "CVE-2017-16808", ["CVE-2017-16808"]),
136+
("CVE-2018-14468 (FrameRelay)", "CVE-2018-14468", ["CVE-2018-14468"]),
137+
("CVE-2018-14469 (IKEv1)", "CVE-2018-14469", ["CVE-2018-14469"]),
138+
("CVE-2018-14470 (BABEL)", "CVE-2018-14470", ["CVE-2018-14470"]),
139+
("CVE-2018-14466 (AFS/RX)", "CVE-2018-14466", ["CVE-2018-14466"]),
140+
("CVE-2018-14461 (LDP)", "CVE-2018-14461", ["CVE-2018-14461"]),
141+
("CVE-2018-14462 (ICMP)", "CVE-2018-14462", ["CVE-2018-14462"]),
142+
("CVE-2018-14465 (RSVP)", "CVE-2018-14465", ["CVE-2018-14465"]),
143+
("CVE-2018-14881 (BGP)", "CVE-2018-14881", ["CVE-2018-14881"]),
144+
("CVE-2018-14464 (LMP)", "CVE-2018-14464", ["CVE-2018-14464"]),
145+
("CVE-2018-14463 (VRRP)", "CVE-2018-14463", ["CVE-2018-14463"]),
146+
("CVE-2018-14467 (BGP)", "CVE-2018-14467", ["CVE-2018-14467"]),
147+
(
148+
"CVE-2018-10103 (SMB - partially fixed, but SMB printing disabled)",
149+
"CVE-2018-10103",
150+
["CVE-2018-10103"],
151+
),
152+
(
153+
"CVE-2018-10105 (SMB - too unreliably reproduced, SMB printing disabled)",
154+
"CVE-2018-10105",
155+
["CVE-2018-10105"],
156+
),
157+
("CVE-2018-14880 (OSPF6)", "CVE-2018-14880", ["CVE-2018-14880"]),
158+
("CVE-2018-16451 (SMB)", "CVE-2018-16451", ["CVE-2018-16451"]),
159+
("CVE-2018-14882 (RPL)", "CVE-2018-14882", ["CVE-2018-14882"]),
160+
("CVE-2018-16227 (802.11)", "CVE-2018-16227", ["CVE-2018-16227"]),
161+
("CVE-2018-16229 (DCCP)", "CVE-2018-16229", ["CVE-2018-16229"]),
162+
("CVE-2018-16301 (was fixed in libpcap)", "CVE-2018-16301", ["CVE-2018-16301"]),
163+
("CVE-2018-16230 (BGP)", "CVE-2018-16230", ["CVE-2018-16230"]),
164+
("CVE-2018-16452 (SMB)", "CVE-2018-16452", ["CVE-2018-16452"]),
165+
("CVE-2018-16300 (BGP)", "CVE-2018-16300", ["CVE-2018-16300"]),
166+
("CVE-2018-16228 (HNCP)", "CVE-2018-16228", ["CVE-2018-16228"]),
167+
("CVE-2019-15166 (LMP)", "CVE-2019-15166", ["CVE-2019-15166"]),
168+
("CVE-2019-15167 (VRRP)", "CVE-2019-15167", ["CVE-2019-15167"]),
169+
("CVE-????-????? TS-2024-005", "TS-2024-005", ["TS-2024-005"]),
170+
("CVE-????-????? TS-2024-005", "TS-2024-005", ["TS-2024-005"]),
171+
("CVE-2018-14879 (tcpdump -V)", "CVE-2018-14879", ["CVE-2018-14879"]),
172+
("CVE-46838", None, []), # invalid CVE
173+
],
174+
)
175+
def test_parse_vuln_ids(raw_input, expected_vuln_id, expected_aliases):
176+
vuln_id, aliases = parse_vuln_ids(raw_input)
177+
assert vuln_id == expected_vuln_id
178+
assert aliases == expected_aliases

0 commit comments

Comments
 (0)