diff --git a/lib/resolve-labels.js b/lib/resolve-labels.js
index 1b63f2b..6707874 100644
--- a/lib/resolve-labels.js
+++ b/lib/resolve-labels.js
@@ -84,7 +84,7 @@ function matchAllSubSystem (filepathsChanged, subSystemLabelsMap) {
 
 function matchSubSystemsByRegex (rxLabelsMap, filepathsChanged) {
   const labelsCountLimit = process.env.MAX_LABELS_LIMIT || 4
-  const labelCount = []
+  const labelCount = new Set()
 
   // by putting matched labels into a map, we avoid duplicate labels
   const labelsMap = filepathsChanged.reduce((map, filepath) => {
@@ -102,7 +102,8 @@ function matchSubSystemsByRegex (rxLabelsMap, filepathsChanged) {
     for (let i = 0; i < mappedSubSystems.length; ++i) {
       const mappedSubSystem = mappedSubSystems[i]
       if (hasLibOrSrcChanges(filepathsChanged)) {
-        if (labelCount.length >= labelsCountLimit) {
+        labelCount.add(mappedSubSystem)
+        if (labelCount.size > labelsCountLimit) {
           for (const label of labelCount) {
             // don't delete the `c++` or `needs-ci` labels as we always want those if they have matched
             if (label !== 'c++' && label !== 'needs-ci') delete map[label]
@@ -110,8 +111,6 @@ function matchSubSystemsByRegex (rxLabelsMap, filepathsChanged) {
           map['lib / src'] = true
           // short-circuit
           return map
-        } else {
-          labelCount.push(mappedSubSystem)
         }
       }
 
diff --git a/test/resolve-labels.test.js b/test/resolve-labels.test.js
index 02a927a..75b7efa 100644
--- a/test/resolve-labels.test.js
+++ b/test/resolve-labels.test.js
@@ -265,6 +265,21 @@ tap.test('label: "lib / src" when 4 or more JS sub-systems have been changed', (
   t.end()
 })
 
+tap.test('labels: retain fewer than four distinct subsystems despite repeated matches', (t) => {
+  const labels = resolveLabels([
+    'lib/internal/crypto/keys.js',
+    'lib/internal/crypto/cipher.js',
+    'lib/internal/crypto/hash.js',
+    'lib/dns.js',
+    'lib/repl.js',
+    'lib/v8.js'
+  ])
+
+  t.same(labels, ['needs-ci', 'crypto', 'dns', 'repl', 'v8'])
+
+  t.end()
+})
+
 // https://github.com/nodejs/node/pull/12366 should have been labelled "lib / src"
 // https://github.com/nodejs/github-bot/issues/137
 tap.test('label: "lib / src" when 4 or more native files have been changed', (t) => {