Fix Malicious Site Protection Duplicate PRs (#3136)

* Add rate-limiting to prevent getting blocked by bot detection.

* Add check-for-duplicates step to workflow to prevent duplicate PRs being created.

* Lint fix.

* Fix yaml syntax.

* Remove duplicate check, introduce PR ID instead.

* Use PR_ID in branch to ensure no duplication.

* Add test branch to test PR creation with ID

* Update test branch typo.

* Remove comment from yaml.

* Update triggers for testing.

* Update peter-evans/create-pull-request version

* Revert testing workflow configs - ready for merge.

* Update cron.

Author: not-a-rootkit

.github/workflows/msp-filter-stale.yml

@@ -29,12 +29,14 @@ jobs:
                   echo "TEMPLATE=$TEMPLATE" >> $GITHUB_ENV
                   PR_BODY="${TEMPLATE//\{\{date\}\}/$DATE}"
                   PR_BODY="${PR_BODY//\\n/$'\n'}" # Replace escaped newlines with actual newlines
+                  PR_ID="$(echo "$PR_BODY" | grep 'PR ID: ' | cut -d' ' -f3)"
+                  echo "PR_ID=$PR_ID" >> $GITHUB_ENV
                   echo "PR_BODY<<EOF" >> $GITHUB_ENV
                   echo "$PR_BODY" >> $GITHUB_ENV
                   echo "EOF" >> $GITHUB_ENV
                   npm run lint-fix
             - name: Create PR with updated configs
-              uses: peter-evans/create-pull-request@88bf0de51c7487d91e1abbb4899332e602c58bbf
+              uses: peter-evans/create-pull-request@889dce9eaba7900ce30494f5e1ac7220b27e5c81
               id: create-pr
               with:
                   github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -43,7 +45,7 @@ jobs:
                       ./
                   commit-message: Remove stale exemptions from malicious site protection
                   assignees: not-a-rootkit
-                  branch: remove-stale-exemptions-${{ env.DATE }}
+                  branch: remove-stale-exemptions-${{ env.PR_ID }}
                   title: Remove stale exemptions from malicious site protection - ${{ env.DATE }}
                   body: |
                       ${{ env.PR_BODY }}

scripts/filter-malicious.js

@@ -2,6 +2,35 @@ const fs = require('fs').promises;
 const path = require('path');
 const fetch = require('node-fetch').default;
 const crypto = require('crypto');
+const _ = require('lodash'); // Add this import
+
+// Rate limit fetch to 30 requests per second using a promise queue
+// https://thoughtspile.github.io/2018/07/07/rate-limit-promises/
+const resolveAfter = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+function rateLimit1(fn, msPerOp) {
+    let wait = Promise.resolve();
+    return (...a) => {
+        const res = wait.then(() => fn(...a));
+        wait = wait.then(() => resolveAfter(msPerOp));
+        return res;
+    };
+}
+
+function rateLimit(fn, windowMs, reqInWindow = 1) {
+    // A battery of 1-rate-limiters
+    const queue = _.range(reqInWindow).map(() => rateLimit1(fn, windowMs));
+    // Circular queue cursor
+    let i = 0;
+    return (...a) => {
+        // to enqueue, we move the cursor...
+        i = (i + 1) % reqInWindow;
+        // and return the rate-limited operation.
+        return queue[i](...a);
+    };
+}
+
+const slowFetch = rateLimit(fetch, 1000, 30);
 
 class ConfigProcessor {
     constructor(options = {}) {
@@ -47,7 +76,12 @@ class ConfigProcessor {
         const hashPrefix = this.generateHashPrefix(domain);
         const url = `${this.apiBaseUrl}/${platform}/matches?hashPrefix=${hashPrefix}`;
         try {
-            const response = await fetch(url);
+            const response = await slowFetch(url, {
+                headers: {
+                    'User-Agent':
+                        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3.1 Safari/605.1.15 Ddg/18.3.1',
+                },
+            });
             if (!response.ok) return false;
 
             const data = await response.json();
@@ -101,6 +135,11 @@ class ConfigProcessor {
         removedDomains.forEach((domain) => {
             prBody += ` - ${domain}\\n`;
         });
+
+        // PR ID is a first half of the hash of the sorted list of removed domains
+        const prId = crypto.createHash('sha256').update(Array.from(removedDomains).sort().join(',')).digest('hex').slice(0, 16);
+        prBody += `\\nPR ID: ${prId}\\n`;
+        prBody += `\\nThis PR is auto-generated. Please do not edit it manually.`;
         console.log(prBody);
     }