fix: never make a page wait on WebDecoy Cloud when ingest is slow or down

includes/class-webdecoy-detection-sender.php

@@ -0,0 +1,154 @@
+<?php
+
+declare(strict_types=1);
+
+if (!defined('ABSPATH')) {
+    exit;
+}
+
+/**
+ * Sends detections to WebDecoy Cloud after the visitor has their page
+ * (WebDecoy/app#1245).
+ *
+ * Detections used to be sent inline, during `init`, before the block
+ * decision, with a 10 second timeout and, on installs without a stored
+ * organization id, a second 10 second key lookup in front of it. When ingest
+ * was slow or down, every flagged page view waited up to 20 seconds on it.
+ *
+ * Now a detection is queued during the request and sent from a shutdown
+ * handler, after fastcgi_finish_request() has handed the response back (where
+ * the server supports it). Blocking and logging never wait on the cloud.
+ *
+ * While ingest is refusing work (rate limited, shedding, or unreachable) the
+ * sender backs off for BACKOFF_SECONDS across all requests, so an outage costs
+ * one failed attempt per window instead of one per flagged page view. Those
+ * detections are dropped, not retried: they are already in the local log, and
+ * the cloud copy is best effort.
+ */
+class WebDecoy_Detection_Sender
+{
+    /** Transient that marks ingest as refusing work; shared by every request. */
+    public const BACKOFF_TRANSIENT = 'webdecoy_ingest_backoff';
+
+    /** How long one refusal pauses sending. */
+    public const BACKOFF_SECONDS = 60;
+
+    /** Cap on detections one request may queue. */
+    private const MAX_PER_REQUEST = 5;
+
+    /** @var array<int,callable():void> */
+    private static $queue = [];
+
+    /** @var bool */
+    private static $registered = false;
+
+    /**
+     * Test seam: replaces the transient store. Null uses WordPress transients.
+     *
+     * @var array<string,mixed>|null
+     */
+    public static $store = null;
+
+    /**
+     * Queue one send for after the response. $send performs the request and
+     * throws on failure.
+     *
+     * @param callable():void $send
+     */
+    public static function defer(callable $send): void
+    {
+        if (self::backing_off() || count(self::$queue) >= self::MAX_PER_REQUEST) {
+            return;
+        }
+        self::$queue[] = $send;
+        if (!self::$registered) {
+            self::$registered = true;
+            register_shutdown_function([self::class, 'flush']);
+        }
+    }
+
+    /**
+     * Shutdown handler: release the visitor first, then send.
+     */
+    public static function flush(): void
+    {
+        if (self::$queue === []) {
+            return;
+        }
+        if (function_exists('fastcgi_finish_request')) {
+            @fastcgi_finish_request(); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged
+        }
+        self::send_queued();
+    }
+
+    /**
+     * Send everything queued, stopping at the first refusal. Separate from
+     * flush() so tests can run it without finishing the request.
+     */
+    public static function send_queued(): void
+    {
+        $queue = self::$queue;
+        self::$queue = [];
+        foreach ($queue as $send) {
+            if (self::backing_off()) {
+                return;
+            }
+            try {
+                $send();
+            } catch (\Throwable $e) {
+                if (self::is_refusal($e)) {
+                    self::note_refusal();
+                }
+                error_log('WebDecoy API error: ' . $e->getMessage());
+            }
+        }
+    }
+
+    /** Whether ingest is currently refusing work. */
+    public static function backing_off(): bool
+    {
+        if (self::$store !== null) {
+            return !empty(self::$store[self::BACKOFF_TRANSIENT]);
+        }
+        return (bool) get_transient(self::BACKOFF_TRANSIENT);
+    }
+
+    /**
+     * Record that ingest refused work, pausing every cloud call that checks
+     * backing_off() (detection sends here, IP enrichment) for BACKOFF_SECONDS.
+     * The one answer to "is ingest refusing right now" for the plugin.
+     */
+    public static function note_refusal(): void
+    {
+        if (self::$store !== null) {
+            self::$store[self::BACKOFF_TRANSIENT] = 1;
+            return;
+        }
+        set_transient(self::BACKOFF_TRANSIENT, 1, self::BACKOFF_SECONDS);
+    }
+
+    /**
+     * A refusal means "come back later": rate limited (429), shedding or down
+     * (5xx), or no HTTP answer at all (code 0). A 4xx other than 429 is an
+     * answer about this request, not about ingest, and does not pause sending.
+     */
+    public static function is_refusal(\Throwable $e): bool
+    {
+        $code = (int) $e->getCode();
+        return $code === 0 || $code === 429 || $code >= 500;
+    }
+
+    /** Test seam: forget queued sends and registration. */
+    public static function reset(): void
+    {
+        self::$queue = [];
+        self::$registered = false;
+        self::$store = [];
+    }
+
+    /** Test seam: how many sends are queued. */
+    public static function queued(): int
+    {
+        return count(self::$queue);
+    }
+}

includes/class-webdecoy-ip-enrichment.php

@@ -79,6 +79,12 @@ public function enrich(string $ip): ?array
             return null;
         }
 
+        // While ingest is refusing work, skip the call instead of making this
+        // page wait out the timeout for every new IP (WebDecoy/app#1245).
+        if (class_exists('WebDecoy_Detection_Sender') && WebDecoy_Detection_Sender::backing_off()) {
+            return null;
+        }
+
         $data = $this->fetch($ip);
 
         if ($data === null) {
@@ -110,9 +116,14 @@ private function fetch(string $ip): ?array
         ]);
 
         if (is_wp_error($response)) {
+            self::note_refusal();
             return null;
         }
-        if ((int) wp_remote_retrieve_response_code($response) !== 200) {
+        $code = (int) wp_remote_retrieve_response_code($response);
+        if ($code === 429 || $code >= 500) {
+            self::note_refusal();
+        }
+        if ($code !== 200) {
             return null;
         }
 
@@ -125,4 +136,12 @@ private function fetch(string $ip): ?array
 
         return $data;
     }
+
+    /** Pause cloud calls after an unavailable answer (WebDecoy/app#1245). */
+    private static function note_refusal(): void
+    {
+        if (class_exists('WebDecoy_Detection_Sender')) {
+            WebDecoy_Detection_Sender::note_refusal();
+        }
+    }
 }

tests/DetectionSenderTest.php

@@ -0,0 +1,86 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * Detections are sent after the response, never in the page path
+ * (WebDecoy/app#1245).
+ *
+ * When ingest was slow or down, an inline send with a 10 second timeout (plus a
+ * 10 second key lookup) stalled every flagged page view for up to 20 seconds,
+ * before the block decision ran. These tests pin the deferral, the backoff that
+ * stops an outage costing one failed attempt per page view, and, by reading the
+ * plugin source, that no send path goes around the sender.
+ *
+ * Run: php tests/run.php
+ */
+
+if (!defined('ABSPATH')) {
+    define('ABSPATH', '/tmp/');
+}
+require_once dirname(__DIR__) . '/includes/class-webdecoy-detection-sender.php';
+
+$t = ['TestRunner', 'test'];
+$same = ['TestRunner', 'assertSame'];
+$true = ['TestRunner', 'assertTrue'];
+
+$t('a detection is queued during the request, not sent', function () use ($same) {
+    WebDecoy_Detection_Sender::reset();
+    $sent = 0;
+    WebDecoy_Detection_Sender::defer(function () use (&$sent) { $sent++; });
+    $same(0, $sent, 'nothing is sent while the request is being handled');
+    $same(1, WebDecoy_Detection_Sender::queued());
+    WebDecoy_Detection_Sender::send_queued();
+    $same(1, $sent, 'sent at shutdown');
+    $same(0, WebDecoy_Detection_Sender::queued());
+});
+
+$t('a refusal pauses sending for every later request', function () use ($same, $true) {
+    WebDecoy_Detection_Sender::reset();
+    $calls = 0;
+    WebDecoy_Detection_Sender::defer(function () use (&$calls) { $calls++; throw new \Exception('shed', 503); });
+    WebDecoy_Detection_Sender::defer(function () use (&$calls) { $calls++; });
+    WebDecoy_Detection_Sender::send_queued();
+    $same(1, $calls, 'stops at the first refusal');
+    $true(WebDecoy_Detection_Sender::backing_off());
+    // The next request queues nothing while backing off.
+    WebDecoy_Detection_Sender::defer(function () use (&$calls) { $calls++; });
+    $same(0, WebDecoy_Detection_Sender::queued());
+});
+
+$t('429, 5xx and no answer are refusals; other 4xx are not', function () use ($true) {
+    $true(WebDecoy_Detection_Sender::is_refusal(new \Exception('', 429)));
+    $true(WebDecoy_Detection_Sender::is_refusal(new \Exception('', 503)));
+    $true(WebDecoy_Detection_Sender::is_refusal(new \Exception('timeout', 0)));
+    $true(!WebDecoy_Detection_Sender::is_refusal(new \Exception('', 400)));
+    $true(!WebDecoy_Detection_Sender::is_refusal(new \Exception('', 401)));
+});
+
+$t('a bad request does not pause sending', function () use ($true) {
+    WebDecoy_Detection_Sender::reset();
+    WebDecoy_Detection_Sender::defer(function () { throw new \Exception('bad payload', 400); });
+    WebDecoy_Detection_Sender::send_queued();
+    $true(!WebDecoy_Detection_Sender::backing_off());
+});
+
+$t('one request queues at most five sends', function () use ($same) {
+    WebDecoy_Detection_Sender::reset();
+    for ($i = 0; $i < 20; $i++) {
+        WebDecoy_Detection_Sender::defer(function () {});
+    }
+    $same(5, WebDecoy_Detection_Sender::queued());
+    WebDecoy_Detection_Sender::reset();
+});
+
+$t('every detection send in the plugin goes through the deferred sender', function () use ($same, $true) {
+    $src = (string) file_get_contents(dirname(__DIR__) . '/webdecoy.php');
+    $same(1, substr_count($src, '->submitDetection('), 'exactly one send site');
+    $send = strpos($src, '->submitDetection(');
+    $defer = strrpos(substr($src, 0, (int) $send), 'WebDecoy_Detection_Sender::defer(');
+    $true($defer !== false && $send - $defer < 400, 'the send site is inside a WebDecoy_Detection_Sender::defer closure');
+    // The detection client never falls back to a per-request key lookup when
+    // the organization id is known, and never waits 10 seconds.
+    $true(strpos($src, "'timeout' => 3,") !== false, 'detection client uses a short timeout');
+    $true(strpos($src, "'organization_id' => \$org !== '' ? \$org : null,") !== false, 'detection client is given the organization id');
+});
+

tests/IpEnrichmentBackoffTest.php

@@ -0,0 +1,53 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * IP enrichment pauses with detection sends while ingest is refusing work
+ * (WebDecoy/app#1245). Named to load after HoneytokenTest, whose get_option
+ * stub WebDecoy_Cloud_Connect needs.
+ *
+ * Run: php tests/run.php
+ */
+
+if (!defined('ABSPATH')) {
+    define('ABSPATH', '/tmp/');
+}
+require_once dirname(__DIR__) . '/includes/class-webdecoy-detection-sender.php';
+
+$t = ['TestRunner', 'test'];
+$same = ['TestRunner', 'assertSame'];
+$true = ['TestRunner', 'assertTrue'];
+
+// IP enrichment shares the backoff: during an outage a page must not wait out
+// the enrichment timeout for every new IP.
+if (!function_exists('apply_filters')) {
+    function apply_filters($hook, $value) { return $value; }
+}
+if (!defined('MINUTE_IN_SECONDS')) {
+    define('MINUTE_IN_SECONDS', 60);
+}
+if (!defined('HOUR_IN_SECONDS')) {
+    define('HOUR_IN_SECONDS', 3600);
+}
+if (!function_exists('wp_remote_get')) {
+    $GLOBALS['wd_test_remote'] = ['calls' => 0, 'code' => 503];
+    function wp_remote_get($url, $args = []) { $GLOBALS['wd_test_remote']['calls']++; return ['code' => $GLOBALS['wd_test_remote']['code']]; }
+    function is_wp_error($thing) { return false; }
+    function wp_remote_retrieve_response_code($response) { return $response['code']; }
+    function wp_remote_retrieve_body($response) { return ''; }
+}
+require_once dirname(__DIR__) . '/includes/class-webdecoy-ip-enrichment.php';
+
+$t('IP enrichment makes no call while ingest is refusing, and a 503 starts the pause', function () use ($same, $true) {
+    WebDecoy_Detection_Sender::reset();
+    $GLOBALS['wd_test_remote'] = ['calls' => 0, 'code' => 503];
+    $enricher = new WebDecoy_IP_Enrichment('key');
+    $enricher->enrich('198.51.100.7');
+    $same(1, $GLOBALS['wd_test_remote']['calls'], 'first call is made');
+    $true(WebDecoy_Detection_Sender::backing_off(), 'a 503 pauses cloud calls');
+    $enricher->enrich('198.51.100.8');
+    $enricher->enrich('198.51.100.9');
+    $same(1, $GLOBALS['wd_test_remote']['calls'], 'no further calls while paused');
+    WebDecoy_Detection_Sender::reset();
+});