test: [SDK-5535] add a fault-injection harness for operation retries

examples/demo/app/build.gradle.kts

@@ -134,6 +134,30 @@ android {
     }
 }
 
+// `-PRETRY_TEST_CA=<pem>` makes debug builds trust that CA, so scripts/retry can proxy SDK traffic
+// without installing a certificate on the device.
+demoOverride("RETRY_TEST_CA")?.let { caPath ->
+    val resDir = layout.buildDirectory.dir("generated/retryTestCa/res").get().asFile
+    file(caPath).copyTo(resDir.resolve("raw/retry_test_ca.pem"), overwrite = true)
+    resDir.resolve("xml/network_security_config.xml").apply {
+        parentFile.mkdirs()
+        writeText(
+            """
+            <?xml version="1.0" encoding="utf-8"?>
+            <network-security-config>
+                <debug-overrides>
+                    <trust-anchors>
+                        <certificates src="user" />
+                        <certificates src="@raw/retry_test_ca" />
+                    </trust-anchors>
+                </debug-overrides>
+            </network-security-config>
+            """.trimIndent(),
+        )
+    }
+    android.sourceSets.getByName("debug").res.srcDir(resDir)
+}
+
 dependencies {
     // Kotlin
     implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion")

examples/demo/app/src/main/AndroidManifest.xml

@@ -21,6 +21,7 @@
         android:allowBackup="true"
         android:icon="@mipmap/ic_onesignal_launcher"
         android:label="@string/app_name"
+        android:networkSecurityConfig="@xml/network_security_config"
         android:roundIcon="@mipmap/ic_onesignal_launcher"
         android:supportsRtl="true"
         android:theme="@style/AppTheme"

examples/demo/app/src/main/res/xml/network_security_config.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Debug builds trust user-installed CAs so scripts/retry can proxy SDK traffic. -->
+<network-security-config>
+    <debug-overrides>
+        <trust-anchors>
+            <certificates src="user" />
+        </trust-anchors>
+    </debug-overrides>
+</network-security-config>

examples/demo/scripts/retry/.gitignore

@@ -0,0 +1,2 @@
+.state/
+__pycache__/

examples/demo/scripts/retry/README.md

@@ -0,0 +1,127 @@
+# Retry test harness
+
+Injects failures into the SDK's requests from the demo app, so OperationRepo retries and backoff can
+be watched and checked. A [mitmproxy](https://mitmproxy.org) addon fails only requests that carry
+the SDK's `SDK-Version` header. The demo's own REST calls and all other traffic pass through. Only
+`onesignal.com` traffic is decrypted.
+
+```bash
+brew install mitmproxy mobile-dev-inc/tap/maestro
+cd examples/demo/scripts/retry
+./retry-test.sh help
+```
+
+[Maestro](https://maestro.mobile.dev) drives the demo's UI (`tap`) through the flows in `flows/`.
+
+## Setup
+
+1. Start an emulator, or connect a device. With more than one, pick it with
+   `RETRY_TEST_SERIAL=<adb serial>`.
+2. `./retry-test.sh install` builds the demo against the local SDK source and installs it. The
+   debug build trusts the mitmproxy CA (`-PRETRY_TEST_CA`), so nothing has to be installed on the
+   device. If a demo signed with another key is already installed, `adb uninstall
+   com.onesignal.example` first.
+3. `./retry-test.sh start` starts mitmproxy on port 8899 and sets the device's global HTTP proxy.
+   An emulator reaches it at `10.0.2.2`, a physical device through `adb reverse`.
+4. `./retry-test.sh fresh` clears the demo's data so the SDK starts as a new install.
+5. In another terminal, `./retry-test.sh watch` follows each SDK request.
+6. When done, **`./retry-test.sh stop`** clears the device's proxy. Until then the whole device
+   goes through the proxy.
+
+## Commands
+
+| Command | Does |
+| --- | --- |
+| `fail <route> <status> [--times N] [--retry-after S] [--add]` | Fail matching requests. Status `0` drops the connection, like being offline. Without `--times` it fails until `clear`. `--add` keeps earlier rules. Applies right away. |
+| `clear` / `rules` | Remove or show the rules |
+| `report [regex]` | Per operation: each attempt, the gap since the previous one, and the gap expected. Also lists any SDK request sent during a Retry-After |
+| `reset-log` | Forget recorded attempts, for example between scenarios |
+| `tap <action> [args]` | Use the demo: `add-tag <key> <value>`, `add-alias <label> <id>`, `add-email <email>`, `login <external id> [jwt]`, `logout`, `toggle-push`, `allow-notifications`. With `RETRY_TEST_IV_KEY` set, `login` signs a JWT when none is given |
+| `jwt <external id> [label=id] [email:addr] [sms:number]` | Print an Identity Verification JWT, valid for an hour, that claims those aliases and subscriptions |
+| `install` | Build the demo against the local SDK and install it |
+| `fresh` / `relaunch` | Clear the demo's data and relaunch / kill and relaunch |
+| `background` / `foreground` | Press Home / bring the demo back |
+| `network-flap [s]` | Turn the device's Wi-Fi and mobile data off for `s` seconds (default 10). Turns back on only what was on, even after Ctrl-C. Not over wireless adb |
+| `logs` | Stream the demo's OperationRepo and HttpClient logs |
+
+Routes: `create-user`, `fetch-user`, `update-user` (tags, properties, session data), `identity`
+(add alias and login's Identify User share it), `remove-alias`, `create-sub`, `update-sub`,
+`delete-sub`, `transfer-sub`, `identity-by-sub`, `custom-events`, `all`.
+
+## Reading `report`
+
+```
+POST apps/<app>/users body=b8398c6a
+    #     time status     body      gap         expected  note
+    1     0.0s   408* b8398c6a
+    2     3.9s   408* b8398c6a     3.9s        2.5-10.0s  backoff #1
+    3    11.2s   201  b8398c6a     7.3s        5.0-15.0s  backoff #2
+```
+
+Attempts are grouped per record (full path), and by body for routes without an id in the path
+(Create User, custom events).
+
+What the SDK should do:
+
+- **Backoff per request**: after the Nth retryable failure the request waits 5s × 2^(N-1), capped at
+  300s (`opRepoBaseFailRetryBackoff`, `opRepoMaxFailRetryBackoff`), times a jitter of 0.5 to 1.
+  Operations batched into one request share the wait. `HttpClient` does not re-send on its own.
+  Retryable is anything but 400, 401, 402, 403, 404, 409 and 410.
+- **Retry-After** is the minimum wait for that request, capped at 1h. `HttpClient` also holds
+  every SDK request, on any route, until it passes, and `report` lists any that weren't. Foreground
+  and network return do not cut it short. A 429 without Retry-After, or with an unparsable one,
+  waits 60s.
+- **Other records keep going**: while one request backs off, operations on other records are sent
+  (outside a Retry-After). Operations on the same record wait behind it, so order is kept.
+- **Foreground, and network return while in the foreground,** end the backoff, and the attempt goes
+  out right away without the queue's 5s batching wait. `report` flags those gaps as EARLY with a
+  note. A network return in the background does nothing; the timed retry or the next foreground
+  picks it up.
+- `report` allows 5s of slack past the window (`opRepoExecutionInterval`), since an enqueue near
+  the end of a backoff can add the batching wait.
+- Backoff state is in memory. After a relaunch, queued operations are sent right away.
+
+## Scenarios
+
+Run `reset-log` before each one. Unless noted, the request should succeed after the last failure,
+and nothing should need a relaunch.
+
+1. **Backoff**: `fail create-user 408 --times 3`, then `fresh`. Gaps within 2.5-5s, 5-10s and
+   10-20s (plus slack), then 201.
+2. **Other records are not blocked**: `fail update-user 408`, `tap add-tag k v`, then after the
+   first failure `tap toggle-push`. The subscription PATCH goes out while the tag keeps retrying.
+3. **Retry-After**: `fail update-user 429 --times 1 --retry-after 40`, then `tap add-tag k v`.
+   One retry after about 40s. `background` then `foreground` during the wait does not cut it short,
+   and a `tap toggle-push` during the wait is not sent until it ends.
+4. **Foreground ends the backoff**: `fail update-user 408`, `tap add-tag k v`, wait for four
+   failures (a 20-40s wait), then `background` and `foreground`. An attempt goes out right after
+   foreground.
+5. **Network return**: as in 4, but `network-flap 5` with the demo in the foreground. An attempt
+   goes out right after the network is back. With the demo in the background, the flap changes
+   nothing.
+6. **Order is kept**: `fail create-user 408 --times 2`, then `fresh` and `tap login <id>` during
+   the failures. Identify User (PATCH `.../identity`) is not sent until Create User succeeds.
+7. **Fetch User**: `fail fetch-user 408 --times 2`, then `tap login <id>`. Fetch User retries in
+   the same run.
+8. **Kill during backoff**: `fail update-user 408`, `tap add-tag k v`, wait for two failures,
+   `clear`, then `relaunch`. The cached request is sent right after launch.
+9. **Not retried**: `fail update-user 400 --times 1`, then `tap add-tag k v`. One attempt.
+10. **Offline**: `fail all 0`, then `tap add-tag k v`, `tap add-alias l i` and `tap add-email e`.
+    Wait for a few failures, then `clear`. Everything goes out on its next attempt.
+11. **Login/logout while failing**: `fail all 408`, `tap login b`, `tap logout`, `tap login c`,
+    then `clear`. Requests for each user go out in order.
+12. **Identity Verification**: needs an app with Identity Verification on and its ES256 private
+    key. Ask the SDK team for both and keep the key outside the repo. Set
+    `RETRY_TEST_IV_APP_ID=<app id>` and `RETRY_TEST_IV_KEY=<path to the key>`, then `install` again.
+    `tap login <id>` now signs a JWT. No anonymous Create User is sent, so in 1 and 6 the Create
+    User comes from the login, and 6 checks that a `tap add-tag` during the failures waits for it.
+    Repeat 1, 2, 6 and 10. In 10, log in with a JWT that claims the alias and email:
+    `tap login <id> "$(./retry-test.sh jwt <id> l=i email:e)"`.
+
+## Troubleshooting
+
+- `tap` types nothing: newer emulator images show a stylus handwriting prompt on text fields.
+  `install` turns it off (`settings put secure stylus_handwriting_enabled 0`).
+- Every SDK request fails with status 0 right after `start`: the installed demo was not built by
+  `install`, so it does not trust the mitmproxy CA.
+- The emulator's clock can drift from the Mac's, so compare logcat times with `report` carefully.

examples/demo/scripts/retry/fault_proxy.py

@@ -0,0 +1,115 @@
+"""
+mitmproxy addon that injects failures into OneSignal SDK requests so operation retries can be
+observed in the demo app. Driven by `retry-test.sh`; see README.md.
+
+Only requests carrying the SDK's `SDK-Version` header are touched, so the demo's own REST calls
+pass through. Every SDK request is appended to attempts.jsonl for `retry-test.sh report`.
+
+Rules are re-read whenever rules.json changes. Each rule:
+  {"id": "<unique>", "route": "update-user", "method": "PATCH", "path": "<regex>", "status": 503,
+   "retry_after": "30", "times": 3}
+`status` 0 drops the connection (the SDK sees status 0, like being offline). `times` is how many
+matching requests fail before the rule passes them through; null fails forever. A rule keeps its
+remaining count across reloads, keyed by `id`.
+"""
+
+import hashlib
+import json
+import os
+import re
+import time
+
+from mitmproxy import http
+
+STATE_DIR = os.environ["RETRY_TEST_STATE_DIR"]
+RULES_PATH = os.path.join(STATE_DIR, "rules.json")
+ATTEMPTS_PATH = os.path.join(STATE_DIR, "attempts.jsonl")
+
+API_PATH_PREFIX = re.compile(r"^/(api/v1/)?")
+
+
+class FaultProxy:
+    def __init__(self):
+        self.rules_mtime = None
+        self.rules = []
+        self.remaining = {}
+
+    def load_rules(self):
+        try:
+            mtime = os.stat(RULES_PATH).st_mtime_ns
+        except FileNotFoundError:
+            mtime = None
+        if mtime == self.rules_mtime:
+            return
+        self.rules_mtime = mtime
+        self.rules = []
+        if mtime is not None:
+            with open(RULES_PATH) as f:
+                self.rules = json.load(f)
+        self.remaining = {rule["id"]: self.remaining.get(rule["id"], rule.get("times")) for rule in self.rules}
+        print(f"[fault] rules: {json.dumps(self.rules) if self.rules else 'none, passing everything'}")
+
+    def match(self, flow):
+        path = API_PATH_PREFIX.sub("", flow.request.path.split("?")[0])
+        for rule in self.rules:
+            if rule.get("method") and rule["method"] != flow.request.method:
+                continue
+            if not re.search(rule["path"], path):
+                continue
+            remaining = self.remaining[rule["id"]]
+            if remaining is not None:
+                if remaining <= 0:
+                    continue
+                self.remaining[rule["id"]] = remaining - 1
+            return rule
+        return None
+
+    def request(self, flow: http.HTTPFlow):
+        if "onesignal.com" not in flow.request.pretty_host or "SDK-Version" not in flow.request.headers:
+            return
+        # Timed at the request, so gaps are between sends rather than between responses.
+        flow.metadata["fault_sent_at"] = time.time()
+        self.load_rules()
+        rule = self.match(flow)
+        flow.metadata["fault_rule"] = rule
+        if rule is None:
+            return
+        if rule["status"] == 0:
+            self.record(flow, 0, rule)
+            flow.kill()
+            return
+        headers = {"Content-Type": "application/json"}
+        if rule.get("retry_after") is not None:
+            headers["Retry-After"] = str(rule["retry_after"])
+        body = json.dumps({"errors": [{"title": f"Injected by fault_proxy ({rule['route']})"}]})
+        flow.response = http.Response.make(rule["status"], body, headers)
+
+    def response(self, flow: http.HTTPFlow):
+        if "fault_rule" not in flow.metadata:
+            return
+        self.record(flow, flow.response.status_code, flow.metadata["fault_rule"])
+
+    def error(self, flow: http.HTTPFlow):
+        if "fault_rule" in flow.metadata and flow.metadata["fault_rule"] is None:
+            self.record(flow, 0, None)
+
+    def record(self, flow, status, rule):
+        path = API_PATH_PREFIX.sub("", flow.request.path.split("?")[0])
+        retry_after = flow.response.headers.get("Retry-After") if flow.response else None
+        entry = {
+            "t": flow.metadata["fault_sent_at"],
+            "method": flow.request.method,
+            "path": path,
+            "status": status,
+            "injected": rule is not None,
+            "retry_after": retry_after,
+            "body": hashlib.sha1(flow.request.raw_content or b"").hexdigest()[:8],
+        }
+        with open(ATTEMPTS_PATH, "a") as f:
+            f.write(json.dumps(entry) + "\n")
+        tag = "INJECTED" if rule else "passed"
+        stamp = time.strftime("%H:%M:%S", time.localtime(entry["t"]))
+        print(f"[fault] {stamp} {status:>3} {tag:<8} {flow.request.method:<6} {path} body={entry['body']}", flush=True)
+
+
+addons = [FaultProxy()]

examples/demo/scripts/retry/flows/_scroll-to.yaml

@@ -0,0 +1,13 @@
+appId: com.onesignal.example
+---
+- scrollUntilVisible:
+    element:
+      id: app_section
+    direction: UP
+    visibilityPercentage: 10
+    timeout: 60000
+- scrollUntilVisible:
+    element:
+      id: ${TARGET}
+    centerElement: true
+    timeout: 60000

examples/demo/scripts/retry/flows/add-alias.yaml

@@ -0,0 +1,16 @@
+appId: com.onesignal.example
+---
+- runFlow:
+    file: _scroll-to.yaml
+    env:
+      TARGET: add_alias_button
+- tapOn:
+    id: add_alias_button
+- tapOn:
+    id: alias_label_input
+- inputText: ${KEY}
+- tapOn:
+    id: alias_id_input
+- inputText: ${VALUE}
+- tapOn:
+    id: singlepair_confirm_button

examples/demo/scripts/retry/flows/add-email.yaml

@@ -0,0 +1,13 @@
+appId: com.onesignal.example
+---
+- runFlow:
+    file: _scroll-to.yaml
+    env:
+      TARGET: add_email_button
+- tapOn:
+    id: add_email_button
+- tapOn:
+    id: email_input
+- inputText: ${VALUE}
+- tapOn:
+    id: singleinput_confirm_button

examples/demo/scripts/retry/flows/add-tag.yaml

@@ -0,0 +1,16 @@
+appId: com.onesignal.example
+---
+- runFlow:
+    file: _scroll-to.yaml
+    env:
+      TARGET: add_tag_button
+- tapOn:
+    id: add_tag_button
+- tapOn:
+    id: tag_key_input
+- inputText: ${KEY}
+- tapOn:
+    id: tag_value_input
+- inputText: ${VALUE}
+- tapOn:
+    id: singlepair_confirm_button

examples/demo/scripts/retry/flows/allow-notifications.yaml

@@ -0,0 +1,6 @@
+appId: com.onesignal.example
+---
+- launchApp
+- tapOn:
+    text: Allow
+    optional: true

examples/demo/scripts/retry/flows/login.yaml

@@ -0,0 +1,20 @@
+appId: com.onesignal.example
+---
+- runFlow:
+    file: _scroll-to.yaml
+    env:
+      TARGET: login_user_button
+- tapOn:
+    id: login_user_button
+- tapOn:
+    id: login_user_id_input
+- inputText: ${VALUE}
+- runFlow:
+    when:
+      true: ${JWT != ''}
+    commands:
+      - tapOn:
+          id: login_user_jwt_input
+      - inputText: ${JWT}
+- tapOn:
+    id: singleinput_confirm_button

examples/demo/scripts/retry/flows/logout.yaml

@@ -0,0 +1,8 @@
+appId: com.onesignal.example
+---
+- runFlow:
+    file: _scroll-to.yaml
+    env:
+      TARGET: logout_user_button
+- tapOn:
+    id: logout_user_button