Ask rtibblesbot to validate a contributor-opened issue

.github/workflows/automation-group-issue-comment.yml

@@ -0,0 +1,42 @@
+# This file is generated by scripts/generate-automation.js from automation-registry.yml.
+# Do not edit it by hand - edit the registry and run:
+#   node scripts/generate-automation.js
+#
+# Reusable workflow called from automation.yml: the automations triggered by issue_comment.
+name: "Automation: Issue Comment"
+on:
+  workflow_call:
+    secrets:
+      LE_BOT_APP_ID:
+        description: "Forwarded to: issue-validation-done, contributor-issue-comment"
+        required: true
+      LE_BOT_PRIVATE_KEY:
+        description: "Forwarded to: issue-validation-done, contributor-issue-comment"
+        required: true
+      SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL:
+        description: "Forwarded to: contributor-issue-comment"
+        required: false
+      SLACK_WEBHOOK_URL:
+        description: "Forwarded to: contributor-issue-comment"
+        required: false
+jobs:
+  issue-validation-done:
+    name: Issue Validation Done
+    if: ${{ github.event_name == 'issue_comment' && github.event.action == 'created' }}
+    uses: learningequality/.github/.github/workflows/issue-validation-done.yml@main
+    permissions:
+      contents: read
+    secrets:
+      LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
+      LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+  contributor-issue-comment:
+    name: Contributor Issue Comment
+    if: ${{ github.event_name == 'issue_comment' && github.event.action == 'created' }}
+    uses: learningequality/.github/.github/workflows/contributor-issue-comment.yml@main
+    permissions:
+      contents: read
+    secrets:
+      LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
+      LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL: ${{ secrets.SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL }}

.github/workflows/automation-group-issues.yml

@@ -8,10 +8,10 @@ on:
   workflow_call:
     secrets:
       LE_BOT_APP_ID:
-        description: "Forwarded to: issue-open, issue-label-header, good-first-issue-comment, community-contribution-label"
+        description: "Forwarded to: issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, issue-validation"
         required: true
       LE_BOT_PRIVATE_KEY:
-        description: "Forwarded to: issue-open, issue-label-header, good-first-issue-comment, community-contribution-label"
+        description: "Forwarded to: issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, issue-validation"
         required: true
 jobs:
   issue-open:
@@ -50,3 +50,12 @@ jobs:
     secrets:
       LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
       LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+  issue-validation:
+    name: Issue Validation
+    if: ${{ github.event_name == 'issues' && github.event.action == 'opened' }}
+    uses: learningequality/.github/.github/workflows/issue-validation.yml@main
+    permissions:
+      contents: read
+    secrets:
+      LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
+      LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}

.github/workflows/automation.yml

@@ -18,10 +18,10 @@ on:
         description: Unused - accepted until every consumer stops forwarding it
         required: false
       LE_BOT_APP_ID:
-        description: "Forwarded to: review-requested, pull-request-label, dependabot-reviewer, contributor-pr-reply, pull-request-template, issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, contributor-issue-comment, unassign-inactive-issues"
+        description: "Forwarded to: review-requested, pull-request-label, dependabot-reviewer, contributor-pr-reply, pull-request-template, issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, issue-validation, issue-validation-done, contributor-issue-comment, unassign-inactive-issues"
         required: true
       LE_BOT_PRIVATE_KEY:
-        description: "Forwarded to: review-requested, pull-request-label, dependabot-reviewer, contributor-pr-reply, pull-request-template, issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, contributor-issue-comment, unassign-inactive-issues"
+        description: "Forwarded to: review-requested, pull-request-label, dependabot-reviewer, contributor-pr-reply, pull-request-template, issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, issue-validation, issue-validation-done, contributor-issue-comment, unassign-inactive-issues"
         required: true
       SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL:
         description: "Forwarded to: contributor-pr-reply, contributor-issue-comment, unassign-inactive-issues"
@@ -42,24 +42,24 @@ jobs:
       SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL: ${{ secrets.SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL }}
   issues:
     name: Issues
-    if: ${{ (github.event_name == 'issues' && contains(fromJSON('["opened","reopened"]'), github.event.action)) || (github.event_name == 'issues' && contains(fromJSON('["labeled","unlabeled"]'), github.event.action) && github.event.label.name == 'help wanted') || (github.event_name == 'issues' && github.event.action == 'labeled' && (github.event.label.name == 'good first issue' || github.event.label.name == 'help wanted')) || (github.event_name == 'issues' && contains(fromJSON('["assigned","unassigned"]'), github.event.action)) }}
+    if: ${{ (github.event_name == 'issues' && contains(fromJSON('["opened","reopened"]'), github.event.action)) || (github.event_name == 'issues' && contains(fromJSON('["labeled","unlabeled"]'), github.event.action) && github.event.label.name == 'help wanted') || (github.event_name == 'issues' && github.event.action == 'labeled' && (github.event.label.name == 'good first issue' || github.event.label.name == 'help wanted')) || (github.event_name == 'issues' && contains(fromJSON('["assigned","unassigned"]'), github.event.action)) || (github.event_name == 'issues' && github.event.action == 'opened') }}
     uses: learningequality/.github/.github/workflows/automation-group-issues.yml@main
     permissions:
       contents: read
     secrets:
       LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
       LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
-  contributor-issue-comment:
-    name: Contributor Issue Comment
-    if: ${{ github.event_name == 'issue_comment' && github.event.action == 'created' }}
-    uses: learningequality/.github/.github/workflows/contributor-issue-comment.yml@main
+  issue-comment:
+    name: Issue Comment
+    if: ${{ (github.event_name == 'issue_comment' && github.event.action == 'created') || (github.event_name == 'issue_comment' && github.event.action == 'created') }}
+    uses: learningequality/.github/.github/workflows/automation-group-issue-comment.yml@main
     permissions:
       contents: read
     secrets:
       LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
       LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
       SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL: ${{ secrets.SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL }}
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
   resolve-bot-pr-threads:
     name: Resolve Bot PR Threads
     if: ${{ github.event_name == 'pull_request_review_comment' }}

.github/workflows/issue-validation-done.yml

@@ -0,0 +1,44 @@
+name: Clear the issue review label when the validation lands
+on:
+  workflow_call:
+    secrets:
+      LE_BOT_APP_ID:
+        description: 'GitHub App ID for authentication'
+        required: true
+      LE_BOT_PRIVATE_KEY:
+        description: 'GitHub App Private Key for authentication'
+        required: true
+jobs:
+  clear-review-label:
+    name: Clear review label
+    if: >
+      github.event.comment.user.login == 'rtibblesbot' && contains(github.event.comment.body, '<!-- ISSUE_VALIDATION -->')
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Generate App Token
+        id: generate-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ secrets.LE_BOT_APP_ID }}
+          private-key: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+          permission-issues: write
+      - name: Checkout .github repository
+        uses: actions/checkout@v7
+        with:
+          repository: learningequality/.github
+          ref: main
+      - name: Setup Node.js
+        uses: actions/setup-node@v7
+        with:
+          node-version: '22'
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+      - name: Run script
+        id: script
+        uses: actions/github-script@v9
+        with:
+          github-token: ${{ steps.generate-token.outputs.token }}
+          script: |
+            const script = require('./scripts/issue-validation-done.js');
+            return await script({ github, context, core });

.github/workflows/issue-validation.yml

@@ -0,0 +1,54 @@
+name: Ask rtibblesbot to validate a contributor issue
+on:
+  workflow_call:
+    secrets:
+      LE_BOT_APP_ID:
+        description: 'GitHub App ID for authentication'
+        required: true
+      LE_BOT_PRIVATE_KEY:
+        description: 'GitHub App Private Key for authentication'
+        required: true
+permissions:
+  contents: read
+jobs:
+  check-if-contributor:
+    name: Check if author is contributor
+    uses: learningequality/.github/.github/workflows/is-contributor.yml@main
+    secrets:
+      LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
+      LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+    with:
+      username: ${{ github.event.issue.user.login }}
+      author_association: ${{ github.event.issue.author_association }}
+  request-validation:
+    name: Request validation
+    needs: [check-if-contributor]
+    if: ${{ needs.check-if-contributor.outputs.is_contributor == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Generate App Token
+        id: generate-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ secrets.LE_BOT_APP_ID }}
+          private-key: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+          permission-issues: write
+      - name: Checkout .github repository
+        uses: actions/checkout@v7
+        with:
+          repository: learningequality/.github
+          ref: main
+      - name: Setup Node.js
+        uses: actions/setup-node@v7
+        with:
+          node-version: '22'
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+      - name: Run script
+        id: script
+        uses: actions/github-script@v9
+        with:
+          github-token: ${{ steps.generate-token.outputs.token }}
+          script: |
+            const script = require('./scripts/issue-validation.js');
+            return await script({ github, context, core });

automation-registry.yml

@@ -128,6 +128,26 @@ automations:
     secrets:
       LE_BOT_APP_ID: required
       LE_BOT_PRIVATE_KEY: required
+  - name: issue-validation
+    enabled: true
+    leaf: issue-validation.yml
+    on:
+      issues:
+        types: [opened]
+    if: "github.event_name == 'issues' && github.event.action == 'opened'"
+    secrets:
+      LE_BOT_APP_ID: required
+      LE_BOT_PRIVATE_KEY: required
+  - name: issue-validation-done
+    enabled: true
+    leaf: issue-validation-done.yml
+    on:
+      issue_comment:
+        types: [created]
+    if: "github.event_name == 'issue_comment' && github.event.action == 'created'"
+    secrets:
+      LE_BOT_APP_ID: required
+      LE_BOT_PRIVATE_KEY: required
   - name: contributor-issue-comment
     enabled: true
     leaf: contributor-issue-comment.yml

docs/community-automations.md

@@ -102,6 +102,49 @@ Four states fail, and the log names every one of them rather than stopping at th
 
 The failed check is the only report. Nothing is posted on the pull request.
 
+# `issue-validation` and `issue-validation-done`
+
+These two automations ask `rtibblesbot` to validate claims in contributor-submitted issues and track the review while it is in progress.
+
+## `issue-validation`
+
+Runs on `issues: [opened]`. It proceeds only if the repository is in `ISSUE_VALIDATION_REPOS`, the issue opener is a contributor, and the issue does not already have the `community-issue-review` label.
+
+It posts a comment mentioning `@rtibblesbot` and adds the label. If the label does not exist in the repository, the add-label call creates it.
+
+**The mention is essential.** The agent polls its GitHub notifications and has no inbound endpoint, so a mention is the only way to reach it. Adding a label does not generate a notification, and a handle inside backticks, as in `BOT_MESSAGE_RTIBBLESBOT_REVIEW`, does not trigger one either.
+
+## `issue-validation-done`
+
+Runs on `issue_comment: [created]`. It removes the `community-issue-review` label only when both conditions are met:
+
+- The comment contains the `<!-- ISSUE_VALIDATION -->` marker.
+- The comment author is `rtibblesbot`.
+
+Both checks are necessary. Anyone can include the marker in a comment, and `rtibblesbot` also comments on issues for other reasons.
+
+The leaf workflow tests both conditions in its job `if:`, as `resolve-bot-pr-threads` does, so an ordinary comment starts no runner. The script tests them again.
+
+This logic cannot live in `contributor-issue-comment`, because that automation deliberately ignores bot comments.
+
+## Handling stalled reviews
+
+The weekly `unassign-inactive-issues` run also sweeps stalled validations through `scripts/issue-review-sweep.js`.
+
+If the agent never responds, for example, because the service is down or all available tokens have reached their limits, the label would otherwise remain indefinitely. The sweep removes it when the issue's `labeled` event is older than `ISSUE_REVIEW_MAX_DAYS` days.
+
+## Scope and configuration
+
+Neither automation closes issues or acts on the validation findings. The agent's comment recommends whether the issue should be closed; a maintainer must request that action in a reply.
+
+Configure the following constants in `scripts/constants.js`:
+
+- `ISSUE_VALIDATION_REPOS`: The repositories where validation runs, which are the community repositories plus the sandbox the automations are tested in.
+- `LABEL_COMMUNITY_ISSUE_REVIEW`: The label applied while validation is in progress.
+- `ISSUE_VALIDATION_MARKER`: The marker that signals validation completion, shared with `learningequality/coding-agent`.
+- `ISSUE_REVIEW_MAX_DAYS`: The maximum age of the label before the sweep clears it.
+- `BOT_MESSAGE_ISSUE_VALIDATION`: The comment text posted to request validation.
+
 # `holiday-message`
 
 Sends a holiday message to community pull requests and issue comments.

scripts/constants.js

@@ -87,6 +87,11 @@ const MAX_ASSIGNED_ISSUES = 2;
 const ASSIGN_COOLDOWN_DAYS = 7;
 const ASSIGN_GUIDANCE_MARKER = '<!-- ASSIGN_GUIDANCE -->';
 const LABEL_COMMUNITY_REVIEW = 'community-review';
+const LABEL_COMMUNITY_ISSUE_REVIEW = 'community-issue-review';
+// rtibblesbot opens its validation comment with this marker. The contract is
+// stated from the agent's side in learningequality/coding-agent#1229.
+const ISSUE_VALIDATION_MARKER = '<!-- ISSUE_VALIDATION -->';
+const ISSUE_REVIEW_MAX_DAYS = 3;
 
 // Will be attached to bot messages when not empty
 // const GSOC_NOTE = '';
@@ -154,6 +159,18 @@ const BOT_MESSAGE_RTIBBLESBOT_REVIEW = `📢✨ **Before we assign a reviewer, w
 
 const BOT_MESSAGE_COMMUNITY_REVIEW = `📢✨ **Before we assign a reviewer, we'll invite community pre-review. See the [community review guidance](https://github.com/learningequality/.github/blob/main/docs/community-review.md) for both authors and reviewers.**`;
 
+const BOT_MESSAGE_ISSUE_VALIDATION =
+  `🔍 **Thanks for opening this issue!** @rtibblesbot will check it against the code ` +
+  `and comment with its findings.\n\n` +
+  `**What happens next:**\n\n` +
+  `- The \`${LABEL_COMMUNITY_ISSUE_REVIEW}\` label stays on until the review comment is posted.\n` +
+  `- The comment identifies the files, lines, or commits examined so you can follow the ` +
+  `reasoning and respond if you disagree.\n` +
+  `- If the check confirms the issue, @rtibblesbot may also correct the title and ` +
+  `description. GitHub preserves the edit history, so the original text remains available.\n` +
+  `- @rtibblesbot's comments are LLM-generated, so please evaluate them critically.\n\n` +
+  `A maintainer decides what happens next. This check does not close the issue.`;
+
 // Repositories to include in PR statistics reports
 const PR_STATS_REPOS = [
   'kolibri',
@@ -162,11 +179,16 @@ const PR_STATS_REPOS = [
   'le-utils',
   '.github',
   'ricecooker',
+  'morango',
 ];
 
 // Repositories in which we accept open-source contributions
 const COMMUNITY_REPOS = [...PR_STATS_REPOS];
 
+// No other automation gates on the repo, so all of them already run in the
+// sandbox. See docs/testing-automations.md.
+const ISSUE_VALIDATION_REPOS = [...COMMUNITY_REPOS, 'test-actions'];
+
 // Every place GitHub reads a pull request template from. The contents API is
 // case sensitive, so both spellings are needed.
 const PR_TEMPLATE_PATHS = [
@@ -205,6 +227,11 @@ module.exports = {
   RTIBBLESBOT_USERNAME,
   LABEL_COMMUNITY_REVIEW,
   BOT_MESSAGE_COMMUNITY_REVIEW,
+  LABEL_COMMUNITY_ISSUE_REVIEW,
+  ISSUE_VALIDATION_REPOS,
+  ISSUE_VALIDATION_MARKER,
+  ISSUE_REVIEW_MAX_DAYS,
+  BOT_MESSAGE_ISSUE_VALIDATION,
   TEAMS_WITH_CLOSE_CONTRIBUTORS,
   HOLIDAY_MESSAGE,
   PR_STATS_REPOS,