diff --git a/.github/workflows/collect-prs.yml b/.github/workflows/collect-prs.yml index 4e27796..b420cea 100644 --- a/.github/workflows/collect-prs.yml +++ b/.github/workflows/collect-prs.yml @@ -16,11 +16,11 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - timeout-minutes: 20 + timeout-minutes: 70 # Message Batches: up to 40 min wait (Program.cs) + one-by-one fallback outputs: has_changes: ${{ steps.commit.outputs.has_changes }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Setup .NET uses: prozolic/Actions/.github/actions/setup-dotnet@main with: @@ -48,7 +48,7 @@ jobs: echo "has_changes=true" >> "$GITHUB_OUTPUT" fi - name: Upload artifact - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: ./outputs/ @@ -65,7 +65,7 @@ jobs: steps: - name: Deploy id: deployment - uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 # NOTE: push with GITHUB_TOKEN won't invoke the event, # as a workaround, directly call the action. diff --git a/README.md b/README.md index da509f2..56952da 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,11 @@ The application runs automatically via GitHub Actions every day at midnight (UTC ## AI Model -Claude Haiku 4.5 +Claude Sonnet 5.5 + +### Previous models + +- Claude Haiku 4.5: used from December 21, 2025 to October 8, 2026 ## Development @@ -44,7 +48,7 @@ https://prozolic.github.io/PRDigest.NET/feed.xml #### Markdig -- Version: 1.2.0 +- Version: 1.4.0 - URL: https://github.com/xoofx/markdig - Copyright (c) 2018-2019, Alexandre Mutel - Licensed under the [BSD 2-Clause License](https://github.com/xoofx/markdig/blob/main/license.txt) @@ -58,7 +62,7 @@ https://prozolic.github.io/PRDigest.NET/feed.xml #### Anthropic SDK for C\# -- Version: 12.23.0 +- Version: 12.54.1 - URL: https://github.com/anthropics/anthropic-sdk-csharp - Copyright 2023 Anthropic, PBC. - Licensed under the [MIT License](https://github.com/anthropics/anthropic-sdk-csharp/blob/main/LICENSE) diff --git a/src/PRDigest.NET/Constants.cs b/src/PRDigest.NET/Constants.cs index 0dc1749..811e591 100644 --- a/src/PRDigest.NET/Constants.cs +++ b/src/PRDigest.NET/Constants.cs @@ -1,4 +1,5 @@ - +using Anthropic.Models.Messages; + namespace PRDigest.NET; internal static class Constants @@ -6,4 +7,18 @@ internal static class Constants public const string Owner = "dotnet"; public const string Repository = "runtime"; public const string FullRepository = $"{Owner}/{Repository}"; + + public const Model SummaryModel = Model.ClaudeSonnet5_5; // Claude Sonnet 5.5 + + // Summarization settings shared by the Messages API and the Message Batches API. + // Thinking is turned off with "between_tools" (Sonnet 5.5 rejects "disabled"), so max_tokens covers only the summary. + // The prompt caps the summary at 1000 characters; 1024 tokens cut off about 4% of Haiku 4.5 summaries mid-sentence. + public const int SummaryMaxTokens = 2048; + + // Shown instead of the summary when Claude declines to summarize a PR. It keeps the 概要 heading so that the + // monthly page and the RSS feed, which read only that section, show it too. + public const string RefusedSummary = """ + #### 概要 + この Pull Request は、安全性に関する判定により要約を生成できませんでした。変更内容は元の Pull Request を参照してください。 + """; } \ No newline at end of file diff --git a/src/PRDigest.NET/PRDigest.NET.csproj b/src/PRDigest.NET/PRDigest.NET.csproj index f860c2d..e9bd1af 100644 --- a/src/PRDigest.NET/PRDigest.NET.csproj +++ b/src/PRDigest.NET/PRDigest.NET.csproj @@ -9,9 +9,9 @@ - + - + diff --git a/src/PRDigest.NET/Program.cs b/src/PRDigest.NET/Program.cs index 95ca062..f38a1e2 100644 --- a/src/PRDigest.NET/Program.cs +++ b/src/PRDigest.NET/Program.cs @@ -5,6 +5,7 @@ using PRDigest.NET; using System.Runtime.InteropServices; using System.Text; +using Anthropic.Models.Messages.Batches; if (args.Length == 0) return; @@ -76,7 +77,12 @@ async ValueTask SummarizeCurrentPullRequestAndCreate(string archivesDir, string Console.WriteLine($"{pullRequestInfos.Length} pull requests into {Constants.FullRepository} were merged between {startTargetDate:yyyy/MM/dd HH:mm:ss} and {endTargetDate:yyyy/MM/dd HH:mm:ss}."); // Generate HTML content for each pull request using Anthropic API. - var markdown = await SummarizePullRequestAsync(pullRequestInfos); + // Many PRs (weekdays: 16-56) go through the Message Batches API at half the price. + // A few PRs (mostly weekends) are summarized one by one: the saving is too small to wait for a batch. + const int BatchThreshold = 20; + var markdown = pullRequestInfos.Length >= BatchThreshold + ? await SummarizePullRequestWithBatchAsync(pullRequestInfos) + : await SummarizePullRequestAsync(pullRequestInfos); if (string.IsNullOrEmpty(markdown)) return; // Save markdown and HTML files. @@ -177,75 +183,141 @@ async ValueTask GetAllPullRequestInfoAsync(DateTimeOffset sta async ValueTask SummarizePullRequestAsync(PullRequestInfo[] pullRequestInfos) { - var markdownlBuilder = new StringBuilder(); - var tableOfContentsBuilder = new StringBuilder(); - tableOfContentsBuilder.AppendLine("### 目次 {#table-of-contents}"); + var llmOutputs = new string?[pullRequestInfos.Length]; + await SummarizeSequentiallyAsync(CreateAnthropicClient(), pullRequestInfos, llmOutputs); + return CreateMarkdown(pullRequestInfos, llmOutputs); +} - var index = 1; - var separator = Environment.NewLine + "---" + Environment.NewLine; - var totalInputTokens = 0L; - var totalInputTokensPerMinute = 0L; - var totalOutputTokens = 0L; - var totalOutputTokensPerMinute = 0L; +async ValueTask SummarizePullRequestWithBatchAsync(PullRequestInfo[] pullRequestInfos) +{ + // Most batches end within an hour, but collect-prs.yml caps the job (timeout-minutes), + // so give up after this rather than publish a digest built from a partly finished batch. + var batchTimeout = TimeSpan.FromMinutes(40); + var pollingInterval = TimeSpan.FromSeconds(30); + var client = CreateAnthropicClient(); + + // custom_id must match ^[a-zA-Z0-9_-]{1,64}$. + var indexByCustomId = new Dictionary(pullRequestInfos.Length); + var requests = new Request[pullRequestInfos.Length]; + + for (var i = 0; i < pullRequestInfos.Length; i++) + { + var customId = $"pr-{pullRequestInfos[i].Issue.Number}"; + indexByCustomId[customId] = i; + requests[i] = new Request + { + CustomID = customId, + Params = new Params + { + MaxTokens = Constants.SummaryMaxTokens, + Model = Constants.SummaryModel, + Thinking = new ThinkingConfigBetweenTools(), + System = new ParamsSystem([new TextBlockParam { Text = PromptGenerator.SystemPrompt }]), + Messages = [new() { Role = Role.User, Content = PromptGenerator.GeneratePrompt(pullRequestInfos[i]) }], + }, + }; + } + + var llmOutputs = new string?[pullRequestInfos.Length]; try { - // Generate HTML content for each pull request using Anthropic API. - // Configures ANTHROPIC_API_KEY. - AnthropicClient anthropicClient = new(); + var batch = await client.Messages.Batches.Create(new BatchCreateParams { Requests = requests }); + Console.WriteLine($"[INFO] Batch {batch.ID} created with {requests.Length} requests."); - foreach (var pr in pullRequestInfos) + var startTime = TimeProvider.System.GetTimestamp(); + while (batch.ProcessingStatus != ProcessingStatus.Ended) { - MessageCreateParams parameters = new() + if (TimeProvider.System.GetElapsedTime(startTime) > batchTimeout) { - MaxTokens = 1024, - Model = Model.ClaudeHaiku4_5_20251001, // Claude Haiku 4.5 - System = new MessageCreateParamsSystem([new() { Text = PromptGenerator.SystemPrompt }]), - Messages = [new() { Role = Role.User, Content = PromptGenerator.GeneratePrompt(pr) }], - }; + // Cancel so that the requests not yet processed are neither run nor billed, then fail the run: + // no markdown is written, so the day can be summarized again from scratch. + Console.WriteLine($"[ERROR] Batch {batch.ID} did not end within {batchTimeout.TotalMinutes} minutes. Canceling it."); + await client.Messages.Batches.Cancel(batch.ID); + throw new TimeoutException($"Batch {batch.ID} did not end within {batchTimeout.TotalMinutes} minutes."); + } - var message = await anthropicClient - .WithOptions(options => options with - { - Timeout = TimeSpan.FromMinutes(5), - MaxRetries = 3, - }) - .Messages.Create(parameters); + await Task.Delay(pollingInterval); + batch = await client.Messages.Batches.Retrieve(batch.ID); - Console.WriteLine($"[INFO] #{pr.Issue.Number} input-token:{message.Usage.InputTokens} output-token:{message.Usage.OutputTokens}"); + // request_counts other than processing stay 0 until the whole batch ends, so only processing is worth logging here. + if (batch.ProcessingStatus != ProcessingStatus.Ended) + { + Console.WriteLine($"[INFO] Batch {batch.ID} is still processing {batch.RequestCounts.Processing} requests."); + } + } + + // Processing time as measured by the API (ended_at - created_at), independent of the polling interval. + var counts = batch.RequestCounts; + var processingTime = batch.EndedAt is { } endedAt ? $"{(endedAt - batch.CreatedAt).TotalSeconds:F0}s" : "unknown"; + Console.WriteLine($"[INFO] Batch {batch.ID} ended. processing-time:{processingTime} succeeded:{counts.Succeeded} errored:{counts.Errored} canceled:{counts.Canceled} expired:{counts.Expired}"); + + await foreach (var response in client.Messages.Batches.ResultsStreaming(batch.ID)) + { + if (!indexByCustomId.TryGetValue(response.CustomID, out var i)) continue; - var llmOutput = ""; - foreach (var content in message.Content) + if (response.Result.TryPickSucceeded(out var succeeded)) { - if (content.TryPickText(out var textBlock)) - { - llmOutput += textBlock.Text; - } + var message = succeeded.Message; + Console.WriteLine($"[INFO] #{pullRequestInfos[i].Issue.Number} input-token:{message.Usage.InputTokens} output-token:{message.Usage.OutputTokens}"); + llmOutputs[i] = GetSummaryText(pullRequestInfos[i].Issue.Number, message); } + else if (response.Result.TryPickErrored(out var errored)) + { + Console.WriteLine($"[WARN] #{pullRequestInfos[i].Issue.Number} errored in batch: {errored.Error.Error}"); + } + else if (response.Result.TryPickExpired(out _)) + { + Console.WriteLine($"[WARN] #{pullRequestInfos[i].Issue.Number} expired in batch."); + } + else if (response.Result.TryPickCanceled(out _)) + { + Console.WriteLine($"[WARN] #{pullRequestInfos[i].Issue.Number} canceled in batch."); + } + } + } + catch (AnthropicRateLimitException rle) + { + Console.WriteLine($"[ERROR] AnthropicRateLimitException: {rle.StatusCode}"); + throw; + } + catch (AnthropicBadRequestException bre) + { + Console.WriteLine($"[ERROR] AnthropicBadRequestException: {bre.StatusCode}"); + throw; + } - var title = TitleHelper.EscapedTitle(pr.Issue.Title); - tableOfContentsBuilder.AppendLine($"{index++}. [#{pr.Issue.Number} {title}](#{pr.Issue.Number})"); + // Fills only the entries the batch did not produce (requests that errored in it). + await SummarizeSequentiallyAsync(client, pullRequestInfos, llmOutputs); + return CreateMarkdown(pullRequestInfos, llmOutputs); +} - var labels = pr.PullRequest.Labels; - var labelText = labels.Count > 0 ? - string.Join(" ", labels.Select(label => $"{label.Name}")) : - "指定なし"; +async ValueTask SummarizeSequentiallyAsync(IAnthropicClient client, PullRequestInfo[] pullRequestInfos, string?[] llmOutputs) +{ + var totalInputTokensPerMinute = 0L; - var prHeader = $$""" -### [#{{pr.Issue.Number}}]({{pr.Issue.HtmlUrl}}) {{title}} {#{{pr.Issue.Number}}} -- 作成者: [@{{pr.Issue.User.Login}}]({{pr.Issue.User.HtmlUrl}}) -- 作成日時: {{pr.Issue.CreatedAt:yyyy年MM月dd日 HH:mm:ss}}(UTC) -- マージ日時: {{pr.PullRequest.MergedAt:yyyy年MM月dd日 HH:mm:ss}}(UTC) -- ラベル: {{labelText}} + try + { + for (var i = 0; i < pullRequestInfos.Length; i++) + { + if (llmOutputs[i] is not null) continue; -"""; - markdownlBuilder.AppendLine(prHeader + llmOutput); - markdownlBuilder.Append(separator); + var pr = pullRequestInfos[i]; + MessageCreateParams parameters = new() + { + MaxTokens = Constants.SummaryMaxTokens, + Model = Constants.SummaryModel, + Thinking = new ThinkingConfigBetweenTools(), + System = new MessageCreateParamsSystem([new() { Text = PromptGenerator.SystemPrompt }]), + Messages = [new() { Role = Role.User, Content = PromptGenerator.GeneratePrompt(pr) }], + }; + + var message = await client.Messages.Create(parameters); + + Console.WriteLine($"[INFO] #{pr.Issue.Number} input-token:{message.Usage.InputTokens} output-token:{message.Usage.OutputTokens}"); + llmOutputs[i] = GetSummaryText(pr.Issue.Number, message); - totalInputTokens += message.Usage.InputTokens; totalInputTokensPerMinute += message.Usage.InputTokens; - totalOutputTokens += message.Usage.OutputTokens; - totalOutputTokensPerMinute += message.Usage.OutputTokens; // Since input tokens are variable, wait if it exceeds 30,000 tokens per minute if (totalInputTokensPerMinute >= 30000) @@ -265,6 +337,83 @@ async ValueTask SummarizePullRequestAsync(PullRequestInfo[] pullRequestI Console.WriteLine($"[ERROR] AnthropicBadRequestException: {bre.StatusCode}"); throw; } +} + +IAnthropicClient CreateAnthropicClient() +{ + // Configures ANTHROPIC_API_KEY. + AnthropicClient anthropicClient = new(); + return anthropicClient.WithOptions(options => options with + { + Timeout = TimeSpan.FromMinutes(5), + MaxRetries = 3, + }); +} + +string GetSummaryText(int pullRequestNumber, Message message) +{ + // https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback + // A refused request comes back as a normal response (a "succeeded" result in a batch) with stop_reason "refusal". + // Any partial text is dropped and RefusedSummary is shown instead, so readers can tell the PR was not summarized. + if (message.StopReason == StopReason.Refusal) + { + Console.WriteLine($"[WARN] #{pullRequestNumber} summary was refused by the safety classifiers. category:{message.StopDetails?.Category}"); + return Constants.RefusedSummary; + } + + // A summary cut off at max_tokens is still written as is; the warning only makes it visible in the run log. + if (message.StopReason == StopReason.MaxTokens) + { + Console.WriteLine($"[WARN] #{pullRequestNumber} summary was cut off at max_tokens ({Constants.SummaryMaxTokens})."); + } + + return ExtractText(message); +} + +string ExtractText(Message message) +{ + var builder = new StringBuilder(); + foreach (var content in message.Content) + { + if (content.TryPickText(out var textBlock)) + { + builder.Append(textBlock.Text); + } + } + return builder.ToString(); +} + +// Builds the daily markdown (table of contents + one section per PR) in the original PR order. +string CreateMarkdown(PullRequestInfo[] pullRequestInfos, string?[] llmOutputs) +{ + var markdownlBuilder = new StringBuilder(); + var tableOfContentsBuilder = new StringBuilder(); + tableOfContentsBuilder.AppendLine("### 目次 {#table-of-contents}"); + + var separator = Environment.NewLine + "---" + Environment.NewLine; + + for (var i = 0; i < pullRequestInfos.Length; i++) + { + var pr = pullRequestInfos[i]; + var title = TitleHelper.EscapedTitle(pr.Issue.Title); + tableOfContentsBuilder.AppendLine($"{i + 1}. [#{pr.Issue.Number} {title}](#{pr.Issue.Number})"); + + var labels = pr.PullRequest.Labels; + var labelText = labels.Count > 0 ? + string.Join(" ", labels.Select(label => $"{label.Name}")) : + "指定なし"; + + var prHeader = $$""" +### [#{{pr.Issue.Number}}]({{pr.Issue.HtmlUrl}}) {{title}} {#{{pr.Issue.Number}}} +- 作成者: [@{{pr.Issue.User.Login}}]({{pr.Issue.User.HtmlUrl}}) +- 作成日時: {{pr.Issue.CreatedAt:yyyy年MM月dd日 HH:mm:ss}}(UTC) +- マージ日時: {{pr.PullRequest.MergedAt:yyyy年MM月dd日 HH:mm:ss}}(UTC) +- ラベル: {{labelText}} + +"""; + markdownlBuilder.AppendLine(prHeader + llmOutputs[i]); + markdownlBuilder.Append(separator); + } return $"{tableOfContentsBuilder}{separator}{markdownlBuilder}"; } diff --git a/src/PRDigest.NET/PromptGenerator.cs b/src/PRDigest.NET/PromptGenerator.cs index 2fd4239..376c41a 100644 --- a/src/PRDigest.NET/PromptGenerator.cs +++ b/src/PRDigest.NET/PromptGenerator.cs @@ -75,7 +75,6 @@ public static string GeneratePrompt(PullRequestInfo info) Pull Request: - {info.PullRequest.Title} #{info.PullRequest.Number} - 作成者: {info.PullRequest.User.Login} -- レビュワー: {GenerateReviewersText(info)} 作成者による概要: {GenerateBody(info)} @@ -99,23 +98,6 @@ private static string GenerateBody(PullRequestInfo info) return body; } - private static string GenerateReviewersText(PullRequestInfo info) - { - var reviews = info.Reviews; - var builder = new DefaultInterpolatedStringHandler(0, 0); - for (int i = 0; i < reviews.Count; i++) - { - var pullRequestReview = reviews[i]; - builder.AppendLiteral(pullRequestReview.User.Login); - if (i < reviews.Count - 1) - { - builder.AppendLiteral(", "); - } - } - - return builder.ToStringAndClear(); - } - private static string GenerateCopilotReviewText(PullRequestInfo pullRequestInfo) { var copilotReviews = pullRequestInfo