diff --git a/CHANGELOG.md b/CHANGELOG.md index f75147e..2e955d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ## [Unreleased] +## [0.6.0] - 2026-10-14 + +### Changed + +- Upgrade the official MCP conformance runner to `0.2.0-alpha.12` and pin + the matching TypeScript fixture revision. Refresh packaged baselines from + verified runs against the updated runner and fixture. +- Apply the upstream runner's 30-second server-scenario deadline. Stalled + scenarios now report `scenario-timeout` findings. Regenerate caller-owned + baselines when adopting the upgraded runner. + ## [0.5.1] - 2026-09-21 ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 27cdae0..15e6cac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -201,7 +201,7 @@ dependencies = [ [[package]] name = "cf-integration" -version = "0.5.1" +version = "0.6.0" dependencies = [ "anyhow", "aws-lc-rs", diff --git a/Cargo.toml b/Cargo.toml index 5964071..8b58a84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cf-integration" -version = "0.5.1" +version = "0.6.0" edition = "2024" rust-version = "1.97" license = "Apache-2.0" diff --git a/docker/helpers.Dockerfile b/docker/helpers.Dockerfile index 0e35752..48f2c07 100644 --- a/docker/helpers.Dockerfile +++ b/docker/helpers.Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update \ && apt-get install --yes --no-install-recommends ca-certificates \ && rm -rf /var/lib/apt/lists/* RUN npm install --global --ignore-scripts \ - @modelcontextprotocol/conformance@0.2.0-alpha.11 \ + @modelcontextprotocol/conformance@0.2.0-alpha.12 \ @modelcontextprotocol/inspector@2.2.0 \ && npm cache clean --force COPY --from=build /usr/local/bin/cf-integration /usr/local/bin/cf-integration diff --git a/docker/mcp-conformance-server.Dockerfile b/docker/mcp-conformance-server.Dockerfile index 31f9b8e..adbb4c3 100644 --- a/docker/mcp-conformance-server.Dockerfile +++ b/docker/mcp-conformance-server.Dockerfile @@ -1,6 +1,6 @@ FROM node:22-bookworm-slim -ARG MCP_CONFORMANCE_REVISION=c321dd32035556e6769d3724a8ee97d87c3faaac +ARG MCP_CONFORMANCE_REVISION=f44482ba17df816d3176962a11cdf36aec9bda00 RUN apt-get update \ && apt-get install --yes --no-install-recommends ca-certificates curl git \ @@ -8,6 +8,7 @@ RUN apt-get update \ WORKDIR /opt RUN git clone https://github.com/modelcontextprotocol/conformance.git mcp-conformance \ + && git -C mcp-conformance fetch origin "${MCP_CONFORMANCE_REVISION}" \ && git -C mcp-conformance checkout --detach "${MCP_CONFORMANCE_REVISION}" WORKDIR /opt/mcp-conformance/examples/servers/typescript diff --git a/src/conformance/profile.rs b/src/conformance/profile.rs index 0facfcd..d32d010 100644 --- a/src/conformance/profile.rs +++ b/src/conformance/profile.rs @@ -2,12 +2,12 @@ /// Published official CLI package used as the conformance client. pub(crate) const OFFICIAL_CONFORMANCE_PACKAGE: &str = - "@modelcontextprotocol/conformance@0.2.0-alpha.11"; + "@modelcontextprotocol/conformance@0.2.0-alpha.12"; /// Official repository containing the matching TypeScript fixture server. pub(crate) const OFFICIAL_CONFORMANCE_REPOSITORY: &str = "https://github.com/modelcontextprotocol/conformance"; /// Exact source revision behind the published CLI and TypeScript fixture. -pub(crate) const OFFICIAL_CONFORMANCE_REVISION: &str = "c321dd32035556e6769d3724a8ee97d87c3faaac"; +pub(crate) const OFFICIAL_CONFORMANCE_REVISION: &str = "f44482ba17df816d3176962a11cdf36aec9bda00"; /// Default draft protocol revision exercised by official conformance commands. pub(crate) const DEFAULT_MCP_SPEC_VERSION: &str = "2026-07-28"; /// Previous stable revision supported by the pinned official conformance package. diff --git a/src/conformance/results.rs b/src/conformance/results.rs index f274e5b..6248720 100644 --- a/src/conformance/results.rs +++ b/src/conformance/results.rs @@ -195,9 +195,9 @@ pub(crate) fn is_trusted_official_fixture(fixture: &ConformanceFixtureMetadata) && fixture.server_id == OFFICIAL_CONFORMANCE_SERVER_ID } -// Exact server catalogs emitted by @modelcontextprotocol/conformance@0.2.0-alpha.11. -// Keep these coupled to OFFICIAL_CONFORMANCE_PACKAGE and verify the pin with -// the ignored package-backed test before updating either. +// Exact server catalogs emitted by @modelcontextprotocol/conformance@0.2.0-alpha.12. +// Keep these coupled to OFFICIAL_CONFORMANCE_PACKAGE and verify each catalog +// against the published CLI before updating either. const SERVER_SCENARIOS_2025_06_18: [&str; 27] = [ "server-initialize", "server-session-lifecycle", diff --git a/src/conformance/results_tests.rs b/src/conformance/results_tests.rs index 6d38111..796ae13 100644 --- a/src/conformance/results_tests.rs +++ b/src/conformance/results_tests.rs @@ -246,7 +246,7 @@ fn official_command_is_pinned_complete_and_ordered() { assert_eq!( OFFICIAL_CONFORMANCE_PACKAGE, - "@modelcontextprotocol/conformance@0.2.0-alpha.11" + "@modelcontextprotocol/conformance@0.2.0-alpha.12" ); assert_eq!(DEFAULT_MCP_SPEC_VERSION, "2026-07-28"); assert_eq!(spec.program(), "conformance"); diff --git a/src/infrastructure/compose_integration_tests.rs b/src/infrastructure/compose_integration_tests.rs index 9f4882a..5c9b41d 100644 --- a/src/infrastructure/compose_integration_tests.rs +++ b/src/infrastructure/compose_integration_tests.rs @@ -862,7 +862,7 @@ fn conformance_container_inputs_pin_the_runner_revision_and_protocol_fixture() { assert!(dockerfile.contains("FROM node:22-bookworm-slim")); assert!( dockerfile - .contains("ARG MCP_CONFORMANCE_REVISION=c321dd32035556e6769d3724a8ee97d87c3faaac") + .contains("ARG MCP_CONFORMANCE_REVISION=f44482ba17df816d3176962a11cdf36aec9bda00") ); assert!(dockerfile.contains( "git clone https://github.com/modelcontextprotocol/conformance.git mcp-conformance" @@ -871,6 +871,9 @@ fn conformance_container_inputs_pin_the_runner_revision_and_protocol_fixture() { dockerfile .contains("git -C mcp-conformance checkout --detach \"${MCP_CONFORMANCE_REVISION}\"") ); + assert!( + dockerfile.contains("git -C mcp-conformance fetch origin \"${MCP_CONFORMANCE_REVISION}\"") + ); assert!(dockerfile.contains("WORKDIR /opt/mcp-conformance/examples/servers/typescript")); assert!(dockerfile.contains("npm ci")); assert!(dockerfile.contains( diff --git a/tests/conformance/baselines/2026-07-28/legacy/built-in-data-plane.yml b/tests/conformance/baselines/2026-07-28/legacy/built-in-data-plane.yml index dc3ec18..6e49b57 100644 --- a/tests/conformance/baselines/2026-07-28/legacy/built-in-data-plane.yml +++ b/tests/conformance/baselines/2026-07-28/legacy/built-in-data-plane.yml @@ -67,10 +67,6 @@ findings: check: wire-schema-valid name: WireSchemaValid status: FAILURE -- scenario: input-required-result-validate-input - check: wire-schema-valid - name: WireSchemaValid - status: FAILURE - scenario: json-schema-2020-12 check: wire-schema-valid name: WireSchemaValid @@ -111,10 +107,6 @@ findings: check: wire-schema-valid name: WireSchemaValid status: FAILURE -- scenario: sep-2164-resource-not-found - check: sep-2164-data-uri - name: ResourcesNotFoundDataUri - status: FAILURE - scenario: sep-2164-resource-not-found check: wire-schema-valid name: WireSchemaValid diff --git a/tests/conformance/baselines/2026-07-28/legacy/external-data-plane.yml b/tests/conformance/baselines/2026-07-28/legacy/external-data-plane.yml index 0fe42ce..7052f43 100644 --- a/tests/conformance/baselines/2026-07-28/legacy/external-data-plane.yml +++ b/tests/conformance/baselines/2026-07-28/legacy/external-data-plane.yml @@ -1,33 +1,65 @@ findings: -- scenario: input-required-result-unsupported-methods - check: sep-2322-not-on-unsupported-requests - name: NotOnUnsupportedRequests - status: FAILURE -- scenario: input-required-result-validate-input - check: sep-2322-validate-input-responses - name: ValidateInputResponses - status: WARNING -- scenario: sep-2164-resource-not-found - check: sep-2164-data-uri - name: ResourcesNotFoundDataUri +- scenario: completion-complete + check: wire-schema-valid + name: WireSchemaValid status: FAILURE -- scenario: server-stateless - check: sep-2575-discover-capabilities-match-handlers - name: DiscoverCapabilitiesMatchHandlers +- scenario: prompts-get-embedded-resource + check: wire-schema-valid + name: WireSchemaValid status: FAILURE -- scenario: server-stateless - check: sep-2575-http-server-error-jsonrpc-id - name: HttpServerErrorJsonrpcId +- scenario: prompts-get-simple + check: wire-schema-valid + name: WireSchemaValid status: FAILURE -- scenario: server-stateless - check: sep-2575-http-server-meta-invalid-400 - name: HttpServerMetaInvalid400 +- scenario: prompts-get-with-args + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: prompts-get-with-image + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: resources-read-binary + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: resources-read-text + check: wire-schema-valid + name: WireSchemaValid status: FAILURE - scenario: server-stateless - check: sep-2575-http-server-unsupported-version-400 - name: HttpServerUnsupportedVersion400 + check: sep-2575-discover-capabilities-match-handlers + name: DiscoverCapabilitiesMatchHandlers status: FAILURE - scenario: server-stateless check: sep-2575-server-declares-prompts-in-discover name: ServerDeclaresPromptsInDiscover status: FAILURE +- scenario: tools-call-audio + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-embedded-resource + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-error + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-image + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-mixed-content + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-simple-text + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-with-progress + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE diff --git a/tests/conformance/baselines/2026-07-28/modern/built-in-data-plane.yml b/tests/conformance/baselines/2026-07-28/modern/built-in-data-plane.yml index 2cb8db3..8a66af4 100644 --- a/tests/conformance/baselines/2026-07-28/modern/built-in-data-plane.yml +++ b/tests/conformance/baselines/2026-07-28/modern/built-in-data-plane.yml @@ -1 +1,429 @@ -findings: [] +findings: +- scenario: caching + check: sep-2549-cache-scope-valid + name: CacheScopeValid + status: FAILURE +- scenario: caching + check: sep-2549-prompts-list-caching-hints + name: PromptsListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-resources-list-caching-hints + name: ResourcesListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-resources-templates-list-caching-hints + name: ResourcesTemplatesListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-tools-list-caching-hints + name: ToolsListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-ttl-non-negative + name: TtlNonNegative + status: FAILURE +- scenario: caching + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: completion-complete + check: completion-complete + name: CompletionComplete + status: FAILURE +- scenario: completion-complete + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: dns-rebinding-protection + check: localhost-host-valid-accepted + name: LocalhostHostAccepted + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-custom-setup + name: HttpCustomHeaderServerValidationSetup + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-decode-base64 + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-reject-invalid-param-chars + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-reject-param-mismatch + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-validate-param-match + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: http-header-validation + check: sep-2243-server-accepts-whitespace-header-value + name: ServerAcceptsWhitespaceHeaderValue + status: FAILURE +- scenario: http-header-validation + check: sep-2243-server-standard-setup + name: HttpHeaderValidationSetup + status: FAILURE +- scenario: http-header-validation + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-basic-elicitation + check: sep-2322-elicitation-incomplete + name: InputRequiredResultElicitationIncomplete + status: FAILURE +- scenario: input-required-result-basic-elicitation + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-basic-list-roots + check: sep-2322-list-roots-incomplete + name: InputRequiredResultListRootsIncomplete + status: FAILURE +- scenario: input-required-result-basic-list-roots + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-basic-sampling + check: sep-2322-sampling-incomplete + name: InputRequiredResultSamplingIncomplete + status: FAILURE +- scenario: input-required-result-basic-sampling + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-capability-check + check: sep-2322-respect-client-capabilities + name: RespectClientCapabilities + status: FAILURE +- scenario: input-required-result-capability-check + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-ignore-extra-params + check: sep-2322-ignore-unexpected-params + name: IgnoreUnexpectedParams + status: WARNING +- scenario: input-required-result-ignore-extra-params + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-missing-input-response + check: sep-2322-missing-response-rerequests + name: InputRequiredResultMissingResponseRerequests + status: WARNING +- scenario: input-required-result-missing-input-response + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-multi-round + check: sep-2322-multi-round-r1 + name: InputRequiredResultMultiRoundR1 + status: FAILURE +- scenario: input-required-result-multi-round + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-multiple-input-requests + check: sep-2322-multiple-inputs-incomplete + name: InputRequiredResultMultipleInputsIncomplete + status: FAILURE +- scenario: input-required-result-multiple-input-requests + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-non-tool-request + check: sep-2322-non-tool-incomplete + name: InputRequiredResultNonToolIncomplete + status: FAILURE +- scenario: input-required-result-non-tool-request + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-request-state + check: sep-2322-request-state-incomplete + name: InputRequiredResultRequestStateIncomplete + status: FAILURE +- scenario: input-required-result-request-state + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-result-type + check: sep-2322-result-type-included + name: ResultTypeIncluded + status: FAILURE +- scenario: input-required-result-result-type + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-tampered-state + check: sep-2322-reject-tampered-state + name: RejectTamperedState + status: FAILURE +- scenario: input-required-result-tampered-state + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: input-required-result-unsupported-methods + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: json-schema-2020-12 + check: json-schema-2020-12-error + name: JsonSchema2020_12Error + status: FAILURE +- scenario: json-schema-2020-12 + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: prompts-get-embedded-resource + check: prompts-get-embedded-resource + name: PromptsGetEmbeddedResource + status: FAILURE +- scenario: prompts-get-embedded-resource + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: prompts-get-simple + check: prompts-get-simple + name: PromptsGetSimple + status: FAILURE +- scenario: prompts-get-simple + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: prompts-get-with-args + check: prompts-get-with-args + name: PromptsGetWithArgs + status: FAILURE +- scenario: prompts-get-with-args + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: prompts-get-with-image + check: prompts-get-with-image + name: PromptsGetWithImage + status: FAILURE +- scenario: prompts-get-with-image + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: prompts-list + check: prompts-list + name: PromptsList + status: FAILURE +- scenario: prompts-list + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: resources-list + check: resources-list + name: ResourcesList + status: FAILURE +- scenario: resources-list + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: resources-read-binary + check: resources-read-binary + name: ResourcesReadBinary + status: FAILURE +- scenario: resources-read-binary + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: resources-read-text + check: resources-read-text + name: ResourcesReadText + status: FAILURE +- scenario: resources-read-text + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: resources-templates-read + check: resources-templates-read + name: ResourcesTemplateRead + status: FAILURE +- scenario: resources-templates-read + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: sep-2164-resource-not-found + check: sep-2164-data-uri + name: ResourcesNotFoundDataUri + status: WARNING +- scenario: sep-2164-resource-not-found + check: sep-2164-error-code + name: ResourcesNotFoundErrorCode + status: WARNING +- scenario: sep-2164-resource-not-found + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: server-sse-multiple-streams + check: server-accepts-multiple-post-streams + name: ServerAcceptsMultiplePostStreams + status: FAILURE +- scenario: server-stateless + check: sep-2575-discover-capabilities-match-handlers + name: DiscoverCapabilitiesMatchHandlers + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-error-jsonrpc-id + name: HttpServerErrorJsonrpcId + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-header-mismatch-400 + name: HttpServerHeaderMismatch400 + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-method-not-found-404 + name: HttpServerMethodNotFound404 + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-method-not-found-404-initialize + name: HttpServerMethodNotFound404initialize + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-method-not-found-404-logging-setlevel + name: HttpServerMethodNotFound404loggingsetLevel + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-method-not-found-404-ping + name: HttpServerMethodNotFound404ping + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-method-not-found-404-resources-subscribe + name: HttpServerMethodNotFound404resourcessubscribe + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-method-not-found-404-resources-unsubscribe + name: HttpServerMethodNotFound404resourcesunsubscribe + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-no-independent-requests-on-stream + name: HttpServerNoIndependentRequestsOnStream + status: FAILURE +- scenario: server-stateless + check: sep-2575-missing-capability-http-400 + name: MissingCapabilityHttp400 + status: FAILURE +- scenario: server-stateless + check: sep-2575-request-meta-client-info-optional + name: RequestMetaClientInfoOptional + status: FAILURE +- scenario: server-stateless + check: sep-2575-request-meta-invalid-missing-client-capabilities + name: RequestMetaInvalid + status: FAILURE +- scenario: server-stateless + check: sep-2575-request-meta-invalid-missing-meta + name: RequestMetaInvalid + status: FAILURE +- scenario: server-stateless + check: sep-2575-request-meta-invalid-missing-protocol-version + name: RequestMetaInvalid + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-declares-prompts-in-discover + name: ServerDeclaresPromptsInDiscover + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-identifies-in-result-meta + name: ServerIdentifiesInResultMeta + status: WARNING +- scenario: server-stateless + check: sep-2575-server-implements-discover + name: ServerImplementsDiscover + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-no-log-without-loglevel + name: ServerNoLogWithoutLogLevel + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-rejects-undeclared-capability + name: ServerRejectsUndeclaredCapability + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-sends-subscription-ack + name: ServerSendsSubscriptionAck + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-tags-subscription-id + name: ServerTagsSubscriptionId + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-unsupported-version-error + name: ServerUnsupportedVersionError + status: FAILURE +- scenario: tools-call-audio + check: tools-call-audio + name: ToolsCallAudio + status: FAILURE +- scenario: tools-call-audio + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-embedded-resource + check: tools-call-embedded-resource + name: ToolsCallEmbeddedResource + status: FAILURE +- scenario: tools-call-embedded-resource + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-error + check: tools-call-error + name: ToolsCallError + status: FAILURE +- scenario: tools-call-error + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-image + check: tools-call-image + name: ToolsCallImage + status: FAILURE +- scenario: tools-call-image + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-mixed-content + check: tools-call-mixed-content + name: ToolsCallMixedContent + status: FAILURE +- scenario: tools-call-mixed-content + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-simple-text + check: tools-call-simple-text + name: ToolsCallSimpleText + status: FAILURE +- scenario: tools-call-simple-text + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-call-with-progress + check: tools-call-with-progress + name: ToolsCallWithProgress + status: FAILURE +- scenario: tools-call-with-progress + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE +- scenario: tools-list + check: tools-list + name: ToolsList + status: FAILURE +- scenario: tools-list + check: wire-schema-valid + name: WireSchemaValid + status: FAILURE diff --git a/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml b/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml index 2cb8db3..72d9878 100644 --- a/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml +++ b/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml @@ -1 +1,205 @@ -findings: [] +findings: +- scenario: caching + check: sep-2549-cache-scope-valid + name: CacheScopeValid + status: FAILURE +- scenario: caching + check: sep-2549-prompts-list-caching-hints + name: PromptsListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-resources-list-caching-hints + name: ResourcesListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-resources-templates-list-caching-hints + name: ResourcesTemplatesListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-tools-list-caching-hints + name: ToolsListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-ttl-non-negative + name: TtlNonNegative + status: FAILURE +- scenario: completion-complete + check: completion-complete + name: CompletionComplete + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-custom-setup + name: HttpCustomHeaderServerValidationSetup + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-decode-base64 + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-reject-invalid-param-chars + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-reject-param-mismatch + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-validate-param-match + name: NotObserved + status: FAILURE +- scenario: http-header-validation + check: sep-2243-server-accepts-whitespace-header-value + name: ServerAcceptsWhitespaceHeaderValue + status: FAILURE +- scenario: http-header-validation + check: sep-2243-server-standard-setup + name: HttpHeaderValidationSetup + status: FAILURE +- scenario: input-required-result-basic-elicitation + check: sep-2322-elicitation-incomplete + name: InputRequiredResultElicitationIncomplete + status: FAILURE +- scenario: input-required-result-basic-list-roots + check: sep-2322-list-roots-incomplete + name: InputRequiredResultListRootsIncomplete + status: FAILURE +- scenario: input-required-result-basic-sampling + check: sep-2322-sampling-incomplete + name: InputRequiredResultSamplingIncomplete + status: FAILURE +- scenario: input-required-result-capability-check + check: sep-2322-respect-client-capabilities + name: RespectClientCapabilities + status: FAILURE +- scenario: input-required-result-ignore-extra-params + check: sep-2322-ignore-unexpected-params + name: IgnoreUnexpectedParams + status: WARNING +- scenario: input-required-result-missing-input-response + check: sep-2322-missing-response-rerequests + name: InputRequiredResultMissingResponseRerequests + status: WARNING +- scenario: input-required-result-multi-round + check: sep-2322-multi-round-r1 + name: InputRequiredResultMultiRoundR1 + status: FAILURE +- scenario: input-required-result-multiple-input-requests + check: sep-2322-multiple-inputs-incomplete + name: InputRequiredResultMultipleInputsIncomplete + status: FAILURE +- scenario: input-required-result-non-tool-request + check: sep-2322-non-tool-incomplete + name: InputRequiredResultNonToolIncomplete + status: FAILURE +- scenario: input-required-result-request-state + check: sep-2322-request-state-incomplete + name: InputRequiredResultRequestStateIncomplete + status: FAILURE +- scenario: input-required-result-result-type + check: sep-2322-result-type-included + name: ResultTypeIncluded + status: FAILURE +- scenario: input-required-result-tampered-state + check: sep-2322-reject-tampered-state + name: RejectTamperedState + status: FAILURE +- scenario: json-schema-2020-12 + check: json-schema-2020-12-error + name: JsonSchema2020_12Error + status: FAILURE +- scenario: prompts-get-embedded-resource + check: prompts-get-embedded-resource + name: PromptsGetEmbeddedResource + status: FAILURE +- scenario: prompts-get-simple + check: prompts-get-simple + name: PromptsGetSimple + status: FAILURE +- scenario: prompts-get-with-args + check: prompts-get-with-args + name: PromptsGetWithArgs + status: FAILURE +- scenario: prompts-get-with-image + check: prompts-get-with-image + name: PromptsGetWithImage + status: FAILURE +- scenario: prompts-list + check: prompts-list + name: PromptsList + status: FAILURE +- scenario: resources-list + check: resources-list + name: ResourcesList + status: FAILURE +- scenario: resources-read-binary + check: resources-read-binary + name: ResourcesReadBinary + status: FAILURE +- scenario: resources-read-text + check: resources-read-text + name: ResourcesReadText + status: FAILURE +- scenario: resources-templates-read + check: resources-templates-read + name: ResourcesTemplateRead + status: FAILURE +- scenario: sep-2164-resource-not-found + check: sep-2164-data-uri + name: ResourcesNotFoundDataUri + status: WARNING +- scenario: server-stateless + check: sep-2575-discover-capabilities-match-handlers + name: DiscoverCapabilitiesMatchHandlers + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-no-independent-requests-on-stream + name: HttpServerNoIndependentRequestsOnStream + status: FAILURE +- scenario: server-stateless + check: sep-2575-missing-capability-http-400 + name: MissingCapabilityHttp400 + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-declares-prompts-in-discover + name: ServerDeclaresPromptsInDiscover + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-no-log-without-loglevel + name: ServerNoLogWithoutLogLevel + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-rejects-undeclared-capability + name: ServerRejectsUndeclaredCapability + status: FAILURE +- scenario: tools-call-audio + check: tools-call-audio + name: ToolsCallAudio + status: FAILURE +- scenario: tools-call-embedded-resource + check: tools-call-embedded-resource + name: ToolsCallEmbeddedResource + status: FAILURE +- scenario: tools-call-error + check: tools-call-error + name: ToolsCallError + status: FAILURE +- scenario: tools-call-image + check: tools-call-image + name: ToolsCallImage + status: FAILURE +- scenario: tools-call-mixed-content + check: tools-call-mixed-content + name: ToolsCallMixedContent + status: FAILURE +- scenario: tools-call-simple-text + check: tools-call-simple-text + name: ToolsCallSimpleText + status: FAILURE +- scenario: tools-call-with-progress + check: tools-call-with-progress + name: ToolsCallWithProgress + status: FAILURE +- scenario: tools-list + check: tools-list + name: ToolsList + status: FAILURE diff --git a/tests/conformance/baselines/2026-07-28/modern/fixture-direct.yml b/tests/conformance/baselines/2026-07-28/modern/fixture-direct.yml index efee5f0..f1eee4f 100644 --- a/tests/conformance/baselines/2026-07-28/modern/fixture-direct.yml +++ b/tests/conformance/baselines/2026-07-28/modern/fixture-direct.yml @@ -63,31 +63,3 @@ findings: check: sep-2322-validate-input-responses name: ValidateInputResponses status: WARNING -- scenario: tools-call-audio - check: wire-schema-valid - name: WireSchemaValid - status: FAILURE -- scenario: tools-call-embedded-resource - check: wire-schema-valid - name: WireSchemaValid - status: FAILURE -- scenario: tools-call-error - check: wire-schema-valid - name: WireSchemaValid - status: FAILURE -- scenario: tools-call-image - check: wire-schema-valid - name: WireSchemaValid - status: FAILURE -- scenario: tools-call-mixed-content - check: wire-schema-valid - name: WireSchemaValid - status: FAILURE -- scenario: tools-call-simple-text - check: wire-schema-valid - name: WireSchemaValid - status: FAILURE -- scenario: tools-call-with-progress - check: wire-schema-valid - name: WireSchemaValid - status: FAILURE