diff --git a/README.md b/README.md index e736d33..cd16f71 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ This is a working Database Agent that lets you connect a SQL database, ask questions in natural language, inspect generated SQL, explore results, and create visualizations. +**Research → product change: [S64 — unsupported data was reported as zero](docs/improvements/S64-unsupported-data.md).** +[Original published finding](https://aidemos.com/benchmarks/ai-database-agents/results/futuresmart-ai-database-agent/the-connected-data-cannot-answer-the-question): **Fail, unchanged** · Engineering correction: **released** · Independently accepted benchmark retest: **pending**. +This retrospective connects the finding to the code, regression controls and recorded validation; the correction predates this repository's public history. + But the purpose of this project goes beyond providing another text-to-SQL application. ## Why we're building this publicly diff --git a/docs/improvements/README.md b/docs/improvements/README.md index 8e6c26e..305e3f0 100644 --- a/docs/improvements/README.md +++ b/docs/improvements/README.md @@ -4,6 +4,7 @@ Recorded on 21 October 2026 (IST). This is a public-facing history of our own reference application. It preserves failures, decisions and limited validation alongside successful fixes. It is not a leaderboard or a current accuracy score. +- [S64 retrospective: unsupported data was reported as zero](S64-unsupported-data.md) — original Fail unchanged; correction released; independent benchmark retest pending. - [Original evaluation baseline and published sources](baseline.md) - [Dated 17-result documented baseline and unchanged verdicts](published-results.md) - [Engineering history, including unfinished and declined work](history.md) diff --git a/docs/improvements/S64-unsupported-data.md b/docs/improvements/S64-unsupported-data.md new file mode 100644 index 0000000..7470e6c --- /dev/null +++ b/docs/improvements/S64-unsupported-data.md @@ -0,0 +1,124 @@ +# S64: Unsupported data was reported as zero + +Retrospective case study, documented 22 October 2026 IST. Engineering record: +[DA-001](history.md#da-001-unsupported-data-must-not-become-a-false-zero). This document explains an existing released correction; +it does not implement a new fix or report a new benchmark run. + +| Status | Recorded outcome | +|---|---| +| Original AI Demos result | **Fail — unchanged**; one failed test case | +| Engineering correction | **Released**, recorded 20 October 2026 IST | +| Independent engineering validation | Recorded staging/production controls passed, within the limits below | +| Independently accepted S64 benchmark retest | **Pending**; no new benchmark Pass is claimed | + +## The published finding + +The [original AI Demos result](https://aidemos.com/benchmarks/ai-database-agents/results/futuresmart-ai-database-agent/the-connected-data-cannot-answer-the-question) +concerns a connected dataset that did not support the requested topic. The user asked: + +> How many 5-star reviews have we received this month? + +The product used `action_log` and returned zero instead of explaining that the +available data could not answer the question. The published expectation was an +unavailable answer with no number. The page records **Fail**, with a screenshot +showing the question, answer, returned row and SQL panel. See the +[original public proof](https://aidemos.com/benchmarks/ai-database-agents/results/futuresmart-ai-database-agent/the-connected-data-cannot-answer-the-question#proof-3fd7d42ff0a3). + +The page dates the evidence to **8 September 2026**, its grade to **12 October**, +and first publication to **21 October**. It identifies Cedarline commerce +`customer_support_db`, version 1; the original hosted build was not exposed, +and the original model is unknown. The engineering record is dated **20 October**: +research informed the correction before that finding was publicly published. +This is a retrospective connection, not a claim that a public issue or PR +initiated the historical work. + +## What went wrong, and the decision + +The [approved maintainer record](records.json) diagnoses a semantic support gap: +a query could execute successfully against an available table even though that +table did not represent the requested business metric. An aggregate result of +zero then looked like a factual answer. Successful SQL execution alone does not +establish that the schema supports the question. That is the recorded engineering +diagnosis, not a reconstruction of the unknown historical hosted build. + +The decision was to require an explicit schema-support decision and preserve +three different outcomes: + +| Outcome | Meaning and required behavior | +|---|---| +| Supported metric, result zero | A valid query against the right data returned zero. Keep that answer. | +| Unsupported metric | The available schema does not establish the requested information. Explain unavailability; do not substitute a proxy table or number. | +| Restricted data or execution failure | Preserve the access denial or query error; neither establishes zero or universal data absence. | + +## How the released code carries that decision + +These links point to actual code already present in this public repository; +this documentation change does not alter it. + +1. [SQL-generation instructions](../../backend/agent/utils/prompts.py#L186) require + schema support before generating SQL, reject unrelated proxies, and preserve + legitimate zero aggregates. +2. [SQLQuerySchema and apply_sql_decision](../../backend/agent/utils/answerability.py#L5) + require a strict Boolean `answerable` decision. Missing or malformed decisions + cannot approve SQL. An explicit unsupported decision clears surrogate SQL and + stale rows, sets an answerability error, and returns no query. A supported + decision must contain a nonempty query. +3. [SQL generation](../../backend/agent/nodes/sql_gen.py#L553) applies that decision; + [SQL execution](../../backend/agent/nodes/sql_exec.py#L55) returns before database + execution when an answerability error is present. Existing table-access guards + remain separate. +4. [Final-answer handling](../../backend/agent/nodes/final_answer.py#L149) uses + [failure_response](../../backend/agent/utils/answerability.py#L30) before invoking + the presentation model. Unsupported input yields `is_answered=false`, cleared + result/chart state, and an explanation that no count of zero was established. +5. [Cross-database merging](../../backend/agent/nodes/cross_db_merge.py#L95) retains + unavailability when all required subresults are unsupported. Mixed required-source + outcomes retain the combination/error contract rather than asserting absence + across every database. + +The [public initial commit](https://github.com/FutureSmart-AI/database-agent/commit/37922276843b0db86c275b786d56db6dd322c3f4) +already contains this behavior. DA-001 retains historical implementation +identifiers `174b06e` and `97fdcf4`; those private commits and their development +history were not imported. They are not public fix commits or evidence of a +historical public issue/PR. The current documentation PR is retrospective only. + +## Regression controls and recorded independent validation + +The existing [answerability regression tests](../../tests/backend/test_answerability.py) +cover these boundaries with mocked database/model calls: + +| Existing control | Contract checked | +|---|---| +| `test_missing_schema_discards_surrogate_sql_and_stale_rows` | Unsupported decisions cannot retain proxy SQL or old results. | +| `test_correct_count_query_is_not_rejected_based_on_its_result` and `test_valid_zero_reaches_database_and_remains_answerable` | Genuine zero and positive counts remain supported; zero is not an abstention heuristic. | +| `test_missing_or_malformed_support_decision_cannot_approve_raw_sql` | Missing decisions, string Booleans and empty supported SQL fail validation. | +| `test_unsupported_sql_never_reaches_database_or_final_answer_model` | Explicit unsupported state skips source execution and presentation-model work. | +| `test_existing_table_guard_still_blocks_before_execution` and `test_malformed_sql_and_unavailable_connection_do_not_become_zero` | Restrictions, syntax errors and connection failures do not become numerical answers. | +| Both cross-database tests in `PipelineTests` | All-unsupported and mixed required-source outcomes remain distinct. | + +The [synthetic answerability fixture](../../tests/accuracy/fixtures/answerability.sql) +contains products, orders and administrative activity events, rather than a review +dataset. It is a regression asset, not a reconstruction of the original database. + +The pre-existing [DA-001 history](history.md#da-001-unsupported-data-must-not-become-a-false-zero) records independent staged +browser controls for missing data, a real count of zero and a count of two; it +also records a production missing-data check and a separate real-source positive +SQL/rows/chart check. These are **recorded historical engineering validations**. +Their operator receipts are not published here, and they are not a new independent +S64 benchmark retest. Preparing this retrospective did not rerun those historical controls or request +a new model or benchmark evaluation. Routine documentation and packaging checks +do not constitute a benchmark retest. + +## Limits and the remaining benchmark gate + +The correction carries an explicit support decision through the pipeline; it does +not prove that a model will always classify every schema and question correctly. +Metadata/schema-hint omissions remain separate. The current public source also +contains later changes, so it cannot identify the unknown original hosted build. + +An independently accepted benchmark retest remains **pending**. Such a retest +would need a pinned newer build/configuration, an appropriate bound dataset and +independent grading of the original behavior, with genuine-zero controls. This +case study does not schedule or execute it. The original **Fail** remains visible +and unchanged; a released engineering correction, regression tests, or green CI +does not turn that historical result into Pass. diff --git a/docs/improvements/history.md b/docs/improvements/history.md index dcfb1f8..9a29451 100644 --- a/docs/improvements/history.md +++ b/docs/improvements/history.md @@ -19,6 +19,8 @@ Documented 2026-10-21 (IST); earlier dates are dates supported by the saved work Recorded date: **2026-10-20 IST**. State: **released**. +[Read the complete S64 retrospective](S64-unsupported-data.md): finding → code/tests → recorded validation. Original **Fail unchanged**; independently accepted benchmark retest **pending**. + **Research source.** The [unsupported-data Result](https://aidemos.com/benchmarks/ai-database-agents/results/futuresmart-ai-database-agent/the-connected-data-cannot-answer-the-question) is now published. Our later reproduction and validation remain separate from that historical run. **Original benchmark result.** Published **Fail**, unchanged. The released fix below does not regrade that run. diff --git a/docs/source-export.json b/docs/source-export.json index 7a00d3a..9919b21 100644 --- a/docs/source-export.json +++ b/docs/source-export.json @@ -315,12 +315,12 @@ "docs/improvements/evidence/2026-10-22-fresh-policy-writes.json": "8fa020fd8a8157fcb19b24f79c12893ea5cd20daad34cf9aa8431268a26ae177", "docs/improvements/evidence/2026-10-22-model-profile-offline.json": "85597e95af24bd29dc35f1054e57ef4d56d8b2e5ea6b4862fa4953b700fccf30", "docs/improvements/evidence/2026-10-22-one-key-qualification.json": "35f1a417e0736836b7d412fe4f257736de1874a2584ab64d83b82e25865db408", - "docs/improvements/history.md": "a33475cb4375d73613dc2b716f3ac4c9ab9f2db6fa54b34be40c507dd30aed21", + "docs/improvements/history.md": "f62c47b1241210efb80f4f069a072847b8c155372eee034a3ce5e41259f882a4", "docs/improvements/MAINTAINING.md": "06562772971dd8fdca511a15370b59eb9d7b4c5999d621d0904a79d8466945c8", "docs/improvements/published-observations.json": "0546f15602447774cc7b08e7e3c6a331aba996afd849abc1f146799203ffbe6a", "docs/improvements/published-results.json": "6951d846a5f8a968264e948f4a8001779f5a709a50aafa799049e60316cb807b", "docs/improvements/published-results.md": "71648bfb948cd190b4a46322ccc45c3e9da7bfe78e275f41434a0de0bfe4fae1", - "docs/improvements/README.md": "23c2e3f40e5e11a1ba9d1c9649020da2dd7cfedfe61729b7f8cb864f94c5b93a", + "docs/improvements/README.md": "4748c623602ad259667e7b11b095fb00e58ae25dae3c084cc5dfed0abc484764", "docs/improvements/records.json": "feadc6540f7632b6471bdcc2708bf4b487e8930542ca0cc26e1fb818f5425670", "docs/model-configuration.md": "0caab9561ad7c4fc8a7f340113cb23c2169e740271b0e9c299f8e5d954df5455", "docs/publication.md": "c354454f30da314d6837dc8a717eda323ec1182bdbde964654c8bd40f96a55b5", @@ -789,7 +789,7 @@ "frontend/vitest.setup.ts": "b9fc49f524436b712003dfaf8c89659a986f8a08acde4eb2b21922253d0067fb", "LICENSE": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", "NOTICE": "3c880ad6a14f915c63684d5cf8a085dccbfc1c0d3cabd65015be2382ab3012f8", - "README.md": "6cfea0e515e05fcb267521c61a328590869e31b883536cda0832805dfa3153ef", + "README.md": "ce80e546f99cbb92bbb228905b4e80860ba5b708d16a6425675e1bf49b23a620", "tests/accuracy/fixtures/answerability.sql": "124ef5ae989bea644445dd9f2115a5a9f15bfb58395d80cc2d0a41e64e59fa13", "tests/accuracy/fixtures/reviews-positive.sql": "759dda31bcdc4ec460df6ebba2e06bcbd047b164327600841c70f6f98dc60d2e", "tests/accuracy/fixtures/reviews-zero.sql": "2a7736327df80803b096d5eb5de92fe5379c9b0d76c77acbbee261201076e6e1", @@ -831,6 +831,7 @@ "tests/evaluation/reliability/oracle.py": "15d595fa32dbd8fc5d5e8ece159d448a57e15ce81c13e22bf95901a491eef762", "tests/evaluation/run.py": "2f02324a55690b4d3f5f3cfa438ce672a4957a3ba2f1c06492af0fb15fedd9ef", "tests/evaluation/test_reliability.py": "8857966248f09a0c2b549b0e1f6ddcf4e8340719ea43c249c9bc09f5efcc39b7", - "tests/run.py": "022a8d19cdbbc1fba2ecc9a948394eb66cab2d95450f9032921d569fb8db7e0e" + "tests/run.py": "022a8d19cdbbc1fba2ecc9a948394eb66cab2d95450f9032921d569fb8db7e0e", + "docs/improvements/S64-unsupported-data.md": "f2d415768fbe38542adbed6cfc78c932fc4ec548aac56536484aaf958dc8e010" } }