From cfff95836c2c25af6fb0eb9761d7b662497c4509 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 3 Oct 2026 21:20:19 +0000 Subject: [PATCH 1/2] fix(v18): use pg_query_scan_tokens for scan (libpg_query dropped protobuf-c) --- templates/full/wasm_wrapper.c | 37 +++++++++++----------------------- versions/18/README.md | 4 ++-- versions/18/src/wasm_wrapper.c | 37 +++++++++++----------------------- versions/18/test/pg18.test.js | 6 +++--- 4 files changed, 29 insertions(+), 55 deletions(-) diff --git a/templates/full/wasm_wrapper.c b/templates/full/wasm_wrapper.c index 268d2893..6ec5ab03 100644 --- a/templates/full/wasm_wrapper.c +++ b/templates/full/wasm_wrapper.c @@ -1,5 +1,4 @@ #include "pg_query.h" -#include "protobuf/pg_query.pb-c.h" #include #include #include @@ -230,7 +229,7 @@ void wasm_free_detailed_result(WasmDetailedResult* result) { } } -static const char* get_token_name(PgQuery__Token token_type) { +static const char* get_token_name(PgQueryToken token_type) { // Map some common token types to readable names // Note: This is a simplified mapping - full enum lookup would require more complexity switch(token_type) { @@ -255,7 +254,7 @@ static const char* get_token_name(PgQuery__Token token_type) { } } -static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) { +static const char* get_keyword_name(PgQueryKeywordKind keyword_kind) { switch(keyword_kind) { case 0: return "NO_KEYWORD"; case 1: return "UNRESERVED_KEYWORD"; @@ -266,7 +265,7 @@ static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) { } } -static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* original_sql) { +static char* build_scan_json(PgQueryScanTokensResult *scan_result, const char* original_sql) { if (!scan_result || !original_sql) { return safe_strdup("{\"version\":0,\"tokens\":[]}"); } @@ -275,8 +274,8 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi // contributes at most ~170 bytes of fixed JSON plus up to 2 bytes per input // byte once escaped. size_t estimated_size = 1024; - for (size_t i = 0; i < scan_result->n_tokens; i++) { - PgQuery__ScanToken *token = scan_result->tokens[i]; + for (int i = 0; i < scan_result->n_tokens; i++) { + PgQueryScanToken *token = &scan_result->tokens[i]; int token_length = token->end - token->start; if (token_length < 0) token_length = 0; estimated_size += (size_t) token_length * 2 + 200; @@ -287,10 +286,10 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi } // Start building JSON - size_t pos = snprintf(json, estimated_size, "{\"version\":%d,\"tokens\":[", scan_result->version); + size_t pos = snprintf(json, estimated_size, "{\"version\":%d,\"tokens\":[", PG_VERSION_NUM); - for (size_t i = 0; i < scan_result->n_tokens; i++) { - PgQuery__ScanToken *token = scan_result->tokens[i]; + for (int i = 0; i < scan_result->n_tokens; i++) { + PgQueryScanToken *token = &scan_result->tokens[i]; // Extract token text from original SQL int token_length = token->end - token->start; @@ -369,29 +368,17 @@ char* wasm_scan(const char* input) { return safe_strdup("Invalid input: query cannot be null or empty"); } - PgQueryScanResult result = pg_query_scan(input); + PgQueryScanTokensResult result = pg_query_scan_tokens(input); if (result.error) { char* error_msg = safe_strdup(result.error->message); - pg_query_free_scan_result(result); + pg_query_free_scan_tokens_result(result); return error_msg ? error_msg : safe_strdup("Memory allocation failed"); } - // Unpack protobuf data - PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( - NULL, result.pbuf.len, (void *) result.pbuf.data); + char* json_result = build_scan_json(&result, input); - if (!scan_result) { - pg_query_free_scan_result(result); - return safe_strdup("Failed to unpack scan result"); - } - - // Convert to JSON - char* json_result = build_scan_json(scan_result, input); - - // Clean up - pg_query__scan_result__free_unpacked(scan_result, NULL); - pg_query_free_scan_result(result); + pg_query_free_scan_tokens_result(result); return json_result ? json_result : safe_strdup("{\"version\":0,\"tokens\":[]}"); } diff --git a/versions/18/README.md b/versions/18/README.md index be93bf2f..a5df49bd 100644 --- a/versions/18/README.md +++ b/versions/18/README.md @@ -49,7 +49,7 @@ npm install libpg-query import { parse } from 'libpg-query'; const result = await parse('SELECT * FROM users WHERE active = true'); -// {"version":180004,"stmts":[{"stmt":{"SelectStmt":{"targetList":[{"ResTarget" ... "op":"SETOP_NONE"}}}]} +// {"version":180006,"stmts":[{"stmt":{"SelectStmt":{"targetList":[{"ResTarget" ... "op":"SETOP_NONE"}}}]} ``` ## Versions @@ -116,7 +116,7 @@ Tokenizes the SQL, returning token positions, text, types, and keyword kinds. Se import { scan } from 'libpg-query'; const result = await scan('SELECT id FROM users'); -// { version: 180004, tokens: [{ start, end, text, tokenType, tokenName, keywordKind, keywordName }, ...] } +// { version: 180006, tokens: [{ start, end, text, tokenType, tokenName, keywordKind, keywordName }, ...] } ``` ### `fingerprint(sql: string): Promise` / `fingerprintSync` diff --git a/versions/18/src/wasm_wrapper.c b/versions/18/src/wasm_wrapper.c index 63ae5dab..408e8891 100644 --- a/versions/18/src/wasm_wrapper.c +++ b/versions/18/src/wasm_wrapper.c @@ -6,7 +6,6 @@ */ #include "pg_query.h" -#include "protobuf/pg_query.pb-c.h" #include #include #include @@ -237,7 +236,7 @@ void wasm_free_detailed_result(WasmDetailedResult* result) { } } -static const char* get_token_name(PgQuery__Token token_type) { +static const char* get_token_name(PgQueryToken token_type) { // Map some common token types to readable names // Note: This is a simplified mapping - full enum lookup would require more complexity switch(token_type) { @@ -262,7 +261,7 @@ static const char* get_token_name(PgQuery__Token token_type) { } } -static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) { +static const char* get_keyword_name(PgQueryKeywordKind keyword_kind) { switch(keyword_kind) { case 0: return "NO_KEYWORD"; case 1: return "UNRESERVED_KEYWORD"; @@ -273,7 +272,7 @@ static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) { } } -static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* original_sql) { +static char* build_scan_json(PgQueryScanTokensResult *scan_result, const char* original_sql) { if (!scan_result || !original_sql) { return safe_strdup("{\"version\":0,\"tokens\":[]}"); } @@ -282,8 +281,8 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi // contributes at most ~170 bytes of fixed JSON plus up to 2 bytes per input // byte once escaped. size_t estimated_size = 1024; - for (size_t i = 0; i < scan_result->n_tokens; i++) { - PgQuery__ScanToken *token = scan_result->tokens[i]; + for (int i = 0; i < scan_result->n_tokens; i++) { + PgQueryScanToken *token = &scan_result->tokens[i]; int token_length = token->end - token->start; if (token_length < 0) token_length = 0; estimated_size += (size_t) token_length * 2 + 200; @@ -294,10 +293,10 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi } // Start building JSON - size_t pos = snprintf(json, estimated_size, "{\"version\":%d,\"tokens\":[", scan_result->version); + size_t pos = snprintf(json, estimated_size, "{\"version\":%d,\"tokens\":[", PG_VERSION_NUM); - for (size_t i = 0; i < scan_result->n_tokens; i++) { - PgQuery__ScanToken *token = scan_result->tokens[i]; + for (int i = 0; i < scan_result->n_tokens; i++) { + PgQueryScanToken *token = &scan_result->tokens[i]; // Extract token text from original SQL int token_length = token->end - token->start; @@ -376,29 +375,17 @@ char* wasm_scan(const char* input) { return safe_strdup("Invalid input: query cannot be null or empty"); } - PgQueryScanResult result = pg_query_scan(input); + PgQueryScanTokensResult result = pg_query_scan_tokens(input); if (result.error) { char* error_msg = safe_strdup(result.error->message); - pg_query_free_scan_result(result); + pg_query_free_scan_tokens_result(result); return error_msg ? error_msg : safe_strdup("Memory allocation failed"); } - // Unpack protobuf data - PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( - NULL, result.pbuf.len, (void *) result.pbuf.data); + char* json_result = build_scan_json(&result, input); - if (!scan_result) { - pg_query_free_scan_result(result); - return safe_strdup("Failed to unpack scan result"); - } - - // Convert to JSON - char* json_result = build_scan_json(scan_result, input); - - // Clean up - pg_query__scan_result__free_unpacked(scan_result, NULL); - pg_query_free_scan_result(result); + pg_query_free_scan_tokens_result(result); return json_result ? json_result : safe_strdup("{\"version\":0,\"tokens\":[]}"); } diff --git a/versions/18/test/pg18.test.js b/versions/18/test/pg18.test.js index fb49bae0..ddcd446f 100644 --- a/versions/18/test/pg18.test.js +++ b/versions/18/test/pg18.test.js @@ -7,9 +7,9 @@ describe("PostgreSQL 18 syntax", () => { await loadModule(); }); - it("should report parser version 180004", async () => { + it("should report parser version 180006", async () => { const result = await parse("SELECT 1"); - assert.equal(result.version, 180004); + assert.equal(result.version, 180006); }); it("should parse RETURNING OLD/NEW aliases", async () => { @@ -61,7 +61,7 @@ describe("PostgreSQL 18 syntax", () => { const result = await scan( "UPDATE t SET a = 1 RETURNING OLD.a AS oa, NEW.a AS na" ); - assert.equal(result.version, 180004); + assert.equal(result.version, 180006); const texts = result.tokens.map((t) => t.text); assert.ok(texts.includes("OLD")); assert.ok(texts.includes("NEW")); From 6e21f9d8943a1ab173dec6e0bf40c15e7e56245d Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 3 Oct 2026 22:20:41 +0000 Subject: [PATCH 2/2] docs(v18): document pg_query_scan_tokens; mark protobuf-c scan examples historical --- versions/18/libpg_query.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/versions/18/libpg_query.md b/versions/18/libpg_query.md index 2b4e4e8b..d86d36b0 100644 --- a/versions/18/libpg_query.md +++ b/versions/18/libpg_query.md @@ -63,10 +63,24 @@ PgQueryScanResult pg_query_scan(const char* input); - `stderr_buffer`: Any stderr output during scanning - `error`: Error information if scanning failed -**Usage**: Use this when you need to tokenize SQL without full parsing. +**Usage**: Use this when you need the protobuf-encoded scan result (e.g. to decode in another language). + +#### pg_query_scan_tokens +```c +PgQueryScanTokensResult pg_query_scan_tokens(const char* input); +void pg_query_free_scan_tokens_result(PgQueryScanTokensResult result); +``` +**Description**: Scans SQL input and returns tokens as a plain C array, no protobuf decoding needed. This is what `wasm_scan` uses. + +**Returns**: `PgQueryScanTokensResult` containing: +- `tokens` / `n_tokens`: array of `PgQueryScanToken` (`start`, `end`, `token`, `keyword_kind`) +- `stderr_buffer`: Any stderr output during scanning +- `error`: Error information if scanning failed ## Scanning and Token Processing +> **Note:** libpg_query no longer ships the protobuf-c header `protobuf/pg_query.pb-c.h` (it switched to upb), so the C unpack examples below are historical. In C, use `pg_query_scan_tokens` instead. + ### Working with Scan Results The `pg_query_scan` function returns tokens in protobuf format that need to be unpacked to access individual tokens. Here's the complete workflow: