Skip to content

fix(v18): use pg_query_scan_tokens for scan #171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pyramation wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix(v18): use pg_query_scan_tokens for scan #171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cfff958
fix(v18): use pg_query_scan_tokens for scan (libpg_query dropped prot…
pyramation Oct 3, 2026
6e21f9d
docs(v18): document pg_query_scan_tokens; mark protobuf-c scan exampl…
pyramation Oct 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 12 additions & 25 deletions templates/full/wasm_wrapper.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#include "pg_query.h"
#include "protobuf/pg_query.pb-c.h"
#include <emscripten.h>
#include <stdlib.h>
#include <string.h>
Expand Down Expand Up @@ -230,7 +229,7 @@ void wasm_free_detailed_result(WasmDetailedResult* result) {
}
}

static const char* get_token_name(PgQuery__Token token_type) {
static const char* get_token_name(PgQueryToken token_type) {
// Map some common token types to readable names
// Note: This is a simplified mapping - full enum lookup would require more complexity
switch(token_type) {
Expand All @@ -255,7 +254,7 @@ static const char* get_token_name(PgQuery__Token token_type) {
}
}

static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) {
static const char* get_keyword_name(PgQueryKeywordKind keyword_kind) {
switch(keyword_kind) {
case 0: return "NO_KEYWORD";
case 1: return "UNRESERVED_KEYWORD";
Expand All @@ -266,7 +265,7 @@ static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) {
}
}

static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* original_sql) {
static char* build_scan_json(PgQueryScanTokensResult *scan_result, const char* original_sql) {
if (!scan_result || !original_sql) {
return safe_strdup("{\"version\":0,\"tokens\":[]}");
}
Expand All @@ -275,8 +274,8 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi
// contributes at most ~170 bytes of fixed JSON plus up to 2 bytes per input
// byte once escaped.
size_t estimated_size = 1024;
for (size_t i = 0; i < scan_result->n_tokens; i++) {
PgQuery__ScanToken *token = scan_result->tokens[i];
for (int i = 0; i < scan_result->n_tokens; i++) {
PgQueryScanToken *token = &scan_result->tokens[i];
int token_length = token->end - token->start;
if (token_length < 0) token_length = 0;
estimated_size += (size_t) token_length * 2 + 200;
Expand All @@ -287,10 +286,10 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi
}

// Start building JSON
size_t pos = snprintf(json, estimated_size, "{\"version\":%d,\"tokens\":[", scan_result->version);
size_t pos = snprintf(json, estimated_size, "{\"version\":%d,\"tokens\":[", PG_VERSION_NUM);

for (size_t i = 0; i < scan_result->n_tokens; i++) {
PgQuery__ScanToken *token = scan_result->tokens[i];
for (int i = 0; i < scan_result->n_tokens; i++) {
PgQueryScanToken *token = &scan_result->tokens[i];

// Extract token text from original SQL
int token_length = token->end - token->start;
Expand Down Expand Up @@ -369,29 +368,17 @@ char* wasm_scan(const char* input) {
return safe_strdup("Invalid input: query cannot be null or empty");
}

PgQueryScanResult result = pg_query_scan(input);
PgQueryScanTokensResult result = pg_query_scan_tokens(input);

if (result.error) {
char* error_msg = safe_strdup(result.error->message);
pg_query_free_scan_result(result);
pg_query_free_scan_tokens_result(result);
return error_msg ? error_msg : safe_strdup("Memory allocation failed");
}

// Unpack protobuf data
PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack(
NULL, result.pbuf.len, (void *) result.pbuf.data);
char* json_result = build_scan_json(&result, input);

if (!scan_result) {
pg_query_free_scan_result(result);
return safe_strdup("Failed to unpack scan result");
}

// Convert to JSON
char* json_result = build_scan_json(scan_result, input);

// Clean up
pg_query__scan_result__free_unpacked(scan_result, NULL);
pg_query_free_scan_result(result);
pg_query_free_scan_tokens_result(result);

return json_result ? json_result : safe_strdup("{\"version\":0,\"tokens\":[]}");
}
Expand Down
4 changes: 2 additions & 2 deletions versions/18/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ npm install libpg-query
import { parse } from 'libpg-query';

const result = await parse('SELECT * FROM users WHERE active = true');
// {"version":180004,"stmts":[{"stmt":{"SelectStmt":{"targetList":[{"ResTarget" ... "op":"SETOP_NONE"}}}]}
// {"version":180006,"stmts":[{"stmt":{"SelectStmt":{"targetList":[{"ResTarget" ... "op":"SETOP_NONE"}}}]}
```

## Versions
Expand Down Expand Up @@ -116,7 +116,7 @@ Tokenizes the SQL, returning token positions, text, types, and keyword kinds. Se
import { scan } from 'libpg-query';

const result = await scan('SELECT id FROM users');
// { version: 180004, tokens: [{ start, end, text, tokenType, tokenName, keywordKind, keywordName }, ...] }
// { version: 180006, tokens: [{ start, end, text, tokenType, tokenName, keywordKind, keywordName }, ...] }
```

### `fingerprint(sql: string): Promise<string>` / `fingerprintSync`
Expand Down
16 changes: 15 additions & 1 deletion versions/18/libpg_query.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,24 @@ PgQueryScanResult pg_query_scan(const char* input);
- `stderr_buffer`: Any stderr output during scanning
- `error`: Error information if scanning failed

**Usage**: Use this when you need to tokenize SQL without full parsing.
**Usage**: Use this when you need the protobuf-encoded scan result (e.g. to decode in another language).

#### pg_query_scan_tokens
```c
PgQueryScanTokensResult pg_query_scan_tokens(const char* input);
void pg_query_free_scan_tokens_result(PgQueryScanTokensResult result);
```
**Description**: Scans SQL input and returns tokens as a plain C array, no protobuf decoding needed. This is what `wasm_scan` uses.

**Returns**: `PgQueryScanTokensResult` containing:
- `tokens` / `n_tokens`: array of `PgQueryScanToken` (`start`, `end`, `token`, `keyword_kind`)
- `stderr_buffer`: Any stderr output during scanning
- `error`: Error information if scanning failed

## Scanning and Token Processing

> **Note:** libpg_query no longer ships the protobuf-c header `protobuf/pg_query.pb-c.h` (it switched to upb), so the C unpack examples below are historical. In C, use `pg_query_scan_tokens` instead.

### Working with Scan Results

The `pg_query_scan` function returns tokens in protobuf format that need to be unpacked to access individual tokens. Here's the complete workflow:
Expand Down
37 changes: 12 additions & 25 deletions versions/18/src/wasm_wrapper.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
*/

#include "pg_query.h"
#include "protobuf/pg_query.pb-c.h"
#include <emscripten.h>
#include <stdlib.h>
#include <string.h>
Expand Down Expand Up @@ -237,7 +236,7 @@ void wasm_free_detailed_result(WasmDetailedResult* result) {
}
}

static const char* get_token_name(PgQuery__Token token_type) {
static const char* get_token_name(PgQueryToken token_type) {
// Map some common token types to readable names
// Note: This is a simplified mapping - full enum lookup would require more complexity
switch(token_type) {
Expand All @@ -262,7 +261,7 @@ static const char* get_token_name(PgQuery__Token token_type) {
}
}

static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) {
static const char* get_keyword_name(PgQueryKeywordKind keyword_kind) {
switch(keyword_kind) {
case 0: return "NO_KEYWORD";
case 1: return "UNRESERVED_KEYWORD";
Expand All @@ -273,7 +272,7 @@ static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) {
}
}

static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* original_sql) {
static char* build_scan_json(PgQueryScanTokensResult *scan_result, const char* original_sql) {
if (!scan_result || !original_sql) {
return safe_strdup("{\"version\":0,\"tokens\":[]}");
}
Expand All @@ -282,8 +281,8 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi
// contributes at most ~170 bytes of fixed JSON plus up to 2 bytes per input
// byte once escaped.
size_t estimated_size = 1024;
for (size_t i = 0; i < scan_result->n_tokens; i++) {
PgQuery__ScanToken *token = scan_result->tokens[i];
for (int i = 0; i < scan_result->n_tokens; i++) {
PgQueryScanToken *token = &scan_result->tokens[i];
int token_length = token->end - token->start;
if (token_length < 0) token_length = 0;
estimated_size += (size_t) token_length * 2 + 200;
Expand All @@ -294,10 +293,10 @@ static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* origi
}

// Start building JSON
size_t pos = snprintf(json, estimated_size, "{\"version\":%d,\"tokens\":[", scan_result->version);
size_t pos = snprintf(json, estimated_size, "{\"version\":%d,\"tokens\":[", PG_VERSION_NUM);

for (size_t i = 0; i < scan_result->n_tokens; i++) {
PgQuery__ScanToken *token = scan_result->tokens[i];
for (int i = 0; i < scan_result->n_tokens; i++) {
PgQueryScanToken *token = &scan_result->tokens[i];

// Extract token text from original SQL
int token_length = token->end - token->start;
Expand Down Expand Up @@ -376,29 +375,17 @@ char* wasm_scan(const char* input) {
return safe_strdup("Invalid input: query cannot be null or empty");
}

PgQueryScanResult result = pg_query_scan(input);
PgQueryScanTokensResult result = pg_query_scan_tokens(input);

if (result.error) {
char* error_msg = safe_strdup(result.error->message);
pg_query_free_scan_result(result);
pg_query_free_scan_tokens_result(result);
return error_msg ? error_msg : safe_strdup("Memory allocation failed");
}

// Unpack protobuf data
PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack(
NULL, result.pbuf.len, (void *) result.pbuf.data);
char* json_result = build_scan_json(&result, input);

if (!scan_result) {
pg_query_free_scan_result(result);
return safe_strdup("Failed to unpack scan result");
}

// Convert to JSON
char* json_result = build_scan_json(scan_result, input);

// Clean up
pg_query__scan_result__free_unpacked(scan_result, NULL);
pg_query_free_scan_result(result);
pg_query_free_scan_tokens_result(result);

return json_result ? json_result : safe_strdup("{\"version\":0,\"tokens\":[]}");
}
Expand Down
6 changes: 3 additions & 3 deletions versions/18/test/pg18.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ describe("PostgreSQL 18 syntax", () => {
await loadModule();
});

it("should report parser version 180004", async () => {
it("should report parser version 180006", async () => {
const result = await parse("SELECT 1");
assert.equal(result.version, 180004);
assert.equal(result.version, 180006);
});

it("should parse RETURNING OLD/NEW aliases", async () => {
Expand Down Expand Up @@ -61,7 +61,7 @@ describe("PostgreSQL 18 syntax", () => {
const result = await scan(
"UPDATE t SET a = 1 RETURNING OLD.a AS oa, NEW.a AS na"
);
assert.equal(result.version, 180004);
assert.equal(result.version, 180006);
const texts = result.tokens.map((t) => t.text);
assert.ok(texts.includes("OLD"));
assert.ok(texts.includes("NEW"));
Expand Down
Loading