From 818edaf2f9a7bdb2d5d947a3bf32a091a2996932 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Fri, 16 Oct 2026 13:51:30 +0000 Subject: [PATCH 01/38] feat: use jdv as the data viewer backend when it is installed View() registers data frames, matrices and lists with jdv and relays the jdv protocol through a new jdv_rpc sess request. The existing AG Grid and List Viewer webviews are reused; an extension-side adapter converts AG Grid sort/filter models to jdv queries and maps List Viewer paths to jdv typed paths. Arrow data and options(sess.dataviewer = "sess") keep the sess data viewer. --- sess/DESCRIPTION | 1 + sess/R/handlers.R | 54 +++ sess/R/hooks.R | 3 + sess/R/server.R | 3 +- sess/inst/tinytest/test-ipc.R | 3 + sess/inst/tinytest/test-jdv.R | 95 ++++ sess/inst/tinytest/test-listview.R | 3 + src/dataViewer.ts | 19 + src/interactive/manager.ts | 4 + src/jdvAdapter.ts | 685 +++++++++++++++++++++++++++++ src/jdvViewer.ts | 268 +++++++++++ src/session.ts | 12 +- src/test/node/jdvAdapter.test.ts | 168 +++++++ src/test/suite/dataViewer.test.ts | 24 + 14 files changed, 1340 insertions(+), 2 deletions(-) create mode 100644 sess/inst/tinytest/test-jdv.R create mode 100644 src/jdvAdapter.ts create mode 100644 src/jdvViewer.ts create mode 100644 src/test/node/jdvAdapter.test.ts diff --git a/sess/DESCRIPTION b/sess/DESCRIPTION index 77a5b5f43..7cf6dbf64 100644 --- a/sess/DESCRIPTION +++ b/sess/DESCRIPTION @@ -37,6 +37,7 @@ Suggests: htmlwidgets, systemfonts, httpgd, + jdv, jgd, svglite, tinytest diff --git a/sess/R/handlers.R b/sess/R/handlers.R index ad0f10f50..1f248b34c 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1291,3 +1291,57 @@ handle_dataview_dispose <- function(params) { } TRUE } + +# jdv data viewer backend (experimental) ----------------------------------- +# +# When the jdv package is available, View() registers data frames, matrices and +# lists with jdv instead of the sess data viewer. vscode-R relays the jdv +# protocol through the `jdv_rpc` request. Set `options(sess.dataviewer = "sess")` +# to keep using the sess data viewer. + +.dataview_use_jdv <- function(x) { + backend <- getOption("sess.dataviewer", "auto") + if (identical(backend, "sess")) { + return(FALSE) + } + # Arrow data is not converted to data frames for jdv; the vscode-R client does + # not support jdv's sequential Arrow views yet. + if (inherits(x, c("ArrowTabular", "RecordBatchReader", "nanoarrow_array_stream", + "polars_data_frame"))) { + return(FALSE) + } + requireNamespace("jdv", quietly = TRUE) +} + +.jdv_show_view <- function(x, title, title_key) { + # The title is jdv's display slot: View() with the same title replaces the + # view and increments its generation. + registration <- jdv::jdv_register( + x, + title = if (nzchar(title_key)) title_key else NULL, + mode = "paged" + ) + notify_client("dataview", list( + title = title, + source = "jdv", + type = "json", + open = registration$open, + notifications = I(as.character(registration$notifications)) + )) + registration$id +} + +handle_jdv_rpc <- function(request) { + if (!is.character(request) || length(request) != 1L || is.na(request)) { + stop("Invalid jdv request") + } + if (!requireNamespace("jdv", quietly = TRUE)) { + stop("The jdv package is not available") + } + # Child views opened from lists use jdv's automatic mode; keep them paged so + # sorting and filtering stay in R. + op <- options(jdv.snapshot_max_cells = 0) + on.exit(options(op), add = TRUE) + response <- jdv::jdv_rpc(request) + if (is.null(response)) NULL else as.character(response) +} diff --git a/sess/R/hooks.R b/sess/R/hooks.R index 0d207ea44..1ca593ca9 100644 --- a/sess/R/hooks.R +++ b/sess/R/hooks.R @@ -88,6 +88,9 @@ runtime_start <- function(use_rstudioapi = TRUE, "object" } title_key <- paste(as.character(title), collapse = "\n") + if (view_type %in% c("table", "list") && .dataview_use_jdv(x)) { + return(invisible(.jdv_show_view(x, title, title_key))) + } owner <- owner %||% title_key registry_key <- paste0(view_type, ":", owner) dataview_registry <- .sess_env$dataview_registry diff --git a/sess/R/server.R b/sess/R/server.R index 0fb37b410..b919b636c 100644 --- a/sess/R/server.R +++ b/sess/R/server.R @@ -488,7 +488,8 @@ dispatch_message <- function(line) { "listview_view" = function(p) handle_listview_view(p$view_id, p$index, p$path), "dataview_init" = function(p) handle_dataview_init(p), "dataview_page" = function(p) handle_dataview_page(p), - "dataview_dispose" = function(p) handle_dataview_dispose(p) + "dataview_dispose" = function(p) handle_dataview_dispose(p), + "jdv_rpc" = function(p) handle_jdv_rpc(p$request) ) if (payload$method %in% names(handlers)) { diff --git a/sess/inst/tinytest/test-ipc.R b/sess/inst/tinytest/test-ipc.R index fa7722b2e..1be545483 100644 --- a/sess/inst/tinytest/test-ipc.R +++ b/sess/inst/tinytest/test-ipc.R @@ -1,3 +1,6 @@ +# These tests cover the sess data viewer; keep it when jdv is installed. +options(sess.dataviewer = "sess") + # dispatch_message routes responses to pending_responses local({ .sess_env <- sess:::.sess_env diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R new file mode 100644 index 000000000..e08ddcdbc --- /dev/null +++ b/sess/inst/tinytest/test-jdv.R @@ -0,0 +1,95 @@ +# View() uses jdv when it is installed and relays the jdv protocol through jdv_rpc. +if (!requireNamespace("jdv", quietly = TRUE)) { + exit_file("jdv is not installed") +} + +local({ + runtime <- sess:::.sess_env + previous_con <- runtime$con + previous_plot_path <- runtime$latest_plot_path + pipe <- processx::conn_create_pipepair() + on.exit({ + sess:::runtime_stop() + runtime$con <- previous_con + runtime$latest_plot_path <- previous_plot_path + jdv::jdv_close() + lapply(pipe, close) + }, add = TRUE) + runtime$con <- pipe[[2L]] + # The standard plot backend needs the path that connect() normally sets. + if (is.null(runtime$latest_plot_path)) runtime$latest_plot_path <- tempfile(fileext = ".png") + options(sess.dataviewer = NULL) + sess:::runtime_start(use_rstudioapi = FALSE, plot_backend = "standard") + + read_messages <- function() { + messages <- strsplit(processx::conn_read_chars(pipe[[1L]]), "\n", fixed = TRUE)[[1L]] + lapply(messages[nzchar(messages)], jsonlite::fromJSON, simplifyVector = FALSE) + } + last_view <- function() { + views <- Filter(function(message) identical(message$method, "dataview"), read_messages()) + expect_length(views, 1L) + views[[1L]]$params + } + jdv_request <- function(method, params) { + request <- as.character(jsonlite::toJSON( + list(id = "jdv-test", method = method, params = params), auto_unbox = TRUE + )) + sess:::dispatch_message(as.character(jsonlite::toJSON( + list(jsonrpc = "2.0", id = "sess-test", method = "jdv_rpc", params = list(request = request)), + auto_unbox = TRUE + ))) + replies <- Filter(function(message) identical(message$id, "sess-test"), read_messages()) + expect_length(replies, 1L) + expect_null(replies[[1L]]$error) + jsonlite::fromJSON(replies[[1L]]$result, simplifyVector = FALSE) + } + + sample <- data.frame(x = c(2, 1, NA), y = c("b", "a", "c")) + utils::View(sample) + params <- last_view() + expect_identical(params$source, "jdv") + expect_identical(params$title, "sample") + open <- jsonlite::fromJSON(params$open, simplifyVector = FALSE)$params + expect_identical(open$kind, "grid") + expect_identical(open$access, "random") + expect_identical(open$generation, 1L) + + # Rows are computed in R with jdv sorting. + response <- jdv_request("table/rows", list( + view = open$view, generation = 1L, offset = 0L, limit = 10L, + sort = list(list(col = 0L, desc = FALSE)) + )) + expect_identical(unlist(response$result$data$row_index), c(2L, 1L, 3L)) + + # The same title replaces the view and increments its generation. + utils::View(sample) + reopened <- jsonlite::fromJSON(last_view()$open, simplifyVector = FALSE)$params + expect_identical(reopened$view, open$view) + expect_identical(reopened$generation, 2L) + stale <- jdv_request("table/rows", list(view = open$view, generation = 1L)) + expect_identical(stale$error$code, "stale_generation") + + # Lists open as trees; tables inside them open as paged child views. + nested <- list(a = 1:3, df = sample) + utils::View(nested) + tree <- jsonlite::fromJSON(last_view()$open, simplifyVector = FALSE)$params + expect_identical(tree$kind, "tree") + children <- jdv_request("object/children", list(view = tree$view, generation = 1L, path = list())) + expect_identical(children$result$children[[2L]]$label, "df") + child <- jdv_request("object/open", list( + view = tree$view, generation = 1L, path = children$result$children[[2L]]$path + )) + expect_identical(child$result$kind, "grid") + expect_identical(child$result$access, "random") + expect_identical(child$result$parent, tree$view) + + closed <- jdv_request("view/close", list(view = open$view, generation = 2L)) + expect_true(closed$result$closed) + + # Arrow data and the explicit opt-out keep the sess data viewer. + options(sess.dataviewer = "sess") + utils::View(sample) + expect_identical(last_view()$source, "table") + options(sess.dataviewer = NULL) + expect_false(sess:::.dataview_use_jdv(structure(list(), class = "polars_data_frame"))) +}) diff --git a/sess/inst/tinytest/test-listview.R b/sess/inst/tinytest/test-listview.R index abb071211..a206e2a4b 100644 --- a/sess/inst/tinytest/test-listview.R +++ b/sess/inst/tinytest/test-listview.R @@ -1,3 +1,6 @@ +# These tests cover the sess data viewer; keep it when jdv is installed. +options(sess.dataviewer = "sess") + # Expanding and opening descendants keeps one list/table viewer per root object. local({ runtime <- sess:::.sess_env diff --git a/src/dataViewer.ts b/src/dataViewer.ts index 525e9a811..c08464ce6 100644 --- a/src/dataViewer.ts +++ b/src/dataViewer.ts @@ -132,6 +132,25 @@ export function getDataViewerScript(): string { column.suppressSizeToFit = true; column.initialFlex = 0; } + if (column.jdv) { + // jdv cells hold R-formatted display text, and R evaluates filters. + // Offer only the operators jdv implements for this column. + const naOptions = column.jdv.naOptions ? [ + { displayKey: 'jdvIsNa', displayName: 'Is NA', predicate: () => true, numberOfInputs: 0 }, + { displayKey: 'jdvNotNa', displayName: 'Is not NA', predicate: () => true, numberOfInputs: 0 } + ] : []; + delete column.cellDataType; + if (column.type === 'bigintColumn') { + bigintFields.pop(); + } + column.filterParams = { + ...column.filterParams, + buttons: ['reset', 'apply'], + filterOptions: [...column.jdv.filterOptions, ...naOptions], + inRangeInclusive: true + }; + delete column.jdv; + } if (column.type !== 'numericColumn') { delete column.type; } diff --git a/src/interactive/manager.ts b/src/interactive/manager.ts index 499c820d9..3c3b43f74 100644 --- a/src/interactive/manager.ts +++ b/src/interactive/manager.ts @@ -17,6 +17,7 @@ import { setInteractiveExecutor } from './executionTarget'; import * as session from '../session'; import * as util from '../util'; import type { ListViewNavigation } from '../listViewer'; +import { showJdvNotification } from '../jdvViewer'; import { escapeXml } from './plotSvg'; import { ensureWorkspaceViewer } from '../extension'; import { queryTablePage } from './tableQuery'; @@ -979,6 +980,9 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro void this.prompt(view, event.data); } else if (event.type === 'clientRequest' && view.client.control) { void this.clientRequest(view, event.data); + } else if (event.type === 'viewer' && event.data.method === 'dataview' && + object(event.data.params ?? {}).source === 'jdv') { + await showJdvNotification(object(event.data.params ?? {}), view.target.sessionId); } else if (event.type === 'viewer' && event.data.method === 'dataview') { const params = object(event.data.params ?? {}); const title = typeof params.title === 'string' ? params.title diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts new file mode 100644 index 000000000..1f73cafea --- /dev/null +++ b/src/jdvAdapter.ts @@ -0,0 +1,685 @@ +'use strict'; + +// Adapter between the jdv protocol and the existing Data Viewer and List Viewer +// webviews. It is independent of VS Code so the conversions can be tested in Node. + +export type JsonObject = Record; + +export interface JdvCapabilities { + seek: boolean; + sort: boolean; + filter: boolean; + known_total: boolean; + next: boolean; + children: boolean; +} + +export interface JdvColumnSchema { + position: number; + name: string; + type: string; + class?: string[]; + sortable: boolean; + filterable: boolean; + drillable?: boolean; + levels?: string[]; + ordered?: boolean; + tz?: string; +} + +export interface JdvDescriptor { + view: string; + generation: number; + kind: 'grid' | 'tree'; + access: string; + encoding: string; + capabilities: JdvCapabilities; + title?: string | null; + mode?: string; + nrow?: number | null; + ncol?: number; + columns?: JdvColumnSchema[]; + path?: JdvPathStep[]; + class?: string[]; + parent?: string; +} + +export type JdvPathStep = + | { type: 'index'; index: number } + | { type: 'binding' | 'slot' | 'attr'; name: string }; + +export interface JdvError { + code: string; + message: string; +} + +export class JdvRequestError extends Error { + constructor(readonly code: string, message: string) { + super(message); + } +} + +/** A channel that sends one jdv request and returns its `result`. */ +export interface JdvChannel { + request(method: string, params: JsonObject): Promise; +} + +function isObject(value: unknown): value is JsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Text of a scalar filter operand or channel value; other values become ''. */ +function scalarText(value: unknown): string { + return typeof value === 'string' || typeof value === 'number' || typeof value === 'bigint' || + typeof value === 'boolean' ? String(value) : ''; +} + +/** Parse a `view/open` message or a direct descriptor (as returned by `object/open`). */ +export function parseDescriptor(value: unknown): JdvDescriptor { + const message = typeof value === 'string' ? JSON.parse(value) as unknown : value; + const params = isObject(message) && message.method === 'view/open' ? message.params : message; + if (!isObject(params) || typeof params.view !== 'string' || typeof params.generation !== 'number' || + (params.kind !== 'grid' && params.kind !== 'tree') || !isObject(params.capabilities)) { + throw new Error('Invalid jdv view descriptor'); + } + if (params.kind === 'grid' && !Array.isArray(params.columns)) { + throw new Error('Invalid jdv grid descriptor'); + } + return params as unknown as JdvDescriptor; +} + +/** Return the view ids and generations announced by `view/closed` notifications. */ +export function parseClosedNotifications(values: unknown): { view: string; generation: number }[] { + const list: unknown[] = Array.isArray(values) ? values : typeof values === 'string' ? [values] : []; + const closed: { view: string; generation: number }[] = []; + for (const value of list) { + try { + const message = typeof value === 'string' ? JSON.parse(value) as unknown : value; + if (isObject(message) && message.method === 'view/closed' && isObject(message.params) && + typeof message.params.view === 'string' && typeof message.params.generation === 'number') { + closed.push({ view: message.params.view, generation: message.params.generation }); + } + } catch { + // Ignore malformed notifications; they cannot identify a view. + } + } + return closed; +} + +/** Unwrap a jdv response envelope, throwing its error code and message. */ +export function unwrapResponse(response: unknown): unknown { + const message = typeof response === 'string' ? JSON.parse(response) as unknown : response; + if (!isObject(message)) { + throw new Error('Invalid jdv response'); + } + if (isObject(message.error)) { + const error = message.error as Partial; + throw new JdvRequestError(String(error.code ?? 'unknown'), String(error.message ?? 'jdv request failed')); + } + if (!('result' in message)) { + throw new Error('Invalid jdv response'); + } + return message.result; +} + +// --------------------------------------------------------------------------- +// Grid + +/** Row label column. Data columns use their one-based position as field id. */ +export const ROW_LABEL_FIELD = '0'; + +export interface GridColumnDef { + headerName: string; + headerTooltip?: string; + field: string; + type: string; + filter: string | boolean; + sortable: boolean; + cellDataType?: string; + suppressHeaderMenuButton?: boolean; + lockPosition?: string; + /** Webview-only options; see prepareViewerColumns(). */ + jdv?: { filterOptions: string[]; naOptions: boolean }; +} + +const TEXT_FILTER_OPTIONS = ['contains', 'notContains', 'equals', 'notEqual', 'startsWith', 'endsWith']; +const ORDER_FILTER_OPTIONS = [ + 'equals', 'notEqual', 'lessThan', 'lessThanOrEqual', 'greaterThan', 'greaterThanOrEqual', 'inRange', +]; +// jdv implements only exact equality and inclusive ranges for integer64. +const INT64_FILTER_OPTIONS = ['equals', 'inRange']; +export const NA_FILTER_OPTION = 'jdvIsNa'; +export const NOT_NA_FILTER_OPTION = 'jdvNotNa'; + +function columnTooltip(column: JdvColumnSchema): string { + let tooltip = `${column.name}, class: [${(column.class ?? []).join(', ')}], type: ${column.type}`; + if (column.type === 'factor' && Array.isArray(column.levels)) { + const levels = column.levels.slice(0, 20).join(', '); + tooltip += `, levels: ${levels}${column.levels.length > 20 ? ', ...' : ''}`; + } + if (column.type === 'datetime' && column.tz !== undefined) { + tooltip += `, tz: ${column.tz || '(local)'}`; + } + return tooltip; +} + +export function gridColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { + const columns: GridColumnDef[] = [{ + headerName: '', + field: ROW_LABEL_FIELD, + type: 'textColumn', + filter: false, + sortable: false, + lockPosition: 'left', + suppressHeaderMenuButton: true, + }]; + for (const column of descriptor.columns ?? []) { + const sortable = descriptor.capabilities.sort && column.sortable; + const filterEnabled = descriptor.capabilities.filter; + let type = 'textColumn'; + let filter: string | boolean = false; + let filterOptions: string[] = []; + switch (column.type) { + case 'integer': + case 'number': + type = 'numericColumn'; + filter = 'agNumberColumnFilter'; + filterOptions = ORDER_FILTER_OPTIONS; + break; + case 'int64': + type = 'bigintColumn'; + filter = 'agBigIntColumnFilter'; + filterOptions = INT64_FILTER_OPTIONS; + break; + case 'date': + type = 'dateColumn'; + filter = 'agDateColumnFilter'; + filterOptions = ORDER_FILTER_OPTIONS; + break; + case 'datetime': + type = 'datetimeColumn'; + filter = 'agDateColumnFilter'; + filterOptions = ORDER_FILTER_OPTIONS; + break; + case 'boolean': + // AG Grid's boolean data type supplies its own True/False filter. + type = 'booleanColumn'; + filter = true; + break; + default: + filter = 'agTextColumnFilter'; + filterOptions = TEXT_FILTER_OPTIONS; + } + // Value operators require `filterable`; `is_na` only needs a missingness mask, + // which every column has unless its channel failed. + if (!column.filterable) { + filterOptions = []; + } + const def: GridColumnDef = { + headerName: column.name, + headerTooltip: columnTooltip(column), + field: String(column.position + 1), + type, + filter: filterEnabled ? filter : false, + sortable, + jdv: { filterOptions, naOptions: true }, + }; + if (column.type === 'boolean') { + def.cellDataType = 'boolean'; + delete def.jdv; + if (!column.filterable) { + def.filter = false; + } + } + if (!filterEnabled) { + delete def.jdv; + } + columns.push(def); + } + return columns; +} + +export interface AgSortModelItem { + colId: string; + sort: 'asc' | 'desc'; +} + +export interface JdvSort { + col: number; + desc: boolean; +} + +function fieldPosition(field: string, descriptor: JdvDescriptor): number { + const position = Number(field) - 1; + if (!Number.isSafeInteger(position) || position < 0 || position >= (descriptor.columns?.length ?? 0)) { + throw new JdvRequestError('invalid_request', `Unknown column: ${field}`); + } + return position; +} + +export function toJdvSort(sortModel: unknown, descriptor: JdvDescriptor): JdvSort[] { + if (!Array.isArray(sortModel)) { + return []; + } + return sortModel.map(item => { + const sortItem = item as Partial; + const col = fieldPosition(String(sortItem.colId), descriptor); + if (!descriptor.columns![col].sortable) { + throw new JdvRequestError('unsupported', `Column "${descriptor.columns![col].name}" cannot be sorted`); + } + return { col, desc: sortItem.sort === 'desc' }; + }); +} + +type FilterNode = JsonObject; + +function unsupportedFilter(column: JdvColumnSchema, type: unknown): never { + throw new JdvRequestError('unsupported', + `Filter "${String(type)}" is not supported for column "${column.name}" (${column.type})`); +} + +function canonicalInt64(value: unknown, column: JdvColumnSchema): string { + const text = scalarText(value).trim(); + if (!/^[+-]?\d+$/.test(text)) { + throw new JdvRequestError('invalid_request', `Invalid integer64 value for column "${column.name}"`); + } + return BigInt(text).toString(); +} + +function numberOperand(value: unknown, column: JdvColumnSchema): unknown { + const number = typeof value === 'number' ? value : Number(value); + if (Number.isNaN(number) || value === null || value === undefined || value === '') { + throw new JdvRequestError('invalid_request', `Invalid number for column "${column.name}"`); + } + if (number === Infinity) { return { special: 'pos_inf' }; } + if (number === -Infinity) { return { special: 'neg_inf' }; } + return number; +} + +function dateOperand(value: unknown, column: JdvColumnSchema): string { + // AG Grid date filters use "YYYY-MM-DD HH:mm:ss" in the column's wall time. + const text = scalarText(value); + const match = /^(\d{4}-\d{2}-\d{2})(?:[ T](\d{2}:\d{2}:\d{2}))?/.exec(text); + if (!match) { + throw new JdvRequestError('invalid_request', `Invalid date for column "${column.name}"`); + } + if (column.type === 'date') { + return match[1]; + } + // An operand without offset uses the column's time zone in jdv. + return `${match[1]}T${match[2] ?? '00:00:00'}`; +} + +const ORDER_OPS: Record = { + equals: 'eq', notEqual: 'ne', lessThan: 'lt', lessThanOrEqual: 'le', greaterThan: 'gt', greaterThanOrEqual: 'ge', +}; + +function conditionNode(condition: JsonObject, col: number, column: JdvColumnSchema): FilterNode { + const type = condition.type; + const isNa = { op: 'is_na', col }; + if (type === NA_FILTER_OPTION || type === 'blank') { + return isNa; + } + if (type === NOT_NA_FILTER_OPTION || type === 'notBlank') { + return { op: 'not', arg: isNa }; + } + if (!column.filterable) { + unsupportedFilter(column, type); + } + switch (column.type) { + case 'boolean': + if (type === 'true') { return { op: 'is_true', col }; } + if (type === 'false') { return { op: 'is_false', col }; } + return unsupportedFilter(column, type); + case 'integer': + case 'number': { + if (type === 'inRange') { + return { + op: 'between', col, + lower: numberOperand(condition.filter, column), upper: numberOperand(condition.filterTo, column), + }; + } + const op = ORDER_OPS[String(type)]; + return op ? { op, col, value: numberOperand(condition.filter, column) } : unsupportedFilter(column, type); + } + case 'int64': + if (type === 'equals') { + return { op: 'eq', col, value: canonicalInt64(condition.filter, column) }; + } + if (type === 'inRange') { + return { + op: 'between', col, + lower: canonicalInt64(condition.filter, column), upper: canonicalInt64(condition.filterTo, column), + }; + } + return unsupportedFilter(column, type); + case 'date': + case 'datetime': { + if (type === 'inRange') { + return { + op: 'between', col, + lower: dateOperand(condition.dateFrom, column), upper: dateOperand(condition.dateTo, column), + }; + } + const op = ORDER_OPS[String(type)]; + return op ? { op, col, value: dateOperand(condition.dateFrom, column) } : unsupportedFilter(column, type); + } + default: { + const value = scalarText(condition.filter); + // As in the previous viewer, negated text conditions exclude missing values. + const negate = (node: FilterNode): FilterNode => ({ + op: 'and', args: [{ op: 'not', arg: node }, { op: 'not', arg: isNa }], + }); + switch (type) { + case 'contains': return { op: 'contains', col, value }; + case 'notContains': return negate({ op: 'contains', col, value }); + case 'equals': return { op: 'text_equals', col, value }; + case 'notEqual': return negate({ op: 'text_equals', col, value }); + case 'startsWith': return { op: 'starts_with', col, value }; + case 'endsWith': return { op: 'ends_with', col, value }; + default: return unsupportedFilter(column, type); + } + } + } +} + +/** Convert an AG Grid filter model to the jdv filter AST, or `undefined` for no filter. */ +export function toJdvFilter(filterModel: unknown, descriptor: JdvDescriptor): FilterNode | undefined { + if (!isObject(filterModel)) { + return undefined; + } + const nodes: FilterNode[] = []; + for (const [field, model] of Object.entries(filterModel)) { + if (!isObject(model)) { + continue; + } + const col = fieldPosition(field, descriptor); + const column = descriptor.columns![col]; + const conditions = Array.isArray(model.conditions) ? model.conditions.filter(isObject) : undefined; + if (conditions && conditions.length > 0) { + const args = conditions.map(condition => conditionNode(condition, col, column)); + nodes.push(args.length === 1 ? args[0] : { + op: String(model.operator).toUpperCase() === 'OR' ? 'or' : 'and', args, + }); + } else { + nodes.push(conditionNode(model, col, column)); + } + } + if (nodes.length === 0) { + return undefined; + } + return nodes.length === 1 ? nodes[0] : { op: 'and', args: nodes }; +} + +export interface TableRowsParams extends JsonObject { + view: string; + generation: number; + offset: number; + limit: number; + sort: JdvSort[]; + filter?: FilterNode; +} + +export function tableRowsParams( + descriptor: JdvDescriptor, startRow: number, endRow: number, sortModel: unknown, filterModel: unknown, +): TableRowsParams { + const offset = Math.max(0, Math.floor(startRow)); + const limit = Math.max(1, Math.floor(endRow) - offset); + const params: TableRowsParams = { + view: descriptor.view, + generation: descriptor.generation, + offset, + limit, + sort: toJdvSort(sortModel, descriptor), + }; + const filter = toJdvFilter(filterModel, descriptor); + if (filter) { + params.filter = filter; + } + return params; +} + +interface ColumnChannels { + values?: unknown[]; + display?: (string | null)[]; + na?: boolean[]; +} + +interface PageColumn { + schema?: JdvColumnSchema; + data?: ColumnChannels; + errors?: unknown; +} + +export interface GridPage { + rows: Record[]; + totalRows: number; + totalUnfiltered: number; + lastRow: number; + /** Rows returned; may be fewer than requested when jdv clamps the page. */ + returned: number; +} + +function cellValue(column: JdvColumnSchema, channels: ColumnChannels, row: number): unknown { + if (channels.na?.[row]) { + // Missing cells use the grid's NA renderer, unless R labels them differently + // (NaN, or a class-specific missing label). + const label = channels.display?.[row]; + return typeof label === 'string' && label !== 'NA' ? label : null; + } + // Booleans keep their value for the grid's checkbox renderer. Everything else + // shows the R-formatted display text; comparisons happen in R. + if (column.type === 'boolean' && channels.values) { + return channels.values[row] ?? null; + } + const display = channels.display?.[row]; + if (display !== undefined && display !== null) { + return display; + } + // jdv omits display for strings, factors and int64 when it equals the value. + const value = channels.values?.[row]; + return value === undefined || value === null ? null : scalarText(value); +} + +function columnErrorText(errors: unknown): string | undefined { + if (!isObject(errors)) { + return undefined; + } + const messages = Object.entries(errors).map(([channel, message]) => `${channel}: ${String(message)}`); + return messages.length > 0 ? messages.join('; ') : undefined; +} + +/** Convert a `table/rows` result to row objects keyed by the grid field ids. */ +export function gridPageFromRows(result: unknown, startRow: number, requested: number): GridPage { + if (!isObject(result) || !isObject(result.data) || typeof result.total !== 'number' || + typeof result.total_unfiltered !== 'number' || !Array.isArray(result.data.row_index)) { + throw new Error('Invalid jdv table/rows result'); + } + const data = result.data; + const rowIndex = data.row_index as number[]; + const rowNames = Array.isArray(data.row_names) ? data.row_names as (string | null)[] : undefined; + const columns = Array.isArray(data.columns) ? data.columns as PageColumn[] : []; + const rows: Record[] = rowIndex.map((index, row) => ({ + [ROW_LABEL_FIELD]: rowNames?.[row] ?? String(index), + })); + for (const column of columns) { + if (!column.schema) { + continue; + } + const field = String(column.schema.position + 1); + const channels = column.data ?? {}; + const error = columnErrorText(column.errors); + for (let row = 0; row < rows.length; row++) { + rows[row][field] = channels.display || channels.values || channels.na + ? cellValue(column.schema, channels, row) + : error ? `` : null; + } + } + const total = result.total; + const returned = rows.length; + // When jdv returns fewer rows than requested before the end, report an unknown + // last row so the grid asks again for the remainder of the block. + const lastRow = returned < requested && startRow + returned < total ? -1 : total; + return { rows, totalRows: total, totalUnfiltered: result.total_unfiltered, lastRow, returned }; +} + +// --------------------------------------------------------------------------- +// Tree + +export interface ListViewNavigation { + title: string; + path: number[]; + breadcrumbs: { label: string; path: number[] }[]; + vector?: boolean; +} + +export interface ListViewChild { + label: string; + str: string; + index: number; + viewable?: boolean; + has_children?: boolean; +} + +interface TreeNode { + path: JdvPathStep[]; + label: string; + title: string; + class: string[]; + hasChildren: boolean; + viewable: boolean; +} + +interface TreeChild { + index: number; + label: string; + class?: string[]; + summary?: string; + has_children?: boolean; + viewable?: boolean; + unavailable?: string; + path?: JdvPathStep[]; +} + +export const TREE_PAGE_SIZE = 200; + +const ATOMIC_CLASSES = new Set([ + 'logical', 'integer', 'numeric', 'double', 'character', 'complex', 'raw', + 'factor', 'Date', 'POSIXct', 'difftime', 'integer64', +]); +const TABLE_CLASSES = new Set(['data.frame', 'matrix']); + +/** Atomic vectors are shown as value rows, as in the previous List Viewer. */ +export function isVectorClass(classes: string[]): boolean { + return classes.length > 0 && classes.every(cls => ATOMIC_CLASSES.has(cls) || cls === 'array' || cls === 'ordered') && + !classes.includes('matrix'); +} + +export function isTableClass(classes: string[]): boolean { + return classes.some(cls => TABLE_CLASSES.has(cls)); +} + +function childTitle(parent: TreeNode, child: TreeChild): string { + const step = child.path?.[child.path.length - 1]; + if (step?.type === 'slot') { return `${parent.title}@${step.name}`; } + if (step?.type === 'attr') { return `attr(${parent.title}, "${step.name}")`; } + if (step?.type === 'binding') { return `${parent.title}$${step.name}`; } + const label = child.label; + return /^\[\[?\d+\]\]?$/.test(label) || label === '' + ? `${parent.title}[[${child.index}]]` : `${parent.title}$${label}`; +} + +/** + * Maps the List Viewer's positional paths (child `index` values) to jdv typed + * paths. Typed paths are learned from `object/children` results. + */ +export class TreeAdapter { + private readonly nodes = new Map(); + + constructor(public descriptor: JdvDescriptor, private readonly title: string) { + this.nodes.set('[]', { + path: [], label: title, title, class: descriptor.class ?? [], hasChildren: true, viewable: true, + }); + } + + private node(path: number[]): TreeNode { + const node = this.nodes.get(JSON.stringify(path)); + if (!node) { + throw new JdvRequestError('unavailable', 'This item is no longer available. Reload the parent list.'); + } + return node; + } + + navigation(path: number[] = []): ListViewNavigation { + const node = this.node(path); + const breadcrumbs = []; + for (let depth = 0; depth <= path.length; depth++) { + const prefix = path.slice(0, depth); + breadcrumbs.push({ label: this.node(prefix).label, path: prefix }); + } + return { title: node.title, path, breadcrumbs, vector: isVectorClass(node.class) }; + } + + childrenParams(path: number[], start: number): JsonObject { + const node = this.node(path); + return { + view: this.descriptor.view, + generation: this.descriptor.generation, + path: node.path, + offset: Math.max(0, start - 1), + limit: TREE_PAGE_SIZE, + }; + } + + /** Record the children of `path` and convert them to List Viewer rows. */ + children(path: number[], result: unknown): { children: ListViewChild[]; next_start: number | null } { + if (!isObject(result) || !Array.isArray(result.children) || typeof result.offset !== 'number' || + typeof result.total_children !== 'number') { + throw new Error('Invalid jdv object/children result'); + } + const parent = this.node(path); + const vector = isVectorClass(parent.class); + const children = (result.children as TreeChild[]).map(child => { + const addressable = Array.isArray(child.path) && child.viewable !== false; + if (addressable) { + this.nodes.set(JSON.stringify([...path, child.index]), { + path: child.path!, + label: child.label, + title: childTitle(parent, child), + class: child.class ?? [], + hasChildren: child.has_children === true, + viewable: child.viewable !== false, + }); + } + const summary = child.unavailable ? `(${child.unavailable})` : child.summary ?? ''; + return { + label: child.label, + str: summary, + index: child.index, + // Vector elements are terminal rows in the List Viewer. + viewable: addressable && !vector, + has_children: !vector && child.has_children === true, + }; + }); + const end = result.offset + children.length; + return { children, next_start: end < result.total_children ? end + 1 : null }; + } + + /** What opening the child at `path`/`index` should do. */ + open(path: number[], index: number): { kind: 'navigate'; navigation: ListViewNavigation } + | { kind: 'object'; params: JsonObject; title: string } { + const childPath = [...path, index]; + const node = this.node(childPath); + if (!node.viewable) { + throw new JdvRequestError('unavailable', 'This item cannot be opened.'); + } + if (isTableClass(node.class)) { + return { + kind: 'object', + title: node.title, + params: { view: this.descriptor.view, generation: this.descriptor.generation, path: node.path }, + }; + } + return { kind: 'navigate', navigation: this.navigation(childPath) }; + } +} diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts new file mode 100644 index 000000000..7927567d5 --- /dev/null +++ b/src/jdvViewer.ts @@ -0,0 +1,268 @@ +'use strict'; + +// Data Viewer panels backed by jdv. The webviews are the existing AG Grid and List +// Viewer documents; this module translates their requests to the jdv protocol. + +import * as path from 'path'; +import * as vscode from 'vscode'; +import { Uri, ViewColumn, window } from 'vscode'; +import { + GridPage, JdvChannel, JdvDescriptor, JdvRequestError, JsonObject, TreeAdapter, gridColumnDefs, + gridPageFromRows, parseClosedNotifications, parseDescriptor, tableRowsParams, unwrapResponse, +} from './jdvAdapter'; +import { extensionContext } from './extension'; +import * as session from './session'; +import { config, UriIcon } from './util'; + +interface JdvPanel { + readonly key: string; + readonly panel: vscode.WebviewPanel; + readonly sessionId: string | null; + readonly channel: JdvChannel; + descriptor: JdvDescriptor; + title: string; + tree?: TreeAdapter; + /** Set when R closed or evicted this view generation. */ + closed?: string; +} + +const panels = new Map(); +let requestSequence = 0; + +/** Sends jdv requests through the session's existing sess RPC connection. */ +function sessChannel(sessionId: string | null): JdvChannel { + return { + async request(method: string, params: JsonObject): Promise { + const request = JSON.stringify({ id: `vscode-r-${++requestSequence}`, method, params }); + const response = await session.sessionRequest({ method: 'jdv_rpc', params: { request } }, sessionId); + if (typeof response !== 'string') { + throw new Error('No response from the R session'); + } + return unwrapResponse(response); + }, + }; +} + +function panelKey(sessionId: string | null, view: string): string { + return JSON.stringify([sessionId, view]); +} + +function viewerColumn(): string | undefined { + const viewer = config().get>('session.viewers.viewColumn')?.view ?? 'Two'; + return viewer === 'Disable' ? undefined : viewer; +} + +function errorMessage(error: unknown): string { + if (error instanceof JdvRequestError) { + return `${error.message} (${error.code})`; + } + return error instanceof Error ? error.message : String(error); +} + +/** Handle a sess `dataview` notification whose source is `jdv`. */ +export async function showJdvNotification(params: Record, sessionId: string | null): Promise { + let descriptor: JdvDescriptor; + try { + descriptor = parseDescriptor(params.open); + } catch (e) { + void window.showErrorMessage(`Unable to open the data viewer: ${errorMessage(e)}`); + return; + } + // Notifications precede the view they belong to; see jdv_register(). + for (const closed of parseClosedNotifications(params.notifications)) { + const state = panels.get(panelKey(sessionId, closed.view)); + if (state && state.descriptor.generation === closed.generation) { + state.closed = 'This view was closed in R. Run View() again to refresh it.'; + } + } + const title = typeof params.title === 'string' ? params.title + : Array.isArray(params.title) ? params.title.map(String).join(',') : descriptor.title ?? 'View'; + const viewer = viewerColumn(); + if (viewer) { + await openJdvView(descriptor, title, sessionId, viewer, sessChannel(sessionId)); + } +} + +async function openJdvView( + descriptor: JdvDescriptor, title: string, sessionId: string | null, viewer: string, channel: JdvChannel, +): Promise { + if (descriptor.kind === 'grid' && descriptor.access !== 'random') { + // Snapshot and sequential (Arrow) grids are not integrated yet; release the view. + void channel.request('view/close', { view: descriptor.view, generation: descriptor.generation }).catch(() => undefined); + void window.showWarningMessage(`The data viewer cannot display "${title}" yet (${descriptor.access} view).`); + return; + } + const key = panelKey(sessionId, descriptor.view); + const existing = panels.get(key); + if (existing) { + if (descriptor.generation < existing.descriptor.generation) { + return; + } + existing.descriptor = descriptor; + existing.title = title; + existing.closed = undefined; + existing.tree = descriptor.kind === 'tree' ? new TreeAdapter(descriptor, title) : undefined; + existing.panel.title = title; + existing.panel.reveal(existing.panel.viewColumn, true); + existing.panel.webview.html = await renderPanel(existing); + return; + } + + const resDir = path.join(extensionContext.extensionPath, 'dist', 'resources'); + const panel = window.createWebviewPanel('dataview', title, + { + preserveFocus: true, + viewColumn: ViewColumn[viewer as keyof typeof ViewColumn], + }, + { + enableScripts: true, + enableFindWidget: true, + retainContextWhenHidden: true, + localResourceRoots: [Uri.file(resDir)], + }); + panel.iconPath = new UriIcon(descriptor.kind === 'grid' ? 'open-preview' : 'preview'); + session.attachViewerSessionBridge(panel, sessionId); + const state: JdvPanel = { + key, panel, sessionId, channel, descriptor, title, + tree: descriptor.kind === 'tree' ? new TreeAdapter(descriptor, title) : undefined, + }; + panels.set(key, state); + // The panel's webview getter throws once onDidDispose fires. + const webview = panel.webview; + panel.onDidDispose(() => { + if (panels.get(key) === state) { + panels.delete(key); + } + if (!state.closed) { + const { view, generation } = state.descriptor; + void channel.request('view/close', { view, generation }).catch(() => undefined); + } + }); + webview.onDidReceiveMessage((message: unknown) => { + void handleMessage(state, webview, message as JsonObject); + }); + webview.html = await renderPanel(state); +} + +async function renderPanel(state: JdvPanel): Promise { + const webview = state.panel.webview; + if (state.descriptor.kind === 'grid') { + return session.getTableHtml(webview, undefined, state.title, state.sessionId); + } + return session.getListHtml(webview, state.title, state.tree!.navigation([]), state.sessionId); +} + +function isPositivePath(value: unknown): value is number[] { + return Array.isArray(value) && value.every(index => Number.isSafeInteger(index) && index > 0); +} + +async function handleMessage(state: JdvPanel, webview: vscode.Webview, message: JsonObject): Promise { + const documentGeneration = message.documentGeneration; + if (typeof documentGeneration !== 'number' || documentGeneration !== session.getDocumentGeneration(webview)) { + return; + } + const current = () => documentGeneration === session.getDocumentGeneration(webview); + if (message.message === 'dataview/request' && typeof message.requestId === 'number') { + const reply = (ok: boolean, result?: unknown, error?: string) => { + if (current()) { + void webview.postMessage({ + message: 'dataview/response', documentGeneration, requestId: message.requestId, ok, result, error, + }); + } + }; + try { + if (state.closed) { + throw new Error(state.closed); + } + if (message.action === 'init') { + reply(true, { columns: gridColumnDefs(state.descriptor), totalRows: state.descriptor.nrow ?? 0 }); + } else if (message.action === 'page') { + reply(true, await fetchGridPage(state, message)); + } else { + reply(false, undefined, `Unsupported dataview action: ${String(message.action)}`); + } + } catch (e) { + reply(false, undefined, errorMessage(e)); + } + return; + } + + const tree = state.tree; + if (!tree || !isPositivePath(message.path)) { + return; + } + const treePath = message.path; + if (message.message === 'listview/navigate' || message.message === 'listview/view') { + let navigation; + let error: string | undefined; + try { + if (state.closed) { + throw new Error(state.closed); + } + if (message.message === 'listview/navigate') { + navigation = tree.navigation(treePath); + } else if (Number.isSafeInteger(message.index)) { + const action = tree.open(treePath, message.index as number); + if (action.kind === 'navigate') { + navigation = action.navigation; + } else { + const child = parseDescriptor(await state.channel.request('object/open', action.params)); + const viewer = viewerColumn(); + if (viewer) { + await openJdvView(child, action.title, state.sessionId, viewer, state.channel); + } + } + } + } catch (e) { + error = `Unable to open this item: ${errorMessage(e)}`; + } + if (navigation) { + state.panel.title = navigation.title; + } + if (current()) { + void webview.postMessage({ + message: 'listview/navigation', documentGeneration, requestId: message.requestId, navigation, error, + }); + } + } else if (message.message === 'listview/page' && Number.isSafeInteger(message.start) && + typeof message.requestId === 'number') { + let page; + let error: string | undefined; + try { + if (state.closed) { + throw new Error(state.closed); + } + const result = await state.channel.request('object/children', tree.childrenParams(treePath, message.start as number)); + page = tree.children(treePath, result); + } catch (e) { + error = `Unable to load items: ${errorMessage(e)}`; + } + if (current()) { + void webview.postMessage({ + message: 'listview/page', documentGeneration, requestId: message.requestId, ...page, error, + }); + } + } +} + +/** Fetch one AG Grid block, issuing further pages if jdv clamps the page size. */ +async function fetchGridPage(state: JdvPanel, message: JsonObject): Promise> { + const startRow = Number(message.startRow ?? 0); + const endRow = Number(message.endRow ?? 0); + const descriptor = state.descriptor; + let page: GridPage | undefined; + let offset = startRow; + while (offset < endRow) { + const params = tableRowsParams(descriptor, offset, endRow, message.sortModel, message.filterModel); + const next = gridPageFromRows(await state.channel.request('table/rows', params), offset, params.limit); + page = page ? { ...next, rows: [...page.rows, ...next.rows] } : next; + offset += next.returned; + if (next.returned === 0 || offset >= next.totalRows) { + break; + } + } + if (!page) { + throw new JdvRequestError('invalid_request', 'Empty page request'); + } + return { rows: page.rows, totalRows: page.totalRows, totalUnfiltered: page.totalUnfiltered, lastRow: page.totalRows }; +} diff --git a/src/session.ts b/src/session.ts index 04edee001..79131a507 100644 --- a/src/session.ts +++ b/src/session.ts @@ -35,6 +35,7 @@ import type { import { showWebView } from './webViewer'; import { getListViewerScript, listViewerStyle, ListViewNavigation } from './listViewer'; import { getDataViewerScript, getDataViewerStyle, getDataViewerToolbarHtml } from './dataViewer'; +import { showJdvNotification } from './jdvViewer'; import { getDataViewerColumnPanelHtml, getDataViewerColumnPanelScript, getDataViewerColumnPanelStyle } from './dataViewerColumnPanel'; import { createViewerSessionContext, formatSessionLabel, getViewerSessionScript, ViewerSessionContext, ViewerSessionSource } from './viewerSession'; import { SessConsentChoice, SessConsentService } from './sessConsent'; @@ -518,6 +519,11 @@ const dynamicDataViewStateGenerations = new WeakMap const documentGenerations = new WeakMap(); let documentGenerationRevision = 0; +/** The generation of the HTML document currently loaded in a viewer webview. */ +export function getDocumentGeneration(webview: Webview): number | undefined { + return documentGenerations.get(webview); +} + function escapeHtml(text: string): string { const map: Record = { '&': '&', @@ -1982,7 +1988,7 @@ function getViewerSessionHtml(sessionId: string | null): string { return getViewerSessionContext(sessionId)?.getHtml() ?? ''; } -function attachViewerSessionBridge(panel: vscode.WebviewPanel, sessionId: string | null): void { +export function attachViewerSessionBridge(panel: vscode.WebviewPanel, sessionId: string | null): void { getViewerSessionContext(sessionId)?.attach(panel); } @@ -3114,6 +3120,10 @@ async function handleNotification(message: Record, socket: IpcS break; } case 'dataview': { + if (params.source === 'jdv') { + await showJdvNotification(params, socket._sessionId ?? null); + break; + } // R's deparse() can return several title lines. Preserve the // comma-separated title previously produced by String(array). const title = typeof params.title === 'string' ? params.title diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts new file mode 100644 index 000000000..28d6342fa --- /dev/null +++ b/src/test/node/jdvAdapter.test.ts @@ -0,0 +1,168 @@ +import * as assert from 'node:assert/strict'; +import { + JdvDescriptor, JdvRequestError, TreeAdapter, gridColumnDefs, gridPageFromRows, isVectorClass, + parseClosedNotifications, parseDescriptor, tableRowsParams, toJdvFilter, unwrapResponse, +} from '../../jdvAdapter'; + +const capabilities = { seek: true, sort: true, filter: true, known_total: true, next: false, children: false }; + +const grid: JdvDescriptor = parseDescriptor(JSON.stringify({ + method: 'view/open', + params: { + view: 'v-1', generation: 2, kind: 'grid', access: 'random', encoding: 'r-json', capabilities, + title: 'df', mode: 'paged', nrow: 3, ncol: 6, row_names: null, + columns: [ + { position: 0, name: 'a', type: 'number', class: ['numeric'], sortable: true, filterable: true }, + { position: 1, name: 'b', type: 'factor', class: ['factor'], sortable: true, filterable: true, levels: ['x', 'y'] }, + { position: 2, name: 'd', type: 'date', class: ['Date'], sortable: true, filterable: true }, + { position: 3, name: 't', type: 'datetime', class: ['POSIXct', 'POSIXt'], sortable: true, filterable: true, tz: 'UTC' }, + { position: 4, name: 'i', type: 'int64', class: ['integer64'], sortable: true, filterable: true }, + { position: 5, name: 'a', type: 'other', class: ['AsIs'], sortable: false, filterable: false, drillable: true }, + ], + }, +})); + +suite('jdv adapter', () => { + test('parses descriptors and closed notifications', () => { + assert.equal(grid.view, 'v-1'); + assert.equal(grid.generation, 2); + assert.throws(() => parseDescriptor('{"method":"view/open","params":{}}')); + assert.deepEqual(parseClosedNotifications([ + '{"method":"view/closed","params":{"view":"v-3","generation":1}}', 'not json', + ]), [{ view: 'v-3', generation: 1 }]); + }); + + test('unwraps responses and keeps error codes', () => { + assert.deepEqual(unwrapResponse('{"id":"a","result":{"x":1},"warnings":[]}'), { x: 1 }); + assert.throws(() => unwrapResponse('{"id":"a","error":{"code":"stale_generation","message":"old"},"warnings":[]}'), + (e: unknown) => e instanceof JdvRequestError && e.code === 'stale_generation'); + }); + + test('builds column definitions from the schema', () => { + const columns = gridColumnDefs(grid); + assert.deepEqual(columns.map(column => column.field), ['0', '1', '2', '3', '4', '5', '6']); + assert.equal(columns[0].sortable, false); + assert.equal(columns[1].type, 'numericColumn'); + assert.ok(columns[1].jdv?.filterOptions.includes('inRange')); + assert.deepEqual(columns[5].jdv?.filterOptions, ['equals', 'inRange']); + assert.equal(columns[6].sortable, false); + // Non-filterable columns still offer the NA operators. + assert.deepEqual(columns[6].jdv, { filterOptions: [], naOptions: true }); + assert.match(columns[2].headerTooltip ?? '', /levels: x, y/); + }); + + test('converts sort and filter models', () => { + const params = tableRowsParams(grid, 100, 600, + [{ colId: '2', sort: 'desc' }, { colId: '1', sort: 'asc' }], + { + '1': { filterType: 'number', operator: 'OR', conditions: [ + { filterType: 'number', type: 'inRange', filter: 1, filterTo: 3 }, + { filterType: 'number', type: 'jdvIsNa' }, + ] }, + '2': { filterType: 'text', type: 'notContains', filter: 'X' }, + }); + assert.equal(params.offset, 100); + assert.equal(params.limit, 500); + assert.deepEqual(params.sort, [{ col: 1, desc: true }, { col: 0, desc: false }]); + assert.deepEqual(params.filter, { + op: 'and', args: [ + { op: 'or', args: [{ op: 'between', col: 0, lower: 1, upper: 3 }, { op: 'is_na', col: 0 }] }, + { op: 'and', args: [ + { op: 'not', arg: { op: 'contains', col: 1, value: 'X' } }, + { op: 'not', arg: { op: 'is_na', col: 1 } }, + ] }, + ], + }); + }); + + test('converts date, datetime and integer64 operands', () => { + assert.deepEqual(toJdvFilter({ '3': { filterType: 'date', type: 'greaterThan', dateFrom: '2020-01-02 00:00:00' } }, grid), + { op: 'gt', col: 2, value: '2020-01-02' }); + assert.deepEqual(toJdvFilter({ '4': { filterType: 'date', type: 'lessThan', dateFrom: '2020-01-02 10:30:00' } }, grid), + { op: 'lt', col: 3, value: '2020-01-02T10:30:00' }); + assert.deepEqual(toJdvFilter({ '5': { filterType: 'bigint', type: 'equals', filter: '+0012' } }, grid), + { op: 'eq', col: 4, value: '12' }); + assert.throws(() => toJdvFilter({ '5': { filterType: 'bigint', type: 'lessThan', filter: '1' } }, grid), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported'); + assert.throws(() => toJdvFilter({ '6': { filterType: 'text', type: 'contains', filter: '1' } }, grid), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported'); + assert.throws(() => tableRowsParams(grid, 0, 10, [{ colId: '6', sort: 'asc' }], undefined), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported'); + }); + + test('converts pages to rows using display text and the NA mask', () => { + const page = gridPageFromRows({ + view: 'v-1', generation: 2, total: 3, total_unfiltered: 5, offset: 0, limit: 3, + data: { + row_index: [3, 1, 2], + row_names: ['r3', 'r1', 'r2'], + columns: [ + { schema: grid.columns![0], data: { + values: [null, 1, null], display: ['Inf', 'NaN', 'NA'], + special: ['pos_inf', 'nan', 'na'], na: [false, true, true], + }, errors: [] }, + { schema: grid.columns![1], data: { values: [null, 'x', 'y'], na: [true, false, false] }, errors: [] }, + ], + }, + }, 0, 3); + assert.deepEqual(page.rows, [ + { '0': 'r3', '1': 'Inf', '2': null }, + { '0': 'r1', '1': 'NaN', '2': 'x' }, + { '0': 'r2', '1': null, '2': 'y' }, + ]); + assert.equal(page.totalRows, 3); + assert.equal(page.totalUnfiltered, 5); + const unnamed = gridPageFromRows({ + total: 1, total_unfiltered: 1, offset: 0, limit: 1, + data: { row_index: [7], row_names: null, columns: [] }, + }, 0, 1); + assert.deepEqual(unnamed.rows, [{ '0': '7' }]); + }); + + test('maps List Viewer paths to typed tree paths', () => { + const tree = new TreeAdapter(parseDescriptor({ + view: 'v-2', generation: 1, kind: 'tree', access: 'random', encoding: 'r-json', + capabilities: { ...capabilities, sort: false, filter: false, children: true }, path: [], class: ['list'], + }), 'x'); + assert.deepEqual(tree.childrenParams([], 1), { view: 'v-2', generation: 1, path: [], offset: 0, limit: 200 }); + const page = tree.children([], { + path: [], offset: 0, total_children: 4, children: [ + { index: 1, label: 'a', class: ['integer'], summary: 'integer (3): 1, 2, 3', has_children: true, viewable: true, path: [{ type: 'index', index: 1 }] }, + { index: 2, label: 'b', class: ['list'], summary: 'list (1)', has_children: true, viewable: true, path: [{ type: 'index', index: 2 }] }, + { index: 3, label: 'df', class: ['data.frame'], summary: 'data.frame', has_children: true, viewable: true, path: [{ type: 'index', index: 3 }] }, + ], + }); + assert.equal(page.next_start, 4); + assert.deepEqual(page.children[1], { label: 'b', str: 'list (1)', index: 2, viewable: true, has_children: true }); + + const vector = tree.open([], 1); + assert.equal(vector.kind, 'navigate'); + assert.equal(vector.kind === 'navigate' && vector.navigation.vector, true); + const elements = tree.children([1], { + path: [{ type: 'index', index: 1 }], offset: 0, total_children: 1, children: [ + { index: 1, label: '[1]', class: ['integer'], summary: 'integer (1): 1', has_children: true, viewable: true, + path: [{ type: 'index', index: 1 }, { type: 'index', index: 1 }] }, + ], + }); + assert.deepEqual(elements.children[0], { label: '[1]', str: 'integer (1): 1', index: 1, viewable: false, has_children: false }); + + const nested = tree.open([], 2); + assert.deepEqual(nested.kind === 'navigate' && nested.navigation, { + title: 'x$b', path: [2], vector: false, + breadcrumbs: [{ label: 'x', path: [] }, { label: 'b', path: [2] }], + }); + const table = tree.open([], 3); + assert.deepEqual(table, { + kind: 'object', title: 'x$df', params: { view: 'v-2', generation: 1, path: [{ type: 'index', index: 3 }] }, + }); + assert.throws(() => tree.open([], 9), (e: unknown) => e instanceof JdvRequestError && e.code === 'unavailable'); + }); + + test('classifies atomic vectors', () => { + assert.equal(isVectorClass(['integer']), true); + assert.equal(isVectorClass(['ordered', 'factor']), true); + assert.equal(isVectorClass(['matrix', 'array']), false); + assert.equal(isVectorClass(['list']), false); + assert.equal(isVectorClass([]), false); + }); +}); diff --git a/src/test/suite/dataViewer.test.ts b/src/test/suite/dataViewer.test.ts index 9bae4fa57..cdba53f87 100644 --- a/src/test/suite/dataViewer.test.ts +++ b/src/test/suite/dataViewer.test.ts @@ -229,6 +229,30 @@ suite('Data viewer', () => { } }); + test('jdv columns offer only jdv operators and keep display text', () => { + const viewer = viewerScript(); + const jdvColumns = [ + { field: '1', headerName: 'n', type: 'numericColumn', filter: 'agNumberColumnFilter', jdv: { filterOptions: ['equals', 'inRange'], naOptions: true } }, + { field: '2', headerName: 'i', type: 'bigintColumn', filter: 'agBigIntColumnFilter', jdv: { filterOptions: ['equals'], naOptions: false } }, + { field: '3', headerName: 'd', type: 'dateColumn', filter: 'agDateColumnFilter', jdv: { filterOptions: [], naOptions: true } }, + ] as (ColDef & { jdv?: unknown })[]; + const bigintFields = viewer.script.prepareViewerColumns(jdvColumns); + assert.strictEqual(bigintFields.length, 0); + // Arrays created inside the vm context have a different prototype. + const options = (column: ColDef) => Array.from( + (column.filterParams as { filterOptions: (string | { displayKey: string })[] }).filterOptions, + option => typeof option === 'string' ? option : option.displayKey); + assert.deepStrictEqual(options(jdvColumns[0]), ['equals', 'inRange', 'jdvIsNa', 'jdvNotNa']); + assert.deepStrictEqual(options(jdvColumns[1]), ['equals']); + assert.deepStrictEqual(options(jdvColumns[2]), ['jdvIsNa', 'jdvNotNa']); + assert.strictEqual((jdvColumns[0].filterParams as { inRangeInclusive: boolean }).inRangeInclusive, true); + assert.strictEqual((jdvColumns[2].filterParams as { browserDatePicker: boolean }).browserDatePicker, true); + for (const column of jdvColumns) { + assert.strictEqual(column.cellDataType, undefined); + assert.strictEqual(column.jdv, undefined); + } + }); + test('high contrast light uses the light grid theme', () => { assert.strictEqual(viewerScript(undefined, {}, true).script.getAgTheme(), 'light'); assert.strictEqual(viewerScript().script.getAgTheme(), 'dark'); From 29ff428b59f876049406b1ade6646415154f9afe Mon Sep 17 00:00:00 2001 From: eitsupi Date: Fri, 16 Oct 2026 14:39:14 +0000 Subject: [PATCH 02/38] feat: view Arrow data through the jdv transport Arrow tables, record batch readers, nanoarrow streams and Polars data frames are viewed with jdv without converting them to data frames. The extension host listens for jdv's local transport per session; sess asks for the address with request_client() and sets it only while calling jdv_view(). Received Arrow IPC pages are kept in a bounded cache in the extension host, decoded with apache-arrow, and served to the existing AG Grid infinite row model with an unknown row count. Sorting and filtering are disabled for these forward-only streams. --- package.json | 1 + pnpm-lock.yaml | 36 +++ sess/R/handlers.R | 39 ++- sess/R/hooks.R | 3 + sess/inst/tinytest/test-jdv.R | 6 +- src/jdvAdapter.ts | 39 ++- src/jdvArrow.ts | 424 +++++++++++++++++++++++++++++++++ src/jdvTransport.ts | 376 +++++++++++++++++++++++++++++ src/jdvViewer.ts | 119 ++++++++- src/session.ts | 32 ++- src/test/node/jdvArrow.test.ts | 193 +++++++++++++++ 11 files changed, 1247 insertions(+), 21 deletions(-) create mode 100644 src/jdvArrow.ts create mode 100644 src/jdvTransport.ts create mode 100644 src/test/node/jdvArrow.test.ts diff --git a/package.json b/package.json index 7280273da..780360f9c 100644 --- a/package.json +++ b/package.json @@ -2635,6 +2635,7 @@ "dependencies": { "@vscode/codicons": "0.0.46-24", "ag-grid-community": "^36.2.0", + "apache-arrow": "21.2.0", "cheerio": "1.0.0-rc.12", "ejs": "^3.1.10", "highlight.js": "^11.11.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3d53f790a..f5accdb23 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: ag-grid-community: specifier: ^36.2.0 version: 36.2.0 + apache-arrow: + specifier: 21.2.0 + version: 21.2.0 cheerio: specifier: 1.0.0-rc.12 version: 1.0.0-rc.12 @@ -698,6 +701,9 @@ packages: '@types/node@22.20.5': resolution: {integrity: sha512-U2+DNr+wSjpsTS/wZGYHq7GcwfuSmKiKvoPvK22zwTlRhU91yOniN4qRR5KhIjvif7ysw/dz/hKmfDH0Ris4aA==} + '@types/node@25.9.9': + resolution: {integrity: sha512-b4e2xxj/yMeT2hNlD6x7zNFiQK2Dlmkb9CQ6v3/1G2RezKN4YS2lcEv8lzGm1mhs4f2lM4yNAMI/zUDBZBBt7w==} + '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} @@ -945,6 +951,10 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + apache-arrow@21.2.0: + resolution: {integrity: sha512-Hxe6Agq26gQOM954qpzYSllJBPJl+e16U5CkfuMUhLrNba+5nKkttIVlflaovN6oaTratqMGAO8H5u/aNhmHWQ==} + hasBin: true + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -1304,6 +1314,9 @@ packages: resolution: {integrity: sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==} hasBin: true + flatbuffers@25.9.23: + resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} + follow-redirects@1.16.0: resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} engines: {node: '>=4.0'} @@ -1613,6 +1626,9 @@ packages: json-schema-traverse@1.0.0: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + json-with-bigint@3.5.12: + resolution: {integrity: sha512-uwbF/wSSuOgC7qqlq27Xp5B6a2MHVug3t0idZdTqu0JnlFvgJuH7ju+KAk/J06C7GfhoYy2gnb9wz2INqcne7w==} + json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} @@ -2246,6 +2262,9 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici-types@7.24.6: + resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + unicorn-magic@0.1.0: resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==} engines: {node: '>=18'} @@ -2851,6 +2870,10 @@ snapshots: dependencies: undici-types: 6.21.0 + '@types/node@25.9.9': + dependencies: + undici-types: 7.24.6 + '@types/normalize-package-data@2.4.4': {} '@types/sarif@2.1.7': {} @@ -3071,6 +3094,13 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.2 + apache-arrow@21.2.0: + dependencies: + '@types/node': 25.9.9 + flatbuffers: 25.9.23 + json-with-bigint: 3.5.12 + tslib: 2.8.1 + argparse@2.0.1: {} astral-regex@2.0.0: {} @@ -3464,6 +3494,8 @@ snapshots: flat@5.0.2: {} + flatbuffers@25.9.23: {} + follow-redirects@1.16.0(debug@4.4.3(supports-color@8.1.1)): optionalDependencies: debug: 4.4.3(supports-color@8.1.1) @@ -3753,6 +3785,8 @@ snapshots: json-schema-traverse@1.0.0: {} + json-with-bigint@3.5.12: {} + json5@2.2.3: {} jsonc-parser@3.3.1: {} @@ -4490,6 +4524,8 @@ snapshots: undici-types@6.21.0: {} + undici-types@7.24.6: {} + unicorn-magic@0.1.0: {} unicorn-magic@0.3.0: {} diff --git a/sess/R/handlers.R b/sess/R/handlers.R index 1f248b34c..aeb6b9b98 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1296,23 +1296,50 @@ handle_dataview_dispose <- function(params) { # # When the jdv package is available, View() registers data frames, matrices and # lists with jdv instead of the sess data viewer. vscode-R relays the jdv -# protocol through the `jdv_rpc` request. Set `options(sess.dataviewer = "sess")` -# to keep using the sess data viewer. +# protocol through the `jdv_rpc` request. Arrow data is viewed as a forward-only +# Arrow stream over jdv's own transport, whose address VS Code provides per +# session. Set `options(sess.dataviewer = "sess")` to keep using the sess data +# viewer. + +.jdv_arrow_classes <- c("ArrowTabular", "RecordBatchReader", "nanoarrow_array_stream", + "polars_data_frame") + +.jdv_is_arrow <- function(x) { + inherits(x, .jdv_arrow_classes) +} .dataview_use_jdv <- function(x) { backend <- getOption("sess.dataviewer", "auto") if (identical(backend, "sess")) { return(FALSE) } - # Arrow data is not converted to data frames for jdv; the vscode-R client does - # not support jdv's sequential Arrow views yet. - if (inherits(x, c("ArrowTabular", "RecordBatchReader", "nanoarrow_array_stream", - "polars_data_frame"))) { + # Arrow data needs the jdv transport; see .jdv_show_arrow(). + if (.jdv_is_arrow(x)) { return(FALSE) } requireNamespace("jdv", quietly = TRUE) } +# Show Arrow data with jdv without converting it to a data frame. Returns FALSE +# when jdv cannot be used, so the caller falls back to the sess data viewer. +.jdv_show_arrow <- function(x, title_key) { + if (identical(getOption("sess.dataviewer", "auto"), "sess") || + !requireNamespace("jdv", quietly = TRUE) || + !requireNamespace("nanoarrow", quietly = TRUE)) { + return(FALSE) + } + address <- tryCatch(request_client("jdv/address")$address, error = function(e) NULL) + if (!is.character(address) || length(address) != 1L || !nzchar(address)) { + return(FALSE) + } + # jdv reads the address when it opens its connection; do not change the + # option for other jdv hosts the user may run. + op <- options(jdv.address = address) + on.exit(options(op), add = TRUE) + jdv::jdv_view(x, title = if (nzchar(title_key)) title_key else NULL) + TRUE +} + .jdv_show_view <- function(x, title, title_key) { # The title is jdv's display slot: View() with the same title replaces the # view and increments its generation. diff --git a/sess/R/hooks.R b/sess/R/hooks.R index 1ca593ca9..9f74f1fbc 100644 --- a/sess/R/hooks.R +++ b/sess/R/hooks.R @@ -88,6 +88,9 @@ runtime_start <- function(use_rstudioapi = TRUE, "object" } title_key <- paste(as.character(title), collapse = "\n") + if (.jdv_is_arrow(x) && .jdv_show_arrow(x, title_key)) { + return(invisible(NULL)) + } if (view_type %in% c("table", "list") && .dataview_use_jdv(x)) { return(invisible(.jdv_show_view(x, title, title_key))) } diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R index e08ddcdbc..e6f7d3260 100644 --- a/sess/inst/tinytest/test-jdv.R +++ b/sess/inst/tinytest/test-jdv.R @@ -90,6 +90,10 @@ local({ options(sess.dataviewer = "sess") utils::View(sample) expect_identical(last_view()$source, "table") + arrow_like <- structure(list(), class = "polars_data_frame") + expect_true(sess:::.jdv_is_arrow(arrow_like)) + expect_false(sess:::.jdv_show_arrow(arrow_like, "arrow")) options(sess.dataviewer = NULL) - expect_false(sess:::.dataview_use_jdv(structure(list(), class = "polars_data_frame"))) + # Arrow data never goes through the jdv_rpc relay, which cannot carry Arrow IPC. + expect_false(sess:::.dataview_use_jdv(arrow_like)) }) diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index 1f73cafea..78c05b6f3 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -163,8 +163,10 @@ function columnTooltip(column: JdvColumnSchema): string { return tooltip; } -export function gridColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { - const columns: GridColumnDef[] = [{ +const ARROW_NUMERIC_FORMATS = new Set(['c', 'C', 's', 'S', 'i', 'I', 'l', 'L', 'e', 'f', 'g']); + +function rowLabelColumn(): GridColumnDef { + return { headerName: '', field: ROW_LABEL_FIELD, type: 'textColumn', @@ -172,7 +174,38 @@ export function gridColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { sortable: false, lockPosition: 'left', suppressHeaderMenuButton: true, - }]; + }; +} + +/** Columns of a sequential Arrow view; they can be neither sorted nor filtered. */ +function sequentialColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { + const columns = [rowLabelColumn()]; + const schema = (descriptor as unknown as { schema?: { children?: { dictionary?: unknown }[] } }).schema; + for (const column of (descriptor.columns ?? []) as unknown as { + index?: number; position: number; name: string; type: string; unsupported: string | null; + }[]) { + // A dictionary column's format is its index type; its values are the dictionary's. + const dictionary = schema?.children?.[column.index ?? column.position]?.dictionary; + const numeric = !dictionary && (ARROW_NUMERIC_FORMATS.has(column.type) || column.type.startsWith('d:')); + columns.push({ + headerName: column.name, + headerTooltip: column.unsupported + ? `${column.name}, unsupported: ${column.unsupported}` + : `${column.name}, Arrow format: ${column.type}`, + field: String(column.position + 1), + type: numeric ? 'numericColumn' : 'textColumn', + filter: false, + sortable: false, + }); + } + return columns; +} + +export function gridColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { + if (descriptor.access === 'sequential') { + return sequentialColumnDefs(descriptor); + } + const columns: GridColumnDef[] = [rowLabelColumn()]; for (const column of descriptor.columns ?? []) { const sortable = descriptor.capabilities.sort && column.sortable; const filterEnabled = descriptor.capabilities.filter; diff --git a/src/jdvArrow.ts b/src/jdvArrow.ts new file mode 100644 index 000000000..18ff5ec6c --- /dev/null +++ b/src/jdvArrow.ts @@ -0,0 +1,424 @@ +'use strict'; + +// Sequential (forward-only) jdv Arrow views. The extension host keeps the +// received Arrow IPC pages, decodes them with apache-arrow, and serves the +// existing AG Grid infinite row model with formatted rows. + +import { tableFromIPC } from 'apache-arrow'; +import { JdvDescriptor, JdvRequestError, JsonObject, ROW_LABEL_FIELD } from './jdvAdapter'; + +// --------------------------------------------------------------------------- +// Formatting. Values are formatted from physical buffers where JavaScript +// number getters lose precision. Ported from jdv's reference viewer +// (server/web/sequential.js, MIT). + +export const CELL_TEXT_LIMIT = 4096; +const NANOSECONDS = [1_000_000_000n, 1_000_000n, 1_000n, 1n]; +const UNIT_NAMES = ['s', 'ms', 'us', 'ns']; + +// apache-arrow Type ids used below. +const TYPE_DICTIONARY = -1; +const TYPE_NULL = 1; +const TYPE_INT = 2; +const TYPE_DECIMAL = 7; +const TYPE_DATE = 8; +const TYPE_TIME = 9; +const TYPE_TIMESTAMP = 10; +const TYPE_LIST = 12; +const TYPE_STRUCT = 13; +const TYPE_FIXED_SIZE_LIST = 16; +const TYPE_MAP = 17; +const TYPE_DURATION = 18; +const TYPE_LARGE_LIST = 21; + +interface ArrowType { + typeId: number; + bitWidth?: number; + scale?: number; + unit?: number; + timezone?: string | null; + listSize?: number; + children?: unknown[]; +} + +interface ArrowData { + length: number; + stride: number; + values: ArrayLike; + valueOffsets?: ArrayLike; + dictionary?: ArrowVector; +} + +interface ArrowVector { + length: number; + type: ArrowType; + data: ArrowData[]; + isValid(index: number): boolean; + get(index: number): unknown; +} + +interface ArrowTable { + numRows: number; + numCols: number; + getChildAt(index: number): unknown; +} + +export function formatScaled(coefficient: bigint, scale: number): string { + if (!Number.isSafeInteger(scale)) { + throw new Error('Invalid Arrow decimal scale: expected a safe integer.'); + } + const negative = coefficient < 0n; + const digits = (negative ? -coefficient : coefficient).toString(); + const sign = negative ? '-' : ''; + const fixedLength = sign.length + (scale <= 0 ? digits.length - scale : Math.max(digits.length, scale + 1) + 1); + // Arrow decimal coefficients have at most 76 digits; never expand a huge scale. + if (fixedLength > CELL_TEXT_LIMIT) { + return `${sign}${digits}e${-scale}`; + } + if (scale <= 0) { + return sign + digits + '0'.repeat(-scale); + } + const padded = digits.padStart(scale + 1, '0'); + return `${sign}${padded.slice(0, -scale)}.${padded.slice(-scale)}`; +} + +function signedWords(values: ArrayLike, start: number, width: number): bigint { + let result = 0n; + for (let word = 0; word < width; word++) { + result |= BigInt(Number(values[start + word]) >>> 0) << BigInt(word * 32); + } + return BigInt.asIntN(width * 32, result); +} + +function floorDiv(value: bigint, divisor: bigint): bigint { + const quotient = value / divisor; + return value % divisor < 0n ? quotient - 1n : quotient; +} + +export function formatTimestamp(ticks: bigint, unit: number, timezone: string | null = null): string { + const nanoseconds = ticks * NANOSECONDS[unit]; + const seconds = floorDiv(nanoseconds, 1_000_000_000n); + const fraction = nanoseconds - seconds * 1_000_000_000n; + const date = new Date(Number(seconds) * 1000); + if (!Number.isSafeInteger(Number(seconds)) || !Number.isFinite(date.getTime())) { + return `${ticks} ${UNIT_NAMES[unit]}${timezone ? ` [${timezone}]` : ''}`; + } + const digits = [0, 3, 6, 9][unit]; + const suffix = digits ? `.${fraction.toString().padStart(9, '0').slice(0, digits)}` : ''; + if (!timezone || timezone === 'UTC' || timezone === 'Etc/UTC') { + return date.toISOString().replace(/\.\d{3}Z$/, suffix + (timezone ? 'Z' : '')); + } + try { + const formatter = new Intl.DateTimeFormat('en-US-u-ca-iso8601-nu-latn', { + timeZone: timezone, year: 'numeric', month: '2-digit', day: '2-digit', + hour: '2-digit', minute: '2-digit', second: '2-digit', hourCycle: 'h23', timeZoneName: 'longOffset', + }); + const parts = Object.fromEntries(formatter.formatToParts(date).map(({ type, value }) => [type, value])); + const offset = (parts.timeZoneName ?? '').replace(/^GMT/, '') || '+00:00'; + return `${parts.year}-${parts.month}-${parts.day}T${parts.hour}:${parts.minute}:${parts.second}${suffix}${offset} [${timezone}]`; + } catch { + return `${date.toISOString().replace(/\.\d{3}Z$/, suffix + 'Z')} [${timezone}]`; + } +} + +function chunkAt(vector: ArrowVector, row: number): { data: ArrowData; row: number } { + for (const data of vector.data) { + if (row < data.length) { + return { data, row }; + } + row -= data.length; + } + throw new Error('Arrow row is outside its buffers.'); +} + +function boundedText(value: string): string { + return value.length > CELL_TEXT_LIMIT ? value.slice(0, CELL_TEXT_LIMIT) + '… [truncated]' : value; +} + +/** Display text of one Arrow value, or `null` for a missing value. */ +export function formatArrowValue(vector: ArrowVector, row: number, dictionaries = new Set()): string | null { + if (!Number.isSafeInteger(row) || row < 0 || row >= vector.length) { + throw new Error('Arrow row is outside its buffers.'); + } + const type = vector.type; + if (type.typeId === TYPE_NULL || !vector.isValid(row)) { + return null; + } + const { data, row: local } = chunkAt(vector, row); + // Arrow Data slices already adjust values; offset belongs to the validity bitmap. + const start = local * data.stride; + if (type.typeId === TYPE_DICTIONARY) { + const index = Number(data.values[start]); + if (!data.dictionary || !Number.isSafeInteger(index) || index < 0 || index >= data.dictionary.length) { + throw new Error('Invalid Arrow dictionary index.'); + } + if (dictionaries.has(data) || dictionaries.size >= 32) { + throw new Error('Arrow dictionary nesting is cyclic or exceeds 32 levels.'); + } + dictionaries.add(data); + return formatArrowValue(data.dictionary, index, dictionaries); + } + if (type.typeId === TYPE_INT && type.bitWidth === 64) { + return BigInt(data.values[start]).toString(); + } + if (type.typeId === TYPE_DECIMAL) { + return formatScaled(signedWords(data.values, start, (type.bitWidth ?? 128) / 32), type.scale ?? 0); + } + if (type.typeId === TYPE_TIMESTAMP) { + return formatTimestamp(BigInt(data.values[start]), type.unit ?? 0, type.timezone ?? null); + } + if (type.typeId === TYPE_TIME || type.typeId === TYPE_DURATION) { + return `${String(data.values[start])} ${UNIT_NAMES[type.unit ?? 0]}`; + } + if (type.typeId === TYPE_LIST || type.typeId === TYPE_FIXED_SIZE_LIST || type.typeId === TYPE_LARGE_LIST) { + const size = type.listSize ?? Number(data.valueOffsets![local + 1]) - Number(data.valueOffsets![local]); + return ``; + } + if (type.typeId === TYPE_STRUCT) { + return ``; + } + if (type.typeId === TYPE_MAP) { + return ''; + } + const value = vector.get(row); + if (value instanceof Uint8Array) { + const shown = value.subarray(0, CELL_TEXT_LIMIT / 2); + return Array.from(shown, byte => byte.toString(16).padStart(2, '0')).join('') + + (shown.length < value.length ? '… [truncated]' : ''); + } + if (type.typeId === TYPE_DATE) { + return new Date(Number(value)).toISOString().slice(0, 10); + } + return boundedText(typeof value === 'string' ? value : String(value as number | boolean | bigint)); +} + +// --------------------------------------------------------------------------- +// Page cache + +interface SequentialColumn { + position: number; + name: string; + type: string; + ipc_index: number | null; + unsupported: string | null; +} + +export const VIEW_CACHE_BYTES = 64 * 1024 * 1024; +export const TOTAL_CACHE_BYTES = 256 * 1024 * 1024; +const PAGE_OVERHEAD_BYTES = 1024; +const DECODED_PAGES = 4; +/** Empty non-EOF pages are legal; bound how many one row request may consume. */ +const MAX_EMPTY_PAGES = 64; +export const UNAVAILABLE_TEXT = '(no longer retained)'; + +interface StoredPage { + firstRow: number; + rows: number; + ipc: Uint8Array; + charge: number; + sequence: number; +} + +export interface StreamPageResult { + first_row: number; + rows_fetched: number; + rows_seen: number; + done: boolean; +} + +export type FetchNext = (maxRows: number) => Promise<{ result: unknown; body?: Uint8Array }>; + +let pageSequence = 0; +const streams = new Set(); + +function totalBytes(): number { + let total = 0; + for (const stream of streams) { + total += stream.bytes; + } + return total; +} + +function parseProgress(result: unknown, requested: number, rowsSeen: number): StreamPageResult { + const value = result as Partial | null; + const integer = (field: unknown) => typeof field === 'number' && Number.isSafeInteger(field) && field >= 0; + if (!value || !integer(value.first_row) || !integer(value.rows_fetched) || !integer(value.rows_seen) || + typeof value.done !== 'boolean' || value.rows_seen !== value.first_row! + value.rows_fetched! || + value.rows_fetched! > requested || value.first_row! < rowsSeen) { + throw new JdvRequestError('invalid_request', 'Invalid jdv stream page metadata.'); + } + return value as StreamPageResult; +} + +/** + * Received pages of one view generation. R cannot rewind the stream, so pages + * are kept within byte budgets; evicted rows are reported as unavailable. + */ +export class SequentialStream { + private pages: StoredPage[] = []; + private readonly decoded = new Map(); + private fetching?: Promise; + bytes = 0; + rowsSeen = 0; + done = false; + /** Rows before this index are no longer held. */ + unavailableBefore = 0; + private failed?: Error; + readonly columns: SequentialColumn[]; + readonly pageRows: number; + + constructor( + readonly descriptor: JdvDescriptor, private readonly next: FetchNext, + preferredRows = 1000, private readonly viewCacheBytes = VIEW_CACHE_BYTES, + ) { + this.columns = (descriptor.columns ?? []) as unknown as SequentialColumn[]; + const cap = (descriptor as unknown as JsonObject).max_page_rows; + const maxPageRows = typeof cap === 'number' && Number.isSafeInteger(cap) && cap >= 1 && cap <= 2000 ? cap : 200; + this.pageRows = Math.min(preferredRows, maxPageRows); + streams.add(this); + } + + dispose(): void { + streams.delete(this); + this.pages = []; + this.decoded.clear(); + this.bytes = 0; + } + + /** Fetch pages until `endRow` rows have been seen or the stream ends. */ + async ensure(endRow: number): Promise { + while (!this.done && this.rowsSeen < endRow) { + if (this.failed) { + throw this.failed; + } + // One stream/next per view generation at a time. + this.fetching ??= this.fetchUntil(endRow).finally(() => { this.fetching = undefined; }); + await this.fetching; + } + } + + private async fetchUntil(endRow: number): Promise { + let empty = 0; + while (!this.done && this.rowsSeen < endRow) { + let response; + try { + response = await this.next(this.pageRows); + } catch (e) { + // A producer failure is terminal; R releases the stream. + if (e instanceof JdvRequestError && e.code === 'stream_failed') { + this.failed = e; + } + throw e; + } + const progress = parseProgress(response.result, this.pageRows, this.rowsSeen); + // A gap (first_row > rowsSeen) has no page; rows() reports it as unavailable. + if (progress.rows_fetched > 0) { + if (!response.body) { + throw new JdvRequestError('invalid_request', 'The jdv stream page has no Arrow body.'); + } + this.store(progress, response.body); + empty = 0; + } else if (++empty >= MAX_EMPTY_PAGES && !progress.done) { + return; + } + this.rowsSeen = progress.rows_seen; + this.done = progress.done; + } + } + + private store(progress: StreamPageResult, ipc: Uint8Array): void { + const page: StoredPage = { + firstRow: progress.first_row, rows: progress.rows_fetched, ipc, + charge: ipc.byteLength + PAGE_OVERHEAD_BYTES, sequence: ++pageSequence, + }; + this.pages.push(page); + this.bytes += page.charge; + while (this.bytes > this.viewCacheBytes && this.pages.length > 0) { + this.evictOldest(); + } + // Evict across views, oldest page first. + while (totalBytes() > TOTAL_CACHE_BYTES) { + let victim: SequentialStream | undefined; + for (const stream of streams) { + if (stream.pages.length > 0 && (!victim || stream.pages[0].sequence < victim.pages[0].sequence)) { + victim = stream; + } + } + if (!victim) { + break; + } + victim.evictOldest(); + } + } + + private evictOldest(): void { + const page = this.pages.shift(); + if (page) { + this.bytes -= page.charge; + this.decoded.delete(page); + this.unavailableBefore = Math.max(this.unavailableBefore, page.firstRow + page.rows); + } + } + + private pageFor(row: number): StoredPage | undefined { + let low = 0; + let high = this.pages.length - 1; + while (low <= high) { + const middle = (low + high) >> 1; + const page = this.pages[middle]; + if (row < page.firstRow) { + high = middle - 1; + } else if (row >= page.firstRow + page.rows) { + low = middle + 1; + } else { + return page; + } + } + return undefined; + } + + private table(page: StoredPage): ArrowTable { + let table = this.decoded.get(page); + if (table) { + // Refresh the small decoded LRU. + this.decoded.delete(page); + } else { + table = tableFromIPC(page.ipc) as unknown as ArrowTable; + if (table.numRows !== page.rows && table.numCols !== 0) { + throw new JdvRequestError('invalid_request', 'Arrow rows do not match the stream page metadata.'); + } + } + this.decoded.set(page, table); + while (this.decoded.size > DECODED_PAGES) { + this.decoded.delete(this.decoded.keys().next().value!); + } + return table; + } + + /** Formatted rows in `[startRow, endRow)` that have been seen. */ + rows(startRow: number, endRow: number): Record[] { + const rows: Record[] = []; + const end = Math.min(endRow, this.rowsSeen); + for (let row = Math.max(0, startRow); row < end; row++) { + const output: Record = { [ROW_LABEL_FIELD]: String(row + 1) }; + const page = this.pageFor(row); + const table = page ? this.table(page) : undefined; + for (const column of this.columns) { + const field = String(column.position + 1); + if (column.unsupported || column.ipc_index === null) { + output[field] = ``; + } else if (!page || !table) { + output[field] = UNAVAILABLE_TEXT; + } else { + const vector = table.getChildAt(column.ipc_index) as ArrowVector | null; + if (!vector) { + throw new JdvRequestError('invalid_request', 'Arrow column mapping is outside the IPC schema.'); + } + output[field] = formatArrowValue(vector, row - page.firstRow); + } + } + rows.push(output); + } + return rows; + } +} diff --git a/src/jdvTransport.ts b/src/jdvTransport.ts new file mode 100644 index 000000000..410b0b9ac --- /dev/null +++ b/src/jdvTransport.ts @@ -0,0 +1,376 @@ +'use strict'; + +// Extension-host side of the jdv local transport (see `?jdv_spec`). R connects +// to a listener owned by one vscode-R session and exchanges JSON Lines, with +// Arrow IPC bodies following `stream/next` response headers. + +import * as crypto from 'crypto'; +import * as fs from 'fs'; +import * as net from 'net'; +import * as os from 'os'; +import * as path from 'path'; +import { JdvChannel, JdvRequestError, JsonObject } from './jdvAdapter'; + +const PROTOCOL_MAJOR = 1; +const MAX_LIVE_REQUESTS = 32; +const WAKE_INTERVAL_MS = 50; +const MAX_JSON_LINE_BYTES = 10 * 1024 * 1024; +const MAX_BINARY_HEADER_BYTES = 64 * 1024; +const MAX_BINARY_FRAME_BYTES = 10 * 1024 * 1024; +const BINARY_BODY_TIMEOUT_MS = 30_000; + +export interface JdvResponse { + result: unknown; + /** Arrow IPC body for responses carrying `result.transfer`. */ + body?: Uint8Array; +} + +interface RLimits { + maxReceiveFrames: number; + maxReceiveBytes: number; + maxRequestBytes: number; +} + +interface OutboundFrame { + bytes: Buffer; + wake?: boolean; +} + +interface PendingRequest { + resolve(response: JdvResponse): void; + reject(error: Error): void; + view?: unknown; + generation?: unknown; +} + +function isObject(value: unknown): value is JsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function positiveInteger(value: unknown, fallback: number): number { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : fallback; +} + +/** One R connection. Implements the host obligations of the jdv transport. */ +export class JdvPeer implements JdvChannel { + private buffer: Buffer = Buffer.alloc(0); + private body?: { header: JsonObject; length: number; timer: NodeJS.Timeout }; + private limits?: RLimits; + private readonly outbound: OutboundFrame[] = []; + /** Bytes of frames written but not yet acknowledged, in send order. */ + private readonly unconfirmed: number[] = []; + private unconfirmedBytes = 0; + private sentFrames = 0; + private ackedFrames = 0; + private readonly pending = new Map(); + private wakeTimer?: NodeJS.Timeout; + private sequence = 0; + private closed = false; + session?: string; + pid?: number; + + constructor( + private readonly socket: net.Socket, + private readonly handlers: { + hello(peer: JdvPeer, params: JsonObject): boolean; + notification(peer: JdvPeer, method: string, params: JsonObject): void; + closed(peer: JdvPeer, reason: string): void; + }, + ) { + socket.on('data', chunk => this.receive(chunk)); + socket.on('error', error => this.close(`jdv connection error: ${error.message}`)); + socket.on('close', () => this.close('The R session closed the jdv connection.')); + } + + get isClosed(): boolean { + return this.closed; + } + + async request(method: string, params: JsonObject): Promise { + return (await this.requestWithBody(method, params)).result; + } + + requestWithBody(method: string, params: JsonObject): Promise { + if (this.closed) { + return Promise.reject(new JdvRequestError('view_not_found', 'The jdv connection is closed.')); + } + if (this.pending.size >= MAX_LIVE_REQUESTS) { + return Promise.reject(new JdvRequestError('limit_exceeded', 'Too many pending jdv requests.')); + } + const id = `vscode-r-${++this.sequence}`; + const bytes = Buffer.from(JSON.stringify({ id, method, params }) + '\n', 'utf8'); + if (this.limits && bytes.length > this.limits.maxRequestBytes) { + return Promise.reject(new JdvRequestError('limit_exceeded', 'The jdv request is too large.')); + } + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject, view: params.view, generation: params.generation }); + this.outbound.push({ bytes }); + this.flush(); + this.updateWake(); + }); + } + + close(reason: string): void { + if (this.closed) { + return; + } + this.closed = true; + if (this.body) { + clearTimeout(this.body.timer); + this.body = undefined; + } + this.stopWake(); + for (const pending of this.pending.values()) { + pending.reject(new JdvRequestError('view_not_found', reason)); + } + this.pending.clear(); + this.socket.destroy(); + this.handlers.closed(this, reason); + } + + private receive(chunk: Buffer): void { + this.buffer = this.buffer.length ? Buffer.concat([this.buffer, chunk]) : chunk; + try { + while (!this.closed) { + if (this.body) { + if (this.buffer.length < this.body.length) { + return; + } + const { header, length, timer } = this.body; + clearTimeout(timer); + this.body = undefined; + // Copy so a retained page does not keep the receive buffer alive. + const body = new Uint8Array(this.buffer.subarray(0, length)); + this.buffer = this.buffer.subarray(length); + this.dispatch(header, body); + continue; + } + const newline = this.buffer.indexOf(0x0a); + if (newline < 0) { + if (this.buffer.length > MAX_JSON_LINE_BYTES) { + throw new Error('A jdv message exceeds the size limit.'); + } + return; + } + const line = this.buffer.subarray(0, newline); + this.buffer = this.buffer.subarray(newline + 1); + const message = JSON.parse(line.toString('utf8')) as unknown; + if (!isObject(message)) { + throw new Error('Invalid jdv message.'); + } + const transfer = isObject(message.result) ? message.result.transfer : undefined; + if (isObject(transfer) && transfer.encoding === 'arrow-ipc') { + const length = transfer.byte_length; + if (typeof length !== 'number' || !Number.isSafeInteger(length) || length < 0 || + typeof message.id !== 'string' || line.length > MAX_BINARY_HEADER_BYTES || + line.length + 1 + length > MAX_BINARY_FRAME_BYTES) { + throw new Error('Invalid jdv binary frame.'); + } + const timer = setTimeout(() => this.close('Timed out receiving a jdv Arrow page.'), + BINARY_BODY_TIMEOUT_MS); + this.body = { header: message, length, timer }; + continue; + } + this.dispatch(message); + } + } catch (e) { + this.close(e instanceof Error ? e.message : String(e)); + } + } + + private dispatch(message: JsonObject, body?: Uint8Array): void { + if (typeof message.method === 'string') { + const params = isObject(message.params) ? message.params : {}; + if (message.method === 'hello') { + this.acceptHello(params); + } else if (message.method === 'ack') { + this.acknowledge(params); + } else if (this.limits) { + this.handlers.notification(this, message.method, params); + } + return; + } + if (typeof message.id !== 'string') { + // Best-effort errors for malformed requests carry a null id. + return; + } + const pending = this.pending.get(message.id); + if (!pending) { + return; + } + this.pending.delete(message.id); + this.updateWake(); + if (isObject(message.error)) { + const { code, message: text } = message.error; + pending.reject(new JdvRequestError(typeof code === 'string' ? code : 'unknown', + typeof text === 'string' ? text : 'jdv request failed')); + return; + } + const result = message.result; + if (body && isObject(result) && (result.view !== pending.view || result.generation !== pending.generation)) { + pending.reject(new JdvRequestError('stale_generation', 'The Arrow page belongs to another view generation.')); + return; + } + pending.resolve({ result, body }); + } + + private acceptHello(params: JsonObject): void { + if (this.limits) { + throw new Error('Duplicate jdv hello.'); + } + const version = typeof params.protocol === 'number' ? Math.trunc(params.protocol) : NaN; + if (version !== PROTOCOL_MAJOR) { + throw new Error(`Unsupported jdv protocol: ${String(params.protocol)}`); + } + const limits = isObject(params.limits) ? params.limits : {}; + this.limits = { + maxReceiveFrames: positiveInteger(limits.max_receive_frames, 264), + maxReceiveBytes: positiveInteger(limits.max_receive_bytes, 64 * 1024), + maxRequestBytes: positiveInteger(limits.max_request_bytes, 1024 * 1024), + }; + this.session = typeof params.session === 'string' ? params.session : undefined; + this.pid = typeof params.pid === 'number' ? params.pid : undefined; + if (!this.handlers.hello(this, params)) { + throw new Error('The jdv connection does not belong to this R session.'); + } + this.flush(); + } + + private acknowledge(params: JsonObject): void { + const frames = params.frames; + if (typeof frames !== 'number' || !Number.isSafeInteger(frames) || frames < this.ackedFrames || + frames > this.sentFrames) { + throw new Error('Invalid jdv acknowledgement.'); + } + while (this.ackedFrames < frames) { + this.unconfirmedBytes -= this.unconfirmed.shift() ?? 0; + this.ackedFrames++; + } + this.flush(); + } + + /** Write queued frames while the cumulative unconfirmed output fits R's receive quota. */ + private flush(): void { + const limits = this.limits; + if (!limits || this.closed) { + return; + } + while (this.outbound.length > 0) { + const frame = this.outbound[0]; + if (this.unconfirmed.length + 1 > limits.maxReceiveFrames || + this.unconfirmedBytes + frame.bytes.length > limits.maxReceiveBytes) { + return; + } + this.outbound.shift(); + this.unconfirmed.push(frame.bytes.length); + this.unconfirmedBytes += frame.bytes.length; + this.sentFrames++; + this.socket.write(frame.bytes); + } + } + + /** R dispatches queued requests only when woken; wake while any request is pending. */ + private updateWake(): void { + if (this.pending.size === 0) { + this.stopWake(); + } else if (!this.wakeTimer) { + this.wakeTimer = setInterval(() => { + // At most one queued wake; skip while frames are waiting for credit. + if (this.outbound.length === 0) { + this.outbound.push({ bytes: Buffer.from('{"method":"wake"}\n'), wake: true }); + this.flush(); + } + }, WAKE_INTERVAL_MS); + } + } + + private stopWake(): void { + if (this.wakeTimer) { + clearInterval(this.wakeTimer); + this.wakeTimer = undefined; + } + for (let i = this.outbound.length - 1; i >= 0; i--) { + if (this.outbound[i].wake) { + this.outbound.splice(i, 1); + } + } + } +} + +export interface JdvHostEvents { + /** `view/open` received from R. */ + viewOpen(peer: JdvPeer, params: JsonObject): void; + /** `view/closed` received from R. */ + viewClosed(peer: JdvPeer, view: string, generation: number): void; + /** The R connection closed; its views are gone. */ + disconnected(peer: JdvPeer, reason: string): void; +} + +/** + * A listener owned by one vscode-R session. Its address is handed to that R + * session only, so jdv never connects through a shared discovery file. + */ +export class JdvHost { + private peer?: JdvPeer; + + private constructor( + private readonly server: net.Server, + /** Address in jdv's syntax, for `options(jdv.address = ...)`. */ + readonly address: string, + private readonly socketPath: string, + private readonly expectedPid: number | undefined, + private readonly events: JdvHostEvents, + ) { + server.on('connection', socket => this.accept(socket)); + } + + static async create(expectedPid: number | undefined, events: JdvHostEvents): Promise { + const id = crypto.randomBytes(8).toString('hex'); + // Unix socket paths are limited to about 104-108 bytes; keep the name short. + const socketPath = process.platform === 'win32' + ? `\\\\.\\pipe\\vscode-r-jdv-${id}` + : path.join(os.tmpdir(), `vscr-jdv-${id}.sock`); + const address = process.platform === 'win32' ? `npipe:////./pipe/vscode-r-jdv-${id}` : socketPath; + const server = net.createServer(); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(socketPath, () => { + server.off('error', reject); + resolve(); + }); + }); + if (process.platform !== 'win32') { + fs.chmodSync(socketPath, 0o600); + } + return new JdvHost(server, address, socketPath, expectedPid, events); + } + + get connection(): JdvPeer | undefined { + return this.peer && !this.peer.isClosed ? this.peer : undefined; + } + + dispose(): void { + this.peer?.close('The R session ended.'); + this.server.close(); + if (process.platform !== 'win32') { + fs.rmSync(this.socketPath, { force: true }); + } + } + + private accept(socket: net.Socket): void { + // jdv opens a new connection after a disconnect; only one is active. + this.peer?.close('The R session opened a new jdv connection.'); + this.peer = new JdvPeer(socket, { + hello: (_peer, params) => this.expectedPid === undefined || params.pid === this.expectedPid, + notification: (peer, method, params) => { + if (method === 'view/open') { + this.events.viewOpen(peer, params); + } else if (method === 'view/closed' && typeof params.view === 'string' && + typeof params.generation === 'number') { + this.events.viewClosed(peer, params.view, params.generation); + } + }, + closed: (peer, reason) => this.events.disconnected(peer, reason), + }); + } +} diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index 7927567d5..c9a7cab39 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -10,6 +10,8 @@ import { GridPage, JdvChannel, JdvDescriptor, JdvRequestError, JsonObject, TreeAdapter, gridColumnDefs, gridPageFromRows, parseClosedNotifications, parseDescriptor, tableRowsParams, unwrapResponse, } from './jdvAdapter'; +import { SequentialStream } from './jdvArrow'; +import { JdvHost, JdvPeer } from './jdvTransport'; import { extensionContext } from './extension'; import * as session from './session'; import { config, UriIcon } from './util'; @@ -18,17 +20,83 @@ interface JdvPanel { readonly key: string; readonly panel: vscode.WebviewPanel; readonly sessionId: string | null; - readonly channel: JdvChannel; + /** A title can move between the sess relay and the jdv transport. */ + channel: JdvChannel; descriptor: JdvDescriptor; title: string; tree?: TreeAdapter; + /** Received Arrow pages of a sequential view. */ + stream?: SequentialStream; /** Set when R closed or evicted this view generation. */ closed?: string; } const panels = new Map(); +const hosts = new Map>(); let requestSequence = 0; +/** + * The jdv transport address for a session, creating its listener on first use. + * Sequential Arrow views need this transport; `jdv_rpc()` cannot carry Arrow IPC. + */ +export function getJdvAddress(sessionId: string, pid: number | undefined): Promise { + let host = hosts.get(sessionId); + if (!host) { + host = JdvHost.create(pid, { + viewOpen: (peer, params) => { + let descriptor: JdvDescriptor; + try { + descriptor = parseDescriptor(params); + } catch (e) { + void window.showErrorMessage(`Unable to open the data viewer: ${errorMessage(e)}`); + return; + } + const viewer = viewerColumn(); + if (viewer) { + void openJdvView(descriptor, descriptor.title ?? 'View', sessionId, viewer, peer); + } else { + void peer.request('view/close', { view: descriptor.view, generation: descriptor.generation }) + .catch(() => undefined); + } + }, + viewClosed: (_peer, view, generation) => markClosed(sessionId, view, generation), + disconnected: (peer) => { + for (const state of panels.values()) { + if (state.channel === peer && !state.closed) { + state.closed = 'The R session disconnected from the data viewer.'; + } + } + }, + }); + hosts.set(sessionId, host); + host.catch(() => hosts.delete(sessionId)); + } + return host.then(value => value.address); +} + +/** Close the session's jdv listener when the session ends. */ +export function disposeJdvHost(sessionId: string): void { + const host = hosts.get(sessionId); + hosts.delete(sessionId); + void host?.then(value => value.dispose(), () => undefined); +} + +function markClosed(sessionId: string | null, view: string, generation: number): void { + const state = panels.get(panelKey(sessionId, view)); + if (state && state.descriptor.generation === generation) { + state.closed = 'This view was closed in R. Run View() again to refresh it.'; + } +} + +function createStream(descriptor: JdvDescriptor, channel: JdvChannel): SequentialStream | undefined { + if (descriptor.access !== 'sequential' || !(channel instanceof JdvPeer)) { + return undefined; + } + const { view, generation } = descriptor; + return new SequentialStream(descriptor, + maxRows => channel.requestWithBody('stream/next', { view, generation, max_rows: maxRows })); +} + /** Sends jdv requests through the session's existing sess RPC connection. */ function sessChannel(sessionId: string | null): JdvChannel { return { @@ -70,10 +138,7 @@ export async function showJdvNotification(params: Record, sessi } // Notifications precede the view they belong to; see jdv_register(). for (const closed of parseClosedNotifications(params.notifications)) { - const state = panels.get(panelKey(sessionId, closed.view)); - if (state && state.descriptor.generation === closed.generation) { - state.closed = 'This view was closed in R. Run View() again to refresh it.'; - } + markClosed(sessionId, closed.view, closed.generation); } const title = typeof params.title === 'string' ? params.title : Array.isArray(params.title) ? params.title.map(String).join(',') : descriptor.title ?? 'View'; @@ -86,8 +151,9 @@ export async function showJdvNotification(params: Record, sessi async function openJdvView( descriptor: JdvDescriptor, title: string, sessionId: string | null, viewer: string, channel: JdvChannel, ): Promise { - if (descriptor.kind === 'grid' && descriptor.access !== 'random') { - // Snapshot and sequential (Arrow) grids are not integrated yet; release the view. + const sequential = descriptor.access === 'sequential' && channel instanceof JdvPeer; + if (descriptor.kind === 'grid' && descriptor.access !== 'random' && !sequential) { + // Snapshot grids, and sequential grids without the jdv transport, are not supported; release the view. void channel.request('view/close', { view: descriptor.view, generation: descriptor.generation }).catch(() => undefined); void window.showWarningMessage(`The data viewer cannot display "${title}" yet (${descriptor.access} view).`); return; @@ -99,9 +165,12 @@ async function openJdvView( return; } existing.descriptor = descriptor; + existing.channel = channel; existing.title = title; existing.closed = undefined; existing.tree = descriptor.kind === 'tree' ? new TreeAdapter(descriptor, title) : undefined; + existing.stream?.dispose(); + existing.stream = createStream(descriptor, existing.channel); existing.panel.title = title; existing.panel.reveal(existing.panel.viewColumn, true); existing.panel.webview.html = await renderPanel(existing); @@ -125,6 +194,7 @@ async function openJdvView( const state: JdvPanel = { key, panel, sessionId, channel, descriptor, title, tree: descriptor.kind === 'tree' ? new TreeAdapter(descriptor, title) : undefined, + stream: createStream(descriptor, channel), }; panels.set(key, state); // The panel's webview getter throws once onDidDispose fires. @@ -133,9 +203,10 @@ async function openJdvView( if (panels.get(key) === state) { panels.delete(key); } + state.stream?.dispose(); if (!state.closed) { const { view, generation } = state.descriptor; - void channel.request('view/close', { view, generation }).catch(() => undefined); + void state.channel.request('view/close', { view, generation }).catch(() => undefined); } }); webview.onDidReceiveMessage((message: unknown) => { @@ -171,6 +242,15 @@ async function handleMessage(state: JdvPanel, webview: vscode.Webview, message: } }; try { + if (state.stream) { + // Rows already received stay viewable after R closes the view. + if (message.action === 'init') { + reply(true, { columns: gridColumnDefs(state.descriptor), totalRows: 0, sequential: true }); + } else if (message.action === 'page') { + reply(true, await fetchSequentialPage(state, state.stream, message)); + } + return; + } if (state.closed) { throw new Error(state.closed); } @@ -266,3 +346,26 @@ async function fetchGridPage(state: JdvPanel, message: JsonObject): Promise { + const startRow = Math.max(0, Number(message.startRow ?? 0)); + const endRow = Math.max(startRow, Number(message.endRow ?? 0)); + if (!state.closed) { + await stream.ensure(endRow); + } + // After the stream ends, or once R has closed the view, the received rows are final. + const known = stream.done || state.closed !== undefined; + return { + rows: stream.rows(startRow, endRow), + totalRows: known ? stream.rowsSeen : null, + totalUnfiltered: known ? stream.rowsSeen : null, + lastRow: known ? stream.rowsSeen : -1, + loadedRows: stream.rowsSeen, + done: stream.done, + unavailableBefore: stream.unavailableBefore, + notice: state.closed, + }; +} diff --git a/src/session.ts b/src/session.ts index 79131a507..addfedea4 100644 --- a/src/session.ts +++ b/src/session.ts @@ -35,7 +35,7 @@ import type { import { showWebView } from './webViewer'; import { getListViewerScript, listViewerStyle, ListViewNavigation } from './listViewer'; import { getDataViewerScript, getDataViewerStyle, getDataViewerToolbarHtml } from './dataViewer'; -import { showJdvNotification } from './jdvViewer'; +import { disposeJdvHost, getJdvAddress, showJdvNotification } from './jdvViewer'; import { getDataViewerColumnPanelHtml, getDataViewerColumnPanelScript, getDataViewerColumnPanelStyle } from './dataViewerColumnPanel'; import { createViewerSessionContext, formatSessionLabel, getViewerSessionScript, ViewerSessionContext, ViewerSessionSource } from './viewerSession'; import { SessConsentChoice, SessConsentService } from './sessConsent'; @@ -2428,6 +2428,12 @@ export async function getTableHtml( info.title = 'This view retains the full object without a copy. Reference edits may appear here. Reopen from the cell to refresh after edits. Sorting and filtering scan the full data.'; document.querySelector('#viewerControls').append(info); } + if (init.sequential) { + const info = document.createElement('span'); + info.textContent = 'Arrow stream'; + info.title = 'Rows are read from the Arrow stream as you scroll. The total is unknown until the stream ends. Sorting and filtering are not available.'; + document.querySelector('#viewerControls').append(info); + } const bigintFields = prepareViewerColumns(columns); updateViewerRowCount(filteredRows, totalRows); const rowIndexColumn = columns.find(column => column.field === '0'); @@ -2449,10 +2455,17 @@ export async function getTableHtml( sortModel: params.sortModel, filterModel: params.filterModel, }); - filteredRows = result.totalRows; - totalRows = result.totalUnfiltered; + filteredRows = Number.isFinite(result.totalRows) ? result.totalRows : result.loadedRows; + totalRows = Number.isFinite(result.totalUnfiltered) ? result.totalUnfiltered : result.loadedRows; isFiltered = Object.keys(params.filterModel || {}).length > 0; updateViewerRowCount(filteredRows, totalRows); + if (init.sequential && !Number.isFinite(result.totalRows)) { + document.querySelector('#viewerRowCount').textContent = + rowNumberFormatter.format(result.loadedRows) + '+ rows loaded'; + } + if (init.sequential && result.notice) { + document.querySelector('#viewerRowCount').title = result.notice; + } gridApi?.refreshHeader(); updateScrollPosition(); const resolvedLastRow = Number.isFinite(result.totalRows) ? result.totalRows : result.lastRow; @@ -2488,6 +2501,10 @@ export async function getTableHtml( console.log('[dataview] Creating grid with options:', gridOptions); gridApi = window.agGrid.createGrid(gridDiv, gridOptions); initializeViewerToolbar(); + if (init.sequential) { + // Arrow streams are read forward only; jdv offers no sorting or filtering. + document.querySelector('#viewerFilters').disabled = true; + } console.log('[dataview] Grid created successfully'); updateFetchStatusPosition(); } catch (e) { @@ -3193,6 +3210,14 @@ async function handleRequest(message: Record, socket?: IpcSocke try { switch (method) { + case 'jdv/address': { + // The jdv transport for this session; sess sets it only while calling jdv. + const owner = sessions.get(socket?._sessionId ?? ''); + if (!owner) { throw new Error('The jdv transport requires an attached session'); } + const pid = Number(owner.pid); + result = { address: await getJdvAddress(owner.sessionId, Number.isSafeInteger(pid) ? pid : undefined) }; + break; + } case 'rstudioapi/active_editor_context': result = rstudioapi.activeEditorContext(); break; @@ -3270,6 +3295,7 @@ export async function cleanupSession(sessionId: string, closingSocket?: IpcSocke return; } sessions.delete(sessionId); + disposeJdvHost(sessionId); terminalRegistry.releaseSession(session); const disconnected = invalidateSessionConnection(session); if (pipeClient === session.socket) { diff --git a/src/test/node/jdvArrow.test.ts b/src/test/node/jdvArrow.test.ts new file mode 100644 index 000000000..a0c912d4f --- /dev/null +++ b/src/test/node/jdvArrow.test.ts @@ -0,0 +1,193 @@ +import * as assert from 'node:assert/strict'; +import * as net from 'node:net'; +import { tableFromArrays, tableToIPC } from 'apache-arrow'; +import { JdvDescriptor, JdvRequestError, gridColumnDefs, parseDescriptor } from '../../jdvAdapter'; +import { SequentialStream, UNAVAILABLE_TEXT, formatScaled, formatTimestamp } from '../../jdvArrow'; +import { JdvHost, JdvPeer } from '../../jdvTransport'; + +const descriptor: JdvDescriptor = parseDescriptor({ + view: 'v-1', generation: 1, kind: 'grid', access: 'sequential', encoding: 'arrow-ipc', mode: 'sequential', + capabilities: { seek: false, sort: false, filter: false, known_total: false, next: true, children: false }, + nrow: null, ncol: 3, max_page_rows: 2000, + columns: [ + { index: 0, position: 0, name: 'a', type: 'i', ipc_index: 0, unsupported: null, sortable: false, filterable: false }, + { index: 1, position: 1, name: 's', type: 'u', ipc_index: 1, unsupported: null, sortable: false, filterable: false }, + { index: 2, position: 2, name: 'x', type: '+w:2', ipc_index: null, unsupported: 'not supported', sortable: false, filterable: false }, + ], + schema: { format: '+s', children: [{ format: 'i' }, { format: 'u' }, { format: '+w:2' }] }, +}); + +/** A fake producer returning pages of `size` rows out of `total`. */ +function producer(total: number, size: number) { + let cursor = 0; + let calls = 0; + return { + calls: () => calls, + next: (maxRows: number) => { + calls++; + const rows = Math.min(size, maxRows, total - cursor); + const a = Int32Array.from({ length: rows }, (_, i) => cursor + i + 1); + const s = Array.from({ length: rows }, (_, i) => (cursor + i) % 3 === 0 ? null : `s${cursor + i + 1}`); + const body = rows > 0 ? tableToIPC(tableFromArrays({ a, s })) : undefined; + const result = { view: 'v-1', generation: 1, first_row: cursor, rows_fetched: rows, rows_seen: cursor + rows, done: cursor + rows >= total && rows === 0 }; + cursor += rows; + return Promise.resolve({ result, body }); + }, + }; +} + +suite('jdv Arrow views', () => { + test('formats decimals and timestamps exactly', () => { + assert.equal(formatScaled(12345n, 2), '123.45'); + assert.equal(formatScaled(-5n, 3), '-0.005'); + assert.equal(formatScaled(12n, -2), '1200'); + assert.equal(formatTimestamp(1_500n, 1, 'UTC'), '1970-01-01T00:00:01.500Z'); + assert.equal(formatTimestamp(-1n, 3, null), '1969-12-31T23:59:59.999999999'); + assert.equal(formatTimestamp(0n, 0, 'Asia/Tokyo'), '1970-01-01T09:00:00+09:00 [Asia/Tokyo]'); + }); + + test('builds unsortable, unfilterable columns', () => { + const columns = gridColumnDefs(descriptor); + assert.deepEqual(columns.map(column => [column.field, column.type, column.sortable, column.filter]), [ + ['0', 'textColumn', false, false], ['1', 'numericColumn', false, false], + ['2', 'textColumn', false, false], ['3', 'textColumn', false, false], + ]); + assert.match(columns[3].headerTooltip ?? '', /unsupported: not supported/); + }); + + test('reads the stream forward and serves blocks from received pages', async () => { + const source = producer(25, 10); + const stream = new SequentialStream(descriptor, source.next, 10); + await stream.ensure(5); + assert.equal(stream.rowsSeen, 10); + assert.equal(source.calls(), 1); + assert.deepEqual(stream.rows(0, 2), [ + { '0': '1', '1': '1', '2': null, '3': '' }, + { '0': '2', '1': '2', '2': 's2', '3': '' }, + ]); + // Concurrent requests share one stream/next at a time. + await Promise.all([stream.ensure(15), stream.ensure(20)]); + assert.equal(stream.rowsSeen, 20); + await stream.ensure(Infinity); + assert.equal(stream.done, true); + assert.equal(stream.rowsSeen, 25); + assert.deepEqual(stream.rows(23, 100).map(row => row['1']), ['24', '25']); + stream.dispose(); + }); + + test('reports evicted rows as unavailable', async () => { + const source = producer(40, 10); + // Room for about two pages. + const stream = new SequentialStream(descriptor, source.next, 10, 2 * (1024 + 800)); + await stream.ensure(40); + assert.ok(stream.unavailableBefore >= 10); + assert.equal(stream.rows(0, 1)[0]['1'], UNAVAILABLE_TEXT); + assert.equal(stream.rows(39, 40)[0]['1'], '40'); + stream.dispose(); + }); + + test('rejects inconsistent page metadata', async () => { + const stream = new SequentialStream(descriptor, () => Promise.resolve({ + result: { first_row: 0, rows_fetched: 5, rows_seen: 4, done: false }, + }), 10); + await assert.rejects(stream.ensure(1), (e: unknown) => e instanceof JdvRequestError); + stream.dispose(); + }); +}); + +/** Connect to a host as R would, returning helpers to drive the protocol. */ +async function fakeR(address: string) { + const socket = net.connect(address); + await new Promise(resolve => socket.once('connect', () => resolve())); + let buffer = ''; + const lines: Record[] = []; + let waiter: (() => void) | undefined; + socket.on('data', chunk => { + buffer += chunk.toString('utf8'); + let newline; + while ((newline = buffer.indexOf('\n')) >= 0) { + lines.push(JSON.parse(buffer.slice(0, newline)) as Record); + buffer = buffer.slice(newline + 1); + waiter?.(); + } + }); + let frames = 0; + const next = async (predicate: (line: Record) => boolean) => { + for (;;) { + const index = lines.findIndex(predicate); + if (index >= 0) { + frames += index + 1; + const [line] = lines.splice(0, index + 1).slice(-1); + return line; + } + await new Promise(resolve => { waiter = resolve; }); + } + }; + const send = (value: unknown, body?: Uint8Array) => { + socket.write(JSON.stringify(value) + '\n'); + if (body) { socket.write(body); } + }; + return { socket, next, send, ack: () => send({ method: 'ack', params: { frames } }) }; +} + +suite('jdv transport', () => { + test('exchanges requests, wakes and Arrow bodies with R', async () => { + const opened: string[] = []; + let peer: JdvPeer | undefined; + const host = await JdvHost.create(4321, { + viewOpen: (connection, params) => { peer = connection; opened.push(String(params.view)); }, + viewClosed: () => undefined, + disconnected: () => undefined, + }); + try { + const r = await fakeR(host.address); + r.send({ method: 'hello', params: { protocol: 1, session: 'r-4321-1', pid: 4321, limits: { max_receive_frames: 4, max_receive_bytes: 65536 } } }); + r.send({ method: 'view/open', params: { view: 'v-1', generation: 1 } }); + while (!peer) { await new Promise(resolve => setTimeout(resolve, 5)); } + assert.deepEqual(opened, ['v-1']); + + const response = peer.requestWithBody('stream/next', { view: 'v-1', generation: 1, max_rows: 10 }); + const request = await r.next(line => line.method === 'stream/next'); + // Wakes repeat while the request is pending. + await r.next(line => line.method === 'wake'); + r.ack(); + const body = Uint8Array.from([1, 2, 3, 10, 4]); + r.send({ + id: request.id, warnings: [], + result: { view: 'v-1', generation: 1, first_row: 0, rows_fetched: 1, rows_seen: 1, done: false, transfer: { encoding: 'arrow-ipc', byte_length: body.length } }, + }, body); + r.send({ id: 'unknown', result: {}, warnings: [] }); + const { result, body: received } = await response; + assert.deepEqual(Array.from(received!), [1, 2, 3, 10, 4]); + assert.equal((result as { rows_seen: number }).rows_seen, 1); + + const failed = peer.request('table/rows', { view: 'v-1', generation: 1 }); + const second = await r.next(line => line.method === 'table/rows'); + r.send({ id: second.id, error: { code: 'unsupported', message: 'no' }, warnings: [] }); + await assert.rejects(failed, (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported'); + + const pending = peer.request('view/close', { view: 'v-1', generation: 1 }); + r.socket.destroy(); + await assert.rejects(pending, (e: unknown) => e instanceof JdvRequestError && e.code === 'view_not_found'); + } finally { + host.dispose(); + } + }); + + test('rejects a connection from another process', async () => { + let reason = ''; + const host = await JdvHost.create(1, { + viewOpen: () => undefined, + viewClosed: () => undefined, + disconnected: (_peer, why) => { reason = why; }, + }); + try { + const r = await fakeR(host.address); + r.send({ method: 'hello', params: { protocol: 1, pid: 2, limits: {} } }); + await new Promise(resolve => r.socket.once('close', () => resolve())); + assert.match(reason, /does not belong/); + } finally { + host.dispose(); + } + }); +}); From 06c3891745f367756abd50d2c0c2381da34f71b8 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 17 Oct 2026 07:28:00 +0000 Subject: [PATCH 03/38] fix: connect jdv transport tests to the Node-form socket path --- src/jdvTransport.ts | 5 +++-- src/test/node/jdvArrow.test.ts | 13 ++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/jdvTransport.ts b/src/jdvTransport.ts index 410b0b9ac..d2476277d 100644 --- a/src/jdvTransport.ts +++ b/src/jdvTransport.ts @@ -315,9 +315,10 @@ export class JdvHost { private constructor( private readonly server: net.Server, - /** Address in jdv's syntax, for `options(jdv.address = ...)`. */ + /** Address in jdv's syntax (`npipe:////./pipe/...` on Windows), for R. */ readonly address: string, - private readonly socketPath: string, + /** The same endpoint in the form Node's `net` module accepts. */ + readonly socketPath: string, private readonly expectedPid: number | undefined, private readonly events: JdvHostEvents, ) { diff --git a/src/test/node/jdvArrow.test.ts b/src/test/node/jdvArrow.test.ts index a0c912d4f..45ed04d60 100644 --- a/src/test/node/jdvArrow.test.ts +++ b/src/test/node/jdvArrow.test.ts @@ -96,9 +96,12 @@ suite('jdv Arrow views', () => { }); /** Connect to a host as R would, returning helpers to drive the protocol. */ -async function fakeR(address: string) { - const socket = net.connect(address); - await new Promise(resolve => socket.once('connect', () => resolve())); +async function fakeR(socketPath: string) { + const socket = net.connect(socketPath); + await new Promise((resolve, reject) => { + socket.once('connect', () => resolve()); + socket.once('error', reject); + }); let buffer = ''; const lines: Record[] = []; let waiter: (() => void) | undefined; @@ -140,7 +143,7 @@ suite('jdv transport', () => { disconnected: () => undefined, }); try { - const r = await fakeR(host.address); + const r = await fakeR(host.socketPath); r.send({ method: 'hello', params: { protocol: 1, session: 'r-4321-1', pid: 4321, limits: { max_receive_frames: 4, max_receive_bytes: 65536 } } }); r.send({ method: 'view/open', params: { view: 'v-1', generation: 1 } }); while (!peer) { await new Promise(resolve => setTimeout(resolve, 5)); } @@ -182,7 +185,7 @@ suite('jdv transport', () => { disconnected: (_peer, why) => { reason = why; }, }); try { - const r = await fakeR(host.address); + const r = await fakeR(host.socketPath); r.send({ method: 'hello', params: { protocol: 1, pid: 2, limits: {} } }); await new Promise(resolve => r.socket.once('close', () => resolve())); assert.match(reason, /does not belong/); From a6555a8bb175ff4181da81aa7d51c063bc79866a Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 17 Oct 2026 07:37:48 +0000 Subject: [PATCH 04/38] refactor: connect jdv to the session transport for every data viewer view jdv now treats an embedded session and the transport as mutually exclusive, so View() connects jdv to the per-session listener once and registers all views there. The jdv_rpc relay is removed. --- sess/R/handlers.R | 122 ++++++++++++------------ sess/R/hooks.R | 5 +- sess/R/runtime.R | 1 + sess/R/server.R | 3 +- sess/inst/tinytest/test-jdv.R | 172 +++++++++++++++++++++++----------- 5 files changed, 182 insertions(+), 121 deletions(-) diff --git a/sess/R/handlers.R b/sess/R/handlers.R index aeb6b9b98..844845d78 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1294,12 +1294,11 @@ handle_dataview_dispose <- function(params) { # jdv data viewer backend (experimental) ----------------------------------- # -# When the jdv package is available, View() registers data frames, matrices and -# lists with jdv instead of the sess data viewer. vscode-R relays the jdv -# protocol through the `jdv_rpc` request. Arrow data is viewed as a forward-only -# Arrow stream over jdv's own transport, whose address VS Code provides per -# session. Set `options(sess.dataviewer = "sess")` to keep using the sess data -# viewer. +# When the jdv package is available, View() registers data frames, matrices, +# lists and Arrow data with jdv instead of the sess data viewer. jdv talks to the +# viewer over its own local transport, whose address VS Code provides per +# session; sess only registers the views. Set `options(sess.dataviewer = "sess")` +# to keep using the sess data viewer. .jdv_arrow_classes <- c("ArrowTabular", "RecordBatchReader", "nanoarrow_array_stream", "polars_data_frame") @@ -1308,67 +1307,70 @@ handle_dataview_dispose <- function(params) { inherits(x, .jdv_arrow_classes) } -.dataview_use_jdv <- function(x) { - backend <- getOption("sess.dataviewer", "auto") - if (identical(backend, "sess")) { - return(FALSE) - } - # Arrow data needs the jdv transport; see .jdv_show_arrow(). - if (.jdv_is_arrow(x)) { - return(FALSE) - } - requireNamespace("jdv", quietly = TRUE) -} - -# Show Arrow data with jdv without converting it to a data frame. Returns FALSE -# when jdv cannot be used, so the caller falls back to the sess data viewer. -.jdv_show_arrow <- function(x, title_key) { +# Make sure jdv has a persistent connection to this session's viewer host. +# Returns FALSE, leaving the user's own jdv state untouched, when the sess data +# viewer should be used instead: jdv is missing or opted out, VS Code offers no +# transport, or jdv is already connected elsewhere or owned by an embedding host. +.jdv_connect <- function() { if (identical(getOption("sess.dataviewer", "auto"), "sess") || - !requireNamespace("jdv", quietly = TRUE) || - !requireNamespace("nanoarrow", quietly = TRUE)) { - return(FALSE) - } - address <- tryCatch(request_client("jdv/address")$address, error = function(e) NULL) - if (!is.character(address) || length(address) != 1L || !nzchar(address)) { + !requireNamespace("jdv", quietly = TRUE)) { return(FALSE) } - # jdv reads the address when it opens its connection; do not change the - # option for other jdv hosts the user may run. - op <- options(jdv.address = address) - on.exit(options(op), add = TRUE) - jdv::jdv_view(x, title = if (nzchar(title_key)) title_key else NULL) - TRUE + tryCatch({ + connection <- jdv::jdv_connection() + if (identical(connection$state, "connected")) { + return(is.character(.sess_env$jdv_address) && + identical(connection$address, .sess_env$jdv_address)) + } + if (!identical(connection$state, "disconnected")) { + return(FALSE) + } + address <- request_client("jdv/address")$address + if (!is.character(address) || length(address) != 1L || is.na(address) || + !nzchar(address)) { + return(FALSE) + } + # An explicit address leaves discovery.json and the `jdv.address` option of + # other jdv hosts the user may run alone. + jdv::jdv_connect(address) + .sess_env$jdv_address <- address + TRUE + }, error = function(e) FALSE) } -.jdv_show_view <- function(x, title, title_key) { - # The title is jdv's display slot: View() with the same title replaces the - # view and increments its generation. - registration <- jdv::jdv_register( - x, - title = if (nzchar(title_key)) title_key else NULL, - mode = "paged" - ) - notify_client("dataview", list( - title = title, - source = "jdv", - type = "json", - open = registration$open, - notifications = I(as.character(registration$notifications)) - )) - registration$id +# Close the connection that .jdv_connect() opened, with all of its views. +.jdv_release <- function() { + if (is.null(.sess_env$jdv_address)) { + return(invisible(NULL)) + } + address <- .sess_env$jdv_address + .sess_env$jdv_address <- NULL + if (isNamespaceLoaded("jdv")) { + try({ + if (identical(jdv::jdv_connection()$address, address)) { + jdv::jdv_disconnect() + } + }, silent = TRUE) + } + invisible(NULL) } -handle_jdv_rpc <- function(request) { - if (!is.character(request) || length(request) != 1L || is.na(request)) { - stop("Invalid jdv request") +# Show `x` with jdv. Returns FALSE when jdv cannot be used, so the caller falls +# back to the sess data viewer. +# +# The title is jdv's display slot: View() with the same title replaces the view +# and increments its generation. Grids are paged so that sorting and filtering +# stay in R. Views are not bound to a variable (`binding`): View() receives the +# value, not its name, and tracking needs R 4.6. +.jdv_show <- function(x, title_key) { + if (.jdv_is_arrow(x) && !requireNamespace("nanoarrow", quietly = TRUE)) { + return(FALSE) } - if (!requireNamespace("jdv", quietly = TRUE)) { - stop("The jdv package is not available") + if (!.jdv_connect()) { + return(FALSE) } - # Child views opened from lists use jdv's automatic mode; keep them paged so - # sorting and filtering stay in R. - op <- options(jdv.snapshot_max_cells = 0) - on.exit(options(op), add = TRUE) - response <- jdv::jdv_rpc(request) - if (is.null(response)) NULL else as.character(response) + tryCatch({ + jdv::jdv_register(x, title = if (nzchar(title_key)) title_key else NULL, mode = "paged") + TRUE + }, error = function(e) FALSE) } diff --git a/sess/R/hooks.R b/sess/R/hooks.R index 9f74f1fbc..aacf18c07 100644 --- a/sess/R/hooks.R +++ b/sess/R/hooks.R @@ -88,12 +88,9 @@ runtime_start <- function(use_rstudioapi = TRUE, "object" } title_key <- paste(as.character(title), collapse = "\n") - if (.jdv_is_arrow(x) && .jdv_show_arrow(x, title_key)) { + if ((view_type %in% c("table", "list") || .jdv_is_arrow(x)) && .jdv_show(x, title_key)) { return(invisible(NULL)) } - if (view_type %in% c("table", "list") && .dataview_use_jdv(x)) { - return(invisible(.jdv_show_view(x, title, title_key))) - } owner <- owner %||% title_key registry_key <- paste0(view_type, ":", owner) dataview_registry <- .sess_env$dataview_registry diff --git a/sess/R/runtime.R b/sess/R/runtime.R index c62eb22fd..117abae94 100644 --- a/sess/R/runtime.R +++ b/sess/R/runtime.R @@ -16,6 +16,7 @@ } .runtime_clear_viewer_state <- function() { + .jdv_release() # Open dataviews and title-to-id mappings belong to one runtime connection. # Keep an empty registry available, but never restore data from a prior run. .sess_env$dataviews <- list() diff --git a/sess/R/server.R b/sess/R/server.R index b919b636c..0fb37b410 100644 --- a/sess/R/server.R +++ b/sess/R/server.R @@ -488,8 +488,7 @@ dispatch_message <- function(line) { "listview_view" = function(p) handle_listview_view(p$view_id, p$index, p$path), "dataview_init" = function(p) handle_dataview_init(p), "dataview_page" = function(p) handle_dataview_page(p), - "dataview_dispose" = function(p) handle_dataview_dispose(p), - "jdv_rpc" = function(p) handle_jdv_rpc(p$request) + "dataview_dispose" = function(p) handle_dataview_dispose(p) ) if (payload$method %in% names(handlers)) { diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R index e6f7d3260..8be8903a1 100644 --- a/sess/inst/tinytest/test-jdv.R +++ b/sess/inst/tinytest/test-jdv.R @@ -1,18 +1,25 @@ -# View() uses jdv when it is installed and relays the jdv protocol through jdv_rpc. +# View() uses jdv over a persistent connection to the viewer host when jdv is installed. if (!requireNamespace("jdv", quietly = TRUE)) { exit_file("jdv is not installed") } local({ + ns <- asNamespace("sess") runtime <- sess:::.sess_env previous_con <- runtime$con previous_plot_path <- runtime$latest_plot_path + original_request_client <- ns$request_client pipe <- processx::conn_create_pipepair() + host <- NULL on.exit({ sess:::runtime_stop() + jdv::jdv_disconnect() + unlockBinding("request_client", ns) + assign("request_client", original_request_client, envir = ns) + lockBinding("request_client", ns) runtime$con <- previous_con runtime$latest_plot_path <- previous_plot_path - jdv::jdv_close() + if (!is.null(host)) host$kill() lapply(pipe, close) }, add = TRUE) runtime$con <- pipe[[2L]] @@ -21,79 +28,134 @@ local({ options(sess.dataviewer = NULL) sess:::runtime_start(use_rstudioapi = FALSE, plot_backend = "standard") + # VS Code answers `jdv/address` with the address of its per-session listener. + stub_address <- function(address) { + unlockBinding("request_client", ns) + assign("request_client", function(action, args = list()) { + if (is.null(address)) stop("no jdv transport") else list(address = address) + }, envir = ns) + lockBinding("request_client", ns) + } read_messages <- function() { messages <- strsplit(processx::conn_read_chars(pipe[[1L]]), "\n", fixed = TRUE)[[1L]] lapply(messages[nzchar(messages)], jsonlite::fromJSON, simplifyVector = FALSE) } - last_view <- function() { - views <- Filter(function(message) identical(message$method, "dataview"), read_messages()) - expect_length(views, 1L) - views[[1L]]$params - } - jdv_request <- function(method, params) { - request <- as.character(jsonlite::toJSON( - list(id = "jdv-test", method = method, params = params), auto_unbox = TRUE - )) - sess:::dispatch_message(as.character(jsonlite::toJSON( - list(jsonrpc = "2.0", id = "sess-test", method = "jdv_rpc", params = list(request = request)), - auto_unbox = TRUE - ))) - replies <- Filter(function(message) identical(message$id, "sess-test"), read_messages()) - expect_length(replies, 1L) - expect_null(replies[[1L]]$error) - jsonlite::fromJSON(replies[[1L]]$result, simplifyVector = FALSE) + used_sess_viewer <- function() { + any(vapply(read_messages(), function(message) identical(message$method, "dataview"), logical(1L))) } sample <- data.frame(x = c(2, 1, NA), y = c("b", "a", "c")) + + # The opt-out keeps the sess data viewer. + options(sess.dataviewer = "sess") + stub_address("tcp://127.0.0.1:1") + expect_false(sess:::.jdv_connect()) utils::View(sample) - params <- last_view() - expect_identical(params$source, "jdv") - expect_identical(params$title, "sample") - open <- jsonlite::fromJSON(params$open, simplifyVector = FALSE)$params + expect_true(used_sess_viewer()) + options(sess.dataviewer = NULL) + + # Without a transport, or with an unreachable one, View() falls back and leaves jdv alone. + stub_address(NULL) + expect_false(sess:::.jdv_connect()) + stub_address("tcp://127.0.0.1:1") + expect_false(sess:::.jdv_connect()) + expect_identical(jdv::jdv_connection()$state, "disconnected") + expect_null(runtime$jdv_address) + utils::View(sample) + expect_true(used_sess_viewer()) + + # An embedded session of another jdv user is never taken over. + jdv::jdv_register(sample) + expect_identical(jdv::jdv_connection()$state, "embedded") + stub_address("tcp://127.0.0.1:1") + expect_false(sess:::.jdv_connect()) + utils::View(sample) + expect_true(used_sess_viewer()) + expect_identical(jdv::jdv_connection()$state, "embedded") + jdv::jdv_disconnect() + + # A minimal viewer host: it records every line that jdv sends. + received <- tempfile(fileext = ".jsonl") + port <- NULL + for (candidate in sample(20000:60000, 20L)) { + host <- processx::process$new(file.path(R.home("bin"), "Rscript"), c("--vanilla", "-e", " + args <- commandArgs(TRUE) + server <- serverSocket(as.integer(args[1L])) + connection <- socketAccept(server, blocking = TRUE, open = 'r+') + repeat { + line <- readLines(connection, n = 1L) + if (!length(line)) break + cat(line, '\\n', file = args[2L], append = TRUE, sep = '') + } + ", candidate, received), stderr = "|") + Sys.sleep(1) + if (host$is_alive()) { + port <- candidate + break + } + } + if (is.null(port)) exit_file("no free TCP port for the viewer host") + address <- sprintf("tcp://127.0.0.1:%d", port) + stub_address(address) + host_messages <- function(n, timeout = 10) { + deadline <- Sys.time() + timeout + repeat { + lines <- if (file.exists(received)) readLines(received, warn = FALSE) else character() + if (length(lines) >= n || Sys.time() > deadline) break + Sys.sleep(0.05) + } + lapply(lines, jsonlite::fromJSON, simplifyVector = FALSE) + } + opens <- function() { + Filter(function(message) identical(message$method, "view/open"), host_messages(0L)) + } + + utils::View(sample) + expect_false(used_sess_viewer()) + messages <- host_messages(2L) + expect_identical(messages[[1L]]$method, "hello") + expect_identical(messages[[1L]]$params$pid, Sys.getpid()) + expect_identical(runtime$jdv_address, address) + connection <- jdv::jdv_connection() + expect_identical(connection$state, "connected") + expect_true(connection$persistent) + expect_identical(connection$address, address) + + # Data frames are paged and titled by their expression. + open <- opens()[[1L]]$params expect_identical(open$kind, "grid") expect_identical(open$access, "random") + expect_identical(open$mode, "paged") + expect_identical(open$title, "sample") expect_identical(open$generation, 1L) - # Rows are computed in R with jdv sorting. - response <- jdv_request("table/rows", list( - view = open$view, generation = 1L, offset = 0L, limit = 10L, - sort = list(list(col = 0L, desc = FALSE)) - )) - expect_identical(unlist(response$result$data$row_index), c(2L, 1L, 3L)) - # The same title replaces the view and increments its generation. utils::View(sample) - reopened <- jsonlite::fromJSON(last_view()$open, simplifyVector = FALSE)$params + host_messages(3L) + reopened <- opens()[[2L]]$params expect_identical(reopened$view, open$view) expect_identical(reopened$generation, 2L) - stale <- jdv_request("table/rows", list(view = open$view, generation = 1L)) - expect_identical(stale$error$code, "stale_generation") - # Lists open as trees; tables inside them open as paged child views. + # Lists open as trees on the same connection. nested <- list(a = 1:3, df = sample) utils::View(nested) - tree <- jsonlite::fromJSON(last_view()$open, simplifyVector = FALSE)$params + host_messages(4L) + tree <- opens()[[3L]]$params expect_identical(tree$kind, "tree") - children <- jdv_request("object/children", list(view = tree$view, generation = 1L, path = list())) - expect_identical(children$result$children[[2L]]$label, "df") - child <- jdv_request("object/open", list( - view = tree$view, generation = 1L, path = children$result$children[[2L]]$path - )) - expect_identical(child$result$kind, "grid") - expect_identical(child$result$access, "random") - expect_identical(child$result$parent, tree$view) + expect_identical(tree$title, "nested") + expect_false(used_sess_viewer()) - closed <- jdv_request("view/close", list(view = open$view, generation = 2L)) - expect_true(closed$result$closed) + # Arrow data is a forward-only stream on the same connection. + if (requireNamespace("nanoarrow", quietly = TRUE)) { + utils::View(nanoarrow::as_nanoarrow_array_stream(sample), "stream") + host_messages(5L) + stream <- opens()[[4L]]$params + expect_identical(stream$access, "sequential") + expect_identical(stream$title, "stream") + } - # Arrow data and the explicit opt-out keep the sess data viewer. - options(sess.dataviewer = "sess") - utils::View(sample) - expect_identical(last_view()$source, "table") - arrow_like <- structure(list(), class = "polars_data_frame") - expect_true(sess:::.jdv_is_arrow(arrow_like)) - expect_false(sess:::.jdv_show_arrow(arrow_like, "arrow")) - options(sess.dataviewer = NULL) - # Arrow data never goes through the jdv_rpc relay, which cannot carry Arrow IPC. - expect_false(sess:::.dataview_use_jdv(arrow_like)) + # Releasing the runtime closes the connection that sess opened. + sess:::.jdv_release() + expect_identical(jdv::jdv_connection()$state, "disconnected") + expect_null(runtime$jdv_address) }) From f19ad0d3e837a2f467224e5a851a9cde69057a44 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 17 Oct 2026 07:37:48 +0000 Subject: [PATCH 05/38] refactor: serve all jdv views over the session transport Grids, trees and child views now arrive as view/open notifications on the socket. Tree children use atomic leaves and hide attributes, and child grids are opened in paged mode. --- src/interactive/manager.ts | 4 -- src/jdvAdapter.ts | 90 ++++++++------------------ src/jdvTransport.ts | 4 +- src/jdvViewer.ts | 104 +++++++++++-------------------- src/session.ts | 8 +-- src/test/node/jdvAdapter.test.ts | 88 +++++++++++++------------- 6 files changed, 112 insertions(+), 186 deletions(-) diff --git a/src/interactive/manager.ts b/src/interactive/manager.ts index 3c3b43f74..499c820d9 100644 --- a/src/interactive/manager.ts +++ b/src/interactive/manager.ts @@ -17,7 +17,6 @@ import { setInteractiveExecutor } from './executionTarget'; import * as session from '../session'; import * as util from '../util'; import type { ListViewNavigation } from '../listViewer'; -import { showJdvNotification } from '../jdvViewer'; import { escapeXml } from './plotSvg'; import { ensureWorkspaceViewer } from '../extension'; import { queryTablePage } from './tableQuery'; @@ -980,9 +979,6 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro void this.prompt(view, event.data); } else if (event.type === 'clientRequest' && view.client.control) { void this.clientRequest(view, event.data); - } else if (event.type === 'viewer' && event.data.method === 'dataview' && - object(event.data.params ?? {}).source === 'jdv') { - await showJdvNotification(object(event.data.params ?? {}), view.target.sessionId); } else if (event.type === 'viewer' && event.data.method === 'dataview') { const params = object(event.data.params ?? {}); const title = typeof params.title === 'string' ? params.title diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index 78c05b6f3..7dd7bd76b 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -12,6 +12,7 @@ export interface JdvCapabilities { known_total: boolean; next: boolean; children: boolean; + refresh?: boolean; } export interface JdvColumnSchema { @@ -40,7 +41,9 @@ export interface JdvDescriptor { ncol?: number; columns?: JdvColumnSchema[]; path?: JdvPathStep[]; + /** Tree views: `class()` and `typeof()` of the root object. */ class?: string[]; + type?: string; parent?: string; } @@ -48,22 +51,12 @@ export type JdvPathStep = | { type: 'index'; index: number } | { type: 'binding' | 'slot' | 'attr'; name: string }; -export interface JdvError { - code: string; - message: string; -} - export class JdvRequestError extends Error { constructor(readonly code: string, message: string) { super(message); } } -/** A channel that sends one jdv request and returns its `result`. */ -export interface JdvChannel { - request(method: string, params: JsonObject): Promise; -} - function isObject(value: unknown): value is JsonObject { return typeof value === 'object' && value !== null && !Array.isArray(value); } @@ -74,52 +67,16 @@ function scalarText(value: unknown): string { typeof value === 'boolean' ? String(value) : ''; } -/** Parse a `view/open` message or a direct descriptor (as returned by `object/open`). */ +/** Validate the params of a `view/open` notification. */ export function parseDescriptor(value: unknown): JdvDescriptor { - const message = typeof value === 'string' ? JSON.parse(value) as unknown : value; - const params = isObject(message) && message.method === 'view/open' ? message.params : message; - if (!isObject(params) || typeof params.view !== 'string' || typeof params.generation !== 'number' || - (params.kind !== 'grid' && params.kind !== 'tree') || !isObject(params.capabilities)) { + if (!isObject(value) || typeof value.view !== 'string' || typeof value.generation !== 'number' || + (value.kind !== 'grid' && value.kind !== 'tree') || !isObject(value.capabilities)) { throw new Error('Invalid jdv view descriptor'); } - if (params.kind === 'grid' && !Array.isArray(params.columns)) { + if (value.kind === 'grid' && !Array.isArray(value.columns)) { throw new Error('Invalid jdv grid descriptor'); } - return params as unknown as JdvDescriptor; -} - -/** Return the view ids and generations announced by `view/closed` notifications. */ -export function parseClosedNotifications(values: unknown): { view: string; generation: number }[] { - const list: unknown[] = Array.isArray(values) ? values : typeof values === 'string' ? [values] : []; - const closed: { view: string; generation: number }[] = []; - for (const value of list) { - try { - const message = typeof value === 'string' ? JSON.parse(value) as unknown : value; - if (isObject(message) && message.method === 'view/closed' && isObject(message.params) && - typeof message.params.view === 'string' && typeof message.params.generation === 'number') { - closed.push({ view: message.params.view, generation: message.params.generation }); - } - } catch { - // Ignore malformed notifications; they cannot identify a view. - } - } - return closed; -} - -/** Unwrap a jdv response envelope, throwing its error code and message. */ -export function unwrapResponse(response: unknown): unknown { - const message = typeof response === 'string' ? JSON.parse(response) as unknown : response; - if (!isObject(message)) { - throw new Error('Invalid jdv response'); - } - if (isObject(message.error)) { - const error = message.error as Partial; - throw new JdvRequestError(String(error.code ?? 'unknown'), String(error.message ?? 'jdv request failed')); - } - if (!('result' in message)) { - throw new Error('Invalid jdv response'); - } - return message.result; + return value as unknown as JdvDescriptor; } // --------------------------------------------------------------------------- @@ -579,6 +536,7 @@ interface TreeNode { label: string; title: string; class: string[]; + type?: string; hasChildren: boolean; viewable: boolean; } @@ -587,6 +545,7 @@ interface TreeChild { index: number; label: string; class?: string[]; + type?: string; summary?: string; has_children?: boolean; viewable?: boolean; @@ -596,16 +555,12 @@ interface TreeChild { export const TREE_PAGE_SIZE = 200; -const ATOMIC_CLASSES = new Set([ - 'logical', 'integer', 'numeric', 'double', 'character', 'complex', 'raw', - 'factor', 'Date', 'POSIXct', 'difftime', 'integer64', -]); const TABLE_CLASSES = new Set(['data.frame', 'matrix']); +const ATOMIC_TYPES = new Set(['logical', 'integer', 'double', 'complex', 'character', 'raw']); /** Atomic vectors are shown as value rows, as in the previous List Viewer. */ -export function isVectorClass(classes: string[]): boolean { - return classes.length > 0 && classes.every(cls => ATOMIC_CLASSES.has(cls) || cls === 'array' || cls === 'ordered') && - !classes.includes('matrix'); +export function isVectorType(type: string | undefined, classes: string[]): boolean { + return type !== undefined && ATOMIC_TYPES.has(type) && !classes.includes('matrix'); } export function isTableClass(classes: string[]): boolean { @@ -631,7 +586,7 @@ export class TreeAdapter { constructor(public descriptor: JdvDescriptor, private readonly title: string) { this.nodes.set('[]', { - path: [], label: title, title, class: descriptor.class ?? [], hasChildren: true, viewable: true, + path: [], label: title, title, class: descriptor.class ?? [], type: descriptor.type, hasChildren: true, viewable: true, }); } @@ -650,7 +605,7 @@ export class TreeAdapter { const prefix = path.slice(0, depth); breadcrumbs.push({ label: this.node(prefix).label, path: prefix }); } - return { title: node.title, path, breadcrumbs, vector: isVectorClass(node.class) }; + return { title: node.title, path, breadcrumbs, vector: isVectorType(node.type, node.class) }; } childrenParams(path: number[], start: number): JsonObject { @@ -661,6 +616,8 @@ export class TreeAdapter { path: node.path, offset: Math.max(0, start - 1), limit: TREE_PAGE_SIZE, + // Like the previous List Viewer, show elements without attributes such as `names`. + attributes: false, }; } @@ -671,7 +628,7 @@ export class TreeAdapter { throw new Error('Invalid jdv object/children result'); } const parent = this.node(path); - const vector = isVectorClass(parent.class); + const vector = isVectorType(parent.type, parent.class); const children = (result.children as TreeChild[]).map(child => { const addressable = Array.isArray(child.path) && child.viewable !== false; if (addressable) { @@ -680,18 +637,20 @@ export class TreeAdapter { label: child.label, title: childTitle(parent, child), class: child.class ?? [], + type: child.type, hasChildren: child.has_children === true, viewable: child.viewable !== false, }); } + const hasChildren = child.has_children === true; const summary = child.unavailable ? `(${child.unavailable})` : child.summary ?? ''; return { label: child.label, str: summary, index: child.index, - // Vector elements are terminal rows in the List Viewer. - viewable: addressable && !vector, - has_children: !vector && child.has_children === true, + // Vector elements are terminal rows; other leaves have nothing to open. + viewable: addressable && !vector && (hasChildren || isTableClass(child.class ?? [])), + has_children: hasChildren, }; }); const end = result.offset + children.length; @@ -710,7 +669,8 @@ export class TreeAdapter { return { kind: 'object', title: node.title, - params: { view: this.descriptor.view, generation: this.descriptor.generation, path: node.path }, + // Paged keeps sorting and filtering in R, as for top-level grids. + params: { view: this.descriptor.view, generation: this.descriptor.generation, path: node.path, mode: 'paged' }, }; } return { kind: 'navigate', navigation: this.navigation(childPath) }; diff --git a/src/jdvTransport.ts b/src/jdvTransport.ts index d2476277d..d0cfd25cf 100644 --- a/src/jdvTransport.ts +++ b/src/jdvTransport.ts @@ -9,7 +9,7 @@ import * as fs from 'fs'; import * as net from 'net'; import * as os from 'os'; import * as path from 'path'; -import { JdvChannel, JdvRequestError, JsonObject } from './jdvAdapter'; +import { JdvRequestError, JsonObject } from './jdvAdapter'; const PROTOCOL_MAJOR = 1; const MAX_LIVE_REQUESTS = 32; @@ -52,7 +52,7 @@ function positiveInteger(value: unknown, fallback: number): number { } /** One R connection. Implements the host obligations of the jdv transport. */ -export class JdvPeer implements JdvChannel { +export class JdvPeer { private buffer: Buffer = Buffer.alloc(0); private body?: { header: JsonObject; length: number; timer: NodeJS.Timeout }; private limits?: RLimits; diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index c9a7cab39..cc5d63a8d 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -2,13 +2,15 @@ // Data Viewer panels backed by jdv. The webviews are the existing AG Grid and List // Viewer documents; this module translates their requests to the jdv protocol. +// R connects to a listener owned by the session (see JdvHost) and every view, +// including child views opened from trees, arrives there as a `view/open`. import * as path from 'path'; import * as vscode from 'vscode'; import { Uri, ViewColumn, window } from 'vscode'; import { - GridPage, JdvChannel, JdvDescriptor, JdvRequestError, JsonObject, TreeAdapter, gridColumnDefs, - gridPageFromRows, parseClosedNotifications, parseDescriptor, tableRowsParams, unwrapResponse, + GridPage, JdvDescriptor, JdvRequestError, JsonObject, TreeAdapter, gridColumnDefs, + gridPageFromRows, parseDescriptor, tableRowsParams, } from './jdvAdapter'; import { SequentialStream } from './jdvArrow'; import { JdvHost, JdvPeer } from './jdvTransport'; @@ -20,25 +22,23 @@ interface JdvPanel { readonly key: string; readonly panel: vscode.WebviewPanel; readonly sessionId: string | null; - /** A title can move between the sess relay and the jdv transport. */ - channel: JdvChannel; + /** The R connection that owns the view. */ + peer: JdvPeer; descriptor: JdvDescriptor; title: string; tree?: TreeAdapter; /** Received Arrow pages of a sequential view. */ stream?: SequentialStream; + /** Title for the child view that a pending `object/open` is about to open. */ + childTitle?: string; /** Set when R closed or evicted this view generation. */ closed?: string; } const panels = new Map(); const hosts = new Map>(); -let requestSequence = 0; -/** - * The jdv transport address for a session, creating its listener on first use. - * Sequential Arrow views need this transport; `jdv_rpc()` cannot carry Arrow IPC. - */ +/** The jdv transport address for a session, creating its listener on first use. */ export function getJdvAddress(sessionId: string, pid: number | undefined): Promise { let host = hosts.get(sessionId); if (!host) { @@ -53,7 +53,7 @@ export function getJdvAddress(sessionId: string, pid: number | undefined): Promi } const viewer = viewerColumn(); if (viewer) { - void openJdvView(descriptor, descriptor.title ?? 'View', sessionId, viewer, peer); + void openJdvView(descriptor, sessionId, viewer, peer); } else { void peer.request('view/close', { view: descriptor.view, generation: descriptor.generation }) .catch(() => undefined); @@ -62,7 +62,7 @@ export function getJdvAddress(sessionId: string, pid: number | undefined): Promi viewClosed: (_peer, view, generation) => markClosed(sessionId, view, generation), disconnected: (peer) => { for (const state of panels.values()) { - if (state.channel === peer && !state.closed) { + if (state.peer === peer && !state.closed) { state.closed = 'The R session disconnected from the data viewer.'; } } @@ -88,27 +88,13 @@ function markClosed(sessionId: string | null, view: string, generation: number): } } -function createStream(descriptor: JdvDescriptor, channel: JdvChannel): SequentialStream | undefined { - if (descriptor.access !== 'sequential' || !(channel instanceof JdvPeer)) { +function createStream(descriptor: JdvDescriptor, peer: JdvPeer): SequentialStream | undefined { + if (descriptor.access !== 'sequential') { return undefined; } const { view, generation } = descriptor; return new SequentialStream(descriptor, - maxRows => channel.requestWithBody('stream/next', { view, generation, max_rows: maxRows })); -} - -/** Sends jdv requests through the session's existing sess RPC connection. */ -function sessChannel(sessionId: string | null): JdvChannel { - return { - async request(method: string, params: JsonObject): Promise { - const request = JSON.stringify({ id: `vscode-r-${++requestSequence}`, method, params }); - const response = await session.sessionRequest({ method: 'jdv_rpc', params: { request } }, sessionId); - if (typeof response !== 'string') { - throw new Error('No response from the R session'); - } - return unwrapResponse(response); - }, - }; + maxRows => peer.requestWithBody('stream/next', { view, generation, max_rows: maxRows })); } function panelKey(sessionId: string | null, view: string): string { @@ -127,50 +113,32 @@ function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } -/** Handle a sess `dataview` notification whose source is `jdv`. */ -export async function showJdvNotification(params: Record, sessionId: string | null): Promise { - let descriptor: JdvDescriptor; - try { - descriptor = parseDescriptor(params.open); - } catch (e) { - void window.showErrorMessage(`Unable to open the data viewer: ${errorMessage(e)}`); - return; - } - // Notifications precede the view they belong to; see jdv_register(). - for (const closed of parseClosedNotifications(params.notifications)) { - markClosed(sessionId, closed.view, closed.generation); - } - const title = typeof params.title === 'string' ? params.title - : Array.isArray(params.title) ? params.title.map(String).join(',') : descriptor.title ?? 'View'; - const viewer = viewerColumn(); - if (viewer) { - await openJdvView(descriptor, title, sessionId, viewer, sessChannel(sessionId)); - } -} - async function openJdvView( - descriptor: JdvDescriptor, title: string, sessionId: string | null, viewer: string, channel: JdvChannel, + descriptor: JdvDescriptor, sessionId: string | null, viewer: string, peer: JdvPeer, ): Promise { - const sequential = descriptor.access === 'sequential' && channel instanceof JdvPeer; - if (descriptor.kind === 'grid' && descriptor.access !== 'random' && !sequential) { - // Snapshot grids, and sequential grids without the jdv transport, are not supported; release the view. - void channel.request('view/close', { view: descriptor.view, generation: descriptor.generation }).catch(() => undefined); + const key = panelKey(sessionId, descriptor.view); + const existing = panels.get(key); + // A child view opened from a tree carries no title; its parent knows the item. + const parent = descriptor.parent === undefined ? undefined : panels.get(panelKey(sessionId, descriptor.parent)); + const title = parent?.childTitle ?? descriptor.title ?? existing?.title ?? 'View'; + if (descriptor.kind === 'grid' && descriptor.access !== 'random' && descriptor.access !== 'sequential') { + // Snapshot grids are not supported; release the view. + void peer.request('view/close', { view: descriptor.view, generation: descriptor.generation }).catch(() => undefined); void window.showWarningMessage(`The data viewer cannot display "${title}" yet (${descriptor.access} view).`); return; } - const key = panelKey(sessionId, descriptor.view); - const existing = panels.get(key); if (existing) { if (descriptor.generation < existing.descriptor.generation) { return; } + // The same view with a newer generation (View() again, or a refresh) updates the panel. existing.descriptor = descriptor; - existing.channel = channel; + existing.peer = peer; existing.title = title; existing.closed = undefined; existing.tree = descriptor.kind === 'tree' ? new TreeAdapter(descriptor, title) : undefined; existing.stream?.dispose(); - existing.stream = createStream(descriptor, existing.channel); + existing.stream = createStream(descriptor, peer); existing.panel.title = title; existing.panel.reveal(existing.panel.viewColumn, true); existing.panel.webview.html = await renderPanel(existing); @@ -192,9 +160,9 @@ async function openJdvView( panel.iconPath = new UriIcon(descriptor.kind === 'grid' ? 'open-preview' : 'preview'); session.attachViewerSessionBridge(panel, sessionId); const state: JdvPanel = { - key, panel, sessionId, channel, descriptor, title, + key, panel, sessionId, peer, descriptor, title, tree: descriptor.kind === 'tree' ? new TreeAdapter(descriptor, title) : undefined, - stream: createStream(descriptor, channel), + stream: createStream(descriptor, peer), }; panels.set(key, state); // The panel's webview getter throws once onDidDispose fires. @@ -206,7 +174,7 @@ async function openJdvView( state.stream?.dispose(); if (!state.closed) { const { view, generation } = state.descriptor; - void state.channel.request('view/close', { view, generation }).catch(() => undefined); + void state.peer.request('view/close', { view, generation }).catch(() => undefined); } }); webview.onDidReceiveMessage((message: unknown) => { @@ -286,10 +254,12 @@ async function handleMessage(state: JdvPanel, webview: vscode.Webview, message: if (action.kind === 'navigate') { navigation = action.navigation; } else { - const child = parseDescriptor(await state.channel.request('object/open', action.params)); - const viewer = viewerColumn(); - if (viewer) { - await openJdvView(child, action.title, state.sessionId, viewer, state.channel); + // The child arrives as a `view/open` notification before the response. + state.childTitle = action.title; + try { + await state.peer.request('object/open', action.params); + } finally { + state.childTitle = undefined; } } } @@ -312,7 +282,7 @@ async function handleMessage(state: JdvPanel, webview: vscode.Webview, message: if (state.closed) { throw new Error(state.closed); } - const result = await state.channel.request('object/children', tree.childrenParams(treePath, message.start as number)); + const result = await state.peer.request('object/children', tree.childrenParams(treePath, message.start as number)); page = tree.children(treePath, result); } catch (e) { error = `Unable to load items: ${errorMessage(e)}`; @@ -334,7 +304,7 @@ async function fetchGridPage(state: JdvPanel, message: JsonObject): Promise= next.totalRows) { diff --git a/src/session.ts b/src/session.ts index addfedea4..abd632040 100644 --- a/src/session.ts +++ b/src/session.ts @@ -35,7 +35,7 @@ import type { import { showWebView } from './webViewer'; import { getListViewerScript, listViewerStyle, ListViewNavigation } from './listViewer'; import { getDataViewerScript, getDataViewerStyle, getDataViewerToolbarHtml } from './dataViewer'; -import { disposeJdvHost, getJdvAddress, showJdvNotification } from './jdvViewer'; +import { disposeJdvHost, getJdvAddress } from './jdvViewer'; import { getDataViewerColumnPanelHtml, getDataViewerColumnPanelScript, getDataViewerColumnPanelStyle } from './dataViewerColumnPanel'; import { createViewerSessionContext, formatSessionLabel, getViewerSessionScript, ViewerSessionContext, ViewerSessionSource } from './viewerSession'; import { SessConsentChoice, SessConsentService } from './sessConsent'; @@ -3137,10 +3137,6 @@ async function handleNotification(message: Record, socket: IpcS break; } case 'dataview': { - if (params.source === 'jdv') { - await showJdvNotification(params, socket._sessionId ?? null); - break; - } // R's deparse() can return several title lines. Preserve the // comma-separated title previously produced by String(array). const title = typeof params.title === 'string' ? params.title @@ -3211,7 +3207,7 @@ async function handleRequest(message: Record, socket?: IpcSocke try { switch (method) { case 'jdv/address': { - // The jdv transport for this session; sess sets it only while calling jdv. + // The per-session jdv transport; sess connects jdv to it when View() is used. const owner = sessions.get(socket?._sessionId ?? ''); if (!owner) { throw new Error('The jdv transport requires an attached session'); } const pid = Number(owner.pid); diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts index 28d6342fa..91f9a235b 100644 --- a/src/test/node/jdvAdapter.test.ts +++ b/src/test/node/jdvAdapter.test.ts @@ -1,41 +1,30 @@ import * as assert from 'node:assert/strict'; import { - JdvDescriptor, JdvRequestError, TreeAdapter, gridColumnDefs, gridPageFromRows, isVectorClass, - parseClosedNotifications, parseDescriptor, tableRowsParams, toJdvFilter, unwrapResponse, + JdvDescriptor, JdvRequestError, TreeAdapter, gridColumnDefs, gridPageFromRows, isVectorType, + parseDescriptor, tableRowsParams, toJdvFilter, } from '../../jdvAdapter'; -const capabilities = { seek: true, sort: true, filter: true, known_total: true, next: false, children: false }; +const capabilities = { seek: true, sort: true, filter: true, known_total: true, next: false, children: false, refresh: true }; -const grid: JdvDescriptor = parseDescriptor(JSON.stringify({ - method: 'view/open', - params: { - view: 'v-1', generation: 2, kind: 'grid', access: 'random', encoding: 'r-json', capabilities, - title: 'df', mode: 'paged', nrow: 3, ncol: 6, row_names: null, - columns: [ - { position: 0, name: 'a', type: 'number', class: ['numeric'], sortable: true, filterable: true }, - { position: 1, name: 'b', type: 'factor', class: ['factor'], sortable: true, filterable: true, levels: ['x', 'y'] }, - { position: 2, name: 'd', type: 'date', class: ['Date'], sortable: true, filterable: true }, - { position: 3, name: 't', type: 'datetime', class: ['POSIXct', 'POSIXt'], sortable: true, filterable: true, tz: 'UTC' }, - { position: 4, name: 'i', type: 'int64', class: ['integer64'], sortable: true, filterable: true }, - { position: 5, name: 'a', type: 'other', class: ['AsIs'], sortable: false, filterable: false, drillable: true }, - ], - }, -})); +const grid: JdvDescriptor = parseDescriptor({ + view: 'v-1', generation: 2, kind: 'grid', access: 'random', encoding: 'r-json', capabilities, + title: 'df', mode: 'paged', nrow: 3, ncol: 6, row_names: null, + columns: [ + { position: 0, name: 'a', type: 'number', class: ['numeric'], sortable: true, filterable: true }, + { position: 1, name: 'b', type: 'factor', class: ['factor'], sortable: true, filterable: true, levels: ['x', 'y'] }, + { position: 2, name: 'd', type: 'date', class: ['Date'], sortable: true, filterable: true }, + { position: 3, name: 't', type: 'datetime', class: ['POSIXct', 'POSIXt'], sortable: true, filterable: true, tz: 'UTC' }, + { position: 4, name: 'i', type: 'int64', class: ['integer64'], sortable: true, filterable: true }, + { position: 5, name: 'a', type: 'other', class: ['AsIs'], sortable: false, filterable: false, drillable: true }, + ], +}); suite('jdv adapter', () => { - test('parses descriptors and closed notifications', () => { + test('validates descriptors', () => { assert.equal(grid.view, 'v-1'); assert.equal(grid.generation, 2); - assert.throws(() => parseDescriptor('{"method":"view/open","params":{}}')); - assert.deepEqual(parseClosedNotifications([ - '{"method":"view/closed","params":{"view":"v-3","generation":1}}', 'not json', - ]), [{ view: 'v-3', generation: 1 }]); - }); - - test('unwraps responses and keeps error codes', () => { - assert.deepEqual(unwrapResponse('{"id":"a","result":{"x":1},"warnings":[]}'), { x: 1 }); - assert.throws(() => unwrapResponse('{"id":"a","error":{"code":"stale_generation","message":"old"},"warnings":[]}'), - (e: unknown) => e instanceof JdvRequestError && e.code === 'stale_generation'); + assert.throws(() => parseDescriptor({})); + assert.throws(() => parseDescriptor({ ...grid, columns: undefined })); }); test('builds column definitions from the schema', () => { @@ -122,14 +111,14 @@ suite('jdv adapter', () => { test('maps List Viewer paths to typed tree paths', () => { const tree = new TreeAdapter(parseDescriptor({ view: 'v-2', generation: 1, kind: 'tree', access: 'random', encoding: 'r-json', - capabilities: { ...capabilities, sort: false, filter: false, children: true }, path: [], class: ['list'], + capabilities: { ...capabilities, sort: false, filter: false, children: true }, path: [], class: ['list'], type: 'list', }), 'x'); - assert.deepEqual(tree.childrenParams([], 1), { view: 'v-2', generation: 1, path: [], offset: 0, limit: 200 }); + assert.deepEqual(tree.childrenParams([], 1), { view: 'v-2', generation: 1, path: [], offset: 0, limit: 200, attributes: false }); const page = tree.children([], { path: [], offset: 0, total_children: 4, children: [ - { index: 1, label: 'a', class: ['integer'], summary: 'integer (3): 1, 2, 3', has_children: true, viewable: true, path: [{ type: 'index', index: 1 }] }, - { index: 2, label: 'b', class: ['list'], summary: 'list (1)', has_children: true, viewable: true, path: [{ type: 'index', index: 2 }] }, - { index: 3, label: 'df', class: ['data.frame'], summary: 'data.frame', has_children: true, viewable: true, path: [{ type: 'index', index: 3 }] }, + { index: 1, label: 'a', class: ['integer'], type: 'integer', summary: 'integer (3): 1, 2, 3', has_children: true, viewable: true, path: [{ type: 'index', index: 1 }] }, + { index: 2, label: 'b', class: ['list'], type: 'list', summary: 'list (1)', has_children: true, viewable: true, path: [{ type: 'index', index: 2 }] }, + { index: 3, label: 'df', class: ['data.frame'], type: 'list', summary: 'data.frame', has_children: true, viewable: true, path: [{ type: 'index', index: 3 }] }, ], }); assert.equal(page.next_start, 4); @@ -140,7 +129,7 @@ suite('jdv adapter', () => { assert.equal(vector.kind === 'navigate' && vector.navigation.vector, true); const elements = tree.children([1], { path: [{ type: 'index', index: 1 }], offset: 0, total_children: 1, children: [ - { index: 1, label: '[1]', class: ['integer'], summary: 'integer (1): 1', has_children: true, viewable: true, + { index: 1, label: '[1]', class: ['integer'], type: 'integer', summary: 'integer (1): 1', has_children: false, viewable: true, path: [{ type: 'index', index: 1 }, { type: 'index', index: 1 }] }, ], }); @@ -153,16 +142,31 @@ suite('jdv adapter', () => { }); const table = tree.open([], 3); assert.deepEqual(table, { - kind: 'object', title: 'x$df', params: { view: 'v-2', generation: 1, path: [{ type: 'index', index: 3 }] }, + kind: 'object', title: 'x$df', + params: { view: 'v-2', generation: 1, path: [{ type: 'index', index: 3 }], mode: 'paged' }, }); assert.throws(() => tree.open([], 9), (e: unknown) => e instanceof JdvRequestError && e.code === 'unavailable'); }); - test('classifies atomic vectors', () => { - assert.equal(isVectorClass(['integer']), true); - assert.equal(isVectorClass(['ordered', 'factor']), true); - assert.equal(isVectorClass(['matrix', 'array']), false); - assert.equal(isVectorClass(['list']), false); - assert.equal(isVectorClass([]), false); + test('classifies atomic vectors by type', () => { + assert.equal(isVectorType('integer', ['factor']), true); + assert.equal(isVectorType('double', ['numeric']), true); + assert.equal(isVectorType('integer', ['matrix', 'array']), false); + assert.equal(isVectorType('list', ['list']), false); + assert.equal(isVectorType(undefined, []), false); + }); + + test('leaves have nothing to open unless they are tables', () => { + const tree = new TreeAdapter(parseDescriptor({ + view: 'v-2', generation: 1, kind: 'tree', access: 'random', encoding: 'r-json', + capabilities: { ...capabilities, children: true }, path: [], class: ['list'], type: 'list', + }), 'x'); + const page = tree.children([], { + path: [], offset: 0, total_children: 2, children: [ + { index: 1, label: 'n', class: ['numeric'], type: 'double', summary: 'double (1): 1', has_children: false, viewable: true, path: [{ type: 'index', index: 1 }] }, + { index: 2, label: 'm', class: ['matrix', 'array'], type: 'integer', summary: 'matrix', has_children: false, viewable: true, path: [{ type: 'index', index: 2 }] }, + ], + }); + assert.deepEqual(page.children.map(child => child.viewable), [false, true]); }); }); From 7136f965485c95d8a1ce0a2cf5ac1a980c8fd4cb Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 17 Oct 2026 07:49:51 +0000 Subject: [PATCH 06/38] feat: follow the variable passed to View() with jdv on R 4.6 When View() is called with a single symbol, bind the jdv view to the variable that holds the value so that reassigning it updates the view. The binding is only used on R 4.6.0 or later, can be disabled with options(sess.dataviewer.track = FALSE), and registration falls back to an untracked view if jdv rejects the binding. --- sess/R/handlers.R | 51 ++++++++++++++++++++++++++++++++--- sess/R/hooks.R | 5 +++- sess/inst/tinytest/test-jdv.R | 50 ++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/sess/R/handlers.R b/sess/R/handlers.R index 844845d78..a9338ac5d 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1355,22 +1355,65 @@ handle_dataview_dispose <- function(params) { invisible(NULL) } +# The variable that View() was called with, for jdv to follow. +# Returns list(name, env) when the expression is a single symbol bound in the +# caller's environment or one of its parents, and NULL otherwise. Nothing is +# evaluated: exists() does not force promises and active bindings are skipped. +# Tracking needs R 4.6.0 or later and can be disabled with +# `options(sess.dataviewer.track = FALSE)`. +.jdv_track_binding <- function(expression, env) { + if (!is.symbol(expression) || !nzchar(as.character(expression)) || + !isTRUE(getOption("sess.dataviewer.track", TRUE)) || + getRversion() < "4.6.0") { + return(NULL) + } + name <- as.character(expression) + while (!identical(env, emptyenv())) { + if (exists(name, envir = env, inherits = FALSE)) { + if (bindingIsActive(name, env)) { + return(NULL) + } + return(list(name = name, env = env)) + } + env <- parent.env(env) + } + NULL +} + # Show `x` with jdv. Returns FALSE when jdv cannot be used, so the caller falls # back to the sess data viewer. # # The title is jdv's display slot: View() with the same title replaces the view # and increments its generation. Grids are paged so that sorting and filtering -# stay in R. Views are not bound to a variable (`binding`): View() receives the -# value, not its name, and tracking needs R 4.6. -.jdv_show <- function(x, title_key) { +# stay in R. When `binding` names the variable that holds `x`, jdv follows it and +# rebuilds the view as the next generation after the variable is reassigned. It +# fails if the binding cannot be read without evaluation or does not hold `x` +# (for example View(x) called through do.call()); then the view is registered +# without tracking, since View() must not fail because of it. +# +# jdv checks tracked variables from its own task callback and sends the update +# over the transport itself, so sess needs no task callback of its own for it. +# Only an embedding host that collects jdv_notifications() must call +# jdv_rpc(NULL) first. +.jdv_show <- function(x, title_key, binding = NULL) { if (.jdv_is_arrow(x) && !requireNamespace("nanoarrow", quietly = TRUE)) { return(FALSE) } if (!.jdv_connect()) { return(FALSE) } + title <- if (nzchar(title_key)) title_key else NULL + if (!is.null(binding)) { + registered <- tryCatch({ + jdv::jdv_register(x, title = title, mode = "paged", binding = binding) + TRUE + }, error = function(e) FALSE) + if (registered) { + return(TRUE) + } + } tryCatch({ - jdv::jdv_register(x, title = if (nzchar(title_key)) title_key else NULL, mode = "paged") + jdv::jdv_register(x, title = title, mode = "paged") TRUE }, error = function(e) FALSE) } diff --git a/sess/R/hooks.R b/sess/R/hooks.R index aacf18c07..f31124c21 100644 --- a/sess/R/hooks.R +++ b/sess/R/hooks.R @@ -74,6 +74,9 @@ runtime_start <- function(use_rstudioapi = TRUE, owner <- as.character(expression) } force(title) + # Resolve the variable behind View() before x is forced, so jdv can + # follow it. The lookup starts at the caller, never at this function's frame. + tracked_binding <- .jdv_track_binding(original_expression, parent.frame()) if (is.null(.sess_env$view_owner) && isTRUE(.sess_env$interactive_connected) && .interactive_rich_value(x)) { @@ -88,7 +91,7 @@ runtime_start <- function(use_rstudioapi = TRUE, "object" } title_key <- paste(as.character(title), collapse = "\n") - if ((view_type %in% c("table", "list") || .jdv_is_arrow(x)) && .jdv_show(x, title_key)) { + if ((view_type %in% c("table", "list") || .jdv_is_arrow(x)) && .jdv_show(x, title_key, tracked_binding)) { return(invisible(NULL)) } owner <- owner %||% title_key diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R index 8be8903a1..6b26bc825 100644 --- a/sess/inst/tinytest/test-jdv.R +++ b/sess/inst/tinytest/test-jdv.R @@ -154,6 +154,56 @@ local({ expect_identical(stream$title, "stream") } + # View() follows the variable on R 4.6.0 or later, and never on older R. + generations <- function(title) { + vapply(Filter(function(open) identical(open$title, title), lapply(opens(), `[[`, "params")), + function(open) open$generation, integer(1L)) + } + wait_generations <- function(title, n, timeout = 5) { + deadline <- Sys.time() + timeout + repeat { + jdv::jdv_rpc(NULL) + found <- generations(title) + if (length(found) >= n || Sys.time() > deadline) break + Sys.sleep(0.05) + } + found + } + tracking <- data.frame(a = 1:3) + expect_identical(sess:::.jdv_track_binding(quote(tracking), environment()), + if (getRversion() >= "4.6.0") list(name = "tracking", env = environment())) + expect_null(sess:::.jdv_track_binding(quote(head(tracking)), environment())) + expect_null(sess:::.jdv_track_binding(quote(no_such_variable_for_sess), environment())) + options(sess.dataviewer.track = FALSE) + expect_null(sess:::.jdv_track_binding(quote(tracking), environment())) + options(sess.dataviewer.track = NULL) + + utils::View(tracking) + expect_identical(wait_generations("tracking", 1L), 1L) + tracking <- transform(tracking, b = a * 2L) + if (getRversion() >= "4.6.0") { + expect_identical(wait_generations("tracking", 2L), 1:2) + # Assigning the same object again changes nothing. + tracking <- tracking + expect_identical(wait_generations("tracking", 3L, timeout = 0.5), 1:2) + } else { + expect_identical(wait_generations("tracking", 2L, timeout = 0.5), 1L) + } + + # The opt-out and expressions that are not a single variable stay fixed. + options(sess.dataviewer.track = FALSE) + fixed <- data.frame(a = 1:3) + utils::View(fixed) + options(sess.dataviewer.track = NULL) + utils::View(head(fixed), "head") + fixed <- transform(fixed, b = a * 2L) + expect_identical(wait_generations("fixed", 2L, timeout = 0.5), 1L) + expect_identical(wait_generations("head", 2L, timeout = 0.5), 1L) + + # A binding that does not hold the object is dropped instead of failing View(). + expect_true(sess:::.jdv_show(fixed, "mismatch", list(name = "tracking", env = environment()))) + expect_identical(wait_generations("mismatch", 1L), 1L) + # Releasing the runtime closes the connection that sess opened. sess:::.jdv_release() expect_identical(jdv::jdv_connection()$state, "disconnected") From c71d741dc0c09f001f8ad3d8be1916e3df9fda8d Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 17 Oct 2026 07:49:51 +0000 Subject: [PATCH 07/38] feat: report failed jdv view updates in the data viewer Handle the view/update_failed notification that jdv sends when a tracked variable changed but the view could not be rebuilt, and show it as a warning. --- src/jdvTransport.ts | 4 ++++ src/jdvViewer.ts | 18 ++++++++++++++++++ src/test/node/jdvArrow.test.ts | 10 +++++++++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/jdvTransport.ts b/src/jdvTransport.ts index d0cfd25cf..5c4267e7c 100644 --- a/src/jdvTransport.ts +++ b/src/jdvTransport.ts @@ -302,6 +302,8 @@ export interface JdvHostEvents { viewOpen(peer: JdvPeer, params: JsonObject): void; /** `view/closed` received from R. */ viewClosed(peer: JdvPeer, view: string, generation: number): void; + /** `view/update_failed` received from R: a tracked variable changed but the view could not be rebuilt. */ + viewUpdateFailed(peer: JdvPeer, params: JsonObject): void; /** The R connection closed; its views are gone. */ disconnected(peer: JdvPeer, reason: string): void; } @@ -369,6 +371,8 @@ export class JdvHost { } else if (method === 'view/closed' && typeof params.view === 'string' && typeof params.generation === 'number') { this.events.viewClosed(peer, params.view, params.generation); + } else if (method === 'view/update_failed') { + this.events.viewUpdateFailed(peer, params); } }, closed: (peer, reason) => this.events.disconnected(peer, reason), diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index cc5d63a8d..27d767fa4 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -60,6 +60,7 @@ export function getJdvAddress(sessionId: string, pid: number | undefined): Promi } }, viewClosed: (_peer, view, generation) => markClosed(sessionId, view, generation), + viewUpdateFailed: (_peer, params) => warnUpdateFailed(sessionId, params), disconnected: (peer) => { for (const state of panels.values()) { if (state.peer === peer && !state.closed) { @@ -88,6 +89,20 @@ function markClosed(sessionId: string | null, view: string, generation: number): } } +/** A tracked variable changed but jdv could not rebuild the view; the previous generation stays. */ +function warnUpdateFailed(sessionId: string | null, params: JsonObject): void { + if (typeof params.view !== 'string') { + return; + } + const state = panels.get(panelKey(sessionId, params.view)); + // Ignore a failure reported for a generation that is no longer shown. + if (!state || state.closed || params.generation !== state.descriptor.generation) { + return; + } + const reason = typeof params.message === 'string' ? params.message : 'unknown error'; + void window.showWarningMessage(`The data viewer could not update "${state.title}": ${reason}`); +} + function createStream(descriptor: JdvDescriptor, peer: JdvPeer): SequentialStream | undefined { if (descriptor.access !== 'sequential') { return undefined; @@ -140,6 +155,9 @@ async function openJdvView( existing.stream?.dispose(); existing.stream = createStream(descriptor, peer); existing.panel.title = title; + // jdv sends the same notification for View() and for tracked updates, so + // an update cannot be told apart. Reveal without taking focus so that an + // explicit View() always surfaces the panel. existing.panel.reveal(existing.panel.viewColumn, true); existing.panel.webview.html = await renderPanel(existing); return; diff --git a/src/test/node/jdvArrow.test.ts b/src/test/node/jdvArrow.test.ts index 45ed04d60..063240665 100644 --- a/src/test/node/jdvArrow.test.ts +++ b/src/test/node/jdvArrow.test.ts @@ -1,7 +1,7 @@ import * as assert from 'node:assert/strict'; import * as net from 'node:net'; import { tableFromArrays, tableToIPC } from 'apache-arrow'; -import { JdvDescriptor, JdvRequestError, gridColumnDefs, parseDescriptor } from '../../jdvAdapter'; +import { JdvDescriptor, JdvRequestError, JsonObject, gridColumnDefs, parseDescriptor } from '../../jdvAdapter'; import { SequentialStream, UNAVAILABLE_TEXT, formatScaled, formatTimestamp } from '../../jdvArrow'; import { JdvHost, JdvPeer } from '../../jdvTransport'; @@ -136,10 +136,12 @@ async function fakeR(socketPath: string) { suite('jdv transport', () => { test('exchanges requests, wakes and Arrow bodies with R', async () => { const opened: string[] = []; + const failures: JsonObject[] = []; let peer: JdvPeer | undefined; const host = await JdvHost.create(4321, { viewOpen: (connection, params) => { peer = connection; opened.push(String(params.view)); }, viewClosed: () => undefined, + viewUpdateFailed: (_connection, params) => { failures.push(params); }, disconnected: () => undefined, }); try { @@ -149,6 +151,11 @@ suite('jdv transport', () => { while (!peer) { await new Promise(resolve => setTimeout(resolve, 5)); } assert.deepEqual(opened, ['v-1']); + // A tracked update that failed is reported with the displayed generation. + r.send({ method: 'view/update_failed', params: { view: 'v-1', generation: 1, code: 'limit_exceeded', message: 'too large' } }); + while (failures.length === 0) { await new Promise(resolve => setTimeout(resolve, 5)); } + assert.equal(failures[0].message, 'too large'); + const response = peer.requestWithBody('stream/next', { view: 'v-1', generation: 1, max_rows: 10 }); const request = await r.next(line => line.method === 'stream/next'); // Wakes repeat while the request is pending. @@ -182,6 +189,7 @@ suite('jdv transport', () => { const host = await JdvHost.create(1, { viewOpen: () => undefined, viewClosed: () => undefined, + viewUpdateFailed: () => undefined, disconnected: (_peer, why) => { reason = why; }, }); try { From 426083b4ebd534e536f9940bfdcfac3628b6ebde Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 17 Oct 2026 07:52:45 +0000 Subject: [PATCH 08/38] style: wrap long lines in the jdv View() hook and its test --- sess/R/hooks.R | 3 ++- sess/inst/tinytest/test-jdv.R | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sess/R/hooks.R b/sess/R/hooks.R index f31124c21..7c7a3a11d 100644 --- a/sess/R/hooks.R +++ b/sess/R/hooks.R @@ -91,7 +91,8 @@ runtime_start <- function(use_rstudioapi = TRUE, "object" } title_key <- paste(as.character(title), collapse = "\n") - if ((view_type %in% c("table", "list") || .jdv_is_arrow(x)) && .jdv_show(x, title_key, tracked_binding)) { + if ((view_type %in% c("table", "list") || .jdv_is_arrow(x)) && + .jdv_show(x, title_key, tracked_binding)) { return(invisible(NULL)) } owner <- owner %||% title_key diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R index 6b26bc825..8666e084a 100644 --- a/sess/inst/tinytest/test-jdv.R +++ b/sess/inst/tinytest/test-jdv.R @@ -41,7 +41,8 @@ local({ lapply(messages[nzchar(messages)], jsonlite::fromJSON, simplifyVector = FALSE) } used_sess_viewer <- function() { - any(vapply(read_messages(), function(message) identical(message$method, "dataview"), logical(1L))) + any(vapply(read_messages(), function(message) identical(message$method, "dataview"), + logical(1L))) } sample <- data.frame(x = c(2, 1, NA), y = c("b", "a", "c")) From 30091e4ab082b98c4338b17f99b4064ee843ec1c Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 17 Oct 2026 08:07:30 +0000 Subject: [PATCH 09/38] feat: add a setting to follow the variable passed to View() in the data viewer The new r.session.viewers.trackVariables setting (default on) is passed to sess as connect(track_variables =) by the attach script and, for terminals started by the extension, through SESS_TRACK_VARIABLES. options(sess.dataviewer.track) still takes precedence when set. The setting applies to sessions attached after it is changed. --- R/attach_sess.R | 6 ++++++ R/profile.R | 4 +++- package.json | 5 +++++ sess/R/handlers.R | 9 +++++---- sess/R/server.R | 12 ++++++++++-- sess/README.md | 5 ++++- sess/inst/tinytest/test-jdv.R | 14 ++++++++++++++ sess/man/connect.Rd | 9 ++++++++- src/rTerminal.ts | 1 + src/session.ts | 7 ++++++- src/test/suite/session.test.ts | 3 ++- src/test/suite/terminal.test.ts | 1 + 12 files changed, 65 insertions(+), 11 deletions(-) diff --git a/R/attach_sess.R b/R/attach_sess.R index 3b2849a84..9c448d109 100644 --- a/R/attach_sess.R +++ b/R/attach_sess.R @@ -184,6 +184,7 @@ vscode_r_attach_sess <- function(endpoint, pkg_path, managed_root, consent_dir, timeout_seconds = 180, setup_timeout_seconds = 300, use_rstudioapi = TRUE, + track_variables = TRUE, attachment_id = NULL, auto_activate = TRUE) { if (!is.null(attachment_id) && @@ -235,6 +236,11 @@ vscode_r_attach_sess <- function(endpoint, pkg_path, managed_root, consent_dir, connect <- get("connect", envir = ns, inherits = FALSE) args <- list(endpoint = endpoint, plot_backend = plot_backend, use_rstudioapi = use_rstudioapi) + # Tracking is on by default, so only a disabled setting is passed. This + # keeps older sess namespaces working. + if (!isTRUE(track_variables)) { + args$track_variables <- FALSE + } # Preserve compatibility with older sess namespaces and helper stubs # when the caller uses the legacy, untagged attach path. if (!is.null(attachment_id)) { diff --git a/R/profile.R b/R/profile.R index 9ca4c2c3b..c76e168bd 100644 --- a/R/profile.R +++ b/R/profile.R @@ -52,7 +52,9 @@ invisible(local({ plot_backend <- Sys.getenv("SESS_PLOT_BACKEND", "auto") connect <- get("connect", envir = ns, inherits = FALSE) use_rstudioapi <- as.logical(Sys.getenv("SESS_RSTUDIOAPI", "TRUE")) - result <- connect(use_rstudioapi = use_rstudioapi, plot_backend = plot_backend) + track_variables <- as.logical(Sys.getenv("SESS_TRACK_VARIABLES", "TRUE")) + result <- connect(use_rstudioapi = use_rstudioapi, plot_backend = plot_backend, + track_variables = track_variables) isTRUE(result) } diff --git a/package.json b/package.json index 780360f9c..feec5fb29 100644 --- a/package.json +++ b/package.json @@ -2108,6 +2108,11 @@ "default": 500, "markdownDescription": "The maximum number of rows per page in the data viewer. `0` means to disable pagination." }, + "r.session.viewers.trackVariables": { + "type": "boolean", + "default": true, + "markdownDescription": "Update the data viewer when the variable passed to `View()` is changed in R, for example `View(df)` followed by `df <- transform(df, ...)`. Requires R >= 4.6 and the [jdv](https://github.com/eitsupi/jdv) package; otherwise the data viewer shows a fixed snapshot. Applies to R sessions started or attached after the setting is changed. `options(sess.dataviewer.track = TRUE/FALSE)` in R takes precedence over this setting. Requires `#r.sessionWatcher#` to be set to `true`." + }, "r.session.viewers.viewColumn": { "type": "object", "markdownDescription": "Which view column should R-related webviews be displayed? Requires `#r.sessionWatcher#` to be set to `true`.", diff --git a/sess/R/handlers.R b/sess/R/handlers.R index a9338ac5d..4bc5c98b8 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1359,12 +1359,13 @@ handle_dataview_dispose <- function(params) { # Returns list(name, env) when the expression is a single symbol bound in the # caller's environment or one of its parents, and NULL otherwise. Nothing is # evaluated: exists() does not force promises and active bindings are skipped. -# Tracking needs R 4.6.0 or later and can be disabled with -# `options(sess.dataviewer.track = FALSE)`. +# Tracking needs R 4.6.0 or later. It follows the VS Code setting +# `r.session.viewers.trackVariables` (passed to connect() as `track_variables`, +# default on); `options(sess.dataviewer.track = TRUE/FALSE)`, when set, wins. .jdv_track_binding <- function(expression, env) { + track <- getOption("sess.dataviewer.track", .sess_env$track_variables %||% TRUE) if (!is.symbol(expression) || !nzchar(as.character(expression)) || - !isTRUE(getOption("sess.dataviewer.track", TRUE)) || - getRversion() < "4.6.0") { + !isTRUE(track) || getRversion() < "4.6.0") { return(NULL) } name <- as.character(expression) diff --git a/sess/R/server.R b/sess/R/server.R index 0fb37b410..9fe2259eb 100644 --- a/sess/R/server.R +++ b/sess/R/server.R @@ -11,6 +11,11 @@ #' It is not an authentication token or an exclusive ownership claim. #' @param auto_activate Logical. Whether vscode-R should select this attachment #' automatically. `FALSE` requires `attachment_id`. +#' @param track_variables Logical or NULL. Whether the data viewer follows the +#' variable passed to `View()` (R >= 4.6 and jdv). The VS Code extension passes +#' its `r.session.viewers.trackVariables` setting. `NULL` and `TRUE` both +#' enable tracking. `options(sess.dataviewer.track)`, when set, takes +#' precedence over this argument. #' @return Invisibly returns `TRUE` when the IPC connection and runtime startup #' succeed, or `FALSE` when the IPC endpoint is unavailable. #' @details When SESS_DISCOVERY_FILE describes the connected endpoint, an @@ -23,7 +28,8 @@ #' @export connect <- function(endpoint = NULL, use_rstudioapi = TRUE, plot_backend = c("auto", "jgd", "httpgd", "standard", "native"), - attachment_id = NULL, auto_activate = TRUE) { + attachment_id = NULL, auto_activate = TRUE, + track_variables = NULL) { if (!is.null(attachment_id) && !(is.character(attachment_id) && length(attachment_id) == 1L && !is.na(attachment_id) && @@ -43,6 +49,7 @@ connect <- function(endpoint = NULL, use_rstudioapi = TRUE, # Metadata belongs to this connect call, not to the R process. In particular, # a later native attach must not inherit an external terminal's identity. .sess_env$attachment_id <- attachment_id + .sess_env$track_variables <- !isFALSE(track_variables) .sess_env$auto_activate <- auto_activate .sess_env$con <- NULL .sess_env$pending_responses <- list() @@ -72,7 +79,8 @@ connect <- function(endpoint = NULL, use_rstudioapi = TRUE, .sess_env$reconnect <- list( path = discovery_file, endpoint = endpoint, options = list(use_rstudioapi = use_rstudioapi, plot_backend = plot_backend, - attachment_id = attachment_id, auto_activate = auto_activate) + attachment_id = attachment_id, auto_activate = auto_activate, + track_variables = track_variables) ) } diff --git a/sess/README.md b/sess/README.md index 441bbbb12..43023f909 100644 --- a/sess/README.md +++ b/sess/README.md @@ -87,7 +87,8 @@ When you start an R terminal from VS Code, the extension's R profile calls sess::connect( endpoint = NULL, # socket/pipe endpoint; see below use_rstudioapi = TRUE, # emulate rstudioapi functions - plot_backend = "auto" # or "standard", "httpgd", "jgd", "native" + plot_backend = "auto", # or "standard", "httpgd", "jgd", "native" + track_variables = NULL # follow the variable passed to View() (R >= 4.6 with jdv); NULL means TRUE ) ``` @@ -187,6 +188,8 @@ is used. | `SESS_ENDPOINT` | env var | Socket/pipe path used by `connect()`. | | `SESS_RSTUDIOAPI` | env var | `TRUE`/`FALSE`; passed as `use_rstudioapi` by the extension's R profile. | | `SESS_PLOT_BACKEND` | env var | `auto`, `standard`, `httpgd`, `jgd` or `native`; passed as `plot_backend` by the extension's R profile. | +| `SESS_TRACK_VARIABLES` | env var | `TRUE`/`FALSE`; passed as `track_variables` by the extension's R profile (setting `r.session.viewers.trackVariables`). | +| `sess.dataviewer.track` | R option | `TRUE`/`FALSE`; whether `View()` follows later changes of the variable (R >= 4.6 with jdv). When set, it takes precedence over `track_variables`. | | `JGD_SOCKET` | env var | Socket used by the jgd device; set by the extension. | | `sess.quiet` | R option | Set to `TRUE` to suppress the successful connection message. Connection failures remain visible. | diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R index 8666e084a..7a5f83027 100644 --- a/sess/inst/tinytest/test-jdv.R +++ b/sess/inst/tinytest/test-jdv.R @@ -179,6 +179,20 @@ local({ expect_null(sess:::.jdv_track_binding(quote(tracking), environment())) options(sess.dataviewer.track = NULL) + # The connect() setting turns tracking off; the R option overrides it. + sess_env <- sess:::.sess_env + setting <- sess_env$track_variables + sess_env$track_variables <- FALSE + expect_null(sess:::.jdv_track_binding(quote(tracking), environment())) + options(sess.dataviewer.track = TRUE) + expect_identical(sess:::.jdv_track_binding(quote(tracking), environment()), + if (getRversion() >= "4.6.0") list(name = "tracking", env = environment())) + options(sess.dataviewer.track = NULL) + sess_env$track_variables <- TRUE + expect_identical(sess:::.jdv_track_binding(quote(tracking), environment()), + if (getRversion() >= "4.6.0") list(name = "tracking", env = environment())) + sess_env$track_variables <- setting + utils::View(tracking) expect_identical(wait_generations("tracking", 1L), 1L) tracking <- transform(tracking, b = a * 2L) diff --git a/sess/man/connect.Rd b/sess/man/connect.Rd index deb4bfee2..247b437db 100644 --- a/sess/man/connect.Rd +++ b/sess/man/connect.Rd @@ -9,7 +9,8 @@ connect( use_rstudioapi = TRUE, plot_backend = c("auto", "jgd", "httpgd", "standard", "native"), attachment_id = NULL, - auto_activate = TRUE + auto_activate = TRUE, + track_variables = NULL ) } \arguments{ @@ -29,6 +30,12 @@ ownership claim.} \item{auto_activate}{Logical. Whether vscode-R should select this attachment automatically. \code{FALSE} requires \code{attachment_id}.} + +\item{track_variables}{Logical or NULL. Whether the data viewer follows the +variable passed to \code{View()} (R >= 4.6 and jdv). The VS Code extension passes +its \code{r.session.viewers.trackVariables} setting. \code{NULL} and \code{TRUE} both +enable tracking. \code{options(sess.dataviewer.track)}, when set, takes +precedence over this argument.} } \value{ Invisibly returns \code{TRUE} when the IPC connection and runtime startup diff --git a/src/rTerminal.ts b/src/rTerminal.ts index 1c6e3228c..5d899f32c 100644 --- a/src/rTerminal.ts +++ b/src/rTerminal.ts @@ -303,6 +303,7 @@ export async function makeTerminalOptions(resource?: vscode.Uri): Promise('session.emulateRStudioAPI') ? 'TRUE' : 'FALSE', + SESS_TRACK_VARIABLES: config(configResource).get('session.viewers.trackVariables') === false ? 'FALSE' : 'TRUE', SESS_PLOT_BACKEND: backend, }; Object.assign(env, jgdVars); diff --git a/src/session.ts b/src/session.ts index abd632040..f2a59a8e8 100644 --- a/src/session.ts +++ b/src/session.ts @@ -1229,6 +1229,11 @@ function asRStringLiteral(value: string): string { return `"${escaped}"`; } +/** Whether the data viewer should follow the variable passed to View(); on unless disabled. */ +export function trackVariablesSetting(resource?: vscode.Uri): boolean { + return config(resource).get('session.viewers.trackVariables') !== false; +} + function getAttachSessionScriptPath(pipePath: string): string { const scriptBase = path.basename(pipePath).replace(/[^a-zA-Z0-9_.-]/g, '_') || 'attach_session'; return path.join(extensionContext.globalStorageUri.fsPath, 'tmp', 'attach', `${scriptBase}.R`); @@ -1255,7 +1260,7 @@ function buildAttachSessionScript(pipePath: string, sessPath: string, consentDir ]), ' source(startup_helper, local = TRUE)', ' source(attach_helper, local = TRUE)', - ` vscode_r_attach_sess(endpoint, sess_src, managed_root, consent_dir, source_helper, installer_helper, ${asRStringLiteral(backend)}, use_rstudioapi = ${config().get('session.emulateRStudioAPI') ? 'TRUE' : 'FALSE'}${attachment ? `, attachment_id = ${asRStringLiteral(attachment.attachmentId)}, auto_activate = ${attachment.autoActivate ? 'TRUE' : 'FALSE'}` : ''})`, + ` vscode_r_attach_sess(endpoint, sess_src, managed_root, consent_dir, source_helper, installer_helper, ${asRStringLiteral(backend)}, use_rstudioapi = ${config().get('session.emulateRStudioAPI') ? 'TRUE' : 'FALSE'}, track_variables = ${trackVariablesSetting() ? 'TRUE' : 'FALSE'}${attachment ? `, attachment_id = ${asRStringLiteral(attachment.attachmentId)}, auto_activate = ${attachment.autoActivate ? 'TRUE' : 'FALSE'}` : ''})`, ' })', ' invisible(isTRUE(attached))', ' }, error = function(error) {', diff --git a/src/test/suite/session.test.ts b/src/test/suite/session.test.ts index 916f83a0e..2254453cf 100644 --- a/src/test/suite/session.test.ts +++ b/src/test/suite/session.test.ts @@ -779,6 +779,7 @@ suite('Session Communication', () => { assert.match(source, /auto_activate = TRUE/); assert.match(source, /tryCatch\(/); assert.match(source, /use_rstudioapi = (TRUE|FALSE)/); + assert.match(source, /track_variables = (TRUE|FALSE)/); const backgroundSource = await fsp.readFile(backgroundPreparation.scriptPath, 'utf8'); assert.ok(backgroundSource.includes(`attachment_id = "${backgroundPreparation.attachmentId}"`)); assert.match(backgroundSource, /auto_activate = FALSE/); @@ -1075,7 +1076,7 @@ suite('Session Communication', () => { assert.ok(commandMatch); const scriptPath = JSON.parse(commandMatch[1]) as string; const scriptContent = await fsp.readFile(scriptPath, 'utf8'); - assert.match(scriptContent, /vscode_r_attach_sess\(endpoint, sess_src, managed_root, consent_dir, source_helper, installer_helper, "native", use_rstudioapi = (?:TRUE|FALSE)\)/); + assert.match(scriptContent, /vscode_r_attach_sess\(endpoint, sess_src, managed_root, consent_dir, source_helper, installer_helper, "native", use_rstudioapi = (?:TRUE|FALSE), track_variables = (?:TRUE|FALSE)\)/); assert.doesNotMatch(scriptContent, /Sys\.(?:setenv|unsetenv)\(JGD_SOCKET/); }); diff --git a/src/test/suite/terminal.test.ts b/src/test/suite/terminal.test.ts index 78d75bd3e..f5ca46cf2 100644 --- a/src/test/suite/terminal.test.ts +++ b/src/test/suite/terminal.test.ts @@ -233,6 +233,7 @@ suite('R Terminal', () => { assert.ok(discoveryFile); assert.strictEqual(options.env['SESS_ENDPOINT'], null); assert.strictEqual(options.env['SESS_RSTUDIOAPI'], 'TRUE'); + assert.strictEqual(options.env['SESS_TRACK_VARIABLES'], 'TRUE'); assert.strictEqual(options.env['SESS_PLOT_BACKEND'], 'httpgd'); assert.ok(options.env['R_PROFILE_USER']); assert.ok(options.env['R_PROFILE_USER'].endsWith(path.join('R', 'profile.R'))); From 920cfcc19468a11c05d576209337e5a93e231ae4 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 17 Oct 2026 08:07:30 +0000 Subject: [PATCH 10/38] feat: add a Refresh button to the jdv data viewer and keep tracked updates in place When the view descriptor has capabilities.refresh, the grid and tree viewers show a Refresh button that sends view/refresh and reports failures in the status area. A newer generation announced with reason "tracked" or "refresh" updates the panel without revealing it; other reasons keep revealing it. --- src/dataViewer.ts | 3 ++- src/jdvAdapter.ts | 23 +++++++++++++++- src/jdvViewer.ts | 46 ++++++++++++++++++++++++++------ src/listViewer.ts | 20 ++++++++++++-- src/session.ts | 25 +++++++++++++++-- src/test/node/jdvAdapter.test.ts | 23 ++++++++++++++++ 6 files changed, 126 insertions(+), 14 deletions(-) diff --git a/src/dataViewer.ts b/src/dataViewer.ts index c08464ce6..e77f1e0b1 100644 --- a/src/dataViewer.ts +++ b/src/dataViewer.ts @@ -40,7 +40,7 @@ export function getDataViewerStyle(): string { `; } -export function getDataViewerToolbarHtml(sessionHtml = ''): string { +export function getDataViewerToolbarHtml(sessionHtml = '', refreshable = false): string { return `
@@ -50,6 +50,7 @@ export function getDataViewerToolbarHtml(sessionHtml = ''): string { + ${refreshable ? '' : ''}
diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index 7dd7bd76b..04115c8b4 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -45,6 +45,8 @@ export interface JdvDescriptor { class?: string[]; type?: string; parent?: string; + /** Why jdv sent this `view/open`: "register", "tracked", "refresh" or "child". */ + reason?: string; } export type JdvPathStep = @@ -76,7 +78,26 @@ export function parseDescriptor(value: unknown): JdvDescriptor { if (value.kind === 'grid' && !Array.isArray(value.columns)) { throw new Error('Invalid jdv grid descriptor'); } - return value as unknown as JdvDescriptor; + const descriptor = value as unknown as JdvDescriptor; + // `reason` is optional and may gain values; keep only a string. + if (typeof descriptor.reason !== 'string') { + delete descriptor.reason; + } + return descriptor; +} + +/** + * Whether a newer generation of an open view should bring its panel to the front. + * Updates that R made on its own (a tracked variable, a Refresh click) must not + * switch tabs; an explicit View(), an unknown reason or none still reveal. + */ +export function revealsPanel(descriptor: JdvDescriptor): boolean { + return descriptor.reason !== 'tracked' && descriptor.reason !== 'refresh'; +} + +/** Whether the view accepts `view/refresh`. Sequential streams cannot be rewound. */ +export function canRefresh(descriptor: JdvDescriptor): boolean { + return descriptor.capabilities.refresh === true && descriptor.access !== 'sequential'; } // --------------------------------------------------------------------------- diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index 27d767fa4..a9b02bd33 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -9,8 +9,8 @@ import * as path from 'path'; import * as vscode from 'vscode'; import { Uri, ViewColumn, window } from 'vscode'; import { - GridPage, JdvDescriptor, JdvRequestError, JsonObject, TreeAdapter, gridColumnDefs, - gridPageFromRows, parseDescriptor, tableRowsParams, + GridPage, JdvDescriptor, JdvRequestError, JsonObject, TreeAdapter, canRefresh, gridColumnDefs, + gridPageFromRows, parseDescriptor, revealsPanel, tableRowsParams, } from './jdvAdapter'; import { SequentialStream } from './jdvArrow'; import { JdvHost, JdvPeer } from './jdvTransport'; @@ -155,10 +155,12 @@ async function openJdvView( existing.stream?.dispose(); existing.stream = createStream(descriptor, peer); existing.panel.title = title; - // jdv sends the same notification for View() and for tracked updates, so - // an update cannot be told apart. Reveal without taking focus so that an - // explicit View() always surfaces the panel. - existing.panel.reveal(existing.panel.viewColumn, true); + // Updates that R made on its own (a tracked variable, a Refresh click) keep + // the panel where it is. Otherwise, including when jdv sends no reason, + // reveal without taking focus so that an explicit View() surfaces the panel. + if (revealsPanel(descriptor)) { + existing.panel.reveal(existing.panel.viewColumn, true); + } existing.panel.webview.html = await renderPanel(existing); return; } @@ -203,10 +205,11 @@ async function openJdvView( async function renderPanel(state: JdvPanel): Promise { const webview = state.panel.webview; + const refreshable = canRefresh(state.descriptor); if (state.descriptor.kind === 'grid') { - return session.getTableHtml(webview, undefined, state.title, state.sessionId); + return session.getTableHtml(webview, undefined, state.title, state.sessionId, refreshable); } - return session.getListHtml(webview, state.title, state.tree!.navigation([]), state.sessionId); + return session.getListHtml(webview, state.title, state.tree!.navigation([]), state.sessionId, refreshable); } function isPositivePath(value: unknown): value is number[] { @@ -219,6 +222,10 @@ async function handleMessage(state: JdvPanel, webview: vscode.Webview, message: return; } const current = () => documentGeneration === session.getDocumentGeneration(webview); + if (message.message === 'dataview/refresh' || message.message === 'listview/refresh') { + await refreshView(state, webview, documentGeneration); + return; + } if (message.message === 'dataview/request' && typeof message.requestId === 'number') { const reply = (ok: boolean, result?: unknown, error?: string) => { if (current()) { @@ -313,6 +320,29 @@ async function handleMessage(state: JdvPanel, webview: vscode.Webview, message: } } +/** + * Ask R to rebuild the view. On success the next generation arrives as a `view/open` + * that re-renders the panel, so only a failure is reported to the webview. + */ +async function refreshView(state: JdvPanel, webview: vscode.Webview, documentGeneration: number): Promise { + const { view, generation } = state.descriptor; + let error: string | undefined; + try { + if (state.closed) { + throw new Error(state.closed); + } + if (!canRefresh(state.descriptor)) { + throw new Error('This view cannot be refreshed.'); + } + await state.peer.request('view/refresh', { view, generation }); + } catch (e) { + error = `Unable to refresh: ${errorMessage(e)}`; + } + if (error !== undefined && documentGeneration === session.getDocumentGeneration(webview)) { + void webview.postMessage({ message: 'viewer/refreshFailed', documentGeneration, error }); + } +} + /** Fetch one AG Grid block, issuing further pages if jdv clamps the page size. */ async function fetchGridPage(state: JdvPanel, message: JsonObject): Promise> { const startRow = Number(message.startRow ?? 0); diff --git a/src/listViewer.ts b/src/listViewer.ts index 491171fc3..70c005acd 100644 --- a/src/listViewer.ts +++ b/src/listViewer.ts @@ -217,7 +217,7 @@ export function createListViewer( /** Panel wrapper and its interaction tests. Keep panel focus/blur hooks here. */ export function getListViewerScript(documentGeneration: number, initial: ListViewNavigation = { title: '', path: [], breadcrumbs: [{ label: '', path: [] }], -}): string { +}, refreshable = false): string { return ` const vscode = acquireVsCodeApi(); const documentGeneration = ${documentGeneration}; @@ -234,6 +234,21 @@ export function getListViewerScript(documentGeneration: number, initial: ListVie const message = event.data; if (['listview/page', 'listview/navigation'].includes(message.message) && message.documentGeneration === documentGeneration) viewer.reply(message); }); + ${refreshable ? ` + const refreshButton = document.getElementById('refresh'); + refreshButton.addEventListener('click', () => { + refreshButton.disabled = true; + document.getElementById('navigation-status').textContent = ''; + vscode.postMessage({ message: 'listview/refresh', documentGeneration }); + }); + // A successful refresh replaces this document, so only failures arrive. + window.addEventListener('message', event => { + const message = event.data; + if (message && message.message === 'viewer/refreshFailed' && message.documentGeneration === documentGeneration) { + refreshButton.disabled = false; + document.getElementById('navigation-status').textContent = message.error; + } + });` : ''} `; } @@ -250,7 +265,8 @@ export const listViewerStyle = ` } .r-list-viewer .back { gap: 4px; padding: 4px 6px; flex-shrink: 0; border-radius: 3px; } .r-list-viewer .back:disabled { opacity: 0.4; cursor: default; background: transparent; } - .r-list-viewer .reset { gap: 4px; padding: 4px 6px; flex-shrink: 0; border-radius: 3px; } + .r-list-viewer .reset, .r-list-viewer .refresh { gap: 4px; padding: 4px 6px; flex-shrink: 0; border-radius: 3px; } + .r-list-viewer .refresh:disabled { opacity: 0.4; cursor: default; background: transparent; } ${viewerSessionStyle} .r-list-viewer .viewer-session { margin-left: auto; } .r-list-viewer .codicon { flex-shrink: 0; } diff --git a/src/session.ts b/src/session.ts index f2a59a8e8..2691c338b 100644 --- a/src/session.ts +++ b/src/session.ts @@ -2007,6 +2007,7 @@ export function getViewerSessionContext(sessionId: string | null, saved?: Viewer export async function getTableHtml( webview: Webview, file: string | undefined, title: string, sessionId: string | null = null, + refreshable = false, ): Promise { const sessionHtml = getViewerSessionHtml(sessionId); const codicons = webview.asWebviewUri(Uri.file(path.join(resDir, 'codicon.css'))).toString(); @@ -2525,6 +2526,24 @@ export async function getTableHtml( retryBtn.addEventListener('click', retryCurrentPage); } + const refreshBtn = document.querySelector('#viewerRefresh'); + if (refreshBtn) { + refreshBtn.addEventListener('click', () => { + refreshBtn.disabled = true; + setFetchStatus('loading', 'Refreshing...', false); + vscode.postMessage({ message: 'dataview/refresh', documentGeneration }); + }); + // A successful refresh replaces this document, so only failures arrive. + window.addEventListener('message', (event) => { + const data = event.data; + if (data && data.message === 'viewer/refreshFailed' && + data.documentGeneration === documentGeneration) { + refreshBtn.disabled = false; + setFetchStatus('error', data.error || 'Unable to refresh.', false); + } + }); + } + updateTheme(); initialize().catch((e) => { console.error('[dataview] Initialization failed', e); @@ -2545,7 +2564,7 @@ export async function getTableHtml( - ${getDataViewerToolbarHtml(sessionHtml)} + ${getDataViewerToolbarHtml(sessionHtml, refreshable)}
${getDataViewerColumnPanelHtml()} @@ -2732,6 +2751,7 @@ export function getListHtml( title: string, navigation?: ListViewNavigation, sessionId: string | null = null, + refreshable = false, ): string { const documentGeneration = ++documentGenerationRevision; documentGenerations.set(webview, documentGeneration); @@ -2767,6 +2787,7 @@ export function getListHtml( + ${refreshable ? '' : ''} ${getViewerSessionHtml(sessionId)}
@@ -2774,7 +2795,7 @@ export function getListHtml( diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts index 91f9a235b..f3287d284 100644 --- a/src/test/node/jdvAdapter.test.ts +++ b/src/test/node/jdvAdapter.test.ts @@ -1,6 +1,7 @@ import * as assert from 'node:assert/strict'; import { JdvDescriptor, JdvRequestError, TreeAdapter, gridColumnDefs, gridPageFromRows, isVectorType, + canRefresh, revealsPanel, parseDescriptor, tableRowsParams, toJdvFilter, } from '../../jdvAdapter'; @@ -27,6 +28,28 @@ suite('jdv adapter', () => { assert.throws(() => parseDescriptor({ ...grid, columns: undefined })); }); + test('reveals the panel unless the update is tracked or a refresh', () => { + assert.equal(grid.reason, undefined); + assert.equal(revealsPanel(grid), true); + for (const reason of ['register', 'child', 'something-new']) { + assert.equal(revealsPanel(parseDescriptor({ ...grid, reason })), true, reason); + } + for (const reason of ['tracked', 'refresh']) { + const descriptor = parseDescriptor({ ...grid, reason }); + assert.equal(descriptor.reason, reason); + assert.equal(revealsPanel(descriptor), false, reason); + } + // A non-string reason is ignored. + assert.equal(parseDescriptor({ ...grid, reason: 5 }).reason, undefined); + }); + + test('offers refresh only for refreshable, non-sequential views', () => { + assert.equal(canRefresh(grid), true); + assert.equal(canRefresh({ ...grid, access: 'sequential' }), false); + assert.equal(canRefresh({ ...grid, capabilities: { ...capabilities, refresh: false } }), false); + assert.equal(canRefresh({ ...grid, capabilities: { ...capabilities, refresh: undefined } }), false); + }); + test('builds column definitions from the schema', () => { const columns = gridColumnDefs(grid); assert.deepEqual(columns.map(column => column.field), ['0', '1', '2', '3', '4', '5', '6']); From d40fa1916e69728aea3c12f33a9855a6c1bd6942 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 17 Oct 2026 08:07:34 +0000 Subject: [PATCH 11/38] chore: bump sess development version to 3.2.9000.9000 --- sess/DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sess/DESCRIPTION b/sess/DESCRIPTION index 7cf6dbf64..67a923b78 100644 --- a/sess/DESCRIPTION +++ b/sess/DESCRIPTION @@ -1,7 +1,7 @@ Package: sess Type: Package Title: High-Performance IPC Bridge for R Sessions -Version: 3.2.0 +Version: 3.2.9000.9000 Authors@R: c( person(given = "Randy", family = "Lai", From 6233430165b4f9725dcb55e61339ea38ff2ed8b6 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 17 Oct 2026 13:31:51 +0000 Subject: [PATCH 12/38] refactor: use jdv_binding() and jdv error classes in the View() hook Look up the variable to follow with jdv::jdv_binding() instead of a hand-written environment walk, never pass a binding for sequential Arrow sources, and retry without the binding only on jdv errors. --- sess/R/handlers.R | 62 +++++++++++++++++------------------ sess/inst/tinytest/test-jdv.R | 7 ++++ 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/sess/R/handlers.R b/sess/R/handlers.R index 4bc5c98b8..7787180eb 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1311,6 +1311,9 @@ handle_dataview_dispose <- function(params) { # Returns FALSE, leaving the user's own jdv state untouched, when the sess data # viewer should be used instead: jdv is missing or opted out, VS Code offers no # transport, or jdv is already connected elsewhere or owned by an embedding host. +# Every failure, whether a jdv_error (jdv_error_in_use, jdv_error_connect_failed) +# or the transport request itself failing, means the same fallback, so errors +# are not told apart. .jdv_connect <- function() { if (identical(getOption("sess.dataviewer", "auto"), "sess") || !requireNamespace("jdv", quietly = TRUE)) { @@ -1356,29 +1359,21 @@ handle_dataview_dispose <- function(params) { } # The variable that View() was called with, for jdv to follow. -# Returns list(name, env) when the expression is a single symbol bound in the -# caller's environment or one of its parents, and NULL otherwise. Nothing is -# evaluated: exists() does not force promises and active bindings are skipped. -# Tracking needs R 4.6.0 or later. It follows the VS Code setting -# `r.session.viewers.trackVariables` (passed to connect() as `track_variables`, -# default on); `options(sess.dataviewer.track = TRUE/FALSE)`, when set, wins. +# jdv::jdv_binding() does the lookup without evaluating anything: it returns +# list(name, env) for a symbol bound in `env` or one of its parents, and NULL +# for other expressions, unreadable bindings (promises, active bindings), unbound +# names and R older than 4.6.0. A jdv that predates jdv_binding() is not tracked. +# sess only adds the gates: the VS Code setting `r.session.viewers.trackVariables` +# (passed to connect() as `track_variables`, default on), which +# `options(sess.dataviewer.track = TRUE/FALSE)` overrides in both directions. .jdv_track_binding <- function(expression, env) { track <- getOption("sess.dataviewer.track", .sess_env$track_variables %||% TRUE) - if (!is.symbol(expression) || !nzchar(as.character(expression)) || - !isTRUE(track) || getRversion() < "4.6.0") { + if (!is.symbol(expression) || !nzchar(as.character(expression)) || !isTRUE(track) || + !requireNamespace("jdv", quietly = TRUE) || + !exists("jdv_binding", envir = asNamespace("jdv"), inherits = FALSE)) { return(NULL) } - name <- as.character(expression) - while (!identical(env, emptyenv())) { - if (exists(name, envir = env, inherits = FALSE)) { - if (bindingIsActive(name, env)) { - return(NULL) - } - return(list(name = name, env = env)) - } - env <- parent.env(env) - } - NULL + tryCatch(jdv::jdv_binding(expression, env), error = function(e) NULL) } # Show `x` with jdv. Returns FALSE when jdv cannot be used, so the caller falls @@ -1387,10 +1382,11 @@ handle_dataview_dispose <- function(params) { # The title is jdv's display slot: View() with the same title replaces the view # and increments its generation. Grids are paged so that sorting and filtering # stay in R. When `binding` names the variable that holds `x`, jdv follows it and -# rebuilds the view as the next generation after the variable is reassigned. It -# fails if the binding cannot be read without evaluation or does not hold `x` -# (for example View(x) called through do.call()); then the view is registered -# without tracking, since View() must not fail because of it. +# rebuilds the view as the next generation after the variable is reassigned. jdv +# rejects the binding with a jdv_error if it does not hold `x` (for example +# View(x) called through do.call()); then the view is registered without +# tracking, since View() must not fail because of it. Arrow sources are +# sequential and cannot be tracked, so they never get a binding. # # jdv checks tracked variables from its own task callback and sends the update # over the transport itself, so sess needs no task callback of its own for it. @@ -1404,16 +1400,18 @@ handle_dataview_dispose <- function(params) { return(FALSE) } title <- if (nzchar(title_key)) title_key else NULL - if (!is.null(binding)) { - registered <- tryCatch({ - jdv::jdv_register(x, title = title, mode = "paged", binding = binding) - TRUE - }, error = function(e) FALSE) - if (registered) { - return(TRUE) - } - } + # Only jdv errors about the binding are retried without it; anything else + # ends in FALSE so that View() falls back to the sess data viewer. tryCatch({ + if (!is.null(binding) && !.jdv_is_arrow(x)) { + registered <- tryCatch({ + jdv::jdv_register(x, title = title, mode = "paged", binding = binding) + TRUE + }, jdv_error = function(e) FALSE) + if (registered) { + return(TRUE) + } + } jdv::jdv_register(x, title = title, mode = "paged") TRUE }, error = function(e) FALSE) diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R index 7a5f83027..be75326d8 100644 --- a/sess/inst/tinytest/test-jdv.R +++ b/sess/inst/tinytest/test-jdv.R @@ -219,6 +219,13 @@ local({ expect_true(sess:::.jdv_show(fixed, "mismatch", list(name = "tracking", env = environment()))) expect_identical(wait_generations("mismatch", 1L), 1L) + # Sequential sources cannot be tracked, so no binding is passed for them. + if (requireNamespace("nanoarrow", quietly = TRUE)) { + stream <- nanoarrow::as_nanoarrow_array_stream(data.frame(a = 1:3)) + expect_true(sess:::.jdv_show(stream, "arrow", list(name = "stream", env = environment()))) + expect_identical(wait_generations("arrow", 1L), 1L) + } + # Releasing the runtime closes the connection that sess opened. sess:::.jdv_release() expect_identical(jdv::jdv_connection()$state, "disconnected") From 687d2f50df97728865fa36ba34f7d7ba19b30dd7 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 17 Oct 2026 13:31:51 +0000 Subject: [PATCH 13/38] feat: follow jdv API v1 in the data viewer - Offer text filters only for string and factor columns; list, other and unknown column types are display-only. - Support duration columns with numeric filters in the column's units. - Prefer the title jdv gives a child view. - Add a Follow toggle that pauses or resumes variable tracking with view/track and reflects view/tracking notifications. --- src/dataViewer.ts | 3 +- src/jdvAdapter.ts | 72 +++++++++++++++++++++++++--- src/jdvTransport.ts | 4 ++ src/jdvViewer.ts | 81 +++++++++++++++++++++++++++++--- src/listViewer.ts | 10 +++- src/session.ts | 13 ++++- src/test/node/jdvAdapter.test.ts | 56 +++++++++++++++++++++- src/test/node/jdvArrow.test.ts | 13 +++++ src/viewerFollow.ts | 54 +++++++++++++++++++++ 9 files changed, 289 insertions(+), 17 deletions(-) create mode 100644 src/viewerFollow.ts diff --git a/src/dataViewer.ts b/src/dataViewer.ts index e77f1e0b1..7827b6546 100644 --- a/src/dataViewer.ts +++ b/src/dataViewer.ts @@ -40,7 +40,7 @@ export function getDataViewerStyle(): string { `; } -export function getDataViewerToolbarHtml(sessionHtml = '', refreshable = false): string { +export function getDataViewerToolbarHtml(sessionHtml = '', refreshable = false, followable = false): string { return `
@@ -51,6 +51,7 @@ export function getDataViewerToolbarHtml(sessionHtml = '', refreshable = false): ${refreshable ? '' : ''} + ${followable ? '' : ''}
diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index 04115c8b4..b770d8106 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -13,6 +13,8 @@ export interface JdvCapabilities { next: boolean; children: boolean; refresh?: boolean; + /** True only when the view tracks a variable and has not lost it. */ + track?: boolean; } export interface JdvColumnSchema { @@ -26,6 +28,8 @@ export interface JdvColumnSchema { levels?: string[]; ordered?: boolean; tz?: string; + /** `duration` columns: the unit of the values (secs, mins, hours, days or weeks). */ + units?: string; } export interface JdvDescriptor { @@ -45,8 +49,22 @@ export interface JdvDescriptor { class?: string[]; type?: string; parent?: string; - /** Why jdv sent this `view/open`: "register", "tracked", "refresh" or "child". */ + /** + * Why jdv sent this `view/open`: "register", "child", "tracked" or "refresh". + * Missing or unknown values mean "register". + */ reason?: string; + /** Present only on views that can follow a variable. */ + tracking?: JdvTracking; +} + +export interface JdvTracking { + name: string; + state: 'active' | 'paused' | 'lost'; + /** Fixed token when lost: "removed", "collected" or an unknown value. */ + reason?: string; + /** English display text when lost; not to be parsed. */ + message?: string; } export type JdvPathStep = @@ -83,9 +101,27 @@ export function parseDescriptor(value: unknown): JdvDescriptor { if (typeof descriptor.reason !== 'string') { delete descriptor.reason; } + const tracking = parseTracking(descriptor.tracking); + if (tracking) { + descriptor.tracking = tracking; + } else { + delete descriptor.tracking; + } return descriptor; } +/** Validate a `tracking` object; a malformed one is dropped (`undefined`). */ +export function parseTracking(value: unknown): JdvTracking | undefined { + if (!isObject(value) || typeof value.name !== 'string' || + (value.state !== 'active' && value.state !== 'paused' && value.state !== 'lost')) { + return undefined; + } + const tracking: JdvTracking = { name: value.name, state: value.state }; + if (typeof value.reason === 'string') { tracking.reason = value.reason; } + if (typeof value.message === 'string') { tracking.message = value.message; } + return tracking; +} + /** * Whether a newer generation of an open view should bring its panel to the front. * Updates that R made on its own (a tracked variable, a Refresh click) must not @@ -138,6 +174,9 @@ function columnTooltip(column: JdvColumnSchema): string { if (column.type === 'datetime' && column.tz !== undefined) { tooltip += `, tz: ${column.tz || '(local)'}`; } + if (column.type === 'duration' && typeof column.units === 'string') { + tooltip += `, units: ${column.units}`; + } return tooltip; } @@ -193,6 +232,7 @@ export function gridColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { switch (column.type) { case 'integer': case 'number': + case 'duration': type = 'numericColumn'; filter = 'agNumberColumnFilter'; filterOptions = ORDER_FILTER_OPTIONS; @@ -217,9 +257,15 @@ export function gridColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { type = 'booleanColumn'; filter = true; break; - default: + case 'string': + case 'factor': filter = 'agTextColumnFilter'; filterOptions = TEXT_FILTER_OPTIONS; + break; + default: + // `list`, `other` and unknown types are display-only: only the NA + // options apply. Never infer semantics from `class`. + filter = 'agTextColumnFilter'; } // Value operators require `filterable`; `is_na` only needs a missingness mask, // which every column has unless its channel failed. @@ -307,6 +353,14 @@ function numberOperand(value: unknown, column: JdvColumnSchema): unknown { return number; } +/** Duration operand: a number in the column's units, or a typed `[-]H:MM:SS[.fff]` string. */ +function durationOperand(value: unknown, column: JdvColumnSchema): unknown { + if (typeof value === 'string' && /^\s*-?\d+:\d{2}:\d{2}(\.\d+)?\s*$/.test(value)) { + return value.trim(); + } + return numberOperand(value, column); +} + function dateOperand(value: unknown, column: JdvColumnSchema): string { // AG Grid date filters use "YYYY-MM-DD HH:mm:ss" in the column's wall time. const text = scalarText(value); @@ -343,15 +397,17 @@ function conditionNode(condition: JsonObject, col: number, column: JdvColumnSche if (type === 'false') { return { op: 'is_false', col }; } return unsupportedFilter(column, type); case 'integer': - case 'number': { + case 'number': + case 'duration': { + const operand = column.type === 'duration' ? durationOperand : numberOperand; if (type === 'inRange') { return { op: 'between', col, - lower: numberOperand(condition.filter, column), upper: numberOperand(condition.filterTo, column), + lower: operand(condition.filter, column), upper: operand(condition.filterTo, column), }; } const op = ORDER_OPS[String(type)]; - return op ? { op, col, value: numberOperand(condition.filter, column) } : unsupportedFilter(column, type); + return op ? { op, col, value: operand(condition.filter, column) } : unsupportedFilter(column, type); } case 'int64': if (type === 'equals') { @@ -375,7 +431,8 @@ function conditionNode(condition: JsonObject, col: number, column: JdvColumnSche const op = ORDER_OPS[String(type)]; return op ? { op, col, value: dateOperand(condition.dateFrom, column) } : unsupportedFilter(column, type); } - default: { + case 'string': + case 'factor': { const value = scalarText(condition.filter); // As in the previous viewer, negated text conditions exclude missing values. const negate = (node: FilterNode): FilterNode => ({ @@ -391,6 +448,9 @@ function conditionNode(condition: JsonObject, col: number, column: JdvColumnSche default: return unsupportedFilter(column, type); } } + default: + // Display-only types (`list`, `other`, unknown) accept only the NA options above. + return unsupportedFilter(column, type); } } diff --git a/src/jdvTransport.ts b/src/jdvTransport.ts index 5c4267e7c..333b4596b 100644 --- a/src/jdvTransport.ts +++ b/src/jdvTransport.ts @@ -304,6 +304,8 @@ export interface JdvHostEvents { viewClosed(peer: JdvPeer, view: string, generation: number): void; /** `view/update_failed` received from R: a tracked variable changed but the view could not be rebuilt. */ viewUpdateFailed(peer: JdvPeer, params: JsonObject): void; + /** `view/tracking` received from R: the tracking state changed without a rebuild (lost, paused or resumed). */ + viewTracking(peer: JdvPeer, params: JsonObject): void; /** The R connection closed; its views are gone. */ disconnected(peer: JdvPeer, reason: string): void; } @@ -373,6 +375,8 @@ export class JdvHost { this.events.viewClosed(peer, params.view, params.generation); } else if (method === 'view/update_failed') { this.events.viewUpdateFailed(peer, params); + } else if (method === 'view/tracking') { + this.events.viewTracking(peer, params); } }, closed: (peer, reason) => this.events.disconnected(peer, reason), diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index a9b02bd33..ac43b98aa 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -9,8 +9,8 @@ import * as path from 'path'; import * as vscode from 'vscode'; import { Uri, ViewColumn, window } from 'vscode'; import { - GridPage, JdvDescriptor, JdvRequestError, JsonObject, TreeAdapter, canRefresh, gridColumnDefs, - gridPageFromRows, parseDescriptor, revealsPanel, tableRowsParams, + GridPage, JdvDescriptor, JdvRequestError, JdvTracking, JsonObject, TreeAdapter, canRefresh, gridColumnDefs, + gridPageFromRows, parseDescriptor, parseTracking, revealsPanel, tableRowsParams, } from './jdvAdapter'; import { SequentialStream } from './jdvArrow'; import { JdvHost, JdvPeer } from './jdvTransport'; @@ -61,6 +61,7 @@ export function getJdvAddress(sessionId: string, pid: number | undefined): Promi }, viewClosed: (_peer, view, generation) => markClosed(sessionId, view, generation), viewUpdateFailed: (_peer, params) => warnUpdateFailed(sessionId, params), + viewTracking: (_peer, params) => applyTracking(sessionId, params), disconnected: (peer) => { for (const state of panels.values()) { if (state.peer === peer && !state.closed) { @@ -103,6 +104,37 @@ function warnUpdateFailed(sessionId: string | null, params: JsonObject): void { void window.showWarningMessage(`The data viewer could not update "${state.title}": ${reason}`); } +/** + * `view/tracking` changes the state of the shown generation without a rebuild. + * The generation is accepted as is: a pause or loss concerns the view, and a + * newer generation arrives as a `view/open` that carries its own state. + */ +function applyTracking(sessionId: string | null, params: JsonObject): void { + const tracking = parseTracking(params.tracking); + if (typeof params.view !== 'string' || !tracking) { + return; + } + const state = panels.get(panelKey(sessionId, params.view)); + if (!state || !state.descriptor.tracking) { + return; + } + setTracking(state, tracking); +} + +function setTracking(state: JdvPanel, tracking: JdvTracking): void { + state.descriptor.tracking = tracking; + state.descriptor.capabilities.track = tracking.state !== 'lost'; + // The webview is gone once the panel is disposed. + try { + const webview = state.panel.webview; + void webview.postMessage({ + message: 'viewer/tracking', documentGeneration: session.getDocumentGeneration(webview), tracking, + }); + } catch { + // Disposed panel: nothing to update. + } +} + function createStream(descriptor: JdvDescriptor, peer: JdvPeer): SequentialStream | undefined { if (descriptor.access !== 'sequential') { return undefined; @@ -133,9 +165,10 @@ async function openJdvView( ): Promise { const key = panelKey(sessionId, descriptor.view); const existing = panels.get(key); - // A child view opened from a tree carries no title; its parent knows the item. + // jdv titles a child view when its parent has a title; older jdv or an untitled + // parent leave it to the host-side bookkeeping of the item that was opened. const parent = descriptor.parent === undefined ? undefined : panels.get(panelKey(sessionId, descriptor.parent)); - const title = parent?.childTitle ?? descriptor.title ?? existing?.title ?? 'View'; + const title = descriptor.title ?? parent?.childTitle ?? existing?.title ?? 'View'; if (descriptor.kind === 'grid' && descriptor.access !== 'random' && descriptor.access !== 'sequential') { // Snapshot grids are not supported; release the view. void peer.request('view/close', { view: descriptor.view, generation: descriptor.generation }).catch(() => undefined); @@ -206,10 +239,11 @@ async function openJdvView( async function renderPanel(state: JdvPanel): Promise { const webview = state.panel.webview; const refreshable = canRefresh(state.descriptor); + const tracking = state.descriptor.tracking; if (state.descriptor.kind === 'grid') { - return session.getTableHtml(webview, undefined, state.title, state.sessionId, refreshable); + return session.getTableHtml(webview, undefined, state.title, state.sessionId, refreshable, tracking); } - return session.getListHtml(webview, state.title, state.tree!.navigation([]), state.sessionId, refreshable); + return session.getListHtml(webview, state.title, state.tree!.navigation([]), state.sessionId, refreshable, tracking); } function isPositivePath(value: unknown): value is number[] { @@ -226,6 +260,10 @@ async function handleMessage(state: JdvPanel, webview: vscode.Webview, message: await refreshView(state, webview, documentGeneration); return; } + if (message.message === 'viewer/track' && typeof message.enabled === 'boolean') { + await trackView(state, webview, documentGeneration, message.enabled); + return; + } if (message.message === 'dataview/request' && typeof message.requestId === 'number') { const reply = (ok: boolean, result?: unknown, error?: string) => { if (current()) { @@ -343,6 +381,37 @@ async function refreshView(state: JdvPanel, webview: vscode.Webview, documentGen } } +/** + * Ask R to pause or resume following the variable. Resuming may rebuild the view, + * in which case the new generation arrives as a `view/open` that re-renders the panel; + * otherwise the toggle is updated from the result. + */ +async function trackView( + state: JdvPanel, webview: vscode.Webview, documentGeneration: number, enabled: boolean, +): Promise { + const { view, generation } = state.descriptor; + let error: string | undefined; + try { + if (state.closed) { + throw new Error(state.closed); + } + if (!state.descriptor.tracking) { + throw new Error('This view does not follow a variable.'); + } + const result = await state.peer.request('view/track', { view, generation, enabled }); + const tracking = parseTracking((result as JsonObject | undefined)?.tracking); + // A rebuilt view was already applied by its `view/open`. + if (tracking && state.descriptor.tracking && (result as JsonObject).generation === state.descriptor.generation) { + setTracking(state, tracking); + } + } catch (e) { + error = `Unable to ${enabled ? 'follow' : 'pause following'} the variable: ${errorMessage(e)}`; + } + if (error !== undefined && documentGeneration === session.getDocumentGeneration(webview)) { + void webview.postMessage({ message: 'viewer/trackFailed', documentGeneration, error }); + } +} + /** Fetch one AG Grid block, issuing further pages if jdv clamps the page size. */ async function fetchGridPage(state: JdvPanel, message: JsonObject): Promise> { const startRow = Number(message.startRow ?? 0); diff --git a/src/listViewer.ts b/src/listViewer.ts index 70c005acd..d92959833 100644 --- a/src/listViewer.ts +++ b/src/listViewer.ts @@ -1,6 +1,8 @@ /// import { viewerSessionStyle } from './viewerSession'; +import { getFollowScript } from './viewerFollow'; +import type { JdvTracking } from './jdvAdapter'; export interface ListViewNavigation { title: string; @@ -217,7 +219,7 @@ export function createListViewer( /** Panel wrapper and its interaction tests. Keep panel focus/blur hooks here. */ export function getListViewerScript(documentGeneration: number, initial: ListViewNavigation = { title: '', path: [], breadcrumbs: [{ label: '', path: [] }], -}, refreshable = false): string { +}, refreshable = false, tracking?: JdvTracking): string { return ` const vscode = acquireVsCodeApi(); const documentGeneration = ${documentGeneration}; @@ -249,6 +251,9 @@ export function getListViewerScript(documentGeneration: number, initial: ListVie document.getElementById('navigation-status').textContent = message.error; } });` : ''} + ${tracking ? getFollowScript(tracking, 'follow', + "document.getElementById('navigation-status').textContent = '';", + "document.getElementById('navigation-status').textContent = message.error;") : ''} `; } @@ -266,6 +271,9 @@ export const listViewerStyle = ` .r-list-viewer .back { gap: 4px; padding: 4px 6px; flex-shrink: 0; border-radius: 3px; } .r-list-viewer .back:disabled { opacity: 0.4; cursor: default; background: transparent; } .r-list-viewer .reset, .r-list-viewer .refresh { gap: 4px; padding: 4px 6px; flex-shrink: 0; border-radius: 3px; } + .r-list-viewer .follow { gap: 4px; padding: 4px 6px; flex-shrink: 0; border-radius: 3px; } + .r-list-viewer .follow[aria-pressed="true"] { background: var(--vscode-toolbar-activeBackground, var(--vscode-button-secondaryBackground)); } + .r-list-viewer .follow:disabled { opacity: 0.4; cursor: default; background: transparent; } .r-list-viewer .refresh:disabled { opacity: 0.4; cursor: default; background: transparent; } ${viewerSessionStyle} .r-list-viewer .viewer-session { margin-left: auto; } diff --git a/src/session.ts b/src/session.ts index 2691c338b..2a4bf1598 100644 --- a/src/session.ts +++ b/src/session.ts @@ -36,6 +36,8 @@ import { showWebView } from './webViewer'; import { getListViewerScript, listViewerStyle, ListViewNavigation } from './listViewer'; import { getDataViewerScript, getDataViewerStyle, getDataViewerToolbarHtml } from './dataViewer'; import { disposeJdvHost, getJdvAddress } from './jdvViewer'; +import { getFollowScript } from './viewerFollow'; +import type { JdvTracking } from './jdvAdapter'; import { getDataViewerColumnPanelHtml, getDataViewerColumnPanelScript, getDataViewerColumnPanelStyle } from './dataViewerColumnPanel'; import { createViewerSessionContext, formatSessionLabel, getViewerSessionScript, ViewerSessionContext, ViewerSessionSource } from './viewerSession'; import { SessConsentChoice, SessConsentService } from './sessConsent'; @@ -2008,6 +2010,7 @@ export function getViewerSessionContext(sessionId: string | null, saved?: Viewer export async function getTableHtml( webview: Webview, file: string | undefined, title: string, sessionId: string | null = null, refreshable = false, + tracking?: JdvTracking, ): Promise { const sessionHtml = getViewerSessionHtml(sessionId); const codicons = webview.asWebviewUri(Uri.file(path.join(resDir, 'codicon.css'))).toString(); @@ -2544,6 +2547,10 @@ export async function getTableHtml( }); } + ${tracking ? getFollowScript(tracking, 'viewerFollow', + '', + "setFetchStatus('error', message.error || 'Unable to change following.', false);") : ''} + updateTheme(); initialize().catch((e) => { console.error('[dataview] Initialization failed', e); @@ -2564,7 +2571,7 @@ export async function getTableHtml( - ${getDataViewerToolbarHtml(sessionHtml, refreshable)} + ${getDataViewerToolbarHtml(sessionHtml, refreshable, tracking !== undefined)}
${getDataViewerColumnPanelHtml()} @@ -2752,6 +2759,7 @@ export function getListHtml( navigation?: ListViewNavigation, sessionId: string | null = null, refreshable = false, + tracking?: JdvTracking, ): string { const documentGeneration = ++documentGenerationRevision; documentGenerations.set(webview, documentGeneration); @@ -2788,6 +2796,7 @@ export function getListHtml( ${refreshable ? '' : ''} + ${tracking ? '' : ''} ${getViewerSessionHtml(sessionId)}
@@ -2795,7 +2804,7 @@ export function getListHtml( diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts index f3287d284..09cf5229f 100644 --- a/src/test/node/jdvAdapter.test.ts +++ b/src/test/node/jdvAdapter.test.ts @@ -2,7 +2,7 @@ import * as assert from 'node:assert/strict'; import { JdvDescriptor, JdvRequestError, TreeAdapter, gridColumnDefs, gridPageFromRows, isVectorType, canRefresh, revealsPanel, - parseDescriptor, tableRowsParams, toJdvFilter, + parseDescriptor, parseTracking, tableRowsParams, toJdvFilter, } from '../../jdvAdapter'; const capabilities = { seek: true, sort: true, filter: true, known_total: true, next: false, children: false, refresh: true }; @@ -63,6 +63,60 @@ suite('jdv adapter', () => { assert.match(columns[2].headerTooltip ?? '', /levels: x, y/); }); + test('keeps text operators to strings and factors, and handles durations', () => { + const descriptor = parseDescriptor({ + ...grid, + columns: [ + { position: 0, name: 's', type: 'string', class: ['character'], sortable: true, filterable: true }, + { position: 1, name: 'l', type: 'list', class: ['list'], sortable: false, filterable: false }, + { position: 2, name: 'u', type: 'future', class: ['character'], sortable: true, filterable: true }, + { position: 3, name: 'o', type: 'other', class: ['character'], sortable: false, filterable: true }, + { position: 4, name: 'dt', type: 'duration', class: ['difftime'], sortable: true, filterable: true, units: 'mins' }, + ], + }); + const columns = gridColumnDefs(descriptor); + assert.ok(columns[1].jdv?.filterOptions.includes('contains')); + // List, other and unknown types are display-only: NA options but no value operators, + // whatever `class` suggests. They still follow `sortable`. + for (const index of [2, 3, 4]) { + assert.deepEqual(columns[index].jdv, { filterOptions: [], naOptions: true }, String(index)); + } + assert.equal(columns[3].sortable, true); + assert.equal(columns[4].sortable, false); + for (const field of ['2', '3', '4']) { + assert.deepEqual(toJdvFilter({ [field]: { filterType: 'text', type: 'jdvNotNa' } }, descriptor), + { op: 'not', arg: { op: 'is_na', col: Number(field) - 1 } }); + assert.throws(() => toJdvFilter({ [field]: { filterType: 'text', type: 'contains', filter: 'x' } }, descriptor), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported', field); + } + // The server may allow an unknown type to be filtered; the host still never guesses. + assert.throws(() => toJdvFilter({ '3': { filterType: 'text', type: 'contains', filter: 'x' } }, descriptor), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported'); + + assert.equal(columns[5].type, 'numericColumn'); + assert.equal(columns[5].filter, 'agNumberColumnFilter'); + assert.ok(columns[5].jdv?.filterOptions.includes('inRange')); + assert.match(columns[5].headerTooltip ?? '', /units: mins/); + assert.deepEqual(toJdvFilter({ '5': { filterType: 'number', type: 'lessThanOrEqual', filter: 90 } }, descriptor), + { op: 'le', col: 4, value: 90 }); + assert.deepEqual(toJdvFilter({ '5': { filterType: 'number', type: 'inRange', filter: 1.5, filterTo: '2:00:00' } }, descriptor), + { op: 'between', col: 4, lower: 1.5, upper: '2:00:00' }); + assert.throws(() => toJdvFilter({ '5': { filterType: 'number', type: 'equals', filter: 'abc' } }, descriptor), + (e: unknown) => e instanceof JdvRequestError && e.code === 'invalid_request'); + }); + + test('parses tracking and drops a malformed one', () => { + assert.equal(grid.tracking, undefined); + const lost = { name: 'df', state: 'lost', reason: 'removed', message: 'gone' }; + assert.deepEqual(parseDescriptor({ ...grid, tracking: lost }).tracking, lost); + assert.deepEqual(parseDescriptor({ ...grid, tracking: { name: 'df', state: 'active', extra: 1 } }).tracking, + { name: 'df', state: 'active' }); + for (const tracking of [null, 'df', { name: 'df' }, { name: 'df', state: 'weird' }, { state: 'active' }]) { + assert.equal(parseDescriptor({ ...grid, tracking }).tracking, undefined, JSON.stringify(tracking)); + } + assert.equal(parseTracking([]), undefined); + }); + test('converts sort and filter models', () => { const params = tableRowsParams(grid, 100, 600, [{ colId: '2', sort: 'desc' }, { colId: '1', sort: 'asc' }], diff --git a/src/test/node/jdvArrow.test.ts b/src/test/node/jdvArrow.test.ts index 063240665..e850c82ce 100644 --- a/src/test/node/jdvArrow.test.ts +++ b/src/test/node/jdvArrow.test.ts @@ -137,11 +137,13 @@ suite('jdv transport', () => { test('exchanges requests, wakes and Arrow bodies with R', async () => { const opened: string[] = []; const failures: JsonObject[] = []; + const tracking: JsonObject[] = []; let peer: JdvPeer | undefined; const host = await JdvHost.create(4321, { viewOpen: (connection, params) => { peer = connection; opened.push(String(params.view)); }, viewClosed: () => undefined, viewUpdateFailed: (_connection, params) => { failures.push(params); }, + viewTracking: (_connection, params) => { tracking.push(params); }, disconnected: () => undefined, }); try { @@ -156,6 +158,16 @@ suite('jdv transport', () => { while (failures.length === 0) { await new Promise(resolve => setTimeout(resolve, 5)); } assert.equal(failures[0].message, 'too large'); + // A tracking change arrives before the response of the request that caused it. + const tracked = peer.request('view/track', { view: 'v-1', generation: 1, enabled: false }); + const trackRequest = await r.next(line => line.method === 'view/track'); + assert.deepEqual(trackRequest.params, { view: 'v-1', generation: 1, enabled: false }); + const paused = { name: 'df', state: 'paused' }; + r.send({ method: 'view/tracking', params: { view: 'v-1', generation: 1, tracking: paused } }); + r.send({ id: trackRequest.id, result: { view: 'v-1', generation: 1, tracking: paused }, warnings: [] }); + assert.deepEqual(await tracked, { view: 'v-1', generation: 1, tracking: paused }); + assert.deepEqual(tracking, [{ view: 'v-1', generation: 1, tracking: paused }]); + const response = peer.requestWithBody('stream/next', { view: 'v-1', generation: 1, max_rows: 10 }); const request = await r.next(line => line.method === 'stream/next'); // Wakes repeat while the request is pending. @@ -190,6 +202,7 @@ suite('jdv transport', () => { viewOpen: () => undefined, viewClosed: () => undefined, viewUpdateFailed: () => undefined, + viewTracking: () => undefined, disconnected: (_peer, why) => { reason = why; }, }); try { diff --git a/src/viewerFollow.ts b/src/viewerFollow.ts new file mode 100644 index 000000000..d402326b5 --- /dev/null +++ b/src/viewerFollow.ts @@ -0,0 +1,54 @@ +'use strict'; + +// The "Follow" toggle shared by the Data Viewer and List Viewer toolbars. It is shown +// only for jdv views that can follow a variable (`tracking` in the descriptor). + +import type { JdvTracking } from './jdvAdapter'; + +/** Update the toggle for a tracking state. Runs in the webview, so it must be self-contained. */ +export function updateFollowButton(button: HTMLButtonElement, tracking: JdvTracking): void { + const lost = tracking.state === 'lost'; + button.disabled = lost; + button.setAttribute('aria-pressed', String(tracking.state === 'active')); + if (lost) { + button.title = tracking.message || `The variable "${tracking.name}" is no longer available.`; + } else if (tracking.state === 'active') { + button.title = `Following "${tracking.name}": the view updates when the variable changes. Click to pause.`; + } else { + button.title = `Paused. Click to follow "${tracking.name}" again.`; + } +} + +/** + * Webview script that wires the toggle `buttonId`. `onStart` and `onError` are + * source snippets that show a status; `onError` can use `message.error`. + */ +export function getFollowScript( + tracking: JdvTracking, buttonId: string, onStart: string, onError: string, +): string { + return ` + (() => { + const followButton = document.getElementById(${JSON.stringify(buttonId)}); + if (!followButton) { return; } + const updateFollowButton = ${updateFollowButton.toString()}; + let tracking = ${JSON.stringify(tracking).replace(/ { + followButton.disabled = true; + ${onStart} + vscode.postMessage({ message: 'viewer/track', documentGeneration, enabled: tracking.state !== 'active' }); + }); + // A rebuilt view replaces this document, so only state changes and failures arrive. + window.addEventListener('message', event => { + const message = event.data; + if (!message || message.documentGeneration !== documentGeneration) { return; } + if (message.message === 'viewer/tracking') { + tracking = message.tracking; + updateFollowButton(followButton, tracking); + } else if (message.message === 'viewer/trackFailed') { + updateFollowButton(followButton, tracking); + ${onError} + } + }); + })();`; +} From bb69e8b041bbb8d5e034a275777f190ba8aa824e Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 00:14:09 +0000 Subject: [PATCH 14/38] fix: preserve jdv datetime filter operands and clarify duration units --- src/dataViewer.ts | 8 +++++-- src/jdvAdapter.ts | 31 +++++++++++++++++--------- src/test/node/jdvAdapter.test.ts | 37 +++++++++++++++++++++++++++++++ src/test/suite/dataViewer.test.ts | 8 ++++++- 4 files changed, 71 insertions(+), 13 deletions(-) diff --git a/src/dataViewer.ts b/src/dataViewer.ts index 7827b6546..09e77533e 100644 --- a/src/dataViewer.ts +++ b/src/dataViewer.ts @@ -113,7 +113,10 @@ export function getDataViewerScript(): string { if (column.type === 'dateColumn' || column.type === 'datetimeColumn') { column.cellDataType = column.type === 'dateColumn' ? 'dateString' : 'dateTimeString'; column.filter = 'agDateColumnFilter'; - column.filterParams = { browserDatePicker: true }; + column.filterParams = { + browserDatePicker: true, + includeTime: column.type === 'datetimeColumn' + }; column.initialWidth = 200; } else if (column.type === 'bigintColumn') { column.cellDataType = 'bigint'; @@ -149,7 +152,8 @@ export function getDataViewerScript(): string { ...column.filterParams, buttons: ['reset', 'apply'], filterOptions: [...column.jdv.filterOptions, ...naOptions], - inRangeInclusive: true + inRangeInclusive: true, + ...(column.jdv.units ? { filterPlaceholder: 'Value (' + column.jdv.units + ')' } : {}) }; delete column.jdv; } diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index b770d8106..bc89d7681 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -153,7 +153,7 @@ export interface GridColumnDef { suppressHeaderMenuButton?: boolean; lockPosition?: string; /** Webview-only options; see prepareViewerColumns(). */ - jdv?: { filterOptions: string[]; naOptions: boolean }; + jdv?: { filterOptions: string[]; naOptions: boolean; units?: string }; } const TEXT_FILTER_OPTIONS = ['contains', 'notContains', 'equals', 'notEqual', 'startsWith', 'endsWith']; @@ -279,7 +279,11 @@ export function gridColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { type, filter: filterEnabled ? filter : false, sortable, - jdv: { filterOptions, naOptions: true }, + jdv: { + filterOptions, + naOptions: true, + ...(column.type === 'duration' && typeof column.units === 'string' ? { units: column.units } : {}), + }, }; if (column.type === 'boolean') { def.cellDataType = 'boolean'; @@ -362,17 +366,24 @@ function durationOperand(value: unknown, column: JdvColumnSchema): unknown { } function dateOperand(value: unknown, column: JdvColumnSchema): string { - // AG Grid date filters use "YYYY-MM-DD HH:mm:ss" in the column's wall time. + // AG Grid's date filter model uses "YYYY-MM-DD HH:mm:ss". Keep the + // datetime text intact (including fractional seconds and explicit offsets) + // so jdv can interpret it without a JavaScript Date round trip. const text = scalarText(value); - const match = /^(\d{4}-\d{2}-\d{2})(?:[ T](\d{2}:\d{2}:\d{2}))?/.exec(text); - if (!match) { - throw new JdvRequestError('invalid_request', `Invalid date for column "${column.name}"`); - } + const invalid = () => new JdvRequestError('invalid_request', `Invalid date for column "${column.name}"`); if (column.type === 'date') { - return match[1]; - } + const date = /^(\d{4}-\d{2}-\d{2})$(?![\s\S])/.exec(text); + if (date) { return date[1]; } + // Accept AG Grid's historical midnight datetime value, but do not + // silently discard an actual time or timezone from a date operand. + const midnight = /^(\d{4}-\d{2}-\d{2})[ T]00:00:00$(?![\s\S])/.exec(text); + if (midnight) { return midnight[1]; } + throw invalid(); + } + const datetime = /^(\d{4}-\d{2}-\d{2})(?:[ T](\d{2}:\d{2}:\d{2})(\.\d+)?(Z|[+-]\d{2}:\d{2})?)?$(?![\s\S])/.exec(text); + if (!datetime) { throw invalid(); } // An operand without offset uses the column's time zone in jdv. - return `${match[1]}T${match[2] ?? '00:00:00'}`; + return `${datetime[1]}T${datetime[2] ?? '00:00:00'}${datetime[3] ?? ''}${datetime[4] ?? ''}`; } const ORDER_OPS: Record = { diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts index 09cf5229f..f3a50090a 100644 --- a/src/test/node/jdvAdapter.test.ts +++ b/src/test/node/jdvAdapter.test.ts @@ -96,6 +96,7 @@ suite('jdv adapter', () => { assert.equal(columns[5].type, 'numericColumn'); assert.equal(columns[5].filter, 'agNumberColumnFilter'); assert.ok(columns[5].jdv?.filterOptions.includes('inRange')); + assert.equal(columns[5].jdv?.units, 'mins'); assert.match(columns[5].headerTooltip ?? '', /units: mins/); assert.deepEqual(toJdvFilter({ '5': { filterType: 'number', type: 'lessThanOrEqual', filter: 90 } }, descriptor), { op: 'le', col: 4, value: 90 }); @@ -146,6 +147,42 @@ suite('jdv adapter', () => { { op: 'gt', col: 2, value: '2020-01-02' }); assert.deepEqual(toJdvFilter({ '4': { filterType: 'date', type: 'lessThan', dateFrom: '2020-01-02 10:30:00' } }, grid), { op: 'lt', col: 3, value: '2020-01-02T10:30:00' }); + for (const [input, expected] of [ + ['2020-01-02T10:30:00.125Z', '2020-01-02T10:30:00.125Z'], + ['2020-01-02 10:30:00.125+09:00', '2020-01-02T10:30:00.125+09:00'], + ['2020-01-02T10:30:00-05:30', '2020-01-02T10:30:00-05:30'], + ['2020-01-02', '2020-01-02T00:00:00'], + ]) { + assert.deepEqual(toJdvFilter({ '4': { filterType: 'date', type: 'equals', dateFrom: input } }, grid), + { op: 'eq', col: 3, value: expected }); + } + for (const input of ['2020-01-02T10:30:00garbage', '2020-01-02T10:30:00.1Ztail', '2020-01-02 10:30']) { + assert.throws(() => toJdvFilter({ '4': { filterType: 'date', type: 'equals', dateFrom: input } }, grid), + (e: unknown) => e instanceof JdvRequestError && e.code === 'invalid_request', input); + } + for (const input of ['2020-01-02T10:30:00\n', '2020-01-02T10:30:00\r\n']) { + assert.throws(() => toJdvFilter({ '4': { filterType: 'date', type: 'equals', dateFrom: input } }, grid), + (e: unknown) => e instanceof JdvRequestError && e.code === 'invalid_request', JSON.stringify(input)); + } + assert.deepEqual(toJdvFilter({ '3': { filterType: 'date', type: 'equals', dateFrom: '2020-01-02' } }, grid), + { op: 'eq', col: 2, value: '2020-01-02' }); + assert.deepEqual(toJdvFilter({ '3': { filterType: 'date', type: 'equals', dateFrom: '2020-01-02 00:00:00' } }, grid), + { op: 'eq', col: 2, value: '2020-01-02' }); + for (const input of ['2020-01-02 10:30:00', '2020-01-02T00:00:00Z', '2020-01-02garbage']) { + assert.throws(() => toJdvFilter({ '3': { filterType: 'date', type: 'equals', dateFrom: input } }, grid), + (e: unknown) => e instanceof JdvRequestError && e.code === 'invalid_request', input); + } + for (const input of ['2020-01-02\n', '2020-01-02\r\n']) { + assert.throws(() => toJdvFilter({ '3': { filterType: 'date', type: 'equals', dateFrom: input } }, grid), + (e: unknown) => e instanceof JdvRequestError && e.code === 'invalid_request', JSON.stringify(input)); + } + assert.deepEqual(toJdvFilter({ '4': { + filterType: 'date', type: 'inRange', + dateFrom: '2020-01-02 10:30:00.125+09:00', dateTo: '2020-01-03T11:45:00.875-05:30', + } }, grid), { + op: 'between', col: 3, + lower: '2020-01-02T10:30:00.125+09:00', upper: '2020-01-03T11:45:00.875-05:30', + }); assert.deepEqual(toJdvFilter({ '5': { filterType: 'bigint', type: 'equals', filter: '+0012' } }, grid), { op: 'eq', col: 4, value: '12' }); assert.throws(() => toJdvFilter({ '5': { filterType: 'bigint', type: 'lessThan', filter: '1' } }, grid), diff --git a/src/test/suite/dataViewer.test.ts b/src/test/suite/dataViewer.test.ts index cdba53f87..2b93517d6 100644 --- a/src/test/suite/dataViewer.test.ts +++ b/src/test/suite/dataViewer.test.ts @@ -192,6 +192,8 @@ suite('Data viewer', () => { assert.strictEqual(defs[3].cellDataType, 'dateString'); assert.strictEqual(defs[4].cellDataType, 'dateTimeString'); assert.strictEqual(defs[4].filter, 'agDateColumnFilter'); + assert.strictEqual((defs[3].filterParams as { includeTime: boolean }).includeTime, false); + assert.strictEqual((defs[4].filterParams as { includeTime: boolean }).includeTime, true); assert.strictEqual(defs[5].cellDataType, 'boolean'); assert.strictEqual(defs[6].cellDataType, 'bigint'); assert.strictEqual(defs[6].filter, 'agBigIntColumnFilter'); @@ -232,9 +234,10 @@ suite('Data viewer', () => { test('jdv columns offer only jdv operators and keep display text', () => { const viewer = viewerScript(); const jdvColumns = [ - { field: '1', headerName: 'n', type: 'numericColumn', filter: 'agNumberColumnFilter', jdv: { filterOptions: ['equals', 'inRange'], naOptions: true } }, + { field: '1', headerName: 'n', type: 'numericColumn', filter: 'agNumberColumnFilter', jdv: { filterOptions: ['equals', 'inRange'], naOptions: true, units: 'hours' } }, { field: '2', headerName: 'i', type: 'bigintColumn', filter: 'agBigIntColumnFilter', jdv: { filterOptions: ['equals'], naOptions: false } }, { field: '3', headerName: 'd', type: 'dateColumn', filter: 'agDateColumnFilter', jdv: { filterOptions: [], naOptions: true } }, + { field: '4', headerName: 'dt', type: 'datetimeColumn', filter: 'agDateColumnFilter', jdv: { filterOptions: ['equals', 'inRange'], naOptions: true } }, ] as (ColDef & { jdv?: unknown })[]; const bigintFields = viewer.script.prepareViewerColumns(jdvColumns); assert.strictEqual(bigintFields.length, 0); @@ -246,7 +249,10 @@ suite('Data viewer', () => { assert.deepStrictEqual(options(jdvColumns[1]), ['equals']); assert.deepStrictEqual(options(jdvColumns[2]), ['jdvIsNa', 'jdvNotNa']); assert.strictEqual((jdvColumns[0].filterParams as { inRangeInclusive: boolean }).inRangeInclusive, true); + assert.strictEqual((jdvColumns[0].filterParams as { filterPlaceholder: string }).filterPlaceholder, 'Value (hours)'); assert.strictEqual((jdvColumns[2].filterParams as { browserDatePicker: boolean }).browserDatePicker, true); + assert.strictEqual((jdvColumns[2].filterParams as { includeTime: boolean }).includeTime, false); + assert.strictEqual((jdvColumns[3].filterParams as { includeTime: boolean }).includeTime, true); for (const column of jdvColumns) { assert.strictEqual(column.cellDataType, undefined); assert.strictEqual(column.jdv, undefined); From eb00208f7804476f191050992cc9eb5581268559 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 00:39:51 +0000 Subject: [PATCH 15/38] feat: use jdv for explicit View() calls in Interactive --- sess/R/hooks.R | 11 +- sess/inst/tinytest/test-jdv.R | 11 ++ src/interactive/jdvAddress.ts | 31 +++++ src/interactive/manager.ts | 48 +++++++- src/jdvHostLifecycle.ts | 21 ++++ src/jdvViewer.ts | 23 ++-- src/test/node/interactiveJdvAddress.test.ts | 119 ++++++++++++++++++++ src/test/node/interactiveRuntime.test.ts | 46 ++++++++ 8 files changed, 296 insertions(+), 14 deletions(-) create mode 100644 src/interactive/jdvAddress.ts create mode 100644 src/jdvHostLifecycle.ts create mode 100644 src/test/node/interactiveJdvAddress.test.ts diff --git a/sess/R/hooks.R b/sess/R/hooks.R index 7c7a3a11d..3dbfc7c6d 100644 --- a/sess/R/hooks.R +++ b/sess/R/hooks.R @@ -78,11 +78,6 @@ runtime_start <- function(use_rstudioapi = TRUE, # follow it. The lookup starts at the caller, never at this function's frame. tracked_binding <- .jdv_track_binding(original_expression, parent.frame()) - if (is.null(.sess_env$view_owner) && isTRUE(.sess_env$interactive_connected) && - .interactive_rich_value(x)) { - return(invisible(NULL)) - } - view_type <- if (dataview_is_table(x)) { "table" } else if (listview_supported(x)) { @@ -95,6 +90,12 @@ runtime_start <- function(use_rstudioapi = TRUE, .jdv_show(x, title_key, tracked_binding)) { return(invisible(NULL)) } + # Explicit View() in Interactive should open the normal data viewer when + # jdv is available. Inline rich output remains the fallback when it is not. + if (is.null(.sess_env$view_owner) && isTRUE(.sess_env$interactive_connected) && + .interactive_rich_value(x)) { + return(invisible(NULL)) + } owner <- owner %||% title_key registry_key <- paste0(view_type, ":", owner) dataview_registry <- .sess_env$dataview_registry diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R index be75326d8..6ac83024b 100644 --- a/sess/inst/tinytest/test-jdv.R +++ b/sess/inst/tinytest/test-jdv.R @@ -8,6 +8,7 @@ local({ runtime <- sess:::.sess_env previous_con <- runtime$con previous_plot_path <- runtime$latest_plot_path + previous_interactive_connected <- runtime$interactive_connected original_request_client <- ns$request_client pipe <- processx::conn_create_pipepair() host <- NULL @@ -19,6 +20,7 @@ local({ lockBinding("request_client", ns) runtime$con <- previous_con runtime$latest_plot_path <- previous_plot_path + runtime$interactive_connected <- previous_interactive_connected if (!is.null(host)) host$kill() lapply(pipe, close) }, add = TRUE) @@ -226,6 +228,15 @@ local({ expect_identical(wait_generations("arrow", 1L), 1L) } + # Explicit View() in Interactive opens the regular jdv panel before trying + # the inline rich-output path used by ordinary cell results. + runtime$interactive_connected <- TRUE + utils::View(sample, "interactive explicit") + runtime$interactive_connected <- FALSE + host_messages(1L) + expect_identical(tail(vapply(opens(), function(message) message$params$title, ""), 1L), + "interactive explicit") + # Releasing the runtime closes the connection that sess opened. sess:::.jdv_release() expect_identical(jdv::jdv_connection()$state, "disconnected") diff --git a/src/interactive/jdvAddress.ts b/src/interactive/jdvAddress.ts new file mode 100644 index 000000000..668cf67de --- /dev/null +++ b/src/interactive/jdvAddress.ts @@ -0,0 +1,31 @@ +export interface InteractiveJdvAddressOptions { + sessionId: string; + hostKey: string; + pid?: number; + canConnect(): boolean; + isCurrent(): boolean; + getAddress(sessionId: string, pid: number | undefined, hostKey: string): Promise; + disposeHost(hostKey: string, expectedAddress: string): void | Promise; +} + +/** Acquire a session-scoped listener without returning an address to a stale client. */ +export async function acquireInteractiveJdvAddress(options: InteractiveJdvAddressOptions): Promise { + if (!options.canConnect()) { + throw new Error('Only the controlling Interactive client can connect the data viewer.'); + } + const address = await options.getAddress(options.sessionId, options.pid, options.hostKey); + if (!options.canConnect() || !options.isCurrent()) { + await options.disposeHost(options.hostKey, address); + throw new Error('The Interactive session changed while opening its data viewer.'); + } + return address; +} + +/** Drop a listener when another client takes control of the same R process. */ +export function disposeInteractiveJdvHostOnControlLoss( + previouslyControlled: boolean, currentlyControlled: boolean, disposeHost: () => void, +): boolean { + if (!previouslyControlled || currentlyControlled) { return false; } + disposeHost(); + return true; +} diff --git a/src/interactive/manager.ts b/src/interactive/manager.ts index 499c820d9..1e979d6bb 100644 --- a/src/interactive/manager.ts +++ b/src/interactive/manager.ts @@ -15,6 +15,7 @@ import { Transcript, TranscriptCell } from './transcript'; import { DISPLAY_MIME, InteractiveSerializer, persistentCellMetadata, persistentNotebookMetadata, restoredCellMetadata } from './notebook'; import { setInteractiveExecutor } from './executionTarget'; import * as session from '../session'; +import { disposeJdvHost, getJdvAddress } from '../jdvViewer'; import * as util from '../util'; import type { ListViewNavigation } from '../listViewer'; import { escapeXml } from './plotSvg'; @@ -26,6 +27,7 @@ import { readExecutionRecords, readPreviousJournal } from './journal'; import { sessionAge, sessionPresentation } from './sessionPresentation'; import { runningSessions, stopSession } from './sessionLifecycle'; import { InteractiveInputFeatures } from './inputFeatures'; +import { acquireInteractiveJdvAddress, disposeInteractiveJdvHostOnControlLoss } from './jdvAddress'; interface InteractiveView { client: AgentClient; @@ -51,6 +53,10 @@ interface InteractiveView { interface SessionChoice extends vscode.QuickPickItem { manifest?: SessionManifest; arf?: ArfSession; create?: boolean; terminal?: boolean } interface SavedConnection { id: string; notebookUri?: string } +function interactiveJdvHostKey(id: string, generation: string): string { + return `interactive:${id}:${generation}`; +} + export class InteractiveManager implements vscode.Disposable, vscode.TreeDataProvider { private views = new Map(); private opening = new Map>(); @@ -797,12 +803,23 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro } private listen(view: InteractiveView, client: AgentClient): void { + const hostKey = interactiveJdvHostKey(client.manifest.id, client.manifest.generation); + let hadControl = client.control; client.on('event', (event: SessionEvent) => { if (view.client !== client) { return; } view.chain = view.chain.then(() => this.event(view, event)).catch(error => this.report(error)); }); - client.on('activity', () => { if (view.client === client) { this.updateStatus(); this.changes.fire(undefined); } }); + client.on('activity', () => { + if (view.client !== client) { return; } + const hasControl = client.control; + disposeInteractiveJdvHostOnControlLoss(hadControl, hasControl, + () => disposeJdvHost(hostKey)); + hadControl = hasControl; + this.updateStatus(); this.changes.fire(undefined); + }); client.on('disconnect', () => { + disposeJdvHost(hostKey); + hadControl = false; if (view.client !== client || view.restarting) { return; } this.updateStatus(); this.changes.fire(undefined); this.reconnect(view, 1000); }); @@ -964,6 +981,9 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro if (typeof event.data.runtimeSessionId === 'string') { view.client.manifest.runtimeSessionId = event.data.runtimeSessionId; } view.target.pid = typeof event.data.rPid === 'number' ? String(event.data.rPid) : ''; view.target.rVer = typeof event.data.rVersion === 'string' ? event.data.rVersion : ''; this.refresh(); + if (event.data.status === 'exited') { + disposeJdvHost(interactiveJdvHostKey(view.client.manifest.id, view.client.manifest.generation)); + } if (event.data.status === 'exited' && !view.restarting) { await this.sessionNotice(view, 'stopped'); for (const cell of this.transcriptCells(view)) { @@ -1200,7 +1220,29 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro const client = view.client; const params = object(data.params ?? {}); try { - const result = await session.handleEditorRequest(String(data.method), params); + let result: unknown; + if (data.method === 'jdv/address') { + const sessionId = view.target.sessionId; + const generation = view.model.generation; + const manifestGeneration = client.manifest.generation; + const hostKey = interactiveJdvHostKey(client.manifest.id, manifestGeneration); + const pid = Number(client.manifest.rPid); + const canConnect = () => client.control && !view.target.processExited && + !['stopping', 'exited'].includes(client.manifest.status); + const address = await acquireInteractiveJdvAddress({ + sessionId, hostKey, + pid: Number.isSafeInteger(pid) && pid > 0 ? pid : undefined, + canConnect, + isCurrent: () => !view.disposed && view.client === client && !view.restarting && + view.target.sessionId === sessionId && view.model.generation === generation && + client.manifest.generation === manifestGeneration, + getAddress: getJdvAddress, + disposeHost: disposeJdvHost, + }); + result = { address }; + } else { + result = await session.handleEditorRequest(String(data.method), params); + } await client.request('clientReply', { id: data.id, result }); } catch (error) { await client.request('clientReply', { id: data.id, error: String(error) }).catch(() => undefined); } } @@ -1443,6 +1485,7 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro try { const answer = await vscode.window.showWarningMessage(`Restart R session “${view.client.manifest.label}”? Its in-memory objects will be lost. Code, output, and input will stay in this Interactive window.`, { modal: true }, 'Restart Session'); if (answer !== 'Restart Session' || view.disposed) { return; } + disposeJdvHost(interactiveJdvHostKey(view.client.manifest.id, view.client.manifest.generation)); const storage = path.join(this.root, view.client.manifest.id); let config = JSON.parse(fs.readFileSync(path.join(storage, 'config.json'), 'utf8')) as AgentConfig; // Re-evaluate auto and honor repaired settings; the saved config records @@ -1611,6 +1654,7 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro private disposeView(view: InteractiveView, forget = true): void { if (view.disposed) { return; } view.disposed = true; clearTimeout(view.timer); clearTimeout(view.reconnectTimer); + disposeJdvHost(interactiveJdvHostKey(view.client.manifest.id, view.client.manifest.generation)); view.client.close(); for (const { task } of view.executions.values()) { task.end(undefined); } view.executions.clear(); for (const controller of view.controllers) { controller.dispose(); } diff --git a/src/jdvHostLifecycle.ts b/src/jdvHostLifecycle.ts new file mode 100644 index 000000000..846ae37ae --- /dev/null +++ b/src/jdvHostLifecycle.ts @@ -0,0 +1,21 @@ +/** Dispose only the exact host currently registered at a key and address. */ +export async function disposeJdvHostIfCurrentAddress( + hosts: Map>, + hostKey: string, + expectedAddress: string, + addressOf: (host: T) => string, + dispose: (host: T) => void, +): Promise { + const pending = hosts.get(hostKey); + if (!pending) { return false; } + let host: T; + try { host = await pending; } + catch { + if (hosts.get(hostKey) === pending) { hosts.delete(hostKey); } + return false; + } + if (hosts.get(hostKey) !== pending || addressOf(host) !== expectedAddress) { return false; } + hosts.delete(hostKey); + dispose(host); + return true; +} diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index ac43b98aa..6530b0097 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -14,6 +14,7 @@ import { } from './jdvAdapter'; import { SequentialStream } from './jdvArrow'; import { JdvHost, JdvPeer } from './jdvTransport'; +import { disposeJdvHostIfCurrentAddress } from './jdvHostLifecycle'; import { extensionContext } from './extension'; import * as session from './session'; import { config, UriIcon } from './util'; @@ -39,8 +40,8 @@ const panels = new Map(); const hosts = new Map>(); /** The jdv transport address for a session, creating its listener on first use. */ -export function getJdvAddress(sessionId: string, pid: number | undefined): Promise { - let host = hosts.get(sessionId); +export function getJdvAddress(sessionId: string, pid: number | undefined, hostKey = sessionId): Promise { + let host = hosts.get(hostKey); if (!host) { host = JdvHost.create(pid, { viewOpen: (peer, params) => { @@ -70,16 +71,24 @@ export function getJdvAddress(sessionId: string, pid: number | undefined): Promi } }, }); - hosts.set(sessionId, host); - host.catch(() => hosts.delete(sessionId)); + hosts.set(hostKey, host); + void host.catch(() => { + if (hosts.get(hostKey) === host) { hosts.delete(hostKey); } + }); } return host.then(value => value.address); } /** Close the session's jdv listener when the session ends. */ -export function disposeJdvHost(sessionId: string): void { - const host = hosts.get(sessionId); - hosts.delete(sessionId); +export function disposeJdvHost(hostKey: string): void; +export function disposeJdvHost(hostKey: string, expectedAddress: string): Promise; +export function disposeJdvHost(hostKey: string, expectedAddress?: string): void | Promise { + if (expectedAddress !== undefined) { + return disposeJdvHostIfCurrentAddress(hosts, hostKey, expectedAddress, + host => host.address, host => host.dispose()).then(() => undefined); + } + const host = hosts.get(hostKey); + hosts.delete(hostKey); void host?.then(value => value.dispose(), () => undefined); } diff --git a/src/test/node/interactiveJdvAddress.test.ts b/src/test/node/interactiveJdvAddress.test.ts new file mode 100644 index 000000000..763d3b91e --- /dev/null +++ b/src/test/node/interactiveJdvAddress.test.ts @@ -0,0 +1,119 @@ +import * as assert from 'assert'; +import { acquireInteractiveJdvAddress, disposeInteractiveJdvHostOnControlLoss } from '../../interactive/jdvAddress'; +import { disposeJdvHostIfCurrentAddress } from '../../jdvHostLifecycle'; + +suite('Interactive jdv address', () => { + test('returns a listener address scoped to the captured session and generation', async () => { + const address = await acquireInteractiveJdvAddress({ + sessionId: 'session:g1', hostKey: 'interactive:session:g1', pid: 123, + canConnect: () => true, isCurrent: () => true, + getAddress: (sessionId, pid, hostKey) => { + assert.strictEqual(sessionId, 'session:g1'); + assert.strictEqual(pid, 123); + assert.strictEqual(hostKey, 'interactive:session:g1'); + return Promise.resolve('/tmp/jdv.sock'); + }, + disposeHost: () => assert.fail('A current session must keep its listener'), + }); + assert.strictEqual(address, '/tmp/jdv.sock'); + }); + + test('refuses observers without creating a listener', async () => { + let created = false; + await assert.rejects(acquireInteractiveJdvAddress({ + sessionId: 'session:g1', hostKey: 'interactive:session:g1', + canConnect: () => false, isCurrent: () => true, + getAddress: () => { created = true; return Promise.resolve('/tmp/jdv.sock'); }, + disposeHost: () => assert.fail('No listener was created'), + }), /controlling Interactive client/); + assert.strictEqual(created, false); + }); + + test('disposes an existing listener when control moves to another client', () => { + let disposed = 0; + assert.strictEqual(disposeInteractiveJdvHostOnControlLoss(true, false, () => { disposed++; }), true); + assert.strictEqual(disposed, 1); + assert.strictEqual(disposeInteractiveJdvHostOnControlLoss(false, false, () => { disposed++; }), false); + assert.strictEqual(disposeInteractiveJdvHostOnControlLoss(true, true, () => { disposed++; }), false); + assert.strictEqual(disposed, 1); + }); + + test('closes a listener when a completed restart changes the captured generation', async () => { + let resolveAddress!: (address: string) => void; + const pending = new Promise(resolve => { resolveAddress = resolve; }); + let generation = 'g1'; + const disposed: string[] = []; + const request = acquireInteractiveJdvAddress({ + sessionId: 'session:g1', hostKey: 'interactive:session:g1', + canConnect: () => true, + isCurrent: () => generation === 'g1', + getAddress: () => pending, + disposeHost: hostKey => { disposed.push(hostKey); }, + }); + // A restart may finish before the asynchronous listener creation resumes. + generation = 'g2'; + resolveAddress('/tmp/old-jdv.sock'); + await assert.rejects(request, /session changed/); + assert.deepStrictEqual(disposed, ['interactive:session:g1']); + }); + + test('does not dispose a reattached host when an old address acquisition finishes late', async () => { + const hostKey = 'interactive:session:g1'; + let resolveAddress!: (address: string) => void; + const pendingAddress = new Promise(resolve => { resolveAddress = resolve; }); + let oldHostDisposed = false; + let newHostDisposed = false; + const oldHostPromise = Promise.resolve({ address: '/tmp/old.sock', dispose: () => { oldHostDisposed = true; } }); + const newHost = { address: '/tmp/new.sock', dispose: () => { newHostDisposed = true; } }; + const hosts = new Map>([[hostKey, oldHostPromise]]); + const request = acquireInteractiveJdvAddress({ + sessionId: 'session:g1', hostKey, canConnect: () => true, isCurrent: () => false, + getAddress: () => pendingAddress, + disposeHost: (key, address) => disposeJdvHostIfCurrentAddress( + hosts, key, address, host => host.address, host => host.dispose()).then(() => undefined), + }); + // Detach removed the old host and a same-generation reattach installed a new one. + hosts.delete(hostKey); + hosts.set(hostKey, Promise.resolve(newHost)); + resolveAddress('/tmp/old.sock'); + await assert.rejects(request, /session changed/); + assert.strictEqual(hosts.get(hostKey) !== undefined, true); + assert.strictEqual((await hosts.get(hostKey))?.address, '/tmp/new.sock'); + assert.strictEqual(oldHostDisposed, false); + assert.strictEqual(newHostDisposed, false); + }); + + test('closes a listener when detach wins while its address is being created', async () => { + let resolveAddress!: (address: string) => void; + const pending = new Promise(resolve => { resolveAddress = resolve; }); + let attached = true; + let disposed = false; + const request = acquireInteractiveJdvAddress({ + sessionId: 'session:g1', hostKey: 'interactive:session:g1', + canConnect: () => true, isCurrent: () => attached, + getAddress: () => pending, + disposeHost: () => { disposed = true; }, + }); + attached = false; + resolveAddress('/tmp/detached-jdv.sock'); + await assert.rejects(request, /session changed/); + assert.strictEqual(disposed, true); + }); + + test('closes a listener if R exits while its address is being created', async () => { + let resolveAddress!: (address: string) => void; + const pending = new Promise(resolve => { resolveAddress = resolve; }); + let running = true; + let disposed = false; + const request = acquireInteractiveJdvAddress({ + sessionId: 'session:g1', hostKey: 'interactive:session:g1', + canConnect: () => running, isCurrent: () => true, + getAddress: () => pending, + disposeHost: () => { disposed = true; }, + }); + running = false; + resolveAddress('/tmp/stopped-jdv.sock'); + await assert.rejects(request, /session changed/); + assert.strictEqual(disposed, true); + }); +}); diff --git a/src/test/node/interactiveRuntime.test.ts b/src/test/node/interactiveRuntime.test.ts index 3b18fe319..f2e38415b 100644 --- a/src/test/node/interactiveRuntime.test.ts +++ b/src/test/node/interactiveRuntime.test.ts @@ -17,6 +17,8 @@ import { queryTablePage, TableColumn, tableSchema } from '../../interactive/tabl import { AssetStorageStats, readAsset } from '../../interactive/assets'; import { assertSvgTextVisible } from '../svgAssertions'; import { rString } from '../../interactive/backends/rCode'; +import { JdvHost, JdvPeer } from '../../jdvTransport'; +import { JsonObject } from '../../jdvAdapter'; const run = promisify(execFile); const delay = (ms: number): Promise => new Promise(resolve => setTimeout(resolve, ms)); @@ -84,6 +86,50 @@ const delay = (ms: number): Promise => new Promise(resolve => setTimeout(r return events.filter(event => event.executionId === id && event.type === 'stream').map(event => event.data.text).join(''); } + test('opens jdv from an Interactive cell and serves a page while R is idle', async function () { + const availability = await run('Rscript', ['--vanilla', '-e', "cat(requireNamespace('jdv', quietly=TRUE))"]); + if (availability.stdout.trim() !== 'TRUE') { this.skip(); return; } + + let resolveOpen!: (value: { peer: JdvPeer; params: JsonObject }) => void; + let rejectOpen!: (reason: Error) => void; + const opened = new Promise<{ peer: JdvPeer; params: JsonObject }>((resolve, reject) => { + resolveOpen = resolve; rejectOpen = reject; + }); + const host = await JdvHost.create(manifest.rPid, { + viewOpen: (peer, params) => resolveOpen({ peer, params }), + viewClosed: () => undefined, + viewUpdateFailed: () => undefined, + viewTracking: () => undefined, + disconnected: (_peer, reason) => rejectOpen(new Error(reason)), + }); + const onEvent = (event: SessionEvent): void => { + if (event.type !== 'clientRequest' || event.data.method !== 'jdv/address') { return; } + void client.request('clientReply', { + id: event.data.id, result: { address: host.address }, + }).catch(error => rejectOpen(error instanceof Error ? error : new Error(String(error)))); + }; + client.on('event', onEvent); + try { + const id = await submit('utils::View(data.frame(a = 1:3, b = letters[1:3]), "interactive jdv test")'); + assert.strictEqual((await finished(id)).state, 'success'); + const { peer, params } = await Promise.race([ + opened, + new Promise((_resolve, reject) => setTimeout(() => reject(new Error('Timed out waiting for jdv view/open')), 10000)), + ]); + assert.strictEqual(params.kind, 'grid'); + assert.strictEqual(params.access, 'random'); + const result = await peer.request('table/rows', { + view: params.view, generation: params.generation, offset: 0, limit: 2, + }) as JsonObject; + const data = result.data as JsonObject; + assert.deepStrictEqual(data.row_index, [1, 2]); + assert.strictEqual(result.total, 3); + } finally { + client.off('event', onEvent); + host.dispose(); + } + }); + test('streams output before completion and keeps one R environment', async () => { const id = await submit('answer <- 42; cat("early λ🙂\\n"); Sys.sleep(0.5); cat("late\\n"); answer'); await until(() => text(id).includes('early')); From 8ab8498f21810a13a1b0fd33c115c906a4c40e59 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 14:25:22 +0000 Subject: [PATCH 16/38] feat: follow the latest jdv protocol in the data viewer Use grid/rows, source_row and row_labels, the jdv-json encoding, categorical columns and native_type. Validate grid descriptors, treat omitted capabilities as false, require max_page_rows for Arrow streams, and read the producer hello (opaque session, required unacked limits, wake flag). Unknown error codes become producer_error. The expected R pid check is dropped because the session id is opaque; the 0600 listener address is only given to this session. --- sess/R/handlers.R | 5 +- sess/inst/tinytest/test-jdv.R | 5 +- src/interactive/jdvAddress.ts | 5 +- src/interactive/manager.ts | 2 - src/jdvAdapter.ts | 92 +++++++++++++++------ src/jdvArrow.ts | 7 +- src/jdvTransport.ts | 85 +++++++++++++------ src/jdvViewer.ts | 6 +- src/session.ts | 3 +- src/test/node/interactiveJdvAddress.test.ts | 5 +- src/test/node/interactiveRuntime.test.ts | 6 +- src/test/node/jdvAdapter.test.ts | 56 +++++++++---- src/test/node/jdvArrow.test.ts | 76 ++++++++++++++--- 13 files changed, 245 insertions(+), 108 deletions(-) diff --git a/sess/R/handlers.R b/sess/R/handlers.R index 7787180eb..a62ad8434 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1362,15 +1362,14 @@ handle_dataview_dispose <- function(params) { # jdv::jdv_binding() does the lookup without evaluating anything: it returns # list(name, env) for a symbol bound in `env` or one of its parents, and NULL # for other expressions, unreadable bindings (promises, active bindings), unbound -# names and R older than 4.6.0. A jdv that predates jdv_binding() is not tracked. +# names and R older than 4.6.0. # sess only adds the gates: the VS Code setting `r.session.viewers.trackVariables` # (passed to connect() as `track_variables`, default on), which # `options(sess.dataviewer.track = TRUE/FALSE)` overrides in both directions. .jdv_track_binding <- function(expression, env) { track <- getOption("sess.dataviewer.track", .sess_env$track_variables %||% TRUE) if (!is.symbol(expression) || !nzchar(as.character(expression)) || !isTRUE(track) || - !requireNamespace("jdv", quietly = TRUE) || - !exists("jdv_binding", envir = asNamespace("jdv"), inherits = FALSE)) { + !requireNamespace("jdv", quietly = TRUE)) { return(NULL) } tryCatch(jdv::jdv_binding(expression, env), error = function(e) NULL) diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R index 6ac83024b..f1109abdd 100644 --- a/sess/inst/tinytest/test-jdv.R +++ b/sess/inst/tinytest/test-jdv.R @@ -117,7 +117,7 @@ local({ expect_false(used_sess_viewer()) messages <- host_messages(2L) expect_identical(messages[[1L]]$method, "hello") - expect_identical(messages[[1L]]$params$pid, Sys.getpid()) + expect_identical(messages[[1L]]$params$producer$name, "jdv") expect_identical(runtime$jdv_address, address) connection <- jdv::jdv_connection() expect_identical(connection$state, "connected") @@ -128,7 +128,8 @@ local({ open <- opens()[[1L]]$params expect_identical(open$kind, "grid") expect_identical(open$access, "random") - expect_identical(open$mode, "paged") + expect_identical(open$encoding, "jdv-json") + expect_identical(open$collation, "C") expect_identical(open$title, "sample") expect_identical(open$generation, 1L) diff --git a/src/interactive/jdvAddress.ts b/src/interactive/jdvAddress.ts index 668cf67de..309a2c60f 100644 --- a/src/interactive/jdvAddress.ts +++ b/src/interactive/jdvAddress.ts @@ -1,10 +1,9 @@ export interface InteractiveJdvAddressOptions { sessionId: string; hostKey: string; - pid?: number; canConnect(): boolean; isCurrent(): boolean; - getAddress(sessionId: string, pid: number | undefined, hostKey: string): Promise; + getAddress(sessionId: string, hostKey: string): Promise; disposeHost(hostKey: string, expectedAddress: string): void | Promise; } @@ -13,7 +12,7 @@ export async function acquireInteractiveJdvAddress(options: InteractiveJdvAddres if (!options.canConnect()) { throw new Error('Only the controlling Interactive client can connect the data viewer.'); } - const address = await options.getAddress(options.sessionId, options.pid, options.hostKey); + const address = await options.getAddress(options.sessionId, options.hostKey); if (!options.canConnect() || !options.isCurrent()) { await options.disposeHost(options.hostKey, address); throw new Error('The Interactive session changed while opening its data viewer.'); diff --git a/src/interactive/manager.ts b/src/interactive/manager.ts index 1e979d6bb..bf10679b5 100644 --- a/src/interactive/manager.ts +++ b/src/interactive/manager.ts @@ -1226,12 +1226,10 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro const generation = view.model.generation; const manifestGeneration = client.manifest.generation; const hostKey = interactiveJdvHostKey(client.manifest.id, manifestGeneration); - const pid = Number(client.manifest.rPid); const canConnect = () => client.control && !view.target.processExited && !['stopping', 'exited'].includes(client.manifest.status); const address = await acquireInteractiveJdvAddress({ sessionId, hostKey, - pid: Number.isSafeInteger(pid) && pid > 0 ? pid : undefined, canConnect, isCurrent: () => !view.disposed && view.client === client && !view.restarting && view.target.sessionId === sessionId && view.model.generation === generation && diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index bc89d7681..08e70f285 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -5,6 +5,7 @@ export type JsonObject = Record; +/** Capability flags of a descriptor; parseDescriptor() fills omitted flags with false. */ export interface JdvCapabilities { seek: boolean; sort: boolean; @@ -12,16 +13,21 @@ export interface JdvCapabilities { known_total: boolean; next: boolean; children: boolean; - refresh?: boolean; + refresh: boolean; /** True only when the view tracks a variable and has not lost it. */ - track?: boolean; + track: boolean; } +const CAPABILITY_NAMES = [ + 'seek', 'sort', 'filter', 'known_total', 'next', 'children', 'refresh', 'track', +] as const; + export interface JdvColumnSchema { position: number; name: string; type: string; - class?: string[]; + /** Producer-defined display hint (R: `class()`); never interpreted. */ + native_type?: string[]; sortable: boolean; filterable: boolean; drillable?: boolean; @@ -40,7 +46,9 @@ export interface JdvDescriptor { encoding: string; capabilities: JdvCapabilities; title?: string | null; - mode?: string; + collation?: 'C' | 'locale'; + /** Sequential views: the row cap captured when the view was opened. */ + max_page_rows?: number; nrow?: number | null; ncol?: number; columns?: JdvColumnSchema[]; @@ -89,12 +97,21 @@ function scalarText(value: unknown): string { /** Validate the params of a `view/open` notification. */ export function parseDescriptor(value: unknown): JdvDescriptor { - if (!isObject(value) || typeof value.view !== 'string' || typeof value.generation !== 'number' || - (value.kind !== 'grid' && value.kind !== 'tree') || !isObject(value.capabilities)) { + if (!isObject(value) || typeof value.view !== 'string' || value.view === '' || + typeof value.generation !== 'number' || !Number.isSafeInteger(value.generation) || value.generation < 1 || + (value.kind !== 'grid' && value.kind !== 'tree') || + typeof value.access !== 'string' || typeof value.encoding !== 'string') { throw new Error('Invalid jdv view descriptor'); } - if (value.kind === 'grid' && !Array.isArray(value.columns)) { - throw new Error('Invalid jdv grid descriptor'); + // An omitted capabilities object or flag means false. + const supplied = value.capabilities === undefined ? {} : value.capabilities; + if (!isObject(supplied) || CAPABILITY_NAMES.some(name => name in supplied && typeof supplied[name] !== 'boolean')) { + throw new Error('Invalid jdv view descriptor capabilities'); + } + const capabilities = Object.fromEntries(CAPABILITY_NAMES.map(name => [name, supplied[name] === true])); + value.capabilities = capabilities; + if (value.kind === 'grid') { + validateGridDescriptor(value, capabilities as unknown as JdvCapabilities); } const descriptor = value as unknown as JdvDescriptor; // `reason` is optional and may gain values; keep only a string. @@ -110,6 +127,27 @@ export function parseDescriptor(value: unknown): JdvDescriptor { return descriptor; } +function validateGridDescriptor(value: JsonObject, capabilities: JdvCapabilities): void { + const invalid = (reason: string) => new Error(`Invalid jdv grid descriptor: ${reason}`); + if (!Array.isArray(value.columns)) { throw invalid('columns'); } + if (value.access === 'sequential') { + const cap = value.max_page_rows; + if (value.encoding !== 'arrow-ipc' || typeof cap !== 'number' || !Number.isSafeInteger(cap) || cap < 1 || + !capabilities.next) { + throw invalid('sequential access'); + } + return; + } + if ((value.access !== 'snapshot' && value.access !== 'random') || value.encoding !== 'jdv-json') { + throw invalid('access or encoding'); + } + if (value.collation !== 'C' && value.collation !== 'locale') { throw invalid('collation'); } + if (value.access === 'random' && (typeof value.nrow !== 'number' || !Number.isSafeInteger(value.nrow) || + value.nrow < 0 || !capabilities.seek || !capabilities.known_total)) { + throw invalid('random access'); + } +} + /** Validate a `tracking` object; a malformed one is dropped (`undefined`). */ export function parseTracking(value: unknown): JdvTracking | undefined { if (!isObject(value) || typeof value.name !== 'string' || @@ -166,8 +204,8 @@ export const NA_FILTER_OPTION = 'jdvIsNa'; export const NOT_NA_FILTER_OPTION = 'jdvNotNa'; function columnTooltip(column: JdvColumnSchema): string { - let tooltip = `${column.name}, class: [${(column.class ?? []).join(', ')}], type: ${column.type}`; - if (column.type === 'factor' && Array.isArray(column.levels)) { + let tooltip = `${column.name}, class: [${(column.native_type ?? []).join(', ')}], type: ${column.type}`; + if (column.type === 'categorical' && Array.isArray(column.levels)) { const levels = column.levels.slice(0, 20).join(', '); tooltip += `, levels: ${levels}${column.levels.length > 20 ? ', ...' : ''}`; } @@ -197,13 +235,12 @@ function rowLabelColumn(): GridColumnDef { /** Columns of a sequential Arrow view; they can be neither sorted nor filtered. */ function sequentialColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { const columns = [rowLabelColumn()]; - const schema = (descriptor as unknown as { schema?: { children?: { dictionary?: unknown }[] } }).schema; for (const column of (descriptor.columns ?? []) as unknown as { - index?: number; position: number; name: string; type: string; unsupported: string | null; + position: number; name: string; type: string; unsupported: string | null; }[]) { - // A dictionary column's format is its index type; its values are the dictionary's. - const dictionary = schema?.children?.[column.index ?? column.position]?.dictionary; - const numeric = !dictionary && (ARROW_NUMERIC_FORMATS.has(column.type) || column.type.startsWith('d:')); + // The descriptor no longer carries the Arrow schema, so a dictionary column + // (whose format is its index type) is aligned like a number. + const numeric = ARROW_NUMERIC_FORMATS.has(column.type) || column.type.startsWith('d:'); columns.push({ headerName: column.name, headerTooltip: column.unsupported @@ -258,7 +295,7 @@ export function gridColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { filter = true; break; case 'string': - case 'factor': + case 'categorical': filter = 'agTextColumnFilter'; filterOptions = TEXT_FILTER_OPTIONS; break; @@ -374,8 +411,8 @@ function dateOperand(value: unknown, column: JdvColumnSchema): string { if (column.type === 'date') { const date = /^(\d{4}-\d{2}-\d{2})$(?![\s\S])/.exec(text); if (date) { return date[1]; } - // Accept AG Grid's historical midnight datetime value, but do not - // silently discard an actual time or timezone from a date operand. + // AG Grid's filter model writes a date as "YYYY-MM-DD 00:00:00" (not a jdv + // compatibility concern), but an actual time or timezone is never discarded. const midnight = /^(\d{4}-\d{2}-\d{2})[ T]00:00:00$(?![\s\S])/.exec(text); if (midnight) { return midnight[1]; } throw invalid(); @@ -443,7 +480,7 @@ function conditionNode(condition: JsonObject, col: number, column: JdvColumnSche return op ? { op, col, value: dateOperand(condition.dateFrom, column) } : unsupportedFilter(column, type); } case 'string': - case 'factor': { + case 'categorical': { const value = scalarText(condition.filter); // As in the previous viewer, negated text conditions exclude missing values. const negate = (node: FilterNode): FilterNode => ({ @@ -558,7 +595,7 @@ function cellValue(column: JdvColumnSchema, channels: ColumnChannels, row: numbe if (display !== undefined && display !== null) { return display; } - // jdv omits display for strings, factors and int64 when it equals the value. + // jdv omits display for strings, categoricals and int64 when it equals the value. const value = channels.values?.[row]; return value === undefined || value === null ? null : scalarText(value); } @@ -571,18 +608,19 @@ function columnErrorText(errors: unknown): string | undefined { return messages.length > 0 ? messages.join('; ') : undefined; } -/** Convert a `table/rows` result to row objects keyed by the grid field ids. */ +/** Convert a `grid/rows` result to row objects keyed by the grid field ids. */ export function gridPageFromRows(result: unknown, startRow: number, requested: number): GridPage { if (!isObject(result) || !isObject(result.data) || typeof result.total !== 'number' || - typeof result.total_unfiltered !== 'number' || !Array.isArray(result.data.row_index)) { - throw new Error('Invalid jdv table/rows result'); + typeof result.total_unfiltered !== 'number' || !Array.isArray(result.data.source_row)) { + throw new Error('Invalid jdv grid/rows result'); } const data = result.data; - const rowIndex = data.row_index as number[]; - const rowNames = Array.isArray(data.row_names) ? data.row_names as (string | null)[] : undefined; + const sourceRow = data.source_row as number[]; + const rowLabels = Array.isArray(data.row_labels) ? data.row_labels as (string | null)[] : undefined; const columns = Array.isArray(data.columns) ? data.columns as PageColumn[] : []; - const rows: Record[] = rowIndex.map((index, row) => ({ - [ROW_LABEL_FIELD]: rowNames?.[row] ?? String(index), + // `source_row` is zero-based; the browser shows it plus one. + const rows: Record[] = sourceRow.map((index, row) => ({ + [ROW_LABEL_FIELD]: rowLabels?.[row] ?? String(index + 1), })); for (const column of columns) { if (!column.schema) { diff --git a/src/jdvArrow.ts b/src/jdvArrow.ts index 18ff5ec6c..a48d94ef3 100644 --- a/src/jdvArrow.ts +++ b/src/jdvArrow.ts @@ -5,7 +5,7 @@ // existing AG Grid infinite row model with formatted rows. import { tableFromIPC } from 'apache-arrow'; -import { JdvDescriptor, JdvRequestError, JsonObject, ROW_LABEL_FIELD } from './jdvAdapter'; +import { JdvDescriptor, JdvRequestError, ROW_LABEL_FIELD } from './jdvAdapter'; // --------------------------------------------------------------------------- // Formatting. Values are formatted from physical buffers where JavaScript @@ -272,9 +272,8 @@ export class SequentialStream { preferredRows = 1000, private readonly viewCacheBytes = VIEW_CACHE_BYTES, ) { this.columns = (descriptor.columns ?? []) as unknown as SequentialColumn[]; - const cap = (descriptor as unknown as JsonObject).max_page_rows; - const maxPageRows = typeof cap === 'number' && Number.isSafeInteger(cap) && cap >= 1 && cap <= 2000 ? cap : 200; - this.pageRows = Math.min(preferredRows, maxPageRows); + // parseDescriptor() guarantees a positive safe-integer cap for sequential views. + this.pageRows = Math.min(preferredRows, descriptor.max_page_rows!); streams.add(this); } diff --git a/src/jdvTransport.ts b/src/jdvTransport.ts index 333b4596b..b6562a751 100644 --- a/src/jdvTransport.ts +++ b/src/jdvTransport.ts @@ -25,12 +25,24 @@ export interface JdvResponse { body?: Uint8Array; } -interface RLimits { - maxReceiveFrames: number; - maxReceiveBytes: number; +interface ProducerLimits { + maxUnackedFrames: number; + maxUnackedBytes: number; maxRequestBytes: number; } +/** Error codes of the protocol; any other code is a general producer failure. */ +const ERROR_CODES = new Set([ + 'invalid_request', 'unsupported', 'limit_exceeded', 'schema_changed', 'producer_error', 'interrupted', + 'cancelled', 'expired', 'unavailable', 'stale_generation', 'view_not_found', 'cache_changed', 'in_use', + 'stream_failed', 'method_not_found', +]); +const SESSION_ID = /^[A-Za-z0-9._:-]{1,128}$/; +const LIMIT_NAMES = [ + 'max_unacked_frames', 'max_unacked_bytes', 'queue_messages', 'queue_bytes', 'max_request_bytes', + 'max_message_bytes', +] as const; + interface OutboundFrame { bytes: Buffer; wake?: boolean; @@ -47,15 +59,21 @@ function isObject(value: unknown): value is JsonObject { return typeof value === 'object' && value !== null && !Array.isArray(value); } -function positiveInteger(value: unknown, fallback: number): number { - return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : fallback; +function positiveInteger(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0; +} + +/** Producer or runtime metadata: string `name` and `version` of 1 to 256 UTF-8 bytes. */ +function validIdentity(value: unknown): boolean { + return isObject(value) && [value.name, value.version].every(field => + typeof field === 'string' && Buffer.byteLength(field) >= 1 && Buffer.byteLength(field) <= 256); } -/** One R connection. Implements the host obligations of the jdv transport. */ +/** One producer connection. Implements the host obligations of the jdv transport. */ export class JdvPeer { private buffer: Buffer = Buffer.alloc(0); private body?: { header: JsonObject; length: number; timer: NodeJS.Timeout }; - private limits?: RLimits; + private limits?: ProducerLimits; private readonly outbound: OutboundFrame[] = []; /** Bytes of frames written but not yet acknowledged, in send order. */ private readonly unconfirmed: number[] = []; @@ -66,13 +84,14 @@ export class JdvPeer { private wakeTimer?: NodeJS.Timeout; private sequence = 0; private closed = false; + /** Whether the producer asked for periodic `wake` notifications. */ + private wanted = false; + /** Opaque session id from `hello`; it carries no process id. */ session?: string; - pid?: number; constructor( private readonly socket: net.Socket, private readonly handlers: { - hello(peer: JdvPeer, params: JsonObject): boolean; notification(peer: JdvPeer, method: string, params: JsonObject): void; closed(peer: JdvPeer, reason: string): void; }, @@ -202,8 +221,11 @@ export class JdvPeer { this.updateWake(); if (isObject(message.error)) { const { code, message: text } = message.error; - pending.reject(new JdvRequestError(typeof code === 'string' ? code : 'unknown', - typeof text === 'string' ? text : 'jdv request failed')); + const known = typeof code === 'string' && ERROR_CODES.has(code); + const detail = typeof text === 'string' ? text : 'jdv request failed'; + // An unrecognized code is a general producer failure; keep it visible in the text. + pending.reject(known ? new JdvRequestError(code, detail) + : new JdvRequestError('producer_error', typeof code === 'string' ? `${detail} (${code})` : detail)); return; } const result = message.result; @@ -222,18 +244,26 @@ export class JdvPeer { if (version !== PROTOCOL_MAJOR) { throw new Error(`Unsupported jdv protocol: ${String(params.protocol)}`); } - const limits = isObject(params.limits) ? params.limits : {}; + if (typeof params.session !== 'string' || !SESSION_ID.test(params.session) || + !validIdentity(params.producer) || + (isObject(params.producer) && params.producer.runtime !== undefined && !validIdentity(params.producer.runtime)) || + (params.wake !== undefined && typeof params.wake !== 'boolean')) { + throw new Error('Invalid jdv hello.'); + } + const limits = params.limits; + if (!isObject(limits) || !LIMIT_NAMES.every(name => positiveInteger(limits[name]))) { + throw new Error('Invalid jdv hello limits.'); + } this.limits = { - maxReceiveFrames: positiveInteger(limits.max_receive_frames, 264), - maxReceiveBytes: positiveInteger(limits.max_receive_bytes, 64 * 1024), - maxRequestBytes: positiveInteger(limits.max_request_bytes, 1024 * 1024), + maxUnackedFrames: limits.max_unacked_frames as number, + maxUnackedBytes: limits.max_unacked_bytes as number, + maxRequestBytes: limits.max_request_bytes as number, }; - this.session = typeof params.session === 'string' ? params.session : undefined; - this.pid = typeof params.pid === 'number' ? params.pid : undefined; - if (!this.handlers.hello(this, params)) { - throw new Error('The jdv connection does not belong to this R session.'); - } + this.session = params.session; + // The host sends periodic wakes only when the producer asks for them. + this.wanted = params.wake === true; this.flush(); + this.updateWake(); } private acknowledge(params: JsonObject): void { @@ -257,8 +287,8 @@ export class JdvPeer { } while (this.outbound.length > 0) { const frame = this.outbound[0]; - if (this.unconfirmed.length + 1 > limits.maxReceiveFrames || - this.unconfirmedBytes + frame.bytes.length > limits.maxReceiveBytes) { + if (this.unconfirmed.length + 1 > limits.maxUnackedFrames || + this.unconfirmedBytes + frame.bytes.length > limits.maxUnackedBytes) { return; } this.outbound.shift(); @@ -269,8 +299,11 @@ export class JdvPeer { } } - /** R dispatches queued requests only when woken; wake while any request is pending. */ + /** Wake the producer repeatedly while a request is pending, if it asked for wakes. */ private updateWake(): void { + if (!this.wanted) { + return; + } if (this.pending.size === 0) { this.stopWake(); } else if (!this.wakeTimer) { @@ -323,13 +356,12 @@ export class JdvHost { readonly address: string, /** The same endpoint in the form Node's `net` module accepts. */ readonly socketPath: string, - private readonly expectedPid: number | undefined, private readonly events: JdvHostEvents, ) { server.on('connection', socket => this.accept(socket)); } - static async create(expectedPid: number | undefined, events: JdvHostEvents): Promise { + static async create(events: JdvHostEvents): Promise { const id = crypto.randomBytes(8).toString('hex'); // Unix socket paths are limited to about 104-108 bytes; keep the name short. const socketPath = process.platform === 'win32' @@ -347,7 +379,7 @@ export class JdvHost { if (process.platform !== 'win32') { fs.chmodSync(socketPath, 0o600); } - return new JdvHost(server, address, socketPath, expectedPid, events); + return new JdvHost(server, address, socketPath, events); } get connection(): JdvPeer | undefined { @@ -366,7 +398,6 @@ export class JdvHost { // jdv opens a new connection after a disconnect; only one is active. this.peer?.close('The R session opened a new jdv connection.'); this.peer = new JdvPeer(socket, { - hello: (_peer, params) => this.expectedPid === undefined || params.pid === this.expectedPid, notification: (peer, method, params) => { if (method === 'view/open') { this.events.viewOpen(peer, params); diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index 6530b0097..9b53a6391 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -40,10 +40,10 @@ const panels = new Map(); const hosts = new Map>(); /** The jdv transport address for a session, creating its listener on first use. */ -export function getJdvAddress(sessionId: string, pid: number | undefined, hostKey = sessionId): Promise { +export function getJdvAddress(sessionId: string, hostKey = sessionId): Promise { let host = hosts.get(hostKey); if (!host) { - host = JdvHost.create(pid, { + host = JdvHost.create({ viewOpen: (peer, params) => { let descriptor: JdvDescriptor; try { @@ -430,7 +430,7 @@ async function fetchGridPage(state: JdvPanel, message: JsonObject): Promise= next.totalRows) { diff --git a/src/session.ts b/src/session.ts index 2a4bf1598..34d6290f3 100644 --- a/src/session.ts +++ b/src/session.ts @@ -3245,8 +3245,7 @@ async function handleRequest(message: Record, socket?: IpcSocke // The per-session jdv transport; sess connects jdv to it when View() is used. const owner = sessions.get(socket?._sessionId ?? ''); if (!owner) { throw new Error('The jdv transport requires an attached session'); } - const pid = Number(owner.pid); - result = { address: await getJdvAddress(owner.sessionId, Number.isSafeInteger(pid) ? pid : undefined) }; + result = { address: await getJdvAddress(owner.sessionId) }; break; } case 'rstudioapi/active_editor_context': diff --git a/src/test/node/interactiveJdvAddress.test.ts b/src/test/node/interactiveJdvAddress.test.ts index 763d3b91e..685e986b4 100644 --- a/src/test/node/interactiveJdvAddress.test.ts +++ b/src/test/node/interactiveJdvAddress.test.ts @@ -5,11 +5,10 @@ import { disposeJdvHostIfCurrentAddress } from '../../jdvHostLifecycle'; suite('Interactive jdv address', () => { test('returns a listener address scoped to the captured session and generation', async () => { const address = await acquireInteractiveJdvAddress({ - sessionId: 'session:g1', hostKey: 'interactive:session:g1', pid: 123, + sessionId: 'session:g1', hostKey: 'interactive:session:g1', canConnect: () => true, isCurrent: () => true, - getAddress: (sessionId, pid, hostKey) => { + getAddress: (sessionId, hostKey) => { assert.strictEqual(sessionId, 'session:g1'); - assert.strictEqual(pid, 123); assert.strictEqual(hostKey, 'interactive:session:g1'); return Promise.resolve('/tmp/jdv.sock'); }, diff --git a/src/test/node/interactiveRuntime.test.ts b/src/test/node/interactiveRuntime.test.ts index f2e38415b..5880b897e 100644 --- a/src/test/node/interactiveRuntime.test.ts +++ b/src/test/node/interactiveRuntime.test.ts @@ -95,7 +95,7 @@ const delay = (ms: number): Promise => new Promise(resolve => setTimeout(r const opened = new Promise<{ peer: JdvPeer; params: JsonObject }>((resolve, reject) => { resolveOpen = resolve; rejectOpen = reject; }); - const host = await JdvHost.create(manifest.rPid, { + const host = await JdvHost.create({ viewOpen: (peer, params) => resolveOpen({ peer, params }), viewClosed: () => undefined, viewUpdateFailed: () => undefined, @@ -118,11 +118,11 @@ const delay = (ms: number): Promise => new Promise(resolve => setTimeout(r ]); assert.strictEqual(params.kind, 'grid'); assert.strictEqual(params.access, 'random'); - const result = await peer.request('table/rows', { + const result = await peer.request('grid/rows', { view: params.view, generation: params.generation, offset: 0, limit: 2, }) as JsonObject; const data = result.data as JsonObject; - assert.deepStrictEqual(data.row_index, [1, 2]); + assert.deepStrictEqual(data.source_row, [0, 1]); assert.strictEqual(result.total, 3); } finally { client.off('event', onEvent); diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts index f3a50090a..88adab9b1 100644 --- a/src/test/node/jdvAdapter.test.ts +++ b/src/test/node/jdvAdapter.test.ts @@ -5,18 +5,20 @@ import { parseDescriptor, parseTracking, tableRowsParams, toJdvFilter, } from '../../jdvAdapter'; -const capabilities = { seek: true, sort: true, filter: true, known_total: true, next: false, children: false, refresh: true }; +const capabilities = { + seek: true, sort: true, filter: true, known_total: true, next: false, children: false, refresh: true, track: false, +}; const grid: JdvDescriptor = parseDescriptor({ - view: 'v-1', generation: 2, kind: 'grid', access: 'random', encoding: 'r-json', capabilities, - title: 'df', mode: 'paged', nrow: 3, ncol: 6, row_names: null, + view: 'v-1', generation: 2, kind: 'grid', access: 'random', encoding: 'jdv-json', capabilities, + title: 'df', collation: 'C', nrow: 3, ncol: 6, columns: [ - { position: 0, name: 'a', type: 'number', class: ['numeric'], sortable: true, filterable: true }, - { position: 1, name: 'b', type: 'factor', class: ['factor'], sortable: true, filterable: true, levels: ['x', 'y'] }, - { position: 2, name: 'd', type: 'date', class: ['Date'], sortable: true, filterable: true }, - { position: 3, name: 't', type: 'datetime', class: ['POSIXct', 'POSIXt'], sortable: true, filterable: true, tz: 'UTC' }, - { position: 4, name: 'i', type: 'int64', class: ['integer64'], sortable: true, filterable: true }, - { position: 5, name: 'a', type: 'other', class: ['AsIs'], sortable: false, filterable: false, drillable: true }, + { position: 0, name: 'a', type: 'number', native_type: ['numeric'], sortable: true, filterable: true }, + { position: 1, name: 'b', type: 'categorical', native_type: ['factor'], sortable: true, filterable: true, levels: ['x', 'y'] }, + { position: 2, name: 'd', type: 'date', native_type: ['Date'], sortable: true, filterable: true }, + { position: 3, name: 't', type: 'datetime', native_type: ['POSIXct', 'POSIXt'], sortable: true, filterable: true, tz: 'UTC' }, + { position: 4, name: 'i', type: 'int64', native_type: ['integer64'], sortable: true, filterable: true }, + { position: 5, name: 'a', type: 'other', native_type: ['AsIs'], sortable: false, filterable: false, drillable: true }, ], }); @@ -26,6 +28,28 @@ suite('jdv adapter', () => { assert.equal(grid.generation, 2); assert.throws(() => parseDescriptor({})); assert.throws(() => parseDescriptor({ ...grid, columns: undefined })); + assert.throws(() => parseDescriptor({ ...grid, view: '' })); + assert.throws(() => parseDescriptor({ ...grid, generation: 0 })); + assert.throws(() => parseDescriptor({ ...grid, collation: 'locale-ish' })); + assert.throws(() => parseDescriptor({ ...grid, collation: undefined })); + assert.throws(() => parseDescriptor({ ...grid, encoding: 'r-json' })); + assert.throws(() => parseDescriptor({ ...grid, nrow: null })); + assert.throws(() => parseDescriptor({ ...grid, capabilities: { ...capabilities, seek: false } })); + assert.throws(() => parseDescriptor({ ...grid, capabilities: { ...capabilities, sort: 'yes' } })); + }); + + test('treats omitted capabilities as false', () => { + const bare = parseDescriptor({ ...grid, access: 'snapshot', capabilities: undefined }); + assert.deepEqual(Object.values(bare.capabilities), Array(8).fill(false)); + }); + + test('requires max_page_rows for sequential views', () => { + const sequential = { + view: 'v-3', generation: 1, kind: 'grid', access: 'sequential', encoding: 'arrow-ipc', + capabilities: { next: true }, nrow: null, ncol: 0, columns: [], + }; + assert.throws(() => parseDescriptor(sequential)); + assert.equal(parseDescriptor({ ...sequential, max_page_rows: 10 }).max_page_rows, 10); }); test('reveals the panel unless the update is tracked or a refresh', () => { @@ -47,7 +71,7 @@ suite('jdv adapter', () => { assert.equal(canRefresh(grid), true); assert.equal(canRefresh({ ...grid, access: 'sequential' }), false); assert.equal(canRefresh({ ...grid, capabilities: { ...capabilities, refresh: false } }), false); - assert.equal(canRefresh({ ...grid, capabilities: { ...capabilities, refresh: undefined } }), false); + assert.equal(canRefresh(parseDescriptor({ ...grid, capabilities: { seek: true, known_total: true } })), false); }); test('builds column definitions from the schema', () => { @@ -197,8 +221,8 @@ suite('jdv adapter', () => { const page = gridPageFromRows({ view: 'v-1', generation: 2, total: 3, total_unfiltered: 5, offset: 0, limit: 3, data: { - row_index: [3, 1, 2], - row_names: ['r3', 'r1', 'r2'], + source_row: [2, 0, 1], + row_labels: ['r3', 'r1', 'r2'], columns: [ { schema: grid.columns![0], data: { values: [null, 1, null], display: ['Inf', 'NaN', 'NA'], @@ -217,14 +241,14 @@ suite('jdv adapter', () => { assert.equal(page.totalUnfiltered, 5); const unnamed = gridPageFromRows({ total: 1, total_unfiltered: 1, offset: 0, limit: 1, - data: { row_index: [7], row_names: null, columns: [] }, + data: { source_row: [6], row_labels: null, columns: [] }, }, 0, 1); assert.deepEqual(unnamed.rows, [{ '0': '7' }]); }); test('maps List Viewer paths to typed tree paths', () => { const tree = new TreeAdapter(parseDescriptor({ - view: 'v-2', generation: 1, kind: 'tree', access: 'random', encoding: 'r-json', + view: 'v-2', generation: 1, kind: 'tree', access: 'random', encoding: 'jdv-json', capabilities: { ...capabilities, sort: false, filter: false, children: true }, path: [], class: ['list'], type: 'list', }), 'x'); assert.deepEqual(tree.childrenParams([], 1), { view: 'v-2', generation: 1, path: [], offset: 0, limit: 200, attributes: false }); @@ -272,12 +296,12 @@ suite('jdv adapter', () => { test('leaves have nothing to open unless they are tables', () => { const tree = new TreeAdapter(parseDescriptor({ - view: 'v-2', generation: 1, kind: 'tree', access: 'random', encoding: 'r-json', + view: 'v-2', generation: 1, kind: 'tree', access: 'random', encoding: 'jdv-json', capabilities: { ...capabilities, children: true }, path: [], class: ['list'], type: 'list', }), 'x'); const page = tree.children([], { path: [], offset: 0, total_children: 2, children: [ - { index: 1, label: 'n', class: ['numeric'], type: 'double', summary: 'double (1): 1', has_children: false, viewable: true, path: [{ type: 'index', index: 1 }] }, + { index: 1, label: 'n', native_type: ['numeric'], type: 'double', summary: 'double (1): 1', has_children: false, viewable: true, path: [{ type: 'index', index: 1 }] }, { index: 2, label: 'm', class: ['matrix', 'array'], type: 'integer', summary: 'matrix', has_children: false, viewable: true, path: [{ type: 'index', index: 2 }] }, ], }); diff --git a/src/test/node/jdvArrow.test.ts b/src/test/node/jdvArrow.test.ts index e850c82ce..93bb206a0 100644 --- a/src/test/node/jdvArrow.test.ts +++ b/src/test/node/jdvArrow.test.ts @@ -6,7 +6,7 @@ import { SequentialStream, UNAVAILABLE_TEXT, formatScaled, formatTimestamp } fro import { JdvHost, JdvPeer } from '../../jdvTransport'; const descriptor: JdvDescriptor = parseDescriptor({ - view: 'v-1', generation: 1, kind: 'grid', access: 'sequential', encoding: 'arrow-ipc', mode: 'sequential', + view: 'v-1', generation: 1, kind: 'grid', access: 'sequential', encoding: 'arrow-ipc', capabilities: { seek: false, sort: false, filter: false, known_total: false, next: true, children: false }, nrow: null, ncol: 3, max_page_rows: 2000, columns: [ @@ -14,7 +14,6 @@ const descriptor: JdvDescriptor = parseDescriptor({ { index: 1, position: 1, name: 's', type: 'u', ipc_index: 1, unsupported: null, sortable: false, filterable: false }, { index: 2, position: 2, name: 'x', type: '+w:2', ipc_index: null, unsupported: 'not supported', sortable: false, filterable: false }, ], - schema: { format: '+s', children: [{ format: 'i' }, { format: 'u' }, { format: '+w:2' }] }, }); /** A fake producer returning pages of `size` rows out of `total`. */ @@ -133,13 +132,24 @@ async function fakeR(socketPath: string) { return { socket, next, send, ack: () => send({ method: 'ack', params: { frames } }) }; } +function helloParams(limits: Record = {}): JsonObject { + return { + protocol: 1, session: 'producer-1', wake: true, + producer: { name: 'jdv', version: '0.0.9000', runtime: { name: 'R', version: '4.5.2' } }, + limits: { + max_unacked_frames: 264, max_unacked_bytes: 1048577, queue_messages: 256, queue_bytes: 16777216, + max_request_bytes: 1048576, max_message_bytes: 10485760, ...limits, + }, + }; +} + suite('jdv transport', () => { test('exchanges requests, wakes and Arrow bodies with R', async () => { const opened: string[] = []; const failures: JsonObject[] = []; const tracking: JsonObject[] = []; let peer: JdvPeer | undefined; - const host = await JdvHost.create(4321, { + const host = await JdvHost.create({ viewOpen: (connection, params) => { peer = connection; opened.push(String(params.view)); }, viewClosed: () => undefined, viewUpdateFailed: (_connection, params) => { failures.push(params); }, @@ -148,7 +158,7 @@ suite('jdv transport', () => { }); try { const r = await fakeR(host.socketPath); - r.send({ method: 'hello', params: { protocol: 1, session: 'r-4321-1', pid: 4321, limits: { max_receive_frames: 4, max_receive_bytes: 65536 } } }); + r.send({ method: 'hello', params: helloParams({ max_unacked_frames: 4, max_unacked_bytes: 65536 }) }); r.send({ method: 'view/open', params: { view: 'v-1', generation: 1 } }); while (!peer) { await new Promise(resolve => setTimeout(resolve, 5)); } assert.deepEqual(opened, ['v-1']); @@ -183,8 +193,8 @@ suite('jdv transport', () => { assert.deepEqual(Array.from(received!), [1, 2, 3, 10, 4]); assert.equal((result as { rows_seen: number }).rows_seen, 1); - const failed = peer.request('table/rows', { view: 'v-1', generation: 1 }); - const second = await r.next(line => line.method === 'table/rows'); + const failed = peer.request('grid/rows', { view: 'v-1', generation: 1 }); + const second = await r.next(line => line.method === 'grid/rows'); r.send({ id: second.id, error: { code: 'unsupported', message: 'no' }, warnings: [] }); await assert.rejects(failed, (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported'); @@ -196,20 +206,60 @@ suite('jdv transport', () => { } }); - test('rejects a connection from another process', async () => { + test('rejects an invalid hello and treats unknown error codes as producer failures', async () => { let reason = ''; - const host = await JdvHost.create(1, { - viewOpen: () => undefined, + let peer: JdvPeer | undefined; + const host = await JdvHost.create({ + viewOpen: (connection) => { peer = connection; }, viewClosed: () => undefined, viewUpdateFailed: () => undefined, viewTracking: () => undefined, disconnected: (_peer, why) => { reason = why; }, }); try { - const r = await fakeR(host.socketPath); - r.send({ method: 'hello', params: { protocol: 1, pid: 2, limits: {} } }); - await new Promise(resolve => r.socket.once('close', () => resolve())); - assert.match(reason, /does not belong/); + const bad = await fakeR(host.socketPath); + bad.send({ method: 'hello', params: { ...helloParams(), limits: { max_unacked_frames: 4 } } }); + await new Promise(resolve => bad.socket.once('close', () => resolve())); + assert.match(reason, /Invalid jdv hello/); + + const good = await fakeR(host.socketPath); + good.send({ method: 'hello', params: helloParams() }); + good.send({ method: 'view/open', params: { view: 'v-1', generation: 1 } }); + while (!peer) { await new Promise(resolve => setTimeout(resolve, 5)); } + const failed = peer.request('grid/rows', { view: 'v-1', generation: 1 }); + const request = await good.next(line => line.method === 'grid/rows'); + good.send({ id: request.id, error: { code: 'r_error', message: 'boom' }, warnings: [] }); + await assert.rejects(failed, (e: unknown) => e instanceof JdvRequestError && + e.code === 'producer_error' && /boom \(r_error\)/.test(e.message)); + } finally { + host.dispose(); + } + }); + + test('sends wakes only when the producer asks for them', async () => { + let peer: JdvPeer | undefined; + const host = await JdvHost.create({ + viewOpen: (connection) => { peer = connection; }, + viewClosed: () => undefined, + viewUpdateFailed: () => undefined, + viewTracking: () => undefined, + disconnected: () => undefined, + }); + try { + const quiet = await fakeR(host.socketPath); + quiet.send({ method: 'hello', params: { ...helloParams(), wake: false } }); + quiet.send({ method: 'view/open', params: { view: 'v-1', generation: 1 } }); + while (!peer) { await new Promise(resolve => setTimeout(resolve, 5)); } + void peer.request('view/close', { view: 'v-1', generation: 1 }).catch(() => undefined); + const request = await quiet.next(line => line.method === 'view/close'); + // A later request arrives without a wake in between. + await new Promise(resolve => setTimeout(resolve, 200)); + quiet.send({ id: request.id, result: {}, warnings: [] }); + quiet.ack(); + await assert.rejects(Promise.race([ + quiet.next(line => line.method === 'wake'), + new Promise((_, reject) => setTimeout(() => reject(new Error('no wake')), 150)), + ]), /no wake/); } finally { host.dispose(); } From d075f84e00b6d5386a195730754bec3c82020830 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 14:26:26 +0000 Subject: [PATCH 17/38] refactor: drop notes about older jdv from the data viewer comments --- src/jdvAdapter.ts | 4 ++-- src/jdvViewer.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index 08e70f285..9db48d0cc 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -411,8 +411,8 @@ function dateOperand(value: unknown, column: JdvColumnSchema): string { if (column.type === 'date') { const date = /^(\d{4}-\d{2}-\d{2})$(?![\s\S])/.exec(text); if (date) { return date[1]; } - // AG Grid's filter model writes a date as "YYYY-MM-DD 00:00:00" (not a jdv - // compatibility concern), but an actual time or timezone is never discarded. + // AG Grid's filter model writes a date as "YYYY-MM-DD 00:00:00"; an actual time + // or timezone is never discarded. const midnight = /^(\d{4}-\d{2}-\d{2})[ T]00:00:00$(?![\s\S])/.exec(text); if (midnight) { return midnight[1]; } throw invalid(); diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index 9b53a6391..fb086564d 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -174,8 +174,8 @@ async function openJdvView( ): Promise { const key = panelKey(sessionId, descriptor.view); const existing = panels.get(key); - // jdv titles a child view when its parent has a title; older jdv or an untitled - // parent leave it to the host-side bookkeeping of the item that was opened. + // jdv titles a child view when its parent has a title; an untitled parent leaves it + // to the host-side bookkeeping of the item that was opened. const parent = descriptor.parent === undefined ? undefined : panels.get(panelKey(sessionId, descriptor.parent)); const title = descriptor.title ?? parent?.childTitle ?? existing?.title ?? 'View'; if (descriptor.kind === 'grid' && descriptor.access !== 'random' && descriptor.access !== 'sequential') { From 4ed2c801c0b1d102cf5b7aa587a3ee9cddfb05ee Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 14:31:43 +0000 Subject: [PATCH 18/38] test: express jdv contract checks without naming retired formats --- src/test/node/jdvAdapter.test.ts | 2 +- src/test/node/jdvArrow.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts index 88adab9b1..a90ee6c2d 100644 --- a/src/test/node/jdvAdapter.test.ts +++ b/src/test/node/jdvAdapter.test.ts @@ -32,7 +32,7 @@ suite('jdv adapter', () => { assert.throws(() => parseDescriptor({ ...grid, generation: 0 })); assert.throws(() => parseDescriptor({ ...grid, collation: 'locale-ish' })); assert.throws(() => parseDescriptor({ ...grid, collation: undefined })); - assert.throws(() => parseDescriptor({ ...grid, encoding: 'r-json' })); + assert.throws(() => parseDescriptor({ ...grid, encoding: 'unsupported-encoding' })); assert.throws(() => parseDescriptor({ ...grid, nrow: null })); assert.throws(() => parseDescriptor({ ...grid, capabilities: { ...capabilities, seek: false } })); assert.throws(() => parseDescriptor({ ...grid, capabilities: { ...capabilities, sort: 'yes' } })); diff --git a/src/test/node/jdvArrow.test.ts b/src/test/node/jdvArrow.test.ts index 93bb206a0..ae44f71f7 100644 --- a/src/test/node/jdvArrow.test.ts +++ b/src/test/node/jdvArrow.test.ts @@ -228,9 +228,9 @@ suite('jdv transport', () => { while (!peer) { await new Promise(resolve => setTimeout(resolve, 5)); } const failed = peer.request('grid/rows', { view: 'v-1', generation: 1 }); const request = await good.next(line => line.method === 'grid/rows'); - good.send({ id: request.id, error: { code: 'r_error', message: 'boom' }, warnings: [] }); + good.send({ id: request.id, error: { code: 'some_future_code', message: 'boom' }, warnings: [] }); await assert.rejects(failed, (e: unknown) => e instanceof JdvRequestError && - e.code === 'producer_error' && /boom \(r_error\)/.test(e.message)); + e.code === 'producer_error' && /boom \(some_future_code\)/.test(e.message)); } finally { host.dispose(); } From 4112507659de95f0749608d6a3663a204c5587c1 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 14:35:13 +0000 Subject: [PATCH 19/38] feat: open drillable jdv grid cells as child views --- src/dataViewer.ts | 43 ++++++++++++++++++++++++++++++- src/jdvAdapter.ts | 31 ++++++++++++++++++++++ src/jdvViewer.ts | 27 ++++++++++++++++++- src/session.ts | 8 ++++++ src/test/node/jdvAdapter.test.ts | 27 +++++++++++++++---- src/test/suite/dataViewer.test.ts | 18 +++++++++++++ 6 files changed, 147 insertions(+), 7 deletions(-) diff --git a/src/dataViewer.ts b/src/dataViewer.ts index 09e77533e..096ceaac4 100644 --- a/src/dataViewer.ts +++ b/src/dataViewer.ts @@ -36,6 +36,14 @@ export function getDataViewerStyle(): string { font-size: 12px; color: var(--vscode-descriptionForeground); } ${viewerSessionStyle} + .dataview-drill { display: flex; align-items: center; gap: 4px; } + .dataview-drill-text { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; } + .dataview-drill-open { + flex-shrink: 0; padding: 0 4px; border: 1px solid transparent; border-radius: 3px; + background: transparent; color: var(--vscode-textLink-foreground); font: inherit; cursor: pointer; + line-height: 1.2; + } + .dataview-drill-open:hover { background: var(--vscode-toolbar-hoverBackground); } .dataview-na { color: var(--vscode-descriptionForeground); font-style: italic; opacity: 0.75; } `; } @@ -77,6 +85,30 @@ export function getDataViewerScript(): string { element.textContent = 'NA'; return element; }; + // A drillable jdv cell shows its text and a button that opens the cell as a child view. + // The cell is addressed by its original source row, so it works after sorting and filtering. + const drillCellRenderer = params => { + const element = document.createElement('span'); + element.className = 'dataview-drill'; + const text = document.createElement('span'); + text.className = 'dataview-drill-text'; + text.textContent = params.value; + const button = document.createElement('button'); + button.type = 'button'; + button.className = 'dataview-drill-open'; + button.title = 'Open this cell in a new view'; + button.setAttribute('aria-label', 'Open this cell in a new view'); + button.textContent = '\u2197'; + button.addEventListener('click', event => { + event.stopPropagation(); + vscode.postMessage({ + message: 'dataview/openCell', documentGeneration, + sourceRow: params.data.__sourceRow, column: params.colDef.context.drillColumn + }); + }); + element.append(text, button); + return element; + }; function getAgTheme() { const light = document.body.classList.contains('vscode-light') || @@ -104,11 +136,20 @@ export function getDataViewerScript(): string { } const bigintFields = []; for (const column of columns) { + const drillColumn = column.jdvDrill; + delete column.jdvDrill; + if (Number.isInteger(drillColumn)) { + column.context = { drillColumn }; + } column.cellRendererSelector = params => { if (params.data == null) { return { component: emptyCellRenderer }; } - return params.value == null ? { component: naCellRenderer } : undefined; + if (params.value == null) { + return { component: naCellRenderer }; + } + return Number.isInteger(drillColumn) && params.data.__sourceRow !== undefined + ? { component: drillCellRenderer } : undefined; }; if (column.type === 'dateColumn' || column.type === 'datetimeColumn') { column.cellDataType = column.type === 'dateColumn' ? 'dateString' : 'dateTimeString'; diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index 9db48d0cc..e2aca7d2b 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -179,6 +179,8 @@ export function canRefresh(descriptor: JdvDescriptor): boolean { /** Row label column. Data columns use their one-based position as field id. */ export const ROW_LABEL_FIELD = '0'; +/** Row data key holding the zero-based source row; not a column field. */ +export const SOURCE_ROW_FIELD = '__sourceRow'; export interface GridColumnDef { headerName: string; @@ -190,6 +192,8 @@ export interface GridColumnDef { cellDataType?: string; suppressHeaderMenuButton?: boolean; lockPosition?: string; + /** Drillable grid cells: the zero-based schema position that `object/open` addresses. */ + jdvDrill?: number; /** Webview-only options; see prepareViewerColumns(). */ jdv?: { filterOptions: string[]; naOptions: boolean; units?: string }; } @@ -322,6 +326,9 @@ export function gridColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { ...(column.type === 'duration' && typeof column.units === 'string' ? { units: column.units } : {}), }, }; + if (column.drillable === true && descriptor.capabilities.children) { + def.jdvDrill = column.position; + } if (column.type === 'boolean') { def.cellDataType = 'boolean'; delete def.jdv; @@ -530,6 +537,28 @@ export function toJdvFilter(filterModel: unknown, descriptor: JdvDescriptor): Fi return nodes.length === 1 ? nodes[0] : { op: 'and', args: nodes }; } +/** `object/open` params that open one drillable grid cell as a child view. */ +export function cellOpenParams(descriptor: JdvDescriptor, sourceRow: unknown, column: unknown): JsonObject { + if (descriptor.kind !== 'grid' || !descriptor.capabilities.children) { + throw new JdvRequestError('unsupported', 'This view has no cells that can be opened.'); + } + if (typeof sourceRow !== 'number' || !Number.isSafeInteger(sourceRow) || sourceRow < 0 || + typeof column !== 'number' || !Number.isSafeInteger(column) || column < 0) { + throw new JdvRequestError('invalid_request', 'Invalid cell address.'); + } + const schema = descriptor.columns?.find(candidate => candidate.position === column); + if (!schema?.drillable) { + throw new JdvRequestError('invalid_request', 'This column cannot be opened.'); + } + return { + view: descriptor.view, + generation: descriptor.generation, + path: [{ kind: 'cell', source_row: sourceRow, column }], + // Paged keeps sorting and filtering in R, as for top-level grids. + mode: 'paged', + }; +} + export interface TableRowsParams extends JsonObject { view: string; generation: number; @@ -621,6 +650,8 @@ export function gridPageFromRows(result: unknown, startRow: number, requested: n // `source_row` is zero-based; the browser shows it plus one. const rows: Record[] = sourceRow.map((index, row) => ({ [ROW_LABEL_FIELD]: rowLabels?.[row] ?? String(index + 1), + // Original zero-based row, for opening a drillable cell after sorting or filtering. + [SOURCE_ROW_FIELD]: index, })); for (const column of columns) { if (!column.schema) { diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index fb086564d..132473ac6 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -9,7 +9,7 @@ import * as path from 'path'; import * as vscode from 'vscode'; import { Uri, ViewColumn, window } from 'vscode'; import { - GridPage, JdvDescriptor, JdvRequestError, JdvTracking, JsonObject, TreeAdapter, canRefresh, gridColumnDefs, + GridPage, JdvDescriptor, cellOpenParams, JdvRequestError, JdvTracking, JsonObject, TreeAdapter, canRefresh, gridColumnDefs, gridPageFromRows, parseDescriptor, parseTracking, revealsPanel, tableRowsParams, } from './jdvAdapter'; import { SequentialStream } from './jdvArrow'; @@ -269,6 +269,10 @@ async function handleMessage(state: JdvPanel, webview: vscode.Webview, message: await refreshView(state, webview, documentGeneration); return; } + if (message.message === 'dataview/openCell') { + await openCell(state, webview, documentGeneration, message); + return; + } if (message.message === 'viewer/track' && typeof message.enabled === 'boolean') { await trackView(state, webview, documentGeneration, message.enabled); return; @@ -367,6 +371,27 @@ async function handleMessage(state: JdvPanel, webview: vscode.Webview, message: } } +/** + * Open a drillable grid cell as a child view. The child arrives as a `view/open` + * notification (with jdv's title) before the response, so only a failure is reported. + */ +async function openCell( + state: JdvPanel, webview: vscode.Webview, documentGeneration: number, message: JsonObject, +): Promise { + let error: string | undefined; + try { + if (state.closed) { + throw new Error(state.closed); + } + await state.peer.request('object/open', cellOpenParams(state.descriptor, message.sourceRow, message.column)); + } catch (e) { + error = `Unable to open this cell: ${errorMessage(e)}`; + } + if (error !== undefined && documentGeneration === session.getDocumentGeneration(webview)) { + void webview.postMessage({ message: 'viewer/openFailed', documentGeneration, error }); + } +} + /** * Ask R to rebuild the view. On success the next generation arrives as a `view/open` * that re-renders the panel, so only a failure is reported to the webview. diff --git a/src/session.ts b/src/session.ts index 34d6290f3..2fdcbac11 100644 --- a/src/session.ts +++ b/src/session.ts @@ -2529,6 +2529,14 @@ export async function getTableHtml( retryBtn.addEventListener('click', retryCurrentPage); } + window.addEventListener('message', (event) => { + const data = event.data; + if (data && data.message === 'viewer/openFailed' && + data.documentGeneration === documentGeneration) { + setFetchStatus('error', data.error || 'Unable to open this cell.', false); + } + }); + const refreshBtn = document.querySelector('#viewerRefresh'); if (refreshBtn) { refreshBtn.addEventListener('click', () => { diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts index a90ee6c2d..af5e756ea 100644 --- a/src/test/node/jdvAdapter.test.ts +++ b/src/test/node/jdvAdapter.test.ts @@ -1,7 +1,7 @@ import * as assert from 'node:assert/strict'; import { JdvDescriptor, JdvRequestError, TreeAdapter, gridColumnDefs, gridPageFromRows, isVectorType, - canRefresh, revealsPanel, + canRefresh, cellOpenParams, revealsPanel, parseDescriptor, parseTracking, tableRowsParams, toJdvFilter, } from '../../jdvAdapter'; @@ -233,9 +233,9 @@ suite('jdv adapter', () => { }, }, 0, 3); assert.deepEqual(page.rows, [ - { '0': 'r3', '1': 'Inf', '2': null }, - { '0': 'r1', '1': 'NaN', '2': 'x' }, - { '0': 'r2', '1': null, '2': 'y' }, + { '0': 'r3', __sourceRow: 2, '1': 'Inf', '2': null }, + { '0': 'r1', __sourceRow: 0, '1': 'NaN', '2': 'x' }, + { '0': 'r2', __sourceRow: 1, '1': null, '2': 'y' }, ]); assert.equal(page.totalRows, 3); assert.equal(page.totalUnfiltered, 5); @@ -243,7 +243,24 @@ suite('jdv adapter', () => { total: 1, total_unfiltered: 1, offset: 0, limit: 1, data: { source_row: [6], row_labels: null, columns: [] }, }, 0, 1); - assert.deepEqual(unnamed.rows, [{ '0': '7' }]); + assert.deepEqual(unnamed.rows, [{ '0': '7', __sourceRow: 6 }]); + }); + + test('opens drillable cells by original source row', () => { + // Column 5 is the only drillable column; the grid reports children. + const drillable = parseDescriptor({ ...grid, capabilities: { ...capabilities, children: true } }); + assert.equal(gridColumnDefs(drillable).filter(column => column.jdvDrill !== undefined).length, 1); + assert.equal(gridColumnDefs(drillable)[6].jdvDrill, 5); + assert.equal(gridColumnDefs(grid)[6].jdvDrill, undefined); + assert.deepEqual(cellOpenParams(drillable, 204, 5), { + view: 'v-1', generation: 2, path: [{ kind: 'cell', source_row: 204, column: 5 }], mode: 'paged', + }); + const rejected = (code: string) => (e: unknown) => e instanceof JdvRequestError && e.code === code; + assert.throws(() => cellOpenParams(grid, 0, 5), rejected('unsupported')); + assert.throws(() => cellOpenParams(drillable, 0, 0), rejected('invalid_request')); + assert.throws(() => cellOpenParams(drillable, -1, 5), rejected('invalid_request')); + assert.throws(() => cellOpenParams(drillable, 1.5, 5), rejected('invalid_request')); + assert.throws(() => cellOpenParams(drillable, 0, 9), rejected('invalid_request')); }); test('maps List Viewer paths to typed tree paths', () => { diff --git a/src/test/suite/dataViewer.test.ts b/src/test/suite/dataViewer.test.ts index 2b93517d6..b2dcc3dff 100644 --- a/src/test/suite/dataViewer.test.ts +++ b/src/test/suite/dataViewer.test.ts @@ -259,6 +259,24 @@ suite('Data viewer', () => { } }); + test('drillable jdv cells get an opening control only when they have a source row', () => { + const viewer = viewerScript(); + const columns = [ + { field: '1', headerName: 'x', jdvDrill: 3 }, + { field: '2', headerName: 'y' }, + ] as (ColDef & { jdvDrill?: number })[]; + viewer.script.prepareViewerColumns(columns); + assert.strictEqual(columns[0].jdvDrill, undefined); + assert.strictEqual((columns[0].context as { drillColumn: number }).drillColumn, 3); + const select = (column: ColDef, data: unknown, value: unknown) => (column.cellRendererSelector as + (params: unknown) => { component: unknown } | undefined)({ data, value }); + const drill = select(columns[0], { __sourceRow: 7 }, '1 item'); + assert.strictEqual(typeof drill?.component, 'function'); + assert.strictEqual(select(columns[0], { __sourceRow: 7 }, null)?.component !== drill?.component, true); + assert.strictEqual(select(columns[0], {}, '1 item'), undefined); + assert.strictEqual(select(columns[1], { __sourceRow: 7 }, 'v'), undefined); + }); + test('high contrast light uses the light grid theme', () => { assert.strictEqual(viewerScript(undefined, {}, true).script.getAgTheme(), 'light'); assert.strictEqual(viewerScript().script.getAgTheme(), 'dark'); From 195d5b4d46e6f8d4bfc12f07cd37a079b404386c Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 14:50:38 +0000 Subject: [PATCH 20/38] feat: back Interactive table and list previews with jdv handles in sess Hold one registry-free jdv handle per table or list output, answer paged queries and child pages through inspections, and open the value in the Data Viewer by registering it with jdv over a host-provided address. The dataview registry still serves outputs when jdv is not installed. --- sess/R/handlers.R | 11 +- sess/R/interactive.R | 152 ++++++++++++++++++ sess/R/runtime.R | 1 + sess/R/server.R | 5 +- sess/inst/tinytest/test-interactive-handles.R | 80 +++++++++ 5 files changed, 244 insertions(+), 5 deletions(-) create mode 100644 sess/inst/tinytest/test-interactive-handles.R diff --git a/sess/R/handlers.R b/sess/R/handlers.R index a62ad8434..f3ffbf943 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1314,7 +1314,7 @@ handle_dataview_dispose <- function(params) { # Every failure, whether a jdv_error (jdv_error_in_use, jdv_error_connect_failed) # or the transport request itself failing, means the same fallback, so errors # are not told apart. -.jdv_connect <- function() { +.jdv_connect <- function(address = NULL) { if (identical(getOption("sess.dataviewer", "auto"), "sess") || !requireNamespace("jdv", quietly = TRUE)) { return(FALSE) @@ -1328,7 +1328,10 @@ handle_dataview_dispose <- function(params) { if (!identical(connection$state, "disconnected")) { return(FALSE) } - address <- request_client("jdv/address")$address + # Callers that cannot wait for a reply (an inspection being answered) pass it in. + if (is.null(address)) { + address <- request_client("jdv/address")$address + } if (!is.character(address) || length(address) != 1L || is.na(address) || !nzchar(address)) { return(FALSE) @@ -1391,11 +1394,11 @@ handle_dataview_dispose <- function(params) { # over the transport itself, so sess needs no task callback of its own for it. # Only an embedding host that collects jdv_notifications() must call # jdv_rpc(NULL) first. -.jdv_show <- function(x, title_key, binding = NULL) { +.jdv_show <- function(x, title_key, binding = NULL, address = NULL) { if (.jdv_is_arrow(x) && !requireNamespace("nanoarrow", quietly = TRUE)) { return(FALSE) } - if (!.jdv_connect()) { + if (!.jdv_connect(address)) { return(FALSE) } title <- if (nzchar(title_key)) title_key else NULL diff --git a/sess/R/interactive.R b/sess/R/interactive.R index 03e935025..fc1d57090 100644 --- a/sess/R/interactive.R +++ b/sess/R/interactive.R @@ -369,9 +369,153 @@ interactive_execute <- function(id, code, source = NULL) { list(value = snapshot, source_rows = source_rows, truncated = truncated) } +# jdv handles back the inline table and list previews ------------------------ +# +# Each table or list output holds one registry-free jdv handle (no viewer slot, +# no notifications). The agent sends the descriptor and the first page to the +# extension, which pages, sorts, filters and explores through `jdv_handle_request` +# inspections. Handles are closed when the output is cleared, the registry +# exceeds its bound, or the Interactive bridge stops; jdv also closes +# unreachable handles. Without jdv (or with options(sess.dataviewer = "sess")) +# the previous dataview registry serves the same outputs. + +.interactive_handle_limit <- 64L +.interactive_page_bytes <- 256L * 1024L + +.interactive_jdv_usable <- function() { + !identical(getOption("sess.dataviewer", "auto"), "sess") && + requireNamespace("jdv", quietly = TRUE) +} + +.interactive_handle_store <- function() { + if (is.null(.sess_env$interactive_handles)) { + .sess_env$interactive_handles <- new.env(parent = emptyenv()) + .sess_env$interactive_handle_order <- character() + .sess_env$interactive_handle_seq <- 0L + } + .sess_env$interactive_handles +} + +.interactive_handle_close <- function(id) { + store <- .interactive_handle_store() + if (is.character(id) && length(id) == 1L && exists(id, envir = store, inherits = FALSE)) { + entry <- get(id, envir = store, inherits = FALSE) + rm(list = id, envir = store) + .sess_env$interactive_handle_order <- setdiff(.sess_env$interactive_handle_order, id) + try(jdv::jdv_handle_close(entry$handle), silent = TRUE) + } + invisible(NULL) +} + +.interactive_handle_close_all <- function() { + for (id in .sess_env$interactive_handle_order) .interactive_handle_close(id) + invisible(NULL) +} + +.interactive_handle_entry <- function(id) { + store <- .interactive_handle_store() + if (!is.character(id) || length(id) != 1L || !exists(id, envir = store, inherits = FALSE)) { + stop("This output's data is no longer available. Run its code again.") + } + get(id, envir = store, inherits = FALSE) +} + +# Capture `value` and read its descriptor and first page. Returns NULL, leaving no +# handle behind, when jdv cannot represent the value; the caller then uses the +# dataview registry. +.interactive_handle_preview <- function(value, kind) { + if (!.interactive_jdv_usable()) return(NULL) + handle <- NULL + tryCatch({ + handle <- jdv::jdv_handle(value, mode = "paged") + descriptor <- jdv::jdv_handle_descriptor(handle, max_bytes = .interactive_page_bytes) + page <- if (kind == "table") { + jdv::jdv_handle_request(handle, "grid/rows", list(offset = 0L, limit = 20L), + max_bytes = .interactive_page_bytes) + } else { + jdv::jdv_handle_request( + handle, "object/children", + list(path = list(), offset = 0L, limit = 200L, attributes = FALSE), + max_bytes = .interactive_page_bytes) + } + store <- .interactive_handle_store() + .sess_env$interactive_handle_seq <- .sess_env$interactive_handle_seq + 1L + id <- paste0("h", .sess_env$interactive_handle_seq) + assign(id, list(handle = handle, value = value), envir = store) + .sess_env$interactive_handle_order <- c(.sess_env$interactive_handle_order, id) + while (length(.sess_env$interactive_handle_order) > .interactive_handle_limit) { + .interactive_handle_close(.sess_env$interactive_handle_order[[1L]]) + } + list(handleId = id, descriptor = descriptor, page = page) + }, error = function(e) { + if (!is.null(handle)) try(jdv::jdv_handle_close(handle), silent = TRUE) + NULL + }) +} + +handle_jdv_request <- function(params) { + entry <- .interactive_handle_entry(params$handle_id) + method <- params$method + if (!identical(method, "grid/rows") && !identical(method, "object/children")) { + stop("Unsupported jdv handle method") + } + max_bytes <- suppressWarnings(as.integer(params$max_bytes %||% .interactive_page_bytes)) + if (is.na(max_bytes) || max_bytes < 1L) max_bytes <- .interactive_page_bytes + list(json = jdv::jdv_handle_request(entry$handle, method, params$params %||% list(), + max_bytes = max_bytes)) +} + +handle_jdv_handle_close <- function(params) { + .interactive_handle_close(params$handle_id) + TRUE +} + +# Select the value at a typed jdv path (index, binding, slot or attr steps). +.interactive_resolve_path <- function(value, path) { + for (step in path) { + value <- switch( + step$type, + index = value[[as.integer(step$index)]], + binding = get(step$name, envir = value, inherits = FALSE), + slot = methods::slot(value, step$name), + attr = attr(value, step$name, exact = TRUE), + stop("Unsupported path step") + ) + } + value +} + +# Show the output's value, or the element at `path`, in the Data Viewer panel by +# registering it through the stage A jdv connection. Returns FALSE when the panel +# cannot be opened. +handle_jdv_open_viewer <- function(params) { + entry <- .interactive_handle_entry(params$handle_id) + value <- .interactive_resolve_path(entry$value, params$path %||% list()) + title <- params$title + address <- params$address + .jdv_show(value, if (is.character(title) && length(title) == 1L) title else "", + address = if (is.character(address) && length(address) == 1L) address) +} + .interactive_rich_value <- function(value) { if (dataview_is_table(value)) { snapshot <- .interactive_table_snapshot(value) + preview <- if (is.data.frame(value) || is.matrix(value)) { + .interactive_handle_preview(value, "table") + } + if (!is.null(preview)) { + printed <- .interactive_table_text(snapshot$value) + if (snapshot$truncated && !is.null(printed$printedText)) { + printed$printedText <- paste0(printed$printedText, sprintf( + "\n[Snapshot: first %s of %s rows. Open Data viewer for the full table.]\n", + format(nrow(snapshot$value), big.mark = ",", scientific = FALSE, trim = TRUE), + format(snapshot$source_rows, big.mark = ",", scientific = FALSE, trim = TRUE))) + } + .interactive_event("display", c(list(kind = "table", handleId = preview$handleId, + descriptor = preview$descriptor, page = preview$page, + sourceRows = snapshot$source_rows), printed)) + return(TRUE) + } registration <- dataview_register(snapshot$value) metadata <- handle_dataview_init(list(view_id = registration$view_id)) preview <- handle_dataview_page(list(view_id = registration$view_id, @@ -402,6 +546,13 @@ interactive_execute <- function(id, code, source = NULL) { return(TRUE) } if (is.list(value) && !is.object(value)) { + preview <- .interactive_handle_preview(value, "list") + if (!is.null(preview)) { + .interactive_event("display", c(list(kind = "list", handleId = preview$handleId, + descriptor = preview$descriptor, page = preview$page), + .interactive_table_text(value))) + return(TRUE) + } view_id <- dataview_new_id() # Nested panels reuse the root's owner; separate cell results must stay independent. root <- listview_state(value, "List", paste0("interactive:", view_id)) @@ -455,6 +606,7 @@ display <- function(x, mime = NULL) { #' @export interactive_stop <- function() { .sess_env$interactive_stop <- TRUE + try(.interactive_handle_close_all(), silent = TRUE) if (!is.null(.sess_env$interactive_task)) { removeTaskCallback(.sess_env$interactive_task) .sess_env$interactive_task <- NULL diff --git a/sess/R/runtime.R b/sess/R/runtime.R index 117abae94..69bbdb380 100644 --- a/sess/R/runtime.R +++ b/sess/R/runtime.R @@ -16,6 +16,7 @@ } .runtime_clear_viewer_state <- function() { + try(.interactive_handle_close_all(), silent = TRUE) .jdv_release() # Open dataviews and title-to-id mappings belong to one runtime connection. # Keep an empty registry available, but never restore data from a prior run. diff --git a/sess/R/server.R b/sess/R/server.R index 9fe2259eb..038593f40 100644 --- a/sess/R/server.R +++ b/sess/R/server.R @@ -496,7 +496,10 @@ dispatch_message <- function(line) { "listview_view" = function(p) handle_listview_view(p$view_id, p$index, p$path), "dataview_init" = function(p) handle_dataview_init(p), "dataview_page" = function(p) handle_dataview_page(p), - "dataview_dispose" = function(p) handle_dataview_dispose(p) + "dataview_dispose" = function(p) handle_dataview_dispose(p), + "jdv_handle_request" = function(p) handle_jdv_request(p), + "jdv_handle_close" = function(p) handle_jdv_handle_close(p), + "jdv_open_viewer" = function(p) handle_jdv_open_viewer(p) ) if (payload$method %in% names(handlers)) { diff --git a/sess/inst/tinytest/test-interactive-handles.R b/sess/inst/tinytest/test-interactive-handles.R new file mode 100644 index 000000000..3b73d30f4 --- /dev/null +++ b/sess/inst/tinytest/test-interactive-handles.R @@ -0,0 +1,80 @@ +# Inline table and list previews use registry-free jdv handles when jdv is installed. +if (!requireNamespace("jdv", quietly = TRUE)) { + exit_file("jdv is not installed") +} + +ns <- asNamespace("sess") +env <- sess:::.sess_env +previous <- list(handles = env$interactive_handles, order = env$interactive_handle_order, + seq = env$interactive_handle_seq) +on.exit({ + ns$.interactive_handle_close_all() + env$interactive_handles <- previous$handles + env$interactive_handle_order <- previous$order + env$interactive_handle_seq <- previous$seq + options(sess.dataviewer = NULL) +}) +options(sess.dataviewer = NULL) +preview <- ns$.interactive_handle_preview +request <- function(id, method, params = list()) { + jsonlite::fromJSON(ns$handle_jdv_request(list(handle_id = id, method = method, + params = params))$json, + simplifyVector = FALSE) +} + +# A table yields a descriptor and a bounded first page without using the viewer registry. +table <- preview(data.frame(id = 1:100, name = rep(c("a", "b"), 50)), "table") +expect_true(is.character(table$handleId)) +descriptor <- jsonlite::fromJSON(table$descriptor, simplifyVector = FALSE) +expect_identical(descriptor$kind, "grid") +expect_identical(descriptor$access, "random") +expect_identical(descriptor$nrow, 100L) +first <- jsonlite::fromJSON(table$page, simplifyVector = FALSE) +expect_identical(length(first$data$source_row), 20L) +expect_identical(first$total, 100L) + +# Paging, sorting and filtering run in R through the same handle; ids are zero-based rows. +sorted <- request(table$handleId, "grid/rows", list( + offset = 0L, limit = 3L, sort = list(list(col = 0L, desc = TRUE)), + filter = list(op = "text_equals", col = 1L, value = "a") +)) +expect_identical(sorted$total, 50L) +expect_identical(unlist(sorted$data$source_row), c(98L, 96L, 94L)) + +# A list yields a tree descriptor and its first children page. +tree <- preview(list(a = 1, b = data.frame(z = 1:2), 3), "list") +expect_identical(jsonlite::fromJSON(tree$descriptor, simplifyVector = FALSE)$kind, "tree") +children <- jsonlite::fromJSON(tree$page, simplifyVector = FALSE) +expect_identical(vapply(children$children, function(child) child$label, ""), c("a", "b", "")) +expect_identical(request(tree$handleId, "object/children", list( + path = list(list(type = "index", index = 2L)), offset = 0L, limit = 10L, attributes = FALSE +))$total_children, 1L) + +# The viewer opens the output's value or one typed item of it. +entry <- ns$.interactive_handle_entry(tree$handleId) +expect_identical(ns$.interactive_resolve_path(entry$value, list(list(type = "index", index = 2L))), + data.frame(z = 1:2)) +expect_identical(ns$.interactive_resolve_path(entry$value, list()), entry$value) + +# Only grid rows and object children can be requested. +expect_error(ns$handle_jdv_request(list(handle_id = table$handleId, method = "view/close")), + "Unsupported") + +# Closing releases the handle, idempotently; unknown ids report that the data is gone. +expect_true(ns$handle_jdv_handle_close(list(handle_id = table$handleId))) +expect_true(ns$handle_jdv_handle_close(list(handle_id = table$handleId))) +expect_error(request(table$handleId, "grid/rows"), "no longer available") + +# The number of live handles is bounded; the oldest are closed first. +ns$.interactive_handle_close_all() +ids <- vapply(seq_len(ns$.interactive_handle_limit + 3L), function(i) { + preview(list(i), "list")$handleId +}, "") +expect_identical(length(env$interactive_handle_order), ns$.interactive_handle_limit) +expect_error(request(ids[[1L]], "object/children"), "no longer available") +expect_silent(request(ids[[length(ids)]], "object/children", list(path = list()))) + +# Values jdv cannot represent, or an opt-out, leave the dataview registry in charge. +expect_null(preview(quote(x), "list")) +options(sess.dataviewer = "sess") +expect_null(preview(data.frame(a = 1), "table")) From bbcf33aa9f845f8e3955d2cf1d64b982926c2234 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 14:50:38 +0000 Subject: [PATCH 21/38] feat: serve Interactive inline tables and lists from jdv handles Convert the handle descriptor and first page into the existing renderer data, query pages with sort and filter through the handle, explore lists with typed paths, open tables and list items in the Data Viewer, and release handles when completed cells are cleared. Legacy dataview outputs keep working when jdv is not used. --- src/interactive/agent.ts | 3 + src/interactive/backend.ts | 5 +- src/interactive/inlineTable.ts | 2 +- src/interactive/jdvTable.ts | 195 +++++++++++++++++++++++ src/interactive/manager.ts | 88 +++++++--- src/interactive/renderer.ts | 2 +- src/interactive/tableQuery.ts | 8 +- src/jdvAdapter.ts | 9 +- src/test/node/interactiveRuntime.test.ts | 138 ++++++++++++++-- src/test/node/jdvTable.test.ts | 115 +++++++++++++ 10 files changed, 532 insertions(+), 33 deletions(-) create mode 100644 src/interactive/jdvTable.ts create mode 100644 src/test/node/jdvTable.test.ts diff --git a/src/interactive/agent.ts b/src/interactive/agent.ts index 7dc9e1790..8642f17b9 100644 --- a/src/interactive/agent.ts +++ b/src/interactive/agent.ts @@ -11,6 +11,7 @@ import { BackendEvent, BackendFactory, SessionBackend, inspectionMethod } from ' import { AGENT_PROTOCOL, AgentConfig, AgentSnapshot, ExecutionRecord, SessionEvent, SessionManifest, object, submission, identifier, sessionLabel } from './protocol'; import { searchHistory } from './history'; +import { normalizeJdvDisplay } from './jdvTable'; interface Client { socket: net.Socket; authenticated: boolean; subscribed: boolean; id: string } /** This process never imports vscode and is supervised independently of the editor. */ @@ -450,6 +451,8 @@ export class SessionAgent { } private display(data: Record, executionId?: string): void { + // A jdv handle's descriptor and first page become the renderer's table or list data. + data = normalizeJdvDisplay(data); const display: Record = { ...data, displayId: typeof data.displayId === 'string' ? data.displayId : randomUUID() }; if (data.kind === 'plot' && typeof data.svg === 'string') { display.svg = this.assets.put(data.svg, '.svg'); } if (data.kind === 'html' && typeof data.file === 'string') { diff --git a/src/interactive/backend.ts b/src/interactive/backend.ts index a4fd57403..70681fe30 100644 --- a/src/interactive/backend.ts +++ b/src/interactive/backend.ts @@ -10,7 +10,7 @@ export interface RuntimeMetadata { libraryPaths?: string[]; } export type InspectionMethod = 'workspace' | 'workspace_children' | 'workspace_view' | 'listview_navigate' | 'listview_view' | 'hover' | 'completion' | - 'dataview_init' | 'dataview_page' | 'dataview_dispose'; + 'dataview_init' | 'dataview_page' | 'dataview_dispose' | 'jdv_handle_request' | 'jdv_handle_close' | 'jdv_open_viewer'; export interface InspectionRequest { method: InspectionMethod; params: Record; timeout?: number } export interface InputReply { value: string } export interface ClientReply { result?: unknown; error?: string } @@ -56,7 +56,8 @@ export interface SessionBackend { export type BackendFactory = (settings: AgentSettings) => SessionBackend; export function inspectionMethod(value: unknown): InspectionMethod { - if (!['workspace', 'workspace_children', 'workspace_view', 'listview_navigate', 'listview_view', 'hover', 'completion', 'dataview_init', 'dataview_page', 'dataview_dispose'].includes(String(value))) { + if (!['workspace', 'workspace_children', 'workspace_view', 'listview_navigate', 'listview_view', 'hover', 'completion', 'dataview_init', 'dataview_page', 'dataview_dispose', + 'jdv_handle_request', 'jdv_handle_close', 'jdv_open_viewer'].includes(String(value))) { throw new Error('Unsupported inspection method'); } return value as InspectionMethod; diff --git a/src/interactive/inlineTable.ts b/src/interactive/inlineTable.ts index 79ee01298..293d8ccef 100644 --- a/src/interactive/inlineTable.ts +++ b/src/interactive/inlineTable.ts @@ -28,7 +28,7 @@ export function createInlineTable(parent: HTMLElement, toolbar: HTMLElement, sta let dragged: string | undefined; let error = ''; const available = original.connected !== false && original.archived !== true && original.running !== false; - const hasFull = !!original.fullViewId; + const hasFull = !!original.fullViewId || !!original.handleId; const columns = (): TableColumn[] => (state.data.columns ?? []) as TableColumn[]; const count = (): number => !state.live && hasFull && !state.sort.length && !Object.keys(state.filters).length ? Number(original.sourceRows ?? original.totalRows) : Number(state.data.totalRows ?? 0); diff --git a/src/interactive/jdvTable.ts b/src/interactive/jdvTable.ts new file mode 100644 index 000000000..b6e32ddb7 --- /dev/null +++ b/src/interactive/jdvTable.ts @@ -0,0 +1,195 @@ +'use strict'; + +// Inline table and list previews backed by registry-free jdv handles. The R side +// owns one handle per output and answers `jdv_handle_request` inspections; this +// module converts between jdv's protocol shapes and the renderer's data. It has no +// VS Code dependency, so the agent, the extension host and tests can all use it. + +import { + JdvDescriptor, JdvPathStep, JdvRequestError, JsonObject, ListViewChild, TreeAdapter, gridColumnDefs, + gridPageFromRows, parseDescriptor, tableRowsParams, +} from '../jdvAdapter'; +import { TABLE_PAGE_SIZE, tablePage } from './tablePaging'; +import { columnOperators, filterNeedsValue, TableColumn, TableFilters, TableSort } from './tableQuery'; + +export type Inspect = (request: { method: string; params: Record }) => Promise>; + +/** Largest JSON result requested for one page, matching the R side's preview bound. */ +export const PAGE_BYTES = 256 * 1024; +const PAGE_ATTEMPTS = 20; + +/** Whether a display event came from a jdv handle (as opposed to the dataview registry). */ +export function isJdvOutput(data: Record): boolean { + return typeof data.handleId === 'string' && data.handleId !== ''; +} + +function parseJson(text: unknown, what: string): unknown { + if (typeof text !== 'string') { throw new Error(`Invalid jdv ${what}`); } + return JSON.parse(text); +} + +function descriptorOf(data: Record): JdvDescriptor { + return parseDescriptor(typeof data.descriptor === 'string' ? JSON.parse(data.descriptor) : data.descriptor); +} + +/** + * Agent side: turn the raw descriptor and first page that R sent into the data the + * renderer, exports and transcript already understand (columns and rows for a + * table; navigation and children for a list). The parsed descriptor stays so the + * host can query the same handle later. + */ +export function normalizeJdvDisplay(data: Record): Record { + if (!isJdvOutput(data)) { return data; } + const descriptor = parseDescriptor(parseJson(data.descriptor, 'descriptor')); + const page = parseJson(data.page, 'page'); + const { page: _page, ...rest } = data; + if (data.kind === 'table' && descriptor.kind === 'grid') { + const first = gridPageFromRows(page, 0, TABLE_PAGE_SIZE); + return { ...rest, descriptor, columns: gridColumnDefs(descriptor), rows: first.rows, totalRows: first.totalRows }; + } + if (data.kind === 'list' && descriptor.kind === 'tree') { + const adapter = new TreeAdapter(descriptor, 'List'); + return { ...rest, descriptor, navigation: adapter.navigation([]), ...adapter.children([], page) }; + } + throw new Error('The jdv descriptor does not match the output kind'); +} + +// --------------------------------------------------------------------------- +// Tables + +/** AG Grid names the date operand `dateFrom`; the inline filter UI writes `filter`. */ +function filterModel(filters: TableFilters, columns: TableColumn[]): Record { + const model: Record = {}; + for (const [field, filter] of Object.entries(filters)) { + const type = columns.find(column => column.field === field)?.type; + model[field] = type === 'dateColumn' || type === 'datetimeColumn' ? { ...filter, dateFrom: filter.filter } : filter; + } + return model; +} + +/** Validate a renderer table query, as the dataview path does. */ +function validateQuery(columns: TableColumn[], sort: TableSort[], filters: TableFilters): void { + for (const item of sort) { + if (!item || !['asc', 'desc'].includes(item.sort) || + !columns.some(column => column.field === item.colId && column.field !== '0' && column.sortable !== false)) { + throw new Error('This column cannot be sorted.'); + } + } + for (const [field, filter] of Object.entries(filters)) { + const column = columns.find(column => column.field === field && column.field !== '0' && column.filter !== false); + if (!column || !filter || !columnOperators(column).includes(filter.type) || + (filterNeedsValue(filter.type) && (typeof filter.filter !== 'string' || filter.filter.length > 1000))) { + throw new Error('Invalid column filter.'); + } + } +} + +/** Fetch one inline table page from the output's handle, sorted and filtered in R. */ +export async function queryJdvTablePage( + data: Record, message: Record, inspect: Inspect, +): Promise> { + const size = message.size === undefined ? TABLE_PAGE_SIZE : Number(message.size); + if (![20, 50, 100].includes(size)) { throw new Error('Choose 20, 50 or 100 rows per page.'); } + const offset = Number(message.start ?? 0); + if (!Number.isSafeInteger(offset) || offset < 0 || offset > 2147483646) { throw new Error('Invalid table page.'); } + const sort = (message.sortModel ?? []) as TableSort[]; + const filters = (message.filterModel ?? {}) as TableFilters; + if (!Array.isArray(sort) || sort.length > 1 || !filters || typeof filters !== 'object' || Array.isArray(filters) || + Object.keys(filters).length > 100) { + throw new Error('Invalid table query.'); + } + const columns = data.columns as TableColumn[]; + if (!Array.isArray(columns) || columns.length > 100000) { throw new Error('Invalid table columns.'); } + validateQuery(columns, sort, filters); + const descriptor = descriptorOf(data); + const model = filterModel(filters, columns); + + const fetch = async (from: number) => { + const rows: Record[] = []; + let total = 0; + let unfiltered = 0; + let cursor = from; + for (let attempt = 0; attempt < PAGE_ATTEMPTS && rows.length < size; attempt++) { + const wanted = size - rows.length; + const { view: _view, generation: _generation, ...params } = + tableRowsParams(descriptor, cursor, cursor + wanted, sort, model); + if (!params.sort.length) { delete (params as { sort?: unknown }).sort; } + const reply = await inspect({ method: 'jdv_handle_request', params: { + handle_id: data.handleId, method: 'grid/rows', params, max_bytes: PAGE_BYTES, + } }); + const page = gridPageFromRows(parseJson(reply.json, 'page'), cursor, wanted); + rows.push(...page.rows); + total = page.totalRows; + unfiltered = page.totalUnfiltered; + cursor += page.returned; + // jdv may return fewer rows than requested to fit the byte budget. + if (page.returned === 0 || cursor >= total) { break; } + } + return { rows, total, unfiltered }; + }; + let start = Math.floor(offset / size) * size; + let result = await fetch(start); + if (result.total > 0 && start >= result.total) { + start = tablePage(result.total, start, size).start; + result = await fetch(start); + } + return { + rows: result.rows, totalRows: result.total, totalUnfiltered: result.unfiltered, + startRow: result.total === 0 ? 0 : start, columns, live: true, queryReset: false, + }; +} + +// --------------------------------------------------------------------------- +// Lists + +/** What a list request needs the host to do besides replying. */ +export interface ListOutcome { + result?: unknown; + /** Open this item (typed path and title) in the Data Viewer panel. */ + open?: { path: JdvPathStep[]; title: string }; +} + +/** A fresh adapter for an output, primed with the root page the renderer already shows. */ +export async function createListAdapter(data: Record, inspect: Inspect): Promise { + const adapter = new TreeAdapter(descriptorOf(data), 'List'); + const reply = await inspect({ method: 'jdv_handle_request', params: { + handle_id: data.handleId, method: 'object/children', params: stripOwner(adapter.childrenParams([], 1)), + max_bytes: PAGE_BYTES, + } }); + adapter.children([], parseJson(reply.json, 'children')); + return adapter; +} + +function stripOwner(params: JsonObject): JsonObject { + const { view: _view, generation: _generation, ...rest } = params; + return rest; +} + +/** Serve a renderer list action (`list`, `listPage`, `listNavigate`, `listItem`) from the handle. */ +export async function listRequest( + adapter: TreeAdapter, data: Record, message: Record, inspect: Inspect, +): Promise { + const path = message.path as number[]; + switch (message.action) { + case 'listPage': { + const reply = await inspect({ method: 'jdv_handle_request', params: { + handle_id: data.handleId, method: 'object/children', + params: stripOwner(adapter.childrenParams(path, Number(message.start))), max_bytes: PAGE_BYTES, + } }); + return { result: adapter.children(path, parseJson(reply.json, 'children')) }; + } + case 'listNavigate': + return { result: { navigation: adapter.navigation(path) } }; + case 'listItem': { + const action = adapter.open(path, Number(message.index)); + if (action.kind === 'navigate') { return { result: { navigation: action.navigation } }; } + return { open: adapter.locate([...path, Number(message.index)]) }; + } + case 'list': + return { open: adapter.locate(path) }; + default: + throw new JdvRequestError('invalid_request', 'Unsupported list action'); + } +} + +export type { ListViewChild }; diff --git a/src/interactive/manager.ts b/src/interactive/manager.ts index bf10679b5..da925e62f 100644 --- a/src/interactive/manager.ts +++ b/src/interactive/manager.ts @@ -27,6 +27,8 @@ import { readExecutionRecords, readPreviousJournal } from './journal'; import { sessionAge, sessionPresentation } from './sessionPresentation'; import { runningSessions, stopSession } from './sessionLifecycle'; import { InteractiveInputFeatures } from './inputFeatures'; +import { createListAdapter, isJdvOutput, listRequest, queryJdvTablePage } from './jdvTable'; +import type { TreeAdapter } from '../jdvAdapter'; import { acquireInteractiveJdvAddress, disposeInteractiveJdvHostOnControlLoss } from './jdvAddress'; interface InteractiveView { @@ -376,10 +378,18 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro view.chain = view.chain.then(async () => { if (view.disposed) { return; } const edits: vscode.NotebookEdit[] = []; + const handles: string[] = []; for (const cell of view.notebook.getCells().slice().reverse()) { const id = cell.metadata.rExecutionId as string | undefined; - const record = id && this.transcriptCells(view).find(cell => cell.record.id === id)?.record; + const transcript = id ? this.transcriptCells(view).find(cell => cell.record.id === id) : undefined; + const record = transcript?.record; if (record && !['queued', 'running', 'unknown'].includes(record.state)) { + // Cleared outputs release their jdv handles in R. + if (transcript?.generation === view.model.generation) { + for (const output of transcript.outputs) { + if (output.type === 'display' && isJdvOutput(output.data)) { handles.push(String(output.data.handleId)); } + } + } view.hidden.add(record.id); const adjacent = edits.at(-1); if (adjacent?.range.start === cell.index + 1) { @@ -389,6 +399,11 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro } const edit = new vscode.WorkspaceEdit(); edit.set(view.notebook.uri, edits); await vscode.workspace.applyEdit(edit); + if (!['exited', 'stopping'].includes(view.client.manifest.status)) { + for (const handle of handles) { + await view.client.request('inspect', { method: 'jdv_handle_close', params: { handle_id: handle } }).catch(() => undefined); + } + } }); await view.chain; } @@ -1222,22 +1237,7 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro try { let result: unknown; if (data.method === 'jdv/address') { - const sessionId = view.target.sessionId; - const generation = view.model.generation; - const manifestGeneration = client.manifest.generation; - const hostKey = interactiveJdvHostKey(client.manifest.id, manifestGeneration); - const canConnect = () => client.control && !view.target.processExited && - !['stopping', 'exited'].includes(client.manifest.status); - const address = await acquireInteractiveJdvAddress({ - sessionId, hostKey, - canConnect, - isCurrent: () => !view.disposed && view.client === client && !view.restarting && - view.target.sessionId === sessionId && view.model.generation === generation && - client.manifest.generation === manifestGeneration, - getAddress: getJdvAddress, - disposeHost: disposeJdvHost, - }); - result = { address }; + result = { address: await this.jdvAddress(view) }; } else { result = await session.handleEditorRequest(String(data.method), params); } @@ -1245,6 +1245,49 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro } catch (error) { await client.request('clientReply', { id: data.id, error: String(error) }).catch(() => undefined); } } + /** The session-scoped jdv listener address, created on first use. */ + private jdvAddress(view: InteractiveView): Promise { + const client = view.client; + const sessionId = view.target.sessionId; + const generation = view.model.generation; + const manifestGeneration = client.manifest.generation; + return acquireInteractiveJdvAddress({ + sessionId, hostKey: interactiveJdvHostKey(client.manifest.id, manifestGeneration), + canConnect: () => client.control && !view.target.processExited && + !['stopping', 'exited'].includes(client.manifest.status), + isCurrent: () => !view.disposed && view.client === client && !view.restarting && + view.target.sessionId === sessionId && view.model.generation === generation && + client.manifest.generation === manifestGeneration, + getAddress: getJdvAddress, + disposeHost: disposeJdvHost, + }); + } + + /** Positional-path bookkeeping of an inline list, rebuilt from its handle when missing. */ + private readonly listAdapters = new Map(); + + private async listAdapter(view: InteractiveView, data: Record): Promise { + const key = `${view.client.manifest.id}:${view.model.generation}:${String(data.handleId)}`; + let adapter = this.listAdapters.get(key); + if (!adapter) { + adapter = await createListAdapter(data, request => view.client.request('inspect', request)); + this.listAdapters.set(key, adapter); + while (this.listAdapters.size > 64) { this.listAdapters.delete(this.listAdapters.keys().next().value as string); } + } + return adapter; + } + + /** Register the output's value (or one item of it) with jdv so the Data Viewer panel shows it. */ + private async openHandleViewer(view: InteractiveView, data: Record, path: unknown[], title: string): Promise { + // R cannot wait for a reply to its own request while answering this inspection, + // so the listener address is handed over instead of being requested. + const address = await this.jdvAddress(view); + const opened = await view.client.request('inspect', { + method: 'jdv_open_viewer', params: { handle_id: data.handleId, path, title, address }, + }); + if (opened !== true) { throw new Error('Unable to open the Data Viewer. Check the R session and try again.'); } + } + private async rendererMessage(editor: vscode.NotebookEditor, message: Record): Promise { const view = [...this.views.values()].find(item => item.notebook === editor.notebook); if (!view) { return; } @@ -1275,15 +1318,24 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro switch (message.action) { case 'table': if (data.kind !== 'table') { return; } + if (isJdvOutput(data)) { await this.openHandleViewer(view, data, [], `Cell ${owner.record.order}: table`); break; } await session.showDataView('table', 'json', `${view.client.manifest.label}: ${data.fullViewId ? 'full table' : 'table'}`, '', 'Beside', String(data.fullViewId ?? data.viewId), undefined, view.target?.sessionId ?? null); break; case 'page': { if (data.kind !== 'table') { return; } - result = await queryTablePage(data, message, request => view.client.request('inspect', request)); break; + result = await (isJdvOutput(data) ? queryJdvTablePage : queryTablePage)(data, message, + request => view.client.request('inspect', request)); break; } case 'list': case 'listPage': case 'listNavigate': case 'listItem': { if (data.kind !== 'list') { return; } const listPath = message.path ?? []; if (!Array.isArray(listPath) || !listPath.every(index => Number.isSafeInteger(index) && index > 0)) { throw new Error('Invalid list path'); } + if (isJdvOutput(data)) { + const adapter = await this.listAdapter(view, data); + const outcome = await listRequest(adapter, data, { ...message, path: listPath }, request => view.client.request('inspect', request)); + if (outcome.open) { await this.openHandleViewer(view, data, outcome.open.path, `Cell ${owner.record.order}: ${outcome.open.title}`); } + result = outcome.result; + break; + } const params = { view_id: data.viewId, path: listPath }; if (message.action === 'listPage') { if (!Number.isSafeInteger(message.start) || Number(message.start) < 1) { throw new Error('Invalid list page'); } diff --git a/src/interactive/renderer.ts b/src/interactive/renderer.ts index 3ef312bc3..925d01cc3 100644 --- a/src/interactive/renderer.ts +++ b/src/interactive/renderer.ts @@ -93,7 +93,7 @@ export function activate(context: RendererContext): { renderOutputItem(item: Out printed.textContent = typeof data.printedText === 'string' ? data.printedText : ''; element.append(printed); const open = button('Open data viewer', 'table', () => send(item, data, 'table'), 'Data viewer'); - if (data.fullViewId) { + if (data.fullViewId || data.handleId) { open.title = 'Open the full table without copying it. Reference edits may appear here; reopen to refresh. Sorting and filtering scan the full data.'; } open.disabled = !live; diff --git a/src/interactive/tableQuery.ts b/src/interactive/tableQuery.ts index 3a463cbe5..4e35ba6f6 100644 --- a/src/interactive/tableQuery.ts +++ b/src/interactive/tableQuery.ts @@ -1,6 +1,10 @@ import { TABLE_PAGE_SIZE, tablePage } from './tablePaging'; -export interface TableColumn { field: string; headerName: string; headerTooltip?: string; type?: unknown; sortable?: boolean; filter?: unknown } +export interface TableColumn { + field: string; headerName: string; headerTooltip?: string; type?: unknown; sortable?: boolean; filter?: unknown; + /** jdv columns list the operators their type supports. */ + jdv?: { filterOptions: string[] }; +} export interface TableSort { colId: string; sort: 'asc' | 'desc' } export interface TableFilter { type: string; filter?: string } export type TableFilters = Record; @@ -10,6 +14,8 @@ export const filterOperators = { blank: 'is blank', notBlank: 'is not blank', true: 'is true', false: 'is false', }; export function columnOperators(column: TableColumn): string[] { + // An inline filter is one condition, so the range operator is not offered. + if (column.jdv) { return [...column.jdv.filterOptions.filter(option => option !== 'inRange'), 'blank', 'notBlank']; } if (column.type === 'booleanColumn') { return ['true', 'false', 'blank', 'notBlank']; } return ['numericColumn', 'bigintColumn', 'dateColumn', 'datetimeColumn'].includes(String(column.type)) ? ['equals', 'notEqual', 'lessThan', 'lessThanOrEqual', 'greaterThan', 'greaterThanOrEqual', 'blank', 'notBlank'] diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index e2aca7d2b..ba950e0c9 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -759,6 +759,12 @@ export class TreeAdapter { return node; } + /** The typed jdv path and display title of the item at a positional path. */ + locate(path: number[]): { path: JdvPathStep[]; title: string } { + const node = this.node(path); + return { path: node.path, title: node.title }; + } + navigation(path: number[] = []): ListViewNavigation { const node = this.node(path); const breadcrumbs = []; @@ -806,7 +812,8 @@ export class TreeAdapter { const hasChildren = child.has_children === true; const summary = child.unavailable ? `(${child.unavailable})` : child.summary ?? ''; return { - label: child.label, + // Unnamed elements are labeled by position, as in the previous List Viewer. + label: child.label === '' ? `[[${child.index}]]` : child.label, str: summary, index: child.index, // Vector elements are terminal rows; other leaves have nothing to open. diff --git a/src/test/node/interactiveRuntime.test.ts b/src/test/node/interactiveRuntime.test.ts index 5880b897e..d6142c850 100644 --- a/src/test/node/interactiveRuntime.test.ts +++ b/src/test/node/interactiveRuntime.test.ts @@ -19,6 +19,7 @@ import { assertSvgTextVisible } from '../svgAssertions'; import { rString } from '../../interactive/backends/rCode'; import { JdvHost, JdvPeer } from '../../jdvTransport'; import { JsonObject } from '../../jdvAdapter'; +import { createListAdapter, isJdvOutput, listRequest, queryJdvTablePage } from '../../interactive/jdvTable'; const run = promisify(execFile); const delay = (ms: number): Promise => new Promise(resolve => setTimeout(resolve, ms)); @@ -86,6 +87,15 @@ const delay = (ms: number): Promise => new Promise(resolve => setTimeout(r return events.filter(event => event.executionId === id && event.type === 'stream').map(event => event.data.text).join(''); } + /** The dataview registry serves inline previews when jdv is not used (or not installed). */ + const legacyViewerTest = (name: string, body: (this: Mocha.Context) => Promise): void => { + test(name, async function () { + await finished(await submit('options(sess.dataviewer = "sess")')); + try { await body.call(this); } + finally { await finished(await submit('options(sess.dataviewer = NULL)')).catch(() => undefined); } + }); + }; + test('opens jdv from an Interactive cell and serves a page while R is idle', async function () { const availability = await run('Rscript', ['--vanilla', '-e', "cat(requireNamespace('jdv', quietly=TRUE))"]); if (availability.stdout.trim() !== 'TRUE') { this.skip(); return; } @@ -306,7 +316,117 @@ sess::display("display-${cell}", "text/plain"); cat("stderr-${cell}\\n", file=st } }); - test('data inspection preserves random draws and reproducible resampling across cells', async () => { + const jdvAvailable = async (): Promise => + (await run('Rscript', ['--vanilla', '-e', "cat(requireNamespace('jdv', quietly=TRUE))"])).stdout.trim() === 'TRUE'; + const inspect = (request: { method: string; params: Record }): Promise> => + client.request('inspect', request); + + test('inline tables page, sort and filter through one jdv handle per output', async function () { + if (!(await jdvAvailable())) { this.skip(); return; } + const id = await submit('tbl <- data.frame(id=1:100, name=rep(c("a", "b"), 50), x=seq(0.5, 50, by=0.5)); tbl'); + assert.strictEqual((await finished(id)).state, 'success'); + const table = events.find(event => event.executionId === id && event.data.kind === 'table'); assert.ok(table); + assert.ok(isJdvOutput(table.data)); + assert.strictEqual(table.data.fullViewId, undefined); + assert.strictEqual(table.data.totalRows, 100); + assert.strictEqual(table.data.sourceRows, 100); + assert.strictEqual((table.data.rows as unknown[]).length, 20); + assert.deepStrictEqual((table.data.columns as { field: string }[]).map(column => column.field), ['0', '1', '2', '3']); + assert.match(String(table.data.printedText), /name/); + + const page = await queryJdvTablePage(table.data, { start: 40, size: 20 }, inspect); + assert.strictEqual(page.startRow, 40); + assert.deepStrictEqual((page.rows as Record[]).map(row => row['1']).slice(0, 2), ['41', '42']); + assert.strictEqual((page.rows as Record[])[0]['0'], '41'); + + // Sorting and filtering happen in R and report the filtered count. + const filtered = await queryJdvTablePage(table.data, { + start: 0, size: 20, sortModel: [{ colId: '1', sort: 'desc' }], + filterModel: { '2': { type: 'equals', filter: 'a' }, '1': { type: 'greaterThan', filter: '90' } }, + }, inspect); + assert.strictEqual(filtered.totalRows, 5); + assert.deepStrictEqual((filtered.rows as Record[]).map(row => row['1']), ['99', '97', '95', '93', '91']); + // Row labels show the original row after sorting. + assert.strictEqual((filtered.rows as Record[])[0]['0'], '99'); + + // A page past the end is clamped to the last page. + const clamped = await queryJdvTablePage(table.data, { start: 400, size: 20 }, inspect); + assert.strictEqual(clamped.startRow, 80); + + // The handle outlives reconnects and is released on request. + client.close(); await client.connect(); await client.subscribe(0); + const restored = (await client.snapshot()).events.find(event => event.seq === table.seq); + assert.strictEqual(restored?.data.handleId, table.data.handleId); + await inspect({ method: 'jdv_handle_close', params: { handle_id: table.data.handleId } }); + await assert.rejects(queryJdvTablePage(table.data, { start: 0, size: 20 }, inspect), /no longer available/); + }); + + test('inline jdv tables bound a huge table to its first page', async function () { + if (!(await jdvAvailable())) { this.skip(); return; } + const id = await submit(`n <- 832976871L +huge <- structure(rep(list(seq_len(n)), 23L), names=paste0("x", 1:23), + class="data.frame", row.names=c(NA_integer_, -n)) +huge`); + assert.strictEqual((await finished(id)).state, 'success'); + const table = events.find(event => event.executionId === id && event.data.kind === 'table'); assert.ok(table); + assert.ok(isJdvOutput(table.data)); + assert.strictEqual(table.data.totalRows, 832976871); + assert.strictEqual(table.data.sourceRows, 832976871); + assert.match(String(table.data.printedText), /first 1,000 of 832,976,871/); + const last = await queryJdvTablePage(table.data, { start: 832976860, size: 20 }, inspect); + assert.strictEqual(last.startRow, 832976860); + assert.strictEqual((last.rows as unknown[]).length, 11); + assert.strictEqual((last.rows as Record[])[10]['23'], '832976871'); + }); + + test('inline lists explore typed paths through one jdv handle', async function () { + if (!(await jdvAvailable())) { this.skip(); return; } + const id = await submit('lst <- list(nested=list(value=c(1,2)), table=data.frame(id=1:2), "λ "); lst'); + assert.strictEqual((await finished(id)).state, 'success'); + const output = events.find(event => event.executionId === id && event.data.kind === 'list'); assert.ok(output); + assert.ok(isJdvOutput(output.data)); + const children = output.data.children as { label: string; has_children: boolean }[]; + assert.deepStrictEqual(children.map(child => child.label), ['nested', 'table', '[[3]]']); + assert.strictEqual(children[0].has_children, true); + assert.match(String(output.data.printedText), /\$nested\$value/); + + const adapter = await createListAdapter(output.data, inspect); + const nested = await listRequest(adapter, output.data, { action: 'listItem', path: [], index: 1 }, inspect); + assert.deepStrictEqual((nested.result as { navigation: { path: number[] } }).navigation.path, [1]); + const page = await listRequest(adapter, output.data, { action: 'listPage', path: [1], start: 1 }, inspect); + assert.strictEqual((page.result as { children: { label: string }[] }).children[0].label, 'value'); + // A table inside a list is opened in the Data Viewer rather than navigated into. + const table = await listRequest(adapter, output.data, { action: 'listItem', path: [], index: 2 }, inspect); + assert.deepStrictEqual(table.open, { path: [{ type: 'index', index: 2 }], title: 'List$table' }); + }); + + test('opens an inline table and a list item in the Data Viewer through jdv', async function () { + if (!(await jdvAvailable())) { this.skip(); return; } + const opened: JsonObject[] = []; + const host = await JdvHost.create({ + viewOpen: (_peer, params) => { opened.push(params); }, + viewClosed: () => undefined, viewUpdateFailed: () => undefined, viewTracking: () => undefined, + disconnected: () => undefined, + }); + try { + const id = await submit('opened <- list(a=1, df=data.frame(z=1:3)); opened'); + assert.strictEqual((await finished(id)).state, 'success'); + const output = events.find(event => event.executionId === id && event.data.kind === 'list'); assert.ok(output); + const viewer = await inspect({ method: 'jdv_open_viewer', params: { + handle_id: output.data.handleId, path: [{ type: 'index', index: 2 }], title: 'Cell: List$df', address: host.address, + } }); + assert.strictEqual(viewer as unknown, true); + const deadline = Date.now() + 10000; + while (opened.length === 0 && Date.now() < deadline) { await delay(25); } + assert.strictEqual(opened[0]?.kind, 'grid'); + assert.strictEqual(opened[0]?.title, 'Cell: List$df'); + assert.strictEqual(opened[0]?.nrow, 3); + } finally { + host.dispose(); + } + }); + + legacyViewerTest('data inspection preserves random draws and reproducible resampling across cells', async () => { const first = await submit(`set.seed(2026) expected <- replicate(100, mean(sample(mtcars$mpg, replace=TRUE))) set.seed(2026) @@ -320,7 +440,7 @@ stopifnot(identical(actual, expected))`); assert.strictEqual(events.filter(event => event.type === 'display' && event.data.kind === 'table').length, 3); }); - test('table pages retain their original data after reference-based cleaning and schema changes', async function () { + legacyViewerTest('table pages retain their original data after reference-based cleaning and schema changes', async function () { if (!(await run('Rscript', ['-e', 'cat(requireNamespace("data.table", quietly=TRUE))'])).stdout.includes('TRUE')) { this.skip(); } const id = await submit('dt <- data.table::data.table(id=1:50, value=1:50); dt'); assert.strictEqual((await finished(id)).state, 'success'); @@ -338,7 +458,7 @@ stopifnot(identical(actual, expected))`); assert.strictEqual((await finished(current)).state, 'success'); }); - test('unclassed lists retain printable previews and lazy pages across reconnects', async () => { + legacyViewerTest('unclassed lists retain printable previews and lazy pages across reconnects', async () => { const id = await submit('x <- list(nested=list(value=c(1,2)), table=data.frame(id=1:2), "λ "); x'); assert.strictEqual((await finished(id)).state, 'success'); const output = events.find(event => event.executionId === id && event.data.kind === 'list'); assert.ok(output); @@ -366,7 +486,7 @@ stopifnot(identical(actual, expected))`); assert.deepStrictEqual((await client.snapshot()).events.find(event => event.seq === output.seq)?.data, output.data); }); - test('nested table viewers from separate list cells stay independent', async () => { + legacyViewerTest('nested table viewers from separate list cells stay independent', async () => { const lists: SessionEvent[] = []; for (const value of [11, 21]) { const id = await submit(`list(nested=list(first=data.frame(value=${value}), second=data.frame(value=${value + 1})))`); @@ -408,7 +528,7 @@ stopifnot(identical(actual, expected))`); assert.strictEqual(await tableValue(first), 12, 'The first cell remains independently browsable'); }); - test('huge tables keep bounded snapshots and page the full data without materializing it', async () => { + legacyViewerTest('huge tables keep bounded snapshots and page the full data without materializing it', async () => { const id = await submit(`n <- 832976871L huge <- structure(rep(list(seq_len(n)), 23L), names=paste0("x", 1:23), class="data.frame", row.names=c(NA_integer_, -n)) @@ -434,7 +554,7 @@ huge`); assert.strictEqual(restored?.data.sourceRows, table.data.sourceRows); }); - test('large data.table snapshots stay stable while full-view queries refresh after reference edits', async function () { + legacyViewerTest('large data.table snapshots stay stable while full-view queries refresh after reference edits', async function () { if (!(await run('Rscript', ['-e', 'cat(requireNamespace("data.table", quietly=TRUE))'])).stdout.includes('TRUE')) { this.skip(); } const id = await submit('dt <- data.table::data.table(id=1:2000, value=1:2000); dt'); assert.strictEqual((await finished(id)).state, 'success'); @@ -459,7 +579,7 @@ huge`); assert.strictEqual(saved.rows[19]['2'], 1000); }); - test('inline full-table queries sort, filter, widen pages and recover after schema edits', async function () { + legacyViewerTest('inline full-table queries sort, filter, widen pages and recover after schema edits', async function () { if (!(await run('Rscript', ['-e', 'cat(requireNamespace("data.table", quietly=TRUE))'])).stdout.includes('TRUE')) { this.skip(); } const id = await submit('dt <- data.table::data.table(id=1:2000, name=rep(c("a", "b"), 1000), date=as.Date("2026-01-01")+0:1999, flag=rep(c(TRUE, FALSE),1000)); dt'); assert.strictEqual((await finished(id)).state, 'success'); @@ -540,7 +660,7 @@ dt`); assert.strictEqual((await client.snapshot()).events.find(event => event.seq === table.seq)?.data.printedText, printed); }); - test('retains rich tables, plot assets, and structured errors', async () => { + legacyViewerTest('retains rich tables, plot assets, and structured errors', async () => { const id = await submit('plot(1:3); data.frame(x=1:3, label=c("a", "b", "c")); stop("expected error")'); assert.strictEqual((await finished(id)).state, 'error'); const displays = events.filter(event => event.executionId === id && event.type === 'display'); @@ -578,7 +698,7 @@ dt`); assert.ok(shadowTrace.includes('eval()'), 'Keep user functions even when their name matches an evaluation helper'); }); - test('retains R numeric formatting in previews and pages while preserving raw precision', async () => { + legacyViewerTest('retains R numeric formatting in previews and pages while preserving raw precision', async () => { const id = await submit('options(digits=7, scipen=0, OutDec="."); numeric_table <- data.frame(price_per_carat=rep(c(326/0.23, 326/0.21, 1400), 8)); numeric_table'); assert.strictEqual((await finished(id)).state, 'success'); const table = events.find(event => event.executionId === id && event.type === 'display' && event.data.kind === 'table'); diff --git a/src/test/node/jdvTable.test.ts b/src/test/node/jdvTable.test.ts new file mode 100644 index 000000000..ce11367f7 --- /dev/null +++ b/src/test/node/jdvTable.test.ts @@ -0,0 +1,115 @@ +import * as assert from 'node:assert/strict'; +import { createListAdapter, isJdvOutput, listRequest, normalizeJdvDisplay, queryJdvTablePage } from '../../interactive/jdvTable'; +import { columnOperators, TableColumn } from '../../interactive/tableQuery'; + +const capabilities = { seek: true, sort: true, filter: true, known_total: true, next: false, children: false, refresh: false, track: false }; +const gridDescriptor = { + view: 'h-1', generation: 1, kind: 'grid', access: 'random', encoding: 'jdv-json', capabilities, collation: 'C', nrow: 50, ncol: 3, + columns: [ + { position: 0, name: 'n', type: 'integer', sortable: true, filterable: true }, + { position: 1, name: 'd', type: 'date', sortable: true, filterable: true }, + { position: 2, name: 'i', type: 'int64', sortable: true, filterable: true }, + ], +}; +const rowsPage = (offset: number, count: number, total = 50) => JSON.stringify({ + view: 'h-1', generation: 1, total, total_unfiltered: 50, offset, limit: count, + data: { + source_row: Array.from({ length: count }, (_, i) => offset + i), row_labels: null, + columns: [{ schema: gridDescriptor.columns[0], data: { values: Array.from({ length: count }, (_, i) => offset + i + 1), + display: Array.from({ length: count }, (_, i) => String(offset + i + 1)), na: Array(count).fill(false) }, errors: {} }], + }, +}); +const display = (extra: Record = {}) => ({ + kind: 'table', handleId: 'h1', descriptor: JSON.stringify(gridDescriptor), page: rowsPage(0, 2), sourceRows: 50, ...extra, +}); + +suite('Interactive jdv tables', () => { + test('turns a handle preview into renderer data and keeps the descriptor', () => { + assert.equal(isJdvOutput(display()), true); + assert.equal(isJdvOutput({ kind: 'table', viewId: 'x' }), false); + const data = normalizeJdvDisplay(display()); + assert.equal(data.page, undefined); + assert.equal(data.totalRows, 50); + assert.equal(data.sourceRows, 50); + assert.deepEqual((data.columns as TableColumn[]).map(column => column.field), ['0', '1', '2', '3']); + assert.deepEqual(data.rows, [{ '0': '1', __sourceRow: 0, '1': '1' }, { '0': '2', __sourceRow: 1, '1': '2' }]); + assert.equal((data.descriptor as { view: string }).view, 'h-1'); + // Output that is not from a handle is left alone. + const legacy = { kind: 'table', viewId: 'v' }; + assert.equal(normalizeJdvDisplay(legacy), legacy); + assert.throws(() => normalizeJdvDisplay({ ...display(), kind: 'list' })); + }); + + test('offers the operators of the column type', () => { + const columns = normalizeJdvDisplay(display()).columns as TableColumn[]; + assert.deepEqual(columnOperators(columns[1]).slice(-2), ['blank', 'notBlank']); + assert.ok(columnOperators(columns[1]).includes('greaterThan')); + assert.deepEqual(columnOperators(columns[3]), ['equals', 'blank', 'notBlank']); + assert.ok(!columnOperators(columns[1]).includes('inRange')); + }); + + test('queries the handle for sorted, filtered and refilled pages', async () => { + const data = normalizeJdvDisplay(display()); + const requests: Record[] = []; + // The handle returns at most 3 rows per request, as when a page exceeds its byte budget. + const inspect = (request: { method: string; params: Record }) => { + assert.equal(request.method, 'jdv_handle_request'); + assert.equal(request.params.handle_id, 'h1'); + assert.equal(request.params.method, 'grid/rows'); + const params = request.params.params as Record; + requests.push(params); + return Promise.resolve({ json: rowsPage(params.offset, Math.min(3, params.limit)) }); + }; + const result = await queryJdvTablePage(data, { + start: 20, size: 20, sortModel: [{ colId: '1', sort: 'desc' }], + filterModel: { '2': { type: 'greaterThan', filter: '2026-01-01' } }, + }, inspect); + assert.equal((result.rows as unknown[]).length, 20); + assert.equal(result.startRow, 20); + assert.equal(result.totalRows, 50); + assert.equal(result.live, true); + assert.deepEqual(requests.slice(0, 2).map(params => [params.offset, params.limit]), [[20, 20], [23, 17]]); + assert.deepEqual(requests[0].sort, [{ col: 0, desc: true }]); + assert.deepEqual(requests[0].filter, { op: 'gt', col: 1, value: '2026-01-01' }); + assert.equal('view' in requests[0] || 'generation' in requests[0], false); + + requests.length = 0; + await queryJdvTablePage(data, { start: 0, size: 20 }, inspect); + assert.equal('sort' in requests[0] || 'filter' in requests[0], false); + }); + + test('rejects queries the column does not support', async () => { + const data = normalizeJdvDisplay(display()); + const inspect = () => Promise.reject(new Error('not reached')); + await assert.rejects(queryJdvTablePage(data, { start: 0, filterModel: { '3': { type: 'lessThan', filter: '1' } } }, inspect), /Invalid column filter/); + await assert.rejects(queryJdvTablePage(data, { start: 0, sortModel: [{ colId: '9', sort: 'asc' }] }, inspect), /cannot be sorted/); + await assert.rejects(queryJdvTablePage(data, { start: 0, size: 7 }, inspect), /Choose 20, 50 or 100/); + }); + + test('serves list actions from the handle and opens tables in the Data Viewer', async () => { + const tree = { view: 'h-2', generation: 1, kind: 'tree', access: 'random', encoding: 'jdv-json', capabilities: { ...capabilities, sort: false, filter: false, children: true }, path: [], class: ['list'], type: 'list' }; + const children = (path: unknown[], items: unknown[]) => JSON.stringify({ view: 'h-2', generation: 1, path, offset: 0, children: items, total_children: items.length }); + const root = children([], [ + { index: 1, label: 'inner', class: ['list'], type: 'list', summary: 'list (1)', has_children: true, viewable: true, path: [{ type: 'index', index: 1 }] }, + { index: 2, label: '', class: ['data.frame'], type: 'list', summary: 'data.frame (1)', has_children: true, viewable: true, path: [{ type: 'index', index: 2 }] }, + ]); + const data = normalizeJdvDisplay({ kind: 'list', handleId: 'h2', descriptor: JSON.stringify(tree), page: root }); + assert.deepEqual((data.children as { label: string }[]).map(child => child.label), ['inner', '[[2]]']); + const calls: Record[] = []; + const inspect = (request: { method: string; params: Record }) => { + calls.push(request.params); + return Promise.resolve({ json: (request.params.params as { path: unknown[] }).path.length ? children([{ type: 'index', index: 1 }], [ + { index: 1, label: 'x', class: ['numeric'], type: 'double', summary: 'double (1): 1', has_children: false, viewable: true, path: [{ type: 'index', index: 1 }, { type: 'index', index: 1 }] }, + ]) : root }); + }; + const adapter = await createListAdapter(data, inspect); + const navigate = await listRequest(adapter, data, { action: 'listItem', path: [], index: 1 }, inspect); + assert.deepEqual((navigate.result as { navigation: { path: number[] } }).navigation.path, [1]); + const page = await listRequest(adapter, data, { action: 'listPage', path: [1], start: 1 }, inspect); + assert.equal((page.result as { children: { label: string }[] }).children[0].label, 'x'); + assert.deepEqual(calls.at(-1)?.params, { path: [{ type: 'index', index: 1 }], offset: 0, limit: 200, attributes: false }); + const table = await listRequest(adapter, data, { action: 'listItem', path: [], index: 2 }, inspect); + assert.deepEqual(table.open, { path: [{ type: 'index', index: 2 }], title: 'List[[2]]' }); + assert.deepEqual((await listRequest(adapter, data, { action: 'list', path: [] }, inspect)).open, { path: [], title: 'List' }); + }); +}); From 5599b4ff7204bc451626686e7139122cb23a571b Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 14:55:52 +0000 Subject: [PATCH 22/38] style: fix hanging indents flagged by lintr in interactive previews --- sess/R/interactive.R | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sess/R/interactive.R b/sess/R/interactive.R index fc1d57090..79ca35bd6 100644 --- a/sess/R/interactive.R +++ b/sess/R/interactive.R @@ -436,7 +436,8 @@ interactive_execute <- function(id, code, source = NULL) { jdv::jdv_handle_request( handle, "object/children", list(path = list(), offset = 0L, limit = 200L, attributes = FALSE), - max_bytes = .interactive_page_bytes) + max_bytes = .interactive_page_bytes + ) } store <- .interactive_handle_store() .sess_env$interactive_handle_seq <- .sess_env$interactive_handle_seq + 1L @@ -506,10 +507,12 @@ handle_jdv_open_viewer <- function(params) { if (!is.null(preview)) { printed <- .interactive_table_text(snapshot$value) if (snapshot$truncated && !is.null(printed$printedText)) { - printed$printedText <- paste0(printed$printedText, sprintf( + note <- sprintf( "\n[Snapshot: first %s of %s rows. Open Data viewer for the full table.]\n", format(nrow(snapshot$value), big.mark = ",", scientific = FALSE, trim = TRUE), - format(snapshot$source_rows, big.mark = ",", scientific = FALSE, trim = TRUE))) + format(snapshot$source_rows, big.mark = ",", scientific = FALSE, trim = TRUE) + ) + printed$printedText <- paste0(printed$printedText, note) } .interactive_event("display", c(list(kind = "table", handleId = preview$handleId, descriptor = preview$descriptor, page = preview$page, From ee8c2e63bccbcf11db7a4b398a95b3b79ec6c247 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 15:50:33 +0000 Subject: [PATCH 23/38] fix: stop stalled jdv stream and transport requests --- src/jdvArrow.ts | 17 ++++++++- src/jdvTransport.ts | 26 ++++++++++++- src/test/node/jdvArrow.test.ts | 69 ++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 4 deletions(-) diff --git a/src/jdvArrow.ts b/src/jdvArrow.ts index a48d94ef3..c5412f9dc 100644 --- a/src/jdvArrow.ts +++ b/src/jdvArrow.ts @@ -292,7 +292,14 @@ export class SequentialStream { } // One stream/next per view generation at a time. this.fetching ??= this.fetchUntil(endRow).finally(() => { this.fetching = undefined; }); + const rowsBeforeFetch = this.rowsSeen; await this.fetching; + // fetchUntil() bounds consecutive empty pages. Do not immediately + // start another round when that bound was reached without progress. + // A gap page advances rowsSeen and therefore remains progress. + if (this.rowsSeen === rowsBeforeFetch && !this.done) { + return; + } } } @@ -317,11 +324,17 @@ export class SequentialStream { } this.store(progress, response.body); empty = 0; - } else if (++empty >= MAX_EMPTY_PAGES && !progress.done) { - return; + } else if (progress.rows_seen > this.rowsSeen) { + // A gap has no body, but it advances the stream and is progress. + empty = 0; + } else { + empty++; } this.rowsSeen = progress.rows_seen; this.done = progress.done; + if (empty >= MAX_EMPTY_PAGES && !this.done) { + return; + } } } diff --git a/src/jdvTransport.ts b/src/jdvTransport.ts index b6562a751..ca978b447 100644 --- a/src/jdvTransport.ts +++ b/src/jdvTransport.ts @@ -46,6 +46,7 @@ const LIMIT_NAMES = [ interface OutboundFrame { bytes: Buffer; wake?: boolean; + requestId?: string; } interface PendingRequest { @@ -118,12 +119,12 @@ export class JdvPeer { } const id = `vscode-r-${++this.sequence}`; const bytes = Buffer.from(JSON.stringify({ id, method, params }) + '\n', 'utf8'); - if (this.limits && bytes.length > this.limits.maxRequestBytes) { + if (this.limits && bytes.length > this.requestByteLimit()) { return Promise.reject(new JdvRequestError('limit_exceeded', 'The jdv request is too large.')); } return new Promise((resolve, reject) => { this.pending.set(id, { resolve, reject, view: params.view, generation: params.generation }); - this.outbound.push({ bytes }); + this.outbound.push({ bytes, requestId: id }); this.flush(); this.updateWake(); }); @@ -259,6 +260,7 @@ export class JdvPeer { maxUnackedBytes: limits.max_unacked_bytes as number, maxRequestBytes: limits.max_request_bytes as number, }; + this.rejectOversizedQueuedRequests(); this.session = params.session; // The host sends periodic wakes only when the producer asks for them. this.wanted = params.wake === true; @@ -266,6 +268,26 @@ export class JdvPeer { this.updateWake(); } + private requestByteLimit(): number { + return Math.min(this.limits!.maxRequestBytes, this.limits!.maxUnackedBytes); + } + + /** Reject requests that were queued before hello supplied the byte limits. */ + private rejectOversizedQueuedRequests(): void { + const limit = this.requestByteLimit(); + for (let index = this.outbound.length - 1; index >= 0; index--) { + const frame = this.outbound[index]; + if (frame.requestId && frame.bytes.length > limit) { + this.outbound.splice(index, 1); + const pending = this.pending.get(frame.requestId); + if (pending) { + this.pending.delete(frame.requestId); + pending.reject(new JdvRequestError('limit_exceeded', 'The jdv request is too large.')); + } + } + } + } + private acknowledge(params: JsonObject): void { const frames = params.frames; if (typeof frames !== 'number' || !Number.isSafeInteger(frames) || frames < this.ackedFrames || diff --git a/src/test/node/jdvArrow.test.ts b/src/test/node/jdvArrow.test.ts index ae44f71f7..9335fe002 100644 --- a/src/test/node/jdvArrow.test.ts +++ b/src/test/node/jdvArrow.test.ts @@ -92,6 +92,39 @@ suite('jdv Arrow views', () => { await assert.rejects(stream.ensure(1), (e: unknown) => e instanceof JdvRequestError); stream.dispose(); }); + + test('returns after a round of empty non-EOF pages', async () => { + let calls = 0; + const stream = new SequentialStream(descriptor, () => { + calls++; + assert.ok(calls <= 64, 'empty pages must not start another fetch round'); + return Promise.resolve({ + result: { first_row: 0, rows_fetched: 0, rows_seen: 0, done: false }, + }); + }, 10); + await stream.ensure(10); + assert.equal(calls, 64); + assert.equal(stream.rowsSeen, 0); + assert.equal(stream.done, false); + stream.dispose(); + }); + + test('treats a gap page as progress', async () => { + let calls = 0; + const stream = new SequentialStream(descriptor, () => { + calls++; + const gap = calls === 64; + return Promise.resolve({ + result: { first_row: gap ? 5 : 0, rows_fetched: 0, rows_seen: gap ? 5 : 0, done: false }, + }); + }, 10); + await stream.ensure(1); + assert.equal(calls, 64); + assert.equal(stream.rowsSeen, 5); + assert.equal(stream.rows(0, 5).length, 5); + assert.equal(stream.rows(0, 5)[0]['1'], UNAVAILABLE_TEXT); + stream.dispose(); + }); }); /** Connect to a host as R would, returning helpers to drive the protocol. */ @@ -144,6 +177,42 @@ function helloParams(limits: Record = {}): JsonObject { } suite('jdv transport', () => { + test('rejects requests that exceed byte limits before or after hello', async () => { + for (const limits of [ + { max_unacked_bytes: 160, max_request_bytes: 4096 }, + { max_unacked_bytes: 4096, max_request_bytes: 160 }, + ]) { + let peer: JdvPeer | undefined; + const host = await JdvHost.create({ + viewOpen: connection => { peer = connection; }, + viewClosed: () => undefined, + viewUpdateFailed: () => undefined, + viewTracking: () => undefined, + disconnected: () => undefined, + }); + try { + const r = await fakeR(host.socketPath); + while (!host.connection) { await new Promise(resolve => setTimeout(resolve, 5)); } + peer = host.connection; + + const queuedTooLarge = peer!.request('grid/rows', { text: 'x'.repeat(200) }); + const queuedSmall = peer!.request('view/close', { view: 'v-1' }); + r.send({ method: 'hello', params: helloParams(limits) }); + await assert.rejects(queuedTooLarge, + (e: unknown) => e instanceof JdvRequestError && e.code === 'limit_exceeded'); + const small = await r.next(line => line.method === 'view/close'); + r.send({ id: small.id, result: {}, warnings: [] }); + await queuedSmall; + + await assert.rejects(peer!.request('grid/rows', { text: 'x'.repeat(200) }), + (e: unknown) => e instanceof JdvRequestError && e.code === 'limit_exceeded'); + r.socket.destroy(); + } finally { + host.dispose(); + } + } + }); + test('exchanges requests, wakes and Arrow bodies with R', async () => { const opened: string[] = []; const failures: JsonObject[] = []; From 7389ff4f26d1d736b5aba131199d83e69c261026 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 15:51:12 +0000 Subject: [PATCH 24/38] fix: label jdv missing-value filters as NA --- src/interactive/inlineTable.ts | 9 +++++---- src/interactive/tableQuery.ts | 5 +++++ src/test/node/jdvAdapter.test.ts | 10 ++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/interactive/inlineTable.ts b/src/interactive/inlineTable.ts index 293d8ccef..8197861a4 100644 --- a/src/interactive/inlineTable.ts +++ b/src/interactive/inlineTable.ts @@ -1,7 +1,7 @@ /// import { TABLE_PAGE_SIZE, tablePage } from './tablePaging'; import { tableColumnAlignment, tableDisplayValue, tableSnapshotSummary } from './tableFormatting'; -import { columnOperators, filterNeedsValue, filterOperators, TableColumn, TableFilters, tableSchema, TableSort } from './tableQuery'; +import { columnOperators, filterNeedsValue, filterOperatorLabel, TableColumn, TableFilters, tableSchema, TableSort } from './tableQuery'; import { toolbarButton as button } from './rendererToolbar'; export interface InlineTableState { @@ -157,7 +157,7 @@ export function createInlineTable(parent: HTMLElement, toolbar: HTMLElement, sta const column = columns().find(column => column.field === filterColumn.value); operator.replaceChildren(); if (column) { - for (const op of columnOperators(column)) { operator.add(new Option(filterOperators[op as keyof typeof filterOperators], op)); } + for (const op of columnOperators(column)) { operator.add(new Option(filterOperatorLabel(op, column), op)); } } const filter = state.filters[filterColumn.value]; value.value = ''; @@ -185,9 +185,10 @@ export function createInlineTable(parent: HTMLElement, toolbar: HTMLElement, sta if (!resetDraft && Array.from(filterColumn.options).some(option => option.value === selected)) { filterColumn.value = selected; } updateOperators(!resetDraft && filterColumn.value === selected); chips.replaceChildren(); for (const [field, filter] of Object.entries(state.filters)) { - const name = columns().find(column => column.field === field)?.headerName ?? field; + const column = columns().find(column => column.field === field); + const name = column?.headerName ?? field; const remove = document.createElement('button'); remove.type = 'button'; remove.className = 'r-interactive-button'; - remove.textContent = `${name} ${filterOperators[filter.type as keyof typeof filterOperators]} ${filter.filter ?? ''} ×`; + remove.textContent = `${name} ${filterOperatorLabel(filter.type, column)} ${filter.filter ?? ''} ×`; remove.setAttribute('aria-label', `Remove filter for ${name}`); remove.onclick = () => { const filters = { ...state.filters }; delete filters[field]; request(0, state.sort, filters); }; chips.append(remove); diff --git a/src/interactive/tableQuery.ts b/src/interactive/tableQuery.ts index 4e35ba6f6..45cd35cc9 100644 --- a/src/interactive/tableQuery.ts +++ b/src/interactive/tableQuery.ts @@ -13,6 +13,11 @@ export const filterOperators = { contains: 'contains', notContains: 'does not contain', startsWith: 'starts with', endsWith: 'ends with', blank: 'is blank', notBlank: 'is not blank', true: 'is true', false: 'is false', }; +export function filterOperatorLabel(type: string, column?: TableColumn): string { + if (column?.jdv && type === 'blank') { return 'is NA'; } + if (column?.jdv && type === 'notBlank') { return 'is not NA'; } + return filterOperators[type as keyof typeof filterOperators] ?? type; +} export function columnOperators(column: TableColumn): string[] { // An inline filter is one condition, so the range operator is not offered. if (column.jdv) { return [...column.jdv.filterOptions.filter(option => option !== 'inRange'), 'blank', 'notBlank']; } diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts index af5e756ea..fd73db679 100644 --- a/src/test/node/jdvAdapter.test.ts +++ b/src/test/node/jdvAdapter.test.ts @@ -166,6 +166,16 @@ suite('jdv adapter', () => { }); }); + test('converts boolean filter models', () => { + const booleanGrid = parseDescriptor({ ...grid, ncol: 1, columns: [ + { position: 0, name: 'flag', type: 'boolean', native_type: ['logical'], sortable: true, filterable: true }, + ] }); + assert.deepEqual(toJdvFilter({ '1': { filterType: 'text', type: 'true' } }, booleanGrid), + { op: 'is_true', col: 0 }); + assert.deepEqual(toJdvFilter({ '1': { filterType: 'text', type: 'false' } }, booleanGrid), + { op: 'is_false', col: 0 }); + }); + test('converts date, datetime and integer64 operands', () => { assert.deepEqual(toJdvFilter({ '3': { filterType: 'date', type: 'greaterThan', dateFrom: '2020-01-02 00:00:00' } }, grid), { op: 'gt', col: 2, value: '2020-01-02' }); From 478fc5b941637271ff16eca7a5d494e6aa7273b6 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 15:51:33 +0000 Subject: [PATCH 25/38] fix: fill byte-limited jdv inline table pages --- sess/R/interactive.R | 51 ++++++++++++++++++- sess/inst/tinytest/test-interactive-handles.R | 22 ++++++++ src/interactive/jdvTable.ts | 3 +- src/test/node/jdvTable.test.ts | 45 +++++++++++++++- 4 files changed, 116 insertions(+), 5 deletions(-) diff --git a/sess/R/interactive.R b/sess/R/interactive.R index 79ca35bd6..ab78f55cc 100644 --- a/sess/R/interactive.R +++ b/sess/R/interactive.R @@ -420,6 +420,39 @@ interactive_execute <- function(id, code, source = NULL) { get(id, envir = store, inherits = FALSE) } +# Combine short jdv grid pages without changing JSON object/array shapes. +.interactive_join_grid_pages <- function(pages) { + parsed <- lapply(pages, jsonlite::fromJSON, simplifyVector = FALSE) + result <- parsed[[1L]] + for (page in parsed[-1L]) { + returned <- length(page$data$source_row) + result$data$source_row <- c(result$data$source_row, page$data$source_row) + if (is.list(result$data$row_labels) && is.list(page$data$row_labels)) { + result$data$row_labels <- c(result$data$row_labels, page$data$row_labels) + } else if (is.list(result$data$row_labels) || is.list(page$data$row_labels)) { + stop("jdv row labels changed while filling preview") + } + for (column in seq_along(result$data$columns)) { + first <- result$data$columns[[column]] + next_column <- page$data$columns[[column]] + if (!identical(first$errors, next_column$errors)) { + stop("jdv column errors changed while filling preview") + } + channels <- union(names(first$data), names(next_column$data)) + for (channel in channels) { + left_present <- channel %in% names(first$data) + right_present <- channel %in% names(next_column$data) + previous <- length(result$data$source_row) - returned + left <- if (left_present) first$data[[channel]] else rep(list(NULL), previous) + right <- if (right_present) next_column$data[[channel]] else rep(list(NULL), returned) + first$data[[channel]] <- c(left, right) + } + result$data$columns[[column]] <- first + } + } + jsonlite::toJSON(result, auto_unbox = TRUE, null = "null", digits = NA) +} + # Capture `value` and read its descriptor and first page. Returns NULL, leaving no # handle behind, when jdv cannot represent the value; the caller then uses the # dataview registry. @@ -430,8 +463,22 @@ interactive_execute <- function(id, code, source = NULL) { handle <- jdv::jdv_handle(value, mode = "paged") descriptor <- jdv::jdv_handle_descriptor(handle, max_bytes = .interactive_page_bytes) page <- if (kind == "table") { - jdv::jdv_handle_request(handle, "grid/rows", list(offset = 0L, limit = 20L), - max_bytes = .interactive_page_bytes) + pages <- jdv::jdv_handle_request(handle, "grid/rows", list(offset = 0L, limit = 20L), + max_bytes = .interactive_page_bytes) + parsed <- jsonlite::fromJSON(pages, simplifyVector = FALSE) + offset <- length(parsed$data$source_row) + while (offset > 0L && offset < parsed$total && offset < 20L) { + next_page <- jdv::jdv_handle_request( + handle, "grid/rows", list(offset = offset, limit = 20L - offset), + max_bytes = .interactive_page_bytes + ) + next_parsed <- jsonlite::fromJSON(next_page, simplifyVector = FALSE) + returned <- length(next_parsed$data$source_row) + if (returned == 0L) break + pages <- c(pages, next_page) + offset <- offset + returned + } + if (length(pages) == 1L) pages[[1L]] else .interactive_join_grid_pages(pages) } else { jdv::jdv_handle_request( handle, "object/children", diff --git a/sess/inst/tinytest/test-interactive-handles.R b/sess/inst/tinytest/test-interactive-handles.R index 3b73d30f4..42a5fffe5 100644 --- a/sess/inst/tinytest/test-interactive-handles.R +++ b/sess/inst/tinytest/test-interactive-handles.R @@ -33,6 +33,27 @@ first <- jsonlite::fromJSON(table$page, simplifyVector = FALSE) expect_identical(length(first$data$source_row), 20L) expect_identical(first$total, 100L) +# A byte-limited page is refilled so the renderer can advance by its page size +# without skipping rows. +numeric <- rep(1, 150) +numeric[[3L]] <- Inf +wide <- data.frame( + id = 1:150, text = rep(strrep("a", 100000), 150), + category = factor(rep(c("a", "b"), 75)), missing = rep(c(NA, "x"), 75), + numeric = numeric, row.names = paste0("row-", 0:149) +) +wide_preview <- preview(wide, "table") +wide_first <- jsonlite::fromJSON(wide_preview$page, simplifyVector = FALSE) +expect_identical(length(wide_first$data$source_row), 20L) +expect_identical(unlist(wide_first$data$source_row), 0:19) +expect_identical(length(wide_first$data$columns[[3L]]$data$values), 20L) +expect_identical(unlist(wide_first$data$columns[[4L]]$data$na), rep(c(TRUE, FALSE), 10L)) +expect_identical(is.null(wide_first$data$columns[[5L]]$data$special[[1L]]), TRUE) +expect_identical(is.null(wide_first$data$columns[[5L]]$data$special[[2L]]), TRUE) +expect_identical(wide_first$data$columns[[5L]]$data$special[[3L]], "pos_inf") +expect_identical(unlist(wide_first$data$row_labels), paste0("row-", 0:19)) +expect_identical(names(wide_first$data$columns[[1L]]$errors), character()) + # Paging, sorting and filtering run in R through the same handle; ids are zero-based rows. sorted <- request(table$handleId, "grid/rows", list( offset = 0L, limit = 3L, sort = list(list(col = 0L, desc = TRUE)), @@ -76,5 +97,6 @@ expect_silent(request(ids[[length(ids)]], "object/children", list(path = list()) # Values jdv cannot represent, or an opt-out, leave the dataview registry in charge. expect_null(preview(quote(x), "list")) +expect_null(preview(data.frame(text = strrep("a", 300000)), "table")) options(sess.dataviewer = "sess") expect_null(preview(data.frame(a = 1), "table")) diff --git a/src/interactive/jdvTable.ts b/src/interactive/jdvTable.ts index b6e32ddb7..e198a60ec 100644 --- a/src/interactive/jdvTable.ts +++ b/src/interactive/jdvTable.ts @@ -16,7 +16,6 @@ export type Inspect = (request: { method: string; params: Record): boolean { @@ -109,7 +108,7 @@ export async function queryJdvTablePage( let total = 0; let unfiltered = 0; let cursor = from; - for (let attempt = 0; attempt < PAGE_ATTEMPTS && rows.length < size; attempt++) { + while (rows.length < size) { const wanted = size - rows.length; const { view: _view, generation: _generation, ...params } = tableRowsParams(descriptor, cursor, cursor + wanted, sort, model); diff --git a/src/test/node/jdvTable.test.ts b/src/test/node/jdvTable.test.ts index ce11367f7..d52b8cf59 100644 --- a/src/test/node/jdvTable.test.ts +++ b/src/test/node/jdvTable.test.ts @@ -1,6 +1,6 @@ import * as assert from 'node:assert/strict'; import { createListAdapter, isJdvOutput, listRequest, normalizeJdvDisplay, queryJdvTablePage } from '../../interactive/jdvTable'; -import { columnOperators, TableColumn } from '../../interactive/tableQuery'; +import { columnOperators, filterOperatorLabel, TableColumn } from '../../interactive/tableQuery'; const capabilities = { seek: true, sort: true, filter: true, known_total: true, next: false, children: false, refresh: false, track: false }; const gridDescriptor = { @@ -43,8 +43,12 @@ suite('Interactive jdv tables', () => { test('offers the operators of the column type', () => { const columns = normalizeJdvDisplay(display()).columns as TableColumn[]; assert.deepEqual(columnOperators(columns[1]).slice(-2), ['blank', 'notBlank']); + assert.equal(filterOperatorLabel('blank', columns[1]), 'is NA'); + assert.equal(filterOperatorLabel('notBlank', columns[1]), 'is not NA'); assert.ok(columnOperators(columns[1]).includes('greaterThan')); assert.deepEqual(columnOperators(columns[3]), ['equals', 'blank', 'notBlank']); + assert.equal(filterOperatorLabel('blank', { field: 'legacy', headerName: 'legacy' }), 'is blank'); + assert.equal(filterOperatorLabel('notBlank', { field: 'legacy', headerName: 'legacy' }), 'is not blank'); assert.ok(!columnOperators(columns[1]).includes('inRange')); }); @@ -78,6 +82,45 @@ suite('Interactive jdv tables', () => { assert.equal('sort' in requests[0] || 'filter' in requests[0], false); }); + test('fills a 100-row page after more than twenty short responses and stops on no progress', async () => { + const data = normalizeJdvDisplay(display()); + const offsets: number[] = []; + const inspect = (request: { params: Record }) => { + const params = request.params.params as { offset: number; limit: number }; + offsets.push(params.offset); + return Promise.resolve({ json: rowsPage(params.offset, Math.min(1, params.limit), 150) }); + }; + const result = await queryJdvTablePage(data, { start: 0, size: 100 }, inspect); + assert.equal((result.rows as unknown[]).length, 100); + assert.deepEqual((result.rows as Record[]).map(row => row.__sourceRow), Array.from({ length: 100 }, (_, i) => i)); + assert.equal(offsets.length, 100); + assert.equal(result.totalRows, 150); + assert.deepEqual(offsets.slice(0, 3), [0, 1, 2]); + const following = await queryJdvTablePage(data, { start: 100, size: 100 }, inspect); + assert.equal((following.rows as unknown[]).length, 50); + assert.deepEqual((following.rows as Record[]).map(row => row.__sourceRow), Array.from({ length: 50 }, (_, i) => i + 100)); + assert.deepEqual(offsets.slice(100, 103), [100, 101, 102]); + assert.equal(offsets.at(-1), 149); + + let calls = 0; + const noProgress = await queryJdvTablePage(data, { start: 0, size: 20 }, () => { + calls++; + return Promise.resolve({ json: rowsPage(0, 0) }); + }); + assert.equal(calls, 1); + assert.deepEqual(noProgress.rows, []); + }); + + test('surfaces a jdv limit_exceeded error instead of retrying it', async () => { + const data = normalizeJdvDisplay(display()); + let calls = 0; + await assert.rejects(queryJdvTablePage(data, { start: 0, size: 20 }, () => { + calls++; + return Promise.reject(new Error('limit_exceeded: row is too large')); + }), /limit_exceeded/); + assert.equal(calls, 1); + }); + test('rejects queries the column does not support', async () => { const data = normalizeJdvDisplay(display()); const inspect = () => Promise.reject(new Error('not reached')); From 55460e064ac88e06924b5660c4d661eda61048fa Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sun, 18 Oct 2026 15:57:41 +0000 Subject: [PATCH 26/38] fix: preserve raw jdv preview pages until normalization --- sess/R/interactive.R | 37 +------------- sess/inst/tinytest/test-interactive-handles.R | 51 +++++++++++++++---- src/interactive/jdvTable.ts | 21 ++++++-- src/test/node/jdvTable.test.ts | 13 +++++ 4 files changed, 71 insertions(+), 51 deletions(-) diff --git a/sess/R/interactive.R b/sess/R/interactive.R index ab78f55cc..9a306418c 100644 --- a/sess/R/interactive.R +++ b/sess/R/interactive.R @@ -420,40 +420,7 @@ interactive_execute <- function(id, code, source = NULL) { get(id, envir = store, inherits = FALSE) } -# Combine short jdv grid pages without changing JSON object/array shapes. -.interactive_join_grid_pages <- function(pages) { - parsed <- lapply(pages, jsonlite::fromJSON, simplifyVector = FALSE) - result <- parsed[[1L]] - for (page in parsed[-1L]) { - returned <- length(page$data$source_row) - result$data$source_row <- c(result$data$source_row, page$data$source_row) - if (is.list(result$data$row_labels) && is.list(page$data$row_labels)) { - result$data$row_labels <- c(result$data$row_labels, page$data$row_labels) - } else if (is.list(result$data$row_labels) || is.list(page$data$row_labels)) { - stop("jdv row labels changed while filling preview") - } - for (column in seq_along(result$data$columns)) { - first <- result$data$columns[[column]] - next_column <- page$data$columns[[column]] - if (!identical(first$errors, next_column$errors)) { - stop("jdv column errors changed while filling preview") - } - channels <- union(names(first$data), names(next_column$data)) - for (channel in channels) { - left_present <- channel %in% names(first$data) - right_present <- channel %in% names(next_column$data) - previous <- length(result$data$source_row) - returned - left <- if (left_present) first$data[[channel]] else rep(list(NULL), previous) - right <- if (right_present) next_column$data[[channel]] else rep(list(NULL), returned) - first$data[[channel]] <- c(left, right) - } - result$data$columns[[column]] <- first - } - } - jsonlite::toJSON(result, auto_unbox = TRUE, null = "null", digits = NA) -} - -# Capture `value` and read its descriptor and first page. Returns NULL, leaving no +# Capture `value` and read its descriptor and preview pages. Returns NULL, leaving no # handle behind, when jdv cannot represent the value; the caller then uses the # dataview registry. .interactive_handle_preview <- function(value, kind) { @@ -478,7 +445,7 @@ interactive_execute <- function(id, code, source = NULL) { pages <- c(pages, next_page) offset <- offset + returned } - if (length(pages) == 1L) pages[[1L]] else .interactive_join_grid_pages(pages) + pages } else { jdv::jdv_handle_request( handle, "object/children", diff --git a/sess/inst/tinytest/test-interactive-handles.R b/sess/inst/tinytest/test-interactive-handles.R index 42a5fffe5..1c703326b 100644 --- a/sess/inst/tinytest/test-interactive-handles.R +++ b/sess/inst/tinytest/test-interactive-handles.R @@ -32,10 +32,14 @@ expect_identical(descriptor$nrow, 100L) first <- jsonlite::fromJSON(table$page, simplifyVector = FALSE) expect_identical(length(first$data$source_row), 20L) expect_identical(first$total, 100L) +single_event <- jsonlite::fromJSON(jsonlite::toJSON( + list(page = table$page), auto_unbox = TRUE, null = "null", digits = NA +), simplifyVector = FALSE) +expect_identical(single_event$page, table$page) # A byte-limited page is refilled so the renderer can advance by its page size # without skipping rows. -numeric <- rep(1, 150) +numeric <- rep(pi, 150) numeric[[3L]] <- Inf wide <- data.frame( id = 1:150, text = rep(strrep("a", 100000), 150), @@ -43,16 +47,41 @@ wide <- data.frame( numeric = numeric, row.names = paste0("row-", 0:149) ) wide_preview <- preview(wide, "table") -wide_first <- jsonlite::fromJSON(wide_preview$page, simplifyVector = FALSE) -expect_identical(length(wide_first$data$source_row), 20L) -expect_identical(unlist(wide_first$data$source_row), 0:19) -expect_identical(length(wide_first$data$columns[[3L]]$data$values), 20L) -expect_identical(unlist(wide_first$data$columns[[4L]]$data$na), rep(c(TRUE, FALSE), 10L)) -expect_identical(is.null(wide_first$data$columns[[5L]]$data$special[[1L]]), TRUE) -expect_identical(is.null(wide_first$data$columns[[5L]]$data$special[[2L]]), TRUE) -expect_identical(wide_first$data$columns[[5L]]$data$special[[3L]], "pos_inf") -expect_identical(unlist(wide_first$data$row_labels), paste0("row-", 0:19)) -expect_identical(names(wide_first$data$columns[[1L]]$errors), character()) +expect_true(length(wide_preview$page) > 1L) +wide_entry <- ns$.interactive_handle_entry(wide_preview$handleId) +expected_pages <- character() +offset <- 0L +while (offset < 20L) { + raw_page <- jdv::jdv_handle_request( + wide_entry$handle, "grid/rows", list(offset = offset, limit = 20L - offset), + max_bytes = ns$.interactive_page_bytes + ) + expected_pages <- c(expected_pages, raw_page) + parsed_page <- jsonlite::fromJSON(raw_page, simplifyVector = FALSE) + returned <- length(parsed_page$data$source_row) + if (returned == 0L) break + offset <- offset + returned +} +expect_identical(wide_preview$page, expected_pages) +multi_event <- jsonlite::fromJSON(jsonlite::toJSON( + list(page = wide_preview$page), auto_unbox = TRUE, null = "null", digits = NA +), simplifyVector = FALSE) +expect_identical(unlist(multi_event$page), expected_pages) +wide_pages <- lapply(wide_preview$page, jsonlite::fromJSON, simplifyVector = FALSE) +wide_rows <- unlist(lapply(wide_pages, function(page) page$data$source_row)) +expect_identical(wide_rows, 0:19) +factor_values <- sum(vapply(wide_pages, function(page) { + length(page$data$columns[[3L]]$data$values) +}, 0L)) +missing_values <- unlist(lapply(wide_pages, function(page) page$data$columns[[4L]]$data$na)) +row_labels <- unlist(lapply(wide_pages, function(page) page$data$row_labels)) +empty_errors <- vapply(wide_pages, function(page) { + identical(names(page$data$columns[[1L]]$errors), character()) +}, logical(1L)) +expect_identical(factor_values, 20L) +expect_identical(missing_values, rep(c(TRUE, FALSE), 10L)) +expect_identical(row_labels, paste0("row-", 0:19)) +expect_true(all(empty_errors)) # Paging, sorting and filtering run in R through the same handle; ids are zero-based rows. sorted <- request(table$handleId, "grid/rows", list( diff --git a/src/interactive/jdvTable.ts b/src/interactive/jdvTable.ts index e198a60ec..97f088e35 100644 --- a/src/interactive/jdvTable.ts +++ b/src/interactive/jdvTable.ts @@ -32,7 +32,7 @@ function descriptorOf(data: Record): JdvDescriptor { } /** - * Agent side: turn the raw descriptor and first page that R sent into the data the + * Agent side: turn the raw descriptor and preview pages that R sent into the data the * renderer, exports and transcript already understand (columns and rows for a * table; navigation and children for a list). The parsed descriptor stays so the * host can query the same handle later. @@ -40,15 +40,26 @@ function descriptorOf(data: Record): JdvDescriptor { export function normalizeJdvDisplay(data: Record): Record { if (!isJdvOutput(data)) { return data; } const descriptor = parseDescriptor(parseJson(data.descriptor, 'descriptor')); - const page = parseJson(data.page, 'page'); + const rawPage = data.page; const { page: _page, ...rest } = data; if (data.kind === 'table' && descriptor.kind === 'grid') { - const first = gridPageFromRows(page, 0, TABLE_PAGE_SIZE); - return { ...rest, descriptor, columns: gridColumnDefs(descriptor), rows: first.rows, totalRows: first.totalRows }; + const pages = (Array.isArray(rawPage) ? rawPage : [rawPage]).map(page => parseJson(page, 'page')); + const rows: Record[] = []; + let totalRows = 0; + for (const raw of pages) { + const next = gridPageFromRows(raw, rows.length, TABLE_PAGE_SIZE - rows.length); + rows.push(...next.rows); + totalRows = next.totalRows; + if (rows.length >= TABLE_PAGE_SIZE) { break; } + } + return { ...rest, descriptor, columns: gridColumnDefs(descriptor), rows, totalRows }; } if (data.kind === 'list' && descriptor.kind === 'tree') { const adapter = new TreeAdapter(descriptor, 'List'); - return { ...rest, descriptor, navigation: adapter.navigation([]), ...adapter.children([], page) }; + return { + ...rest, descriptor, navigation: adapter.navigation([]), + ...adapter.children([], parseJson(rawPage, 'page')), + }; } throw new Error('The jdv descriptor does not match the output kind'); } diff --git a/src/test/node/jdvTable.test.ts b/src/test/node/jdvTable.test.ts index d52b8cf59..d6e410f02 100644 --- a/src/test/node/jdvTable.test.ts +++ b/src/test/node/jdvTable.test.ts @@ -40,6 +40,19 @@ suite('Interactive jdv tables', () => { assert.throws(() => normalizeJdvDisplay({ ...display(), kind: 'list' })); }); + test('combines raw short preview pages and keeps their display precision', () => { + const first = JSON.parse(rowsPage(0, 2)) as { data: { columns: { data: { display: string[] } }[] } }; + first.data.columns[0].data.display[0] = '3.141592653589793'; + const onePage = normalizeJdvDisplay(display({ page: [JSON.stringify(first)] })); + assert.equal((onePage.rows as unknown[]).length, 2); + const data = normalizeJdvDisplay(display({ page: [JSON.stringify(first), rowsPage(2, 2)] })); + const rows = data.rows as Record[]; + assert.equal(rows.length, 4); + assert.deepEqual(rows.map(row => row.__sourceRow), [0, 1, 2, 3]); + assert.equal(rows[0]['1'], '3.141592653589793'); + assert.equal(rows[3]['1'], '4'); + }); + test('offers the operators of the column type', () => { const columns = normalizeJdvDisplay(display()).columns as TableColumn[]; assert.deepEqual(columnOperators(columns[1]).slice(-2), ['blank', 'notBlank']); From 55799dfeddb76add0b78243c625044b4298f7f8c Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 00:07:12 +0000 Subject: [PATCH 27/38] fix: open interactive viewer paths through jdv handles --- sess/R/handlers.R | 14 +- sess/R/interactive.R | 35 ++--- sess/inst/tinytest/test-interactive-handles.R | 128 +++++++++++++++++- src/test/node/interactiveRuntime.test.ts | 37 +++++ 4 files changed, 188 insertions(+), 26 deletions(-) diff --git a/sess/R/handlers.R b/sess/R/handlers.R index f3ffbf943..b611f0cec 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1398,13 +1398,10 @@ handle_dataview_dispose <- function(params) { if (.jdv_is_arrow(x) && !requireNamespace("nanoarrow", quietly = TRUE)) { return(FALSE) } - if (!.jdv_connect(address)) { - return(FALSE) - } title <- if (nzchar(title_key)) title_key else NULL # Only jdv errors about the binding are retried without it; anything else # ends in FALSE so that View() falls back to the sess data viewer. - tryCatch({ + .jdv_show_registered(function() { if (!is.null(binding) && !.jdv_is_arrow(x)) { registered <- tryCatch({ jdv::jdv_register(x, title = title, mode = "paged", binding = binding) @@ -1415,6 +1412,15 @@ handle_dataview_dispose <- function(params) { } } jdv::jdv_register(x, title = title, mode = "paged") + }, address) +} + +# Connect and register a view, sharing the same failure contract for values and +# Interactive handles. Registration may return metadata; callers only need success. +.jdv_show_registered <- function(register, address = NULL) { + if (!.jdv_connect(address)) return(FALSE) + tryCatch({ + register() TRUE }, error = function(e) FALSE) } diff --git a/sess/R/interactive.R b/sess/R/interactive.R index 9a306418c..213af5d91 100644 --- a/sess/R/interactive.R +++ b/sess/R/interactive.R @@ -456,7 +456,7 @@ interactive_execute <- function(id, code, source = NULL) { store <- .interactive_handle_store() .sess_env$interactive_handle_seq <- .sess_env$interactive_handle_seq + 1L id <- paste0("h", .sess_env$interactive_handle_seq) - assign(id, list(handle = handle, value = value), envir = store) + assign(id, list(handle = handle), envir = store) .sess_env$interactive_handle_order <- c(.sess_env$interactive_handle_order, id) while (length(.sess_env$interactive_handle_order) > .interactive_handle_limit) { .interactive_handle_close(.sess_env$interactive_handle_order[[1L]]) @@ -485,19 +485,20 @@ handle_jdv_handle_close <- function(params) { TRUE } -# Select the value at a typed jdv path (index, binding, slot or attr steps). -.interactive_resolve_path <- function(value, path) { - for (step in path) { - value <- switch( - step$type, - index = value[[as.integer(step$index)]], - binding = get(step$name, envir = value, inherits = FALSE), - slot = methods::slot(value, step$name), - attr = attr(value, step$name, exact = TRUE), - stop("Unsupported path step") - ) - } - value +# Register the captured value without invoking R subsetting methods or forcing +# bindings. A temporary child is released even when registration fails; the view +# itself has an independent lifetime. +.jdv_show_handle <- function(handle, path, title_key, address = NULL) { + .jdv_show_registered(function() { + if (length(path)) { + handle <- jdv::jdv_handle_open(handle, path, mode = "paged") + on.exit(jdv::jdv_handle_close(handle), add = TRUE) + } + # NULL would reuse the handle's title, hence refresh an unrelated view. The + # handle API requires an explicit unique title for an untitled independent view. + title <- if (nzchar(title_key)) title_key else basename(tempfile("Interactive output ")) + jdv::jdv_handle_register(handle, title = title, mode = "paged") + }, address) } # Show the output's value, or the element at `path`, in the Data Viewer panel by @@ -505,11 +506,11 @@ handle_jdv_handle_close <- function(params) { # cannot be opened. handle_jdv_open_viewer <- function(params) { entry <- .interactive_handle_entry(params$handle_id) - value <- .interactive_resolve_path(entry$value, params$path %||% list()) title <- params$title address <- params$address - .jdv_show(value, if (is.character(title) && length(title) == 1L) title else "", - address = if (is.character(address) && length(address) == 1L) address) + .jdv_show_handle(entry$handle, params$path %||% list(), + if (is.character(title) && length(title) == 1L && !is.na(title)) title else "", + address = if (is.character(address) && length(address) == 1L) address) } .interactive_rich_value <- function(value) { diff --git a/sess/inst/tinytest/test-interactive-handles.R b/sess/inst/tinytest/test-interactive-handles.R index 1c703326b..ac1a9c761 100644 --- a/sess/inst/tinytest/test-interactive-handles.R +++ b/sess/inst/tinytest/test-interactive-handles.R @@ -100,11 +100,129 @@ expect_identical(request(tree$handleId, "object/children", list( path = list(list(type = "index", index = 2L)), offset = 0L, limit = 10L, attributes = FALSE ))$total_children, 1L) -# The viewer opens the output's value or one typed item of it. -entry <- ns$.interactive_handle_entry(tree$handleId) -expect_identical(ns$.interactive_resolve_path(entry$value, list(list(type = "index", index = 2L))), - data.frame(z = 1:2)) -expect_identical(ns$.interactive_resolve_path(entry$value, list()), entry$value) +# Opening an Interactive handle registers the selected value through jdv's handle API. +# The custom [[ method must not run while opening a nested table. +local({ + original_connect <- ns$.jdv_connect + review_handle_id <- NULL + registered_views <- list() + unlockBinding(".jdv_connect", ns) + assign(".jdv_connect", function(address = NULL) TRUE, envir = ns) + lockBinding(".jdv_connect", ns) + on.exit({ + if (!is.null(review_handle_id)) { + ns$handle_jdv_handle_close(list(handle_id = review_handle_id)) + } + for (view in registered_views) { + try(jdv::jdv_rpc(jsonlite::toJSON(list( + id = 1L, method = "view/close", + params = list(view = view$id, generation = view$generation) + ), auto_unbox = TRUE)), silent = TRUE) + } + jdv::jdv_notifications() + unlockBinding(".jdv_connect", ns) + assign(".jdv_connect", original_connect, envir = ns) + lockBinding(".jdv_connect", ns) + }, add = TRUE) + + anchor <- jdv::jdv_register(data.frame(anchor = 1L), title = "interactive handles test anchor") + registered_views <- list(anchor) + jdv::jdv_notifications() + method_calls <- 0L + previous_method <- if (exists("[[.review_list", envir = .GlobalEnv, inherits = FALSE)) { + get("[[.review_list", envir = .GlobalEnv, inherits = FALSE) + } + assign("[[.review_list", function(x, i, ...) { + method_calls <<- method_calls + 1L + stop("review_list [[ method must not be called") + }, envir = .GlobalEnv) + on.exit({ + if (is.null(previous_method)) { + if (exists("[[.review_list", envir = .GlobalEnv, inherits = FALSE)) { + rm("[[.review_list", envir = .GlobalEnv) + } + } else { + assign("[[.review_list", previous_method, envir = .GlobalEnv) + } + }, add = TRUE) + review <- structure(list(table = data.frame(z = 1:2)), class = "review_list") + review_preview <- preview(review, "list") + review_handle_id <- review_preview$handleId + expect_true(!is.null(review_preview)) + + opens <- function() { + messages <- lapply(jdv::jdv_notifications(), jsonlite::fromJSON, + simplifyVector = FALSE) + result <- Filter(function(message) identical(message$method, "view/open"), messages) + registered_views <<- c(registered_views, lapply(result, function(message) { + list(id = message$params$view, generation = message$params$generation) + })) + result + } + opened <- function() lapply(opens(), `[[`, "params") + request <- function(method, params) { + parsed <- jsonlite::fromJSON(jdv::jdv_rpc(jsonlite::toJSON( + list(id = 1L, method = method, params = params), auto_unbox = TRUE + )), simplifyVector = FALSE) + parsed$result + } + + expect_true(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, + path = list(list(type = "index", index = 1L)), + title = "Cell: List$table" + ))) + child <- opened()[[1L]] + expect_identical(child$kind, "grid") + expect_identical(child$title, "Cell: List$table") + rows <- request("grid/rows", list(view = child$view, generation = child$generation, + offset = 0L, limit = 2L)) + expect_identical(length(rows$data$source_row), 2L) + expect_identical(method_calls, 0L) + + # Empty paths register the root handle itself; named titles retain refresh behavior. + expect_true(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, path = list(), title = "Interactive root" + ))) + root_first <- opened()[[1L]] + expect_identical(root_first$kind, "tree") + expect_identical(root_first$title, "Interactive root") + expect_true(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, path = list(), title = "Interactive root" + ))) + root_refresh <- opened()[[1L]] + expect_identical(root_refresh$view, root_first$view) + expect_identical(root_refresh$generation, root_first$generation + 1L) + + # Empty titles remain independent even though jdv handles carry their own fallback title. + expect_true(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, path = list(), title = "" + ))) + expect_true(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, path = list(), title = "" + ))) + untitled <- opened() + expect_identical(length(untitled), 2L) + expect_true(all(vapply(untitled, function(view) startsWith(view$title, "Interactive output "), + logical(1L)))) + expect_false(identical(untitled[[1L]]$view, untitled[[2L]]$view)) + + expect_false(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, + path = list(list(type = "index", index = 99L)), title = "invalid path" + ))) + unlockBinding(".jdv_connect", ns) + assign(".jdv_connect", function(address = NULL) FALSE, envir = ns) + lockBinding(".jdv_connect", ns) + expect_false(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, path = list(), title = "disconnected" + ))) + expect_true(ns$handle_jdv_handle_close(list(handle_id = review_preview$handleId))) + rows_after_handle_close <- request( + "grid/rows", list(view = child$view, generation = child$generation, offset = 0L, limit = 2L) + ) + expect_identical(length(rows_after_handle_close$data$source_row), 2L) +}) # Only grid rows and object children can be requested. expect_error(ns$handle_jdv_request(list(handle_id = table$handleId, method = "view/close")), diff --git a/src/test/node/interactiveRuntime.test.ts b/src/test/node/interactiveRuntime.test.ts index d6142c850..46bd8a427 100644 --- a/src/test/node/interactiveRuntime.test.ts +++ b/src/test/node/interactiveRuntime.test.ts @@ -426,6 +426,43 @@ huge`); } }); + test('opens nested and root jdv handles without dispatching list subsetting methods', async function () { + if (!(await jdvAvailable())) { this.skip(); return; } + const opened: JsonObject[] = []; + const host = await JdvHost.create({ + viewOpen: (_peer, params) => { opened.push(params); }, + viewClosed: () => undefined, viewUpdateFailed: () => undefined, viewTracking: () => undefined, + disconnected: () => undefined, + }); + try { + const id = await submit('`[[.review_list` <- function(x, i, ...) stop("review_list [[ called"); review <- structure(list(table=data.frame(z=1:3)), class="review_list"); outer <- list(review=review); outer'); + assert.strictEqual((await finished(id)).state, 'success'); + const output = events.find(event => event.executionId === id && event.data.kind === 'list'); assert.ok(output); + assert.ok(isJdvOutput(output.data)); + + const nested = await inspect({ method: 'jdv_open_viewer', params: { + handle_id: output.data.handleId, path: [{ type: 'index', index: 1 }, { type: 'index', index: 1 }], title: 'Cell: List$review$table', address: host.address, + } }); + assert.strictEqual(nested as unknown, true); + const deadline = Date.now() + 10000; + while (opened.length === 0 && Date.now() < deadline) { await delay(25); } + assert.strictEqual(opened[0]?.kind, 'grid'); + assert.strictEqual(opened[0]?.title, 'Cell: List$review$table'); + assert.strictEqual(opened[0]?.nrow, 3); + + const root = await inspect({ method: 'jdv_open_viewer', params: { + handle_id: output.data.handleId, path: [], title: 'Interactive root', address: host.address, + } }); + assert.strictEqual(root as unknown, true); + const rootDeadline = Date.now() + 10000; + while (opened.length < 2 && Date.now() < rootDeadline) { await delay(25); } + assert.strictEqual(opened[1]?.kind, 'tree'); + assert.strictEqual(opened[1]?.title, 'Interactive root'); + } finally { + host.dispose(); + } + }); + legacyViewerTest('data inspection preserves random draws and reproducible resampling across cells', async () => { const first = await submit(`set.seed(2026) expected <- replicate(100, mean(sample(mtcars$mpg, replace=TRUE))) From 8fcc2b7a66cf608c38e3701821fd138240b4dfff Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 00:19:18 +0000 Subject: [PATCH 28/38] fix: preserve Arrow numeric display strings --- src/dataViewer.ts | 7 ++++++- src/jdvAdapter.ts | 3 ++- src/test/node/jdvArrow.test.ts | 2 ++ src/test/suite/dataViewer.test.ts | 22 ++++++++++++++++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/dataViewer.ts b/src/dataViewer.ts index 096ceaac4..48f2e3bb9 100644 --- a/src/dataViewer.ts +++ b/src/dataViewer.ts @@ -164,7 +164,12 @@ export function getDataViewerScript(): string { column.filter = 'agBigIntColumnFilter'; bigintFields.push(column.field); } else if (column.type === 'numericColumn') { - column.cellDataType = 'number'; + // Sequential Arrow values are already formatted display strings. + // Keep them as strings to preserve integer/decimal precision while + // retaining numericColumn's right alignment. + if (column.cellDataType !== false) { + column.cellDataType = 'number'; + } } else if (column.type === 'booleanColumn') { column.cellDataType = 'boolean'; column.filterParams = { buttons: ['reset'] }; diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index ba950e0c9..ae1da919f 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -189,7 +189,7 @@ export interface GridColumnDef { type: string; filter: string | boolean; sortable: boolean; - cellDataType?: string; + cellDataType?: string | false; suppressHeaderMenuButton?: boolean; lockPosition?: string; /** Drillable grid cells: the zero-based schema position that `object/open` addresses. */ @@ -252,6 +252,7 @@ function sequentialColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { : `${column.name}, Arrow format: ${column.type}`, field: String(column.position + 1), type: numeric ? 'numericColumn' : 'textColumn', + ...(numeric ? { cellDataType: false } : {}), filter: false, sortable: false, }); diff --git a/src/test/node/jdvArrow.test.ts b/src/test/node/jdvArrow.test.ts index 9335fe002..2070b38c7 100644 --- a/src/test/node/jdvArrow.test.ts +++ b/src/test/node/jdvArrow.test.ts @@ -51,6 +51,8 @@ suite('jdv Arrow views', () => { ['0', 'textColumn', false, false], ['1', 'numericColumn', false, false], ['2', 'textColumn', false, false], ['3', 'textColumn', false, false], ]); + assert.equal(columns[1].cellDataType, false); + assert.equal(columns[2].cellDataType, undefined); assert.match(columns[3].headerTooltip ?? '', /unsupported: not supported/); }); diff --git a/src/test/suite/dataViewer.test.ts b/src/test/suite/dataViewer.test.ts index b2dcc3dff..6d93fb305 100644 --- a/src/test/suite/dataViewer.test.ts +++ b/src/test/suite/dataViewer.test.ts @@ -2,6 +2,7 @@ import * as assert from 'assert'; import * as vm from 'vm'; import type { ColDef, ColumnState, GridOptions, GridState } from 'ag-grid-community'; import { getDataViewerScript } from '../../dataViewer'; +import { gridColumnDefs, parseDescriptor } from '../../jdvAdapter'; interface SavedViewer { schema: string; @@ -203,6 +204,27 @@ suite('Data viewer', () => { assert.strictEqual(defs[0].lockVisible, true); }); + test('keeps sequential Arrow numeric display strings precise and right aligned', () => { + const descriptor = parseDescriptor({ + view: 'arrow-1', generation: 1, kind: 'grid', access: 'sequential', encoding: 'arrow-ipc', + capabilities: { next: true }, nrow: null, ncol: 1, max_page_rows: 100, + columns: [{ position: 0, name: 'precise', type: 'g', unsupported: null, sortable: false, filterable: false }], + }); + const arrowColumns = gridColumnDefs(descriptor) as unknown as ColDef[]; + const viewer = viewerScript().script; + viewer.prepareViewerColumns(arrowColumns); + assert.strictEqual(arrowColumns[1].type, 'numericColumn'); + assert.strictEqual(arrowColumns[1].cellDataType, false); + + const displayValues = ['3.141592653589793238462643383279', '123456789012345678901234567890']; + const rows = viewer.prepareViewerRows(displayValues.map(value => ({ '1': value })), []); + assert.deepStrictEqual(rows.map(row => row['1']), displayValues); + + const legacyNumeric: ColDef[] = [{ field: '2', type: 'numericColumn' }]; + viewer.prepareViewerColumns(legacyNumeric); + assert.strictEqual(legacyNumeric[0].cellDataType, 'number'); + }); + test('keeps integer64 precision and missing values when loading rows', () => { const script = viewerScript().script; const rows = script.prepareViewerRows([ From abb6d0459607079f1d41fd3ec9821865f641162d Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 00:19:18 +0000 Subject: [PATCH 29/38] perf: skip unchanged data viewer header refreshes --- src/session.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/session.ts b/src/session.ts index 2fdcbac11..b35fdb9b5 100644 --- a/src/session.ts +++ b/src/session.ts @@ -2464,9 +2464,14 @@ export async function getTableHtml( sortModel: params.sortModel, filterModel: params.filterModel, }); - filteredRows = Number.isFinite(result.totalRows) ? result.totalRows : result.loadedRows; - totalRows = Number.isFinite(result.totalUnfiltered) ? result.totalUnfiltered : result.loadedRows; - isFiltered = Object.keys(params.filterModel || {}).length > 0; + const nextFilteredRows = Number.isFinite(result.totalRows) ? result.totalRows : result.loadedRows; + const nextTotalRows = Number.isFinite(result.totalUnfiltered) ? result.totalUnfiltered : result.loadedRows; + const nextIsFiltered = Object.keys(params.filterModel || {}).length > 0; + const headerNeedsRefresh = filteredRows !== nextFilteredRows || + totalRows !== nextTotalRows || isFiltered !== nextIsFiltered; + filteredRows = nextFilteredRows; + totalRows = nextTotalRows; + isFiltered = nextIsFiltered; updateViewerRowCount(filteredRows, totalRows); if (init.sequential && !Number.isFinite(result.totalRows)) { document.querySelector('#viewerRowCount').textContent = @@ -2475,7 +2480,9 @@ export async function getTableHtml( if (init.sequential && result.notice) { document.querySelector('#viewerRowCount').title = result.notice; } - gridApi?.refreshHeader(); + if (headerNeedsRefresh) { + gridApi?.refreshHeader(); + } updateScrollPosition(); const resolvedLastRow = Number.isFinite(result.totalRows) ? result.totalRows : result.lastRow; const rows = prepareViewerRows(result.rows || [], bigintFields); From fbe86b74ff7d59b863998c32879c1c36d03b045a Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 00:31:34 +0000 Subject: [PATCH 30/38] feat: show Arrow stream progress and known row totals --- sess/R/handlers.R | 37 ++++- sess/inst/tinytest/test-jdv.R | 63 ++++++++ src/dataViewer.ts | 23 +++ src/interactive/manager.ts | 3 +- src/jdvRowCount.ts | 183 +++++++++++++++++++++++ src/jdvViewer.ts | 66 +++++++- src/session.ts | 75 ++++++++-- src/test/node/interactiveRuntime.test.ts | 46 ++++++ src/test/node/jdvRowCount.test.ts | 101 +++++++++++++ src/test/suite/dataViewer.test.ts | 18 ++- 10 files changed, 589 insertions(+), 26 deletions(-) create mode 100644 src/jdvRowCount.ts create mode 100644 src/test/node/jdvRowCount.test.ts diff --git a/sess/R/handlers.R b/sess/R/handlers.R index b611f0cec..e960070a2 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1307,6 +1307,20 @@ handle_dataview_dispose <- function(params) { inherits(x, .jdv_arrow_classes) } +# Return a row-count hint when an Arrow source exposes dimensions without reading +# its data. Forward-only readers and streams return NULL from nrow(). +.jdv_arrow_nrow <- function(x) { + tryCatch({ + count <- nrow(x) + if (!is.numeric(count) || length(count) != 1L || is.na(count) || + !is.finite(count) || count < 0 || count != floor(count) || + count > 9007199254740991) { + return(NULL) + } + unname(as.numeric(count)) + }, error = function(e) NULL) +} + # Make sure jdv has a persistent connection to this session's viewer host. # Returns FALSE, leaving the user's own jdv state untouched, when the sess data # viewer should be used instead: jdv is missing or opted out, VS Code offers no @@ -1398,6 +1412,7 @@ handle_dataview_dispose <- function(params) { if (.jdv_is_arrow(x) && !requireNamespace("nanoarrow", quietly = TRUE)) { return(FALSE) } + row_count <- if (.jdv_is_arrow(x)) .jdv_arrow_nrow(x) else NULL title <- if (nzchar(title_key)) title_key else NULL # Only jdv errors about the binding are retried without it; anything else # ends in FALSE so that View() falls back to the sess data viewer. @@ -1405,22 +1420,30 @@ handle_dataview_dispose <- function(params) { if (!is.null(binding) && !.jdv_is_arrow(x)) { registered <- tryCatch({ jdv::jdv_register(x, title = title, mode = "paged", binding = binding) - TRUE }, jdv_error = function(e) FALSE) - if (registered) { - return(TRUE) + if (!identical(registered, FALSE)) { + return(registered) } } jdv::jdv_register(x, title = title, mode = "paged") - }, address) + }, address, row_count) } # Connect and register a view, sharing the same failure contract for values and -# Interactive handles. Registration may return metadata; callers only need success. -.jdv_show_registered <- function(register, address = NULL) { +# Interactive handles. A row-count notification is best-effort display metadata. +.jdv_show_registered <- function(register, address = NULL, row_count = NULL) { if (!.jdv_connect(address)) return(FALSE) tryCatch({ - register() + registration <- register() + if (!is.null(row_count)) { + # A notification is only a hint; it must not make a successful View fail. + try(notify_client("jdv/rowCount", list( + view = registration$id, + generation = registration$generation, + nrow = row_count, + address = .sess_env$jdv_address + )), silent = TRUE) + } TRUE }, error = function(e) FALSE) } diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R index f1109abdd..ee20dadfa 100644 --- a/sess/inst/tinytest/test-jdv.R +++ b/sess/inst/tinytest/test-jdv.R @@ -49,6 +49,40 @@ local({ sample <- data.frame(x = c(2, 1, NA), y = c("b", "a", "c")) + # Only sources with an inexpensive, known row count get a hint. Readers and + # malformed counts remain unknown without consuming a stream. + table <- structure(sample, class = c("Table", class(sample))) + batch <- structure(sample, class = c("RecordBatch", class(sample))) + method_name <- "dim.polars_data_frame" + had_method <- exists(method_name, envir = .GlobalEnv, inherits = FALSE) + old_method <- if (had_method) get(method_name, envir = .GlobalEnv) else NULL + assign(method_name, function(x) x$height, envir = .GlobalEnv) + on.exit({ + if (had_method) { + assign(method_name, old_method, envir = .GlobalEnv) + } else if (exists(method_name, envir = .GlobalEnv, inherits = FALSE)) { + rm(list = method_name, envir = .GlobalEnv) + } + }, add = TRUE) + expect_identical(sess:::.jdv_arrow_nrow(table), 3) + expect_identical(sess:::.jdv_arrow_nrow(batch), 3) + polars <- structure(list(height = 4), class = "polars_data_frame") + expect_identical(sess:::.jdv_arrow_nrow(polars), 4) + for (invalid in list(NA_real_, Inf, -Inf, -1, 1.5, 9007199254740992, "3", 1 + 1i)) { + polars <- structure(list(height = invalid), class = "polars_data_frame") + expect_null(sess:::.jdv_arrow_nrow(polars)) + } + expect_null(sess:::.jdv_arrow_nrow(structure(list(), class = "RecordBatchReader"))) + if (requireNamespace("arrow", quietly = TRUE)) { + expect_identical(sess:::.jdv_arrow_nrow(arrow::arrow_table(x = 1:3)), 3) + } + if (requireNamespace("polars", quietly = TRUE)) { + expect_identical(sess:::.jdv_arrow_nrow(polars::pl$DataFrame(x = 1:3)), 3) + } + if (requireNamespace("nanoarrow", quietly = TRUE)) { + expect_null(sess:::.jdv_arrow_nrow(nanoarrow::as_nanoarrow_array_stream(sample))) + } + # The opt-out keeps the sess data viewer. options(sess.dataviewer = "sess") stub_address("tcp://127.0.0.1:1") @@ -124,6 +158,31 @@ local({ expect_true(connection$persistent) expect_identical(connection$address, address) + # A successful registration publishes its row count to the session client. + expect_true(sess:::.jdv_show_registered(function() { + list(id = "row-count-view", generation = 7L) + }, address, 3)) + row_count_messages <- read_messages() + row_count_message <- Filter(function(message) identical(message$method, "jdv/rowCount"), + row_count_messages) + expect_length(row_count_message, 1L) + expect_identical(row_count_message[[1L]]$params$view, "row-count-view") + expect_equal(row_count_message[[1L]]$params$generation, 7) + expect_equal(row_count_message[[1L]]$params$nrow, 3) + expect_identical(row_count_message[[1L]]$params$address, address) + + # A client notification failure does not undo a successful registration. + original_notify_client <- ns$notify_client + unlockBinding("notify_client", ns) + assign("notify_client", function(...) stop("client is unavailable"), envir = ns) + lockBinding("notify_client", ns) + expect_true(sess:::.jdv_show_registered(function() { + list(id = "row-count-view", generation = 7L) + }, address, 3)) + unlockBinding("notify_client", ns) + assign("notify_client", original_notify_client, envir = ns) + lockBinding("notify_client", ns) + # Data frames are paged and titled by their expression. open <- opens()[[1L]]$params expect_identical(open$kind, "grid") @@ -156,6 +215,10 @@ local({ stream <- opens()[[4L]]$params expect_identical(stream$access, "sequential") expect_identical(stream$title, "stream") + stream_notifications <- read_messages() + expect_false(any(vapply(stream_notifications, function(message) { + identical(message$method, "jdv/rowCount") + }, logical(1L)))) } # View() follows the variable on R 4.6.0 or later, and never on older R. diff --git a/src/dataViewer.ts b/src/dataViewer.ts index 48f2e3bb9..80426eb10 100644 --- a/src/dataViewer.ts +++ b/src/dataViewer.ts @@ -284,6 +284,29 @@ export function getDataViewerScript(): string { ? format(total) + ' rows' : format(filtered) + ' of ' + format(total) + ' rows'; } + function formatSequentialRowCount(loaded, knownTotal, complete) { + const format = value => new Intl.NumberFormat().format(value); + if (complete) { + return format(loaded) + ' rows'; + } + return format(loaded) + ' / ' + (Number.isFinite(knownTotal) + ? format(knownTotal) : '???') + ' rows'; + } + + function sequentialRowCountTitle(knownTotal, notice, complete) { + const unknownText = 'The total is unknown until the stream ends.'; + const knownText = 'The total is known from the R object.'; + if (typeof notice === 'string' && notice.length > 0) { + return Number.isFinite(knownTotal) ? notice.replace(unknownText, knownText) : notice; + } + if (complete) { + return 'All rows have been read from the Arrow stream. Sorting and filtering are not available.'; + } + return 'Rows are read from the Arrow stream as you scroll. ' + + (Number.isFinite(knownTotal) ? knownText : unknownText) + + ' Sorting and filtering are not available.'; + } + function updateViewerSizingButtons() { document.querySelector('#viewerFit').setAttribute('aria-pressed', String(viewerSizingMode === 'fit')); document.querySelector('#viewerAutoSize').setAttribute('aria-pressed', String(viewerSizingMode === 'content')); diff --git a/src/interactive/manager.ts b/src/interactive/manager.ts index da925e62f..4760de50d 100644 --- a/src/interactive/manager.ts +++ b/src/interactive/manager.ts @@ -15,7 +15,7 @@ import { Transcript, TranscriptCell } from './transcript'; import { DISPLAY_MIME, InteractiveSerializer, persistentCellMetadata, persistentNotebookMetadata, restoredCellMetadata } from './notebook'; import { setInteractiveExecutor } from './executionTarget'; import * as session from '../session'; -import { disposeJdvHost, getJdvAddress } from '../jdvViewer'; +import { applyJdvRowCount, disposeJdvHost, getJdvAddress } from '../jdvViewer'; import * as util from '../util'; import type { ListViewNavigation } from '../listViewer'; import { escapeXml } from './plotSvg'; @@ -1036,6 +1036,7 @@ export class InteractiveManager implements vscode.Disposable, vscode.TreeDataPro } } else if (event.type === 'notification') { const params = object(event.data.params ?? {}); + if (event.data.method === 'jdv/rowCount') { applyJdvRowCount(view.target.sessionId, params); } if (event.data.method === 'help') { await session.showHelpNotification(params); } if (event.data.method === 'rstudioapi/send_to_console') { if (params.execute) { await this.submit(view, String(params.code)); } diff --git a/src/jdvRowCount.ts b/src/jdvRowCount.ts new file mode 100644 index 000000000..0ca78a18c --- /dev/null +++ b/src/jdvRowCount.ts @@ -0,0 +1,183 @@ +'use strict'; + +interface PendingHint { + readonly generation: number; + readonly nrow: number; +} + +interface ViewState { + readonly scope: string; + /** Highest generation observed from a hint, open, or close. */ + generation: number; + /** A close can arrive before the corresponding view/open notification. */ + closedGeneration?: number; + active?: { + readonly generation: number; + readonly onHint: (nrow: number) => void; + nrow?: number; + }; + pending?: PendingHint; +} + +/** + * Correlates row-count notifications with jdv view/open notifications, which + * travel over separate connections. `scope` should identify both the + * authenticated sess session and its jdv socket address. + */ +export class JdvRowCountHints { + private readonly views = new Map(); + + constructor( + private readonly maxPending = 128, + private readonly maxMetadata = 512, + ) { + if (!Number.isInteger(maxPending) || maxPending < 0 || + !Number.isInteger(maxMetadata) || maxMetadata < 0) { + throw new RangeError('Row-count hint bounds must be non-negative integers.'); + } + } + + /** Receive a hint, buffering it if the matching view has not opened yet. */ + hint(scope: string, view: string, generation: number, nrow: number): void { + if (!validGeneration(generation) || !validRowCount(nrow)) { return; } + const key = viewKey(scope, view); + const state = this.get(key); + if (!state) { + const created: ViewState = { scope, generation, pending: { generation, nrow } }; + this.set(key, created); + this.enforceBounds(); + return; + } + this.touch(key, state); + if (generation < state.generation) { return; } + if (generation === state.generation) { + if (state.active?.generation === generation) { + state.active.nrow = nrow; + state.active.onHint(nrow); + } else if (state.pending?.generation === generation && state.closedGeneration !== generation) { + state.pending = { generation, nrow }; + } + // A hint for a closed generation, or one without a matching open, + // must not revive the view. + return; + } + + state.generation = generation; + state.pending = { generation, nrow }; + this.enforceBounds(); + } + + /** Register an opened view and return a matching hint buffered earlier. */ + open( + scope: string, + view: string, + generation: number, + onHint: (nrow: number) => void, + ): number | undefined { + if (!validGeneration(generation)) { return undefined; } + const key = viewKey(scope, view); + let state = this.get(key); + if (!state) { + state = { scope, generation }; + this.set(key, state); + } else { + this.touch(key, state); + if (generation < state.generation || state.closedGeneration === generation) { + return undefined; + } + if (generation > state.generation) { + state.generation = generation; + } + } + + // Opening a newer generation supersedes its older pending hint. A + // future generation's hint remains buffered until that view opens. + let nrow: number | undefined; + if (state.active?.generation === generation) { nrow = state.active.nrow; } + if (state.pending && state.pending.generation <= generation) { + if (state.pending.generation === generation) { nrow = state.pending.nrow; } + state.pending = undefined; + } + state.active = { generation, onHint, nrow }; + this.enforceBounds(); + return nrow; + } + + /** Close only this generation; a delayed close cannot affect a newer one. */ + close(scope: string, view: string, generation: number): void { + if (!validGeneration(generation)) { return; } + const key = viewKey(scope, view); + let state = this.get(key); + if (!state) { + state = { scope, generation, closedGeneration: generation }; + this.set(key, state); + this.enforceBounds(); + return; + } + this.touch(key, state); + if (generation < state.generation) { + if (state.active?.generation === generation) { state.active = undefined; } + this.enforceBounds(); + return; + } + if (generation > state.generation) { state.generation = generation; } + state.closedGeneration = generation; + if (state.active && state.active.generation <= generation) { state.active = undefined; } + if (state.pending && state.pending.generation <= generation) { state.pending = undefined; } + this.enforceBounds(); + } + + /** Forget all pending hints and generation metadata for one R session. */ + clear(scope: string): void { + for (const [key, state] of this.views) { + if (state.scope === scope) { this.views.delete(key); } + } + } + + private get(key: string): ViewState | undefined { + return this.views.get(key); + } + + private set(key: string, state: ViewState): void { + this.views.set(key, state); + } + + private touch(key: string, state: ViewState): void { + this.views.delete(key); + this.views.set(key, state); + } + + private enforceBounds(): void { + let pending = 0; + for (const state of this.views.values()) { + if (state.pending) { pending++; } + } + while (pending > this.maxPending) { + const oldest = [...this.views.values()].find(state => state.pending !== undefined); + if (!oldest) { break; } + oldest.pending = undefined; + pending--; + } + + // Live panel callbacks are never evicted. Inactive generation + // tombstones are LRU-bounded so closed or unopened views cannot grow + // metadata without limit. + while (this.views.size > this.maxMetadata) { + const inactive = [...this.views].find(([, state]) => !state.active); + if (!inactive) { break; } + this.views.delete(inactive[0]); + } + } +} + +function viewKey(scope: string, view: string): string { + return JSON.stringify([scope, view]); +} + +function validGeneration(value: number): boolean { + return Number.isSafeInteger(value) && value >= 1; +} + +function validRowCount(value: number): boolean { + return Number.isSafeInteger(value) && value >= 0; +} diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index 132473ac6..f2ad5c1e2 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -13,6 +13,7 @@ import { gridPageFromRows, parseDescriptor, parseTracking, revealsPanel, tableRowsParams, } from './jdvAdapter'; import { SequentialStream } from './jdvArrow'; +import { JdvRowCountHints } from './jdvRowCount'; import { JdvHost, JdvPeer } from './jdvTransport'; import { disposeJdvHostIfCurrentAddress } from './jdvHostLifecycle'; import { extensionContext } from './extension'; @@ -27,6 +28,8 @@ interface JdvPanel { peer: JdvPeer; descriptor: JdvDescriptor; title: string; + hintScope: string; + knownTotal?: number; tree?: TreeAdapter; /** Received Arrow pages of a sequential view. */ stream?: SequentialStream; @@ -38,11 +41,47 @@ interface JdvPanel { const panels = new Map(); const hosts = new Map>(); +const rowCountHints = new JdvRowCountHints(); + +function rowCountScope(sessionId: string | null, address: string): string { + return JSON.stringify([sessionId, address]); +} + +/** A display-only hint from the authenticated sess notification channel. */ +export function applyJdvRowCount(sessionId: string | null, params: Record): void { + if (!sessionId || typeof params.address !== 'string' || !params.address || + typeof params.view !== 'string' || !params.view || + !Number.isSafeInteger(params.generation) || Number(params.generation) < 1 || + !Number.isSafeInteger(params.nrow) || Number(params.nrow) < 0) { + return; + } + rowCountHints.hint(rowCountScope(sessionId, params.address), params.view, + Number(params.generation), Number(params.nrow)); +} + +function bindRowCount(state: JdvPanel): void { + state.knownTotal = undefined; + if (!state.stream) { + return; + } + const { view, generation } = state.descriptor; + state.knownTotal = rowCountHints.open(state.hintScope, view, generation, nrow => { + state.knownTotal = nrow; + try { + const webview = state.panel.webview; + void webview.postMessage({ message: 'dataview/rowCount', + documentGeneration: session.getDocumentGeneration(webview), knownTotal: nrow }); + } catch { + // The panel may have been disposed while the notification was in flight. + } + }); +} /** The jdv transport address for a session, creating its listener on first use. */ export function getJdvAddress(sessionId: string, hostKey = sessionId): Promise { let host = hosts.get(hostKey); if (!host) { + let address = ''; host = JdvHost.create({ viewOpen: (peer, params) => { let descriptor: JdvDescriptor; @@ -54,22 +93,29 @@ export function getJdvAddress(sessionId: string, hostKey = sessionId): Promise undefined); } }, - viewClosed: (_peer, view, generation) => markClosed(sessionId, view, generation), + viewClosed: (_peer, view, generation) => { + rowCountHints.close(rowCountScope(sessionId, address), view, generation); + markClosed(sessionId, view, generation); + }, viewUpdateFailed: (_peer, params) => warnUpdateFailed(sessionId, params), viewTracking: (_peer, params) => applyTracking(sessionId, params), disconnected: (peer) => { + rowCountHints.clear(rowCountScope(sessionId, address)); for (const state of panels.values()) { if (state.peer === peer && !state.closed) { state.closed = 'The R session disconnected from the data viewer.'; } } }, + }).then(value => { + address = value.address; + return value; }); hosts.set(hostKey, host); void host.catch(() => { @@ -170,7 +216,7 @@ function errorMessage(error: unknown): string { } async function openJdvView( - descriptor: JdvDescriptor, sessionId: string | null, viewer: string, peer: JdvPeer, + descriptor: JdvDescriptor, sessionId: string | null, viewer: string, peer: JdvPeer, hintScope: string, ): Promise { const key = panelKey(sessionId, descriptor.view); const existing = panels.get(key); @@ -185,10 +231,13 @@ async function openJdvView( return; } if (existing) { - if (descriptor.generation < existing.descriptor.generation) { + if (existing.hintScope === hintScope && descriptor.generation < existing.descriptor.generation) { return; } // The same view with a newer generation (View() again, or a refresh) updates the panel. + if (existing.hintScope !== hintScope || existing.descriptor.generation !== descriptor.generation) { + rowCountHints.close(existing.hintScope, existing.descriptor.view, existing.descriptor.generation); + } existing.descriptor = descriptor; existing.peer = peer; existing.title = title; @@ -196,6 +245,8 @@ async function openJdvView( existing.tree = descriptor.kind === 'tree' ? new TreeAdapter(descriptor, title) : undefined; existing.stream?.dispose(); existing.stream = createStream(descriptor, peer); + existing.hintScope = hintScope; + bindRowCount(existing); existing.panel.title = title; // Updates that R made on its own (a tracked variable, a Refresh click) keep // the panel where it is. Otherwise, including when jdv sends no reason, @@ -222,11 +273,12 @@ async function openJdvView( panel.iconPath = new UriIcon(descriptor.kind === 'grid' ? 'open-preview' : 'preview'); session.attachViewerSessionBridge(panel, sessionId); const state: JdvPanel = { - key, panel, sessionId, peer, descriptor, title, + key, panel, sessionId, peer, descriptor, title, hintScope, tree: descriptor.kind === 'tree' ? new TreeAdapter(descriptor, title) : undefined, stream: createStream(descriptor, peer), }; panels.set(key, state); + bindRowCount(state); // The panel's webview getter throws once onDidDispose fires. const webview = panel.webview; panel.onDidDispose(() => { @@ -234,6 +286,7 @@ async function openJdvView( panels.delete(key); } state.stream?.dispose(); + rowCountHints.close(state.hintScope, state.descriptor.view, state.descriptor.generation); if (!state.closed) { const { view, generation } = state.descriptor; void state.peer.request('view/close', { view, generation }).catch(() => undefined); @@ -289,7 +342,7 @@ async function handleMessage(state: JdvPanel, webview: vscode.Webview, message: if (state.stream) { // Rows already received stay viewable after R closes the view. if (message.action === 'init') { - reply(true, { columns: gridColumnDefs(state.descriptor), totalRows: 0, sequential: true }); + reply(true, { columns: gridColumnDefs(state.descriptor), totalRows: 0, sequential: true, knownTotal: state.knownTotal }); } else if (message.action === 'page') { reply(true, await fetchSequentialPage(state, state.stream, message)); } @@ -485,6 +538,7 @@ async function fetchSequentialPage( totalUnfiltered: known ? stream.rowsSeen : null, lastRow: known ? stream.rowsSeen : -1, loadedRows: stream.rowsSeen, + knownTotal: state.knownTotal, done: stream.done, unavailableBefore: stream.unavailableBefore, notice: state.closed, diff --git a/src/session.ts b/src/session.ts index b35fdb9b5..02b4e2d75 100644 --- a/src/session.ts +++ b/src/session.ts @@ -35,7 +35,7 @@ import type { import { showWebView } from './webViewer'; import { getListViewerScript, listViewerStyle, ListViewNavigation } from './listViewer'; import { getDataViewerScript, getDataViewerStyle, getDataViewerToolbarHtml } from './dataViewer'; -import { disposeJdvHost, getJdvAddress } from './jdvViewer'; +import { applyJdvRowCount, disposeJdvHost, getJdvAddress } from './jdvViewer'; import { getFollowScript } from './viewerFollow'; import type { JdvTracking } from './jdvAdapter'; import { getDataViewerColumnPanelHtml, getDataViewerColumnPanelScript, getDataViewerColumnPanelStyle } from './dataViewerColumnPanel'; @@ -2207,11 +2207,42 @@ export async function getTableHtml( let filteredRows = 0; let totalRows = 0; let isFiltered = false; + let isSequential = false; + let sequentialKnownTotal; + let sequentialLoadedRows = 0; + let sequentialComplete = false; + let sequentialNotice; + let sequentialInfo; let verticalScrollbar; let scrollbarPositionAttached = false; let scrollbarPressed = false; const LONG_FETCH_DELAY_MS = 2000; const rowNumberFormatter = new Intl.NumberFormat(); + function isValidKnownTotal(value) { + return Number.isSafeInteger(value) && value >= 0; + } + + function renderSequentialRowCount() { + if (!isSequential) { + return; + } + const rowCount = document.querySelector('#viewerRowCount'); + rowCount.textContent = formatSequentialRowCount( + sequentialLoadedRows, sequentialKnownTotal, sequentialComplete + ); + rowCount.title = sequentialRowCountTitle(sequentialKnownTotal, sequentialNotice, sequentialComplete); + if (sequentialInfo) { + sequentialInfo.title = sequentialRowCountTitle(sequentialKnownTotal, sequentialNotice, sequentialComplete); + } + } + + function receiveSequentialKnownTotal(value) { + if (!isValidKnownTotal(value)) { + return; + } + sequentialKnownTotal = value; + renderSequentialRowCount(); + } function clearLongFetchTimer() { if (longFetchTimer) { clearTimeout(longFetchTimer); @@ -2409,6 +2440,14 @@ export async function getTableHtml( } }); + window.addEventListener('message', (event) => { + const data = event.data; + if (data && data.message === 'dataview/rowCount' && + data.documentGeneration === documentGeneration) { + receiveSequentialKnownTotal(data.knownTotal); + } + }); + function updateTheme() { if (gridApi) { gridApi.setGridOption('theme', getAgTheme()); @@ -2431,6 +2470,10 @@ export async function getTableHtml( const columns = Array.isArray(init.columns) ? init.columns : []; filteredRows = init.totalRows; totalRows = init.totalRows; + isSequential = init.sequential === true; + if (isSequential && sequentialKnownTotal === undefined && isValidKnownTotal(init.knownTotal)) { + sequentialKnownTotal = init.knownTotal; + } if (init.live) { const info = document.createElement('span'); info.textContent = 'Full data'; @@ -2438,13 +2481,15 @@ export async function getTableHtml( document.querySelector('#viewerControls').append(info); } if (init.sequential) { - const info = document.createElement('span'); - info.textContent = 'Arrow stream'; - info.title = 'Rows are read from the Arrow stream as you scroll. The total is unknown until the stream ends. Sorting and filtering are not available.'; - document.querySelector('#viewerControls').append(info); + sequentialInfo = document.createElement('span'); + sequentialInfo.textContent = 'Arrow stream'; + document.querySelector('#viewerControls').append(sequentialInfo); } const bigintFields = prepareViewerColumns(columns); updateViewerRowCount(filteredRows, totalRows); + if (init.sequential) { + renderSequentialRowCount(); + } const rowIndexColumn = columns.find(column => column.field === '0'); if (rowIndexColumn) { rowIndexColumn.headerValueGetter = () => isFiltered @@ -2467,18 +2512,22 @@ export async function getTableHtml( const nextFilteredRows = Number.isFinite(result.totalRows) ? result.totalRows : result.loadedRows; const nextTotalRows = Number.isFinite(result.totalUnfiltered) ? result.totalUnfiltered : result.loadedRows; const nextIsFiltered = Object.keys(params.filterModel || {}).length > 0; + if (init.sequential && sequentialKnownTotal === undefined && isValidKnownTotal(result.knownTotal)) { + sequentialKnownTotal = result.knownTotal; + } const headerNeedsRefresh = filteredRows !== nextFilteredRows || totalRows !== nextTotalRows || isFiltered !== nextIsFiltered; filteredRows = nextFilteredRows; totalRows = nextTotalRows; isFiltered = nextIsFiltered; updateViewerRowCount(filteredRows, totalRows); - if (init.sequential && !Number.isFinite(result.totalRows)) { - document.querySelector('#viewerRowCount').textContent = - rowNumberFormatter.format(result.loadedRows) + '+ rows loaded'; - } - if (init.sequential && result.notice) { - document.querySelector('#viewerRowCount').title = result.notice; + if (init.sequential) { + sequentialComplete = Number.isFinite(result.totalRows); + sequentialLoadedRows = sequentialComplete + ? result.totalRows + : (Number.isFinite(result.loadedRows) ? result.loadedRows : 0); + sequentialNotice = result.notice; + renderSequentialRowCount(); } if (headerNeedsRefresh) { gridApi?.refreshHeader(); @@ -3215,6 +3264,10 @@ async function handleNotification(message: Record, socket: IpcS } break; } + case 'jdv/rowCount': { + applyJdvRowCount(socket._sessionId ?? null, params); + break; + } case 'plot_updated': { void updatePlot(); break; diff --git a/src/test/node/interactiveRuntime.test.ts b/src/test/node/interactiveRuntime.test.ts index 46bd8a427..98f0d7016 100644 --- a/src/test/node/interactiveRuntime.test.ts +++ b/src/test/node/interactiveRuntime.test.ts @@ -321,6 +321,52 @@ sess::display("display-${cell}", "text/plain"); cat("stderr-${cell}\\n", file=st const inspect = (request: { method: string; params: Record }): Promise> => client.request('inspect', request); + test('jdv Arrow views send display-only totals for sized sources', async function () { + if (!(await jdvAvailable())) { this.skip(); return; } + const available = (await run('Rscript', ['--vanilla', '-e', + "cat(requireNamespace('nanoarrow', quietly=TRUE), requireNamespace('arrow', quietly=TRUE))"])).stdout.trim().split(/\s+/); + if (available[0] !== 'TRUE') { this.skip(); return; } + const sized = available[1] === 'TRUE'; + const opened: { peer: JdvPeer; params: JsonObject }[] = []; + const host = await JdvHost.create({ + viewOpen: (peer, params) => opened.push({ peer, params }), + viewClosed: () => undefined, viewUpdateFailed: () => undefined, + viewTracking: () => undefined, disconnected: () => undefined, + }); + const onEvent = (event: SessionEvent): void => { + if (event.type === 'clientRequest' && event.data.method === 'jdv/address') { + void client.request('clientReply', { id: event.data.id, result: { address: host.address } }); + } + }; + client.on('event', onEvent); + try { + const source = sized ? 'arrow::Table$create(data.frame(a=1:3))' + : 'nanoarrow::as_nanoarrow_array_stream(data.frame(a=1:3))'; + const id = await submit(`utils::View(${source}, "Arrow count test")`); + assert.strictEqual((await finished(id)).state, 'success'); + await until(() => opened.length > 0); + const { peer, params } = opened[0]; + assert.strictEqual(params.access, 'sequential'); + assert.strictEqual(params.nrow, null); + const hints = events.filter(event => event.type === 'notification' && event.data.method === 'jdv/rowCount'); + assert.strictEqual(hints.length, sized ? 1 : 0); + if (sized) { + assert.deepStrictEqual(hints[0].data.params, { + view: params.view, generation: params.generation, nrow: 3, address: host.address, + }); + } + const page = await peer.requestWithBody('stream/next', { + view: params.view, generation: params.generation, max_rows: 2, + }); + assert.strictEqual((page.result as JsonObject).rows_seen, 2); + assert.strictEqual((page.result as JsonObject).done, false); + assert.ok(page.body); + } finally { + client.off('event', onEvent); + host.dispose(); + } + }); + test('inline tables page, sort and filter through one jdv handle per output', async function () { if (!(await jdvAvailable())) { this.skip(); return; } const id = await submit('tbl <- data.frame(id=1:100, name=rep(c("a", "b"), 50), x=seq(0.5, 50, by=0.5)); tbl'); diff --git a/src/test/node/jdvRowCount.test.ts b/src/test/node/jdvRowCount.test.ts new file mode 100644 index 000000000..e015d823e --- /dev/null +++ b/src/test/node/jdvRowCount.test.ts @@ -0,0 +1,101 @@ +import * as assert from 'node:assert/strict'; +import { JdvRowCountHints } from '../../jdvRowCount'; + +suite('jdv row-count hints', () => { + test('applies a hint received before view/open', () => { + const hints = new JdvRowCountHints(); + hints.hint('session-a', 'view-1', 3, 42); + const applied: number[] = []; + + assert.equal(hints.open('session-a', 'view-1', 3, nrow => applied.push(nrow)), 42); + hints.hint('session-a', 'view-1', 3, 43); + assert.deepEqual(applied, [43]); + }); + + test('applies a hint received after view/open', () => { + const hints = new JdvRowCountHints(); + const applied: number[] = []; + + assert.equal(hints.open('session-a', 'view-1', 1, nrow => applied.push(nrow)), undefined); + hints.hint('session-a', 'view-1', 1, 9); + assert.deepEqual(applied, [9]); + assert.equal(hints.open('session-a', 'view-1', 1, nrow => applied.push(nrow)), 9); + }); + + test('replaces the active generation and ignores stale hints and closes', () => { + const hints = new JdvRowCountHints(); + const oldGeneration: number[] = []; + const newGeneration: number[] = []; + + hints.open('session-a', 'view-1', 1, nrow => oldGeneration.push(nrow)); + hints.open('session-a', 'view-1', 2, nrow => newGeneration.push(nrow)); + hints.hint('session-a', 'view-1', 1, 100); + hints.close('session-a', 'view-1', 1); + hints.hint('session-a', 'view-1', 2, 200); + + assert.deepEqual(oldGeneration, []); + assert.deepEqual(newGeneration, [200]); + }); + + test('closing an older active generation releases its callback while retaining newer pending data', () => { + const hints = new JdvRowCountHints(); + const applied: number[] = []; + hints.open('session-a', 'view-1', 1, nrow => applied.push(nrow)); + hints.hint('session-a', 'view-1', 2, 20); + hints.close('session-a', 'view-1', 1); + hints.hint('session-a', 'view-1', 1, 10); + + assert.equal(applied.length, 0); + assert.equal(hints.open('session-a', 'view-1', 2, nrow => applied.push(nrow)), 20); + }); + + test('a close tombstone blocks same-generation delayed open and hints', () => { + const hints = new JdvRowCountHints(); + const applied: number[] = []; + + hints.close('session-a', 'view-1', 7); + hints.hint('session-a', 'view-1', 7, 70); + assert.equal(hints.open('session-a', 'view-1', 7, nrow => applied.push(nrow)), undefined); + hints.hint('session-a', 'view-1', 7, 71); + assert.equal(applied.length, 0); + + hints.hint('session-a', 'view-1', 8, 80); + assert.equal(hints.open('session-a', 'view-1', 8, nrow => applied.push(nrow)), 80); + }); + + test('isolates scopes even when view and generation match', () => { + const hints = new JdvRowCountHints(); + hints.hint('session-a/socket-a', 'view-1', 1, 11); + hints.hint('session-b/socket-b', 'view-1', 1, 22); + hints.clear('session-a/socket-a'); + + assert.equal(hints.open('session-a/socket-a', 'view-1', 1, () => undefined), undefined); + assert.equal(hints.open('session-b/socket-b', 'view-1', 1, () => undefined), 22); + }); + + test('bounds pending hints and preserves live subscriptions', () => { + const hints = new JdvRowCountHints(2, 8); + const applied: number[] = []; + hints.open('session-a', 'live', 1, nrow => applied.push(nrow)); + hints.hint('session-a', 'one', 1, 1); + hints.hint('session-a', 'two', 1, 2); + hints.hint('session-a', 'three', 1, 3); + + assert.equal(hints.open('session-a', 'one', 1, () => undefined), undefined); + assert.equal(hints.open('session-a', 'two', 1, () => undefined), 2); + assert.equal(hints.open('session-a', 'three', 1, () => undefined), 3); + hints.hint('session-a', 'live', 1, 99); + assert.deepEqual(applied, [99]); + }); + + test('bounds inactive metadata without evicting open panel callbacks', () => { + const hints = new JdvRowCountHints(4, 1); + const applied: number[] = []; + hints.open('session-a', 'live', 1, nrow => applied.push(nrow)); + hints.hint('session-a', 'other', 1, 2); + hints.hint('session-a', 'another', 1, 3); + hints.hint('session-a', 'live', 1, 10); + + assert.deepEqual(applied, [10]); + }); +}); diff --git a/src/test/suite/dataViewer.test.ts b/src/test/suite/dataViewer.test.ts index 6d93fb305..fa2e914a2 100644 --- a/src/test/suite/dataViewer.test.ts +++ b/src/test/suite/dataViewer.test.ts @@ -18,6 +18,8 @@ interface ViewerScript { saveViewerState(): void; getAgTheme(): unknown; setViewerSizingMode(mode: 'fit' | 'content'): void; + formatSequentialRowCount(loaded: number, knownTotal: number | undefined, complete: boolean): string; + sequentialRowCountTitle(knownTotal: number | undefined, notice?: string, complete?: boolean): string; } function viewerScript(saved?: SavedViewer, gridState: GridState = {}, light = false): { @@ -30,7 +32,8 @@ function viewerScript(saved?: SavedViewer, gridState: GridState = {}, light = fa let appliedColumns: ColumnState[] = []; let contentSizingCalls = 0; const script = vm.runInNewContext(`${getDataViewerScript()} - ({ prepareViewerColumns, prepareViewerRows, getViewerGridOptions, saveViewerState, getAgTheme, setViewerSizingMode })`, { + ({ prepareViewerColumns, prepareViewerRows, getViewerGridOptions, saveViewerState, getAgTheme, setViewerSizingMode, + formatSequentialRowCount, sequentialRowCountTitle })`, { vscode: { getState: () => saved, setState: (value: SavedViewer) => { stored = value; }, @@ -178,6 +181,19 @@ suite('Data viewer', () => { assert.strictEqual(state?.rowSelection, undefined); }); + test('formats sequential row counts while the total is unknown, known, and finished', () => { + const script = viewerScript().script; + assert.strictEqual(script.formatSequentialRowCount(2000, undefined, false), '2,000 / ??? rows'); + assert.strictEqual(script.formatSequentialRowCount(2000, 1000000, false), '2,000 / 1,000,000 rows'); + assert.strictEqual(script.formatSequentialRowCount(1000000, 1000000, true), '1,000,000 rows'); + assert.match(script.sequentialRowCountTitle(undefined), /total is unknown until the stream ends/); + assert.match(script.sequentialRowCountTitle(undefined, undefined, true), /All rows have been read/); + assert.match(script.sequentialRowCountTitle(1000000), /total is known from the R object/); + assert.match(script.sequentialRowCountTitle(1000000, + 'Rows are read from the Arrow stream as you scroll. The total is unknown until the stream ends.'), + /total is known from the R object/); + }); + test('uses native date, datetime, boolean and bigint types in both viewer modes', () => { const defs: ColDef[] = [ ...columns(), From c4a79cba86551d23a7888b06dc313f6a36e99350 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 00:36:03 +0000 Subject: [PATCH 31/38] refactor: simplify Arrow row-count hints --- sess/R/handlers.R | 12 +- sess/inst/tinytest/test-jdv.R | 2 +- src/dataViewer.ts | 14 --- src/jdvRowCount.ts | 183 ------------------------------ src/jdvViewer.ts | 47 ++++---- src/session.ts | 72 ++++-------- src/test/node/jdvRowCount.test.ts | 152 +++++++++---------------- src/test/suite/dataViewer.test.ts | 18 +-- 8 files changed, 103 insertions(+), 397 deletions(-) delete mode 100644 src/jdvRowCount.ts diff --git a/sess/R/handlers.R b/sess/R/handlers.R index e960070a2..9c8bb48a2 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1310,15 +1310,11 @@ handle_dataview_dispose <- function(params) { # Return a row-count hint when an Arrow source exposes dimensions without reading # its data. Forward-only readers and streams return NULL from nrow(). .jdv_arrow_nrow <- function(x) { - tryCatch({ - count <- nrow(x) - if (!is.numeric(count) || length(count) != 1L || is.na(count) || - !is.finite(count) || count < 0 || count != floor(count) || - count > 9007199254740991) { - return(NULL) - } + count <- tryCatch(nrow(x), error = function(e) NULL) + if (is.numeric(count) && !is.complex(count) && length(count) == 1L && + is.finite(count) && count >= 0 && count == floor(count)) { unname(as.numeric(count)) - }, error = function(e) NULL) + } } # Make sure jdv has a persistent connection to this session's viewer host. diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R index ee20dadfa..43c198603 100644 --- a/sess/inst/tinytest/test-jdv.R +++ b/sess/inst/tinytest/test-jdv.R @@ -68,7 +68,7 @@ local({ expect_identical(sess:::.jdv_arrow_nrow(batch), 3) polars <- structure(list(height = 4), class = "polars_data_frame") expect_identical(sess:::.jdv_arrow_nrow(polars), 4) - for (invalid in list(NA_real_, Inf, -Inf, -1, 1.5, 9007199254740992, "3", 1 + 1i)) { + for (invalid in list(NA_real_, Inf, -Inf, -1, 1.5, "3", 1 + 1i)) { polars <- structure(list(height = invalid), class = "polars_data_frame") expect_null(sess:::.jdv_arrow_nrow(polars)) } diff --git a/src/dataViewer.ts b/src/dataViewer.ts index 80426eb10..4a637fbe5 100644 --- a/src/dataViewer.ts +++ b/src/dataViewer.ts @@ -293,20 +293,6 @@ export function getDataViewerScript(): string { ? format(knownTotal) : '???') + ' rows'; } - function sequentialRowCountTitle(knownTotal, notice, complete) { - const unknownText = 'The total is unknown until the stream ends.'; - const knownText = 'The total is known from the R object.'; - if (typeof notice === 'string' && notice.length > 0) { - return Number.isFinite(knownTotal) ? notice.replace(unknownText, knownText) : notice; - } - if (complete) { - return 'All rows have been read from the Arrow stream. Sorting and filtering are not available.'; - } - return 'Rows are read from the Arrow stream as you scroll. ' + - (Number.isFinite(knownTotal) ? knownText : unknownText) + - ' Sorting and filtering are not available.'; - } - function updateViewerSizingButtons() { document.querySelector('#viewerFit').setAttribute('aria-pressed', String(viewerSizingMode === 'fit')); document.querySelector('#viewerAutoSize').setAttribute('aria-pressed', String(viewerSizingMode === 'content')); diff --git a/src/jdvRowCount.ts b/src/jdvRowCount.ts deleted file mode 100644 index 0ca78a18c..000000000 --- a/src/jdvRowCount.ts +++ /dev/null @@ -1,183 +0,0 @@ -'use strict'; - -interface PendingHint { - readonly generation: number; - readonly nrow: number; -} - -interface ViewState { - readonly scope: string; - /** Highest generation observed from a hint, open, or close. */ - generation: number; - /** A close can arrive before the corresponding view/open notification. */ - closedGeneration?: number; - active?: { - readonly generation: number; - readonly onHint: (nrow: number) => void; - nrow?: number; - }; - pending?: PendingHint; -} - -/** - * Correlates row-count notifications with jdv view/open notifications, which - * travel over separate connections. `scope` should identify both the - * authenticated sess session and its jdv socket address. - */ -export class JdvRowCountHints { - private readonly views = new Map(); - - constructor( - private readonly maxPending = 128, - private readonly maxMetadata = 512, - ) { - if (!Number.isInteger(maxPending) || maxPending < 0 || - !Number.isInteger(maxMetadata) || maxMetadata < 0) { - throw new RangeError('Row-count hint bounds must be non-negative integers.'); - } - } - - /** Receive a hint, buffering it if the matching view has not opened yet. */ - hint(scope: string, view: string, generation: number, nrow: number): void { - if (!validGeneration(generation) || !validRowCount(nrow)) { return; } - const key = viewKey(scope, view); - const state = this.get(key); - if (!state) { - const created: ViewState = { scope, generation, pending: { generation, nrow } }; - this.set(key, created); - this.enforceBounds(); - return; - } - this.touch(key, state); - if (generation < state.generation) { return; } - if (generation === state.generation) { - if (state.active?.generation === generation) { - state.active.nrow = nrow; - state.active.onHint(nrow); - } else if (state.pending?.generation === generation && state.closedGeneration !== generation) { - state.pending = { generation, nrow }; - } - // A hint for a closed generation, or one without a matching open, - // must not revive the view. - return; - } - - state.generation = generation; - state.pending = { generation, nrow }; - this.enforceBounds(); - } - - /** Register an opened view and return a matching hint buffered earlier. */ - open( - scope: string, - view: string, - generation: number, - onHint: (nrow: number) => void, - ): number | undefined { - if (!validGeneration(generation)) { return undefined; } - const key = viewKey(scope, view); - let state = this.get(key); - if (!state) { - state = { scope, generation }; - this.set(key, state); - } else { - this.touch(key, state); - if (generation < state.generation || state.closedGeneration === generation) { - return undefined; - } - if (generation > state.generation) { - state.generation = generation; - } - } - - // Opening a newer generation supersedes its older pending hint. A - // future generation's hint remains buffered until that view opens. - let nrow: number | undefined; - if (state.active?.generation === generation) { nrow = state.active.nrow; } - if (state.pending && state.pending.generation <= generation) { - if (state.pending.generation === generation) { nrow = state.pending.nrow; } - state.pending = undefined; - } - state.active = { generation, onHint, nrow }; - this.enforceBounds(); - return nrow; - } - - /** Close only this generation; a delayed close cannot affect a newer one. */ - close(scope: string, view: string, generation: number): void { - if (!validGeneration(generation)) { return; } - const key = viewKey(scope, view); - let state = this.get(key); - if (!state) { - state = { scope, generation, closedGeneration: generation }; - this.set(key, state); - this.enforceBounds(); - return; - } - this.touch(key, state); - if (generation < state.generation) { - if (state.active?.generation === generation) { state.active = undefined; } - this.enforceBounds(); - return; - } - if (generation > state.generation) { state.generation = generation; } - state.closedGeneration = generation; - if (state.active && state.active.generation <= generation) { state.active = undefined; } - if (state.pending && state.pending.generation <= generation) { state.pending = undefined; } - this.enforceBounds(); - } - - /** Forget all pending hints and generation metadata for one R session. */ - clear(scope: string): void { - for (const [key, state] of this.views) { - if (state.scope === scope) { this.views.delete(key); } - } - } - - private get(key: string): ViewState | undefined { - return this.views.get(key); - } - - private set(key: string, state: ViewState): void { - this.views.set(key, state); - } - - private touch(key: string, state: ViewState): void { - this.views.delete(key); - this.views.set(key, state); - } - - private enforceBounds(): void { - let pending = 0; - for (const state of this.views.values()) { - if (state.pending) { pending++; } - } - while (pending > this.maxPending) { - const oldest = [...this.views.values()].find(state => state.pending !== undefined); - if (!oldest) { break; } - oldest.pending = undefined; - pending--; - } - - // Live panel callbacks are never evicted. Inactive generation - // tombstones are LRU-bounded so closed or unopened views cannot grow - // metadata without limit. - while (this.views.size > this.maxMetadata) { - const inactive = [...this.views].find(([, state]) => !state.active); - if (!inactive) { break; } - this.views.delete(inactive[0]); - } - } -} - -function viewKey(scope: string, view: string): string { - return JSON.stringify([scope, view]); -} - -function validGeneration(value: number): boolean { - return Number.isSafeInteger(value) && value >= 1; -} - -function validRowCount(value: number): boolean { - return Number.isSafeInteger(value) && value >= 0; -} diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index f2ad5c1e2..adf1fbc7f 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -13,7 +13,6 @@ import { gridPageFromRows, parseDescriptor, parseTracking, revealsPanel, tableRowsParams, } from './jdvAdapter'; import { SequentialStream } from './jdvArrow'; -import { JdvRowCountHints } from './jdvRowCount'; import { JdvHost, JdvPeer } from './jdvTransport'; import { disposeJdvHostIfCurrentAddress } from './jdvHostLifecycle'; import { extensionContext } from './extension'; @@ -41,7 +40,7 @@ interface JdvPanel { const panels = new Map(); const hosts = new Map>(); -const rowCountHints = new JdvRowCountHints(); +const pendingRowCounts = new Map(); function rowCountScope(sessionId: string | null, address: string): string { return JSON.stringify([sessionId, address]); @@ -55,26 +54,26 @@ export function applyJdvRowCount(sessionId: string | null, params: Record 64) { + pendingRowCounts.delete(pendingRowCounts.keys().next().value!); + } + } } function bindRowCount(state: JdvPanel): void { - state.knownTotal = undefined; - if (!state.stream) { - return; - } - const { view, generation } = state.descriptor; - state.knownTotal = rowCountHints.open(state.hintScope, view, generation, nrow => { - state.knownTotal = nrow; - try { - const webview = state.panel.webview; - void webview.postMessage({ message: 'dataview/rowCount', - documentGeneration: session.getDocumentGeneration(webview), knownTotal: nrow }); - } catch { - // The panel may have been disposed while the notification was in flight. - } - }); + const key = JSON.stringify([state.hintScope, state.descriptor.view, state.descriptor.generation]); + state.knownTotal = state.stream ? pendingRowCounts.get(key) : undefined; + pendingRowCounts.delete(key); } /** The jdv transport address for a session, creating its listener on first use. */ @@ -99,14 +98,10 @@ export function getJdvAddress(sessionId: string, hostKey = sessionId): Promise undefined); } }, - viewClosed: (_peer, view, generation) => { - rowCountHints.close(rowCountScope(sessionId, address), view, generation); - markClosed(sessionId, view, generation); - }, + viewClosed: (_peer, view, generation) => markClosed(sessionId, view, generation), viewUpdateFailed: (_peer, params) => warnUpdateFailed(sessionId, params), viewTracking: (_peer, params) => applyTracking(sessionId, params), disconnected: (peer) => { - rowCountHints.clear(rowCountScope(sessionId, address)); for (const state of panels.values()) { if (state.peer === peer && !state.closed) { state.closed = 'The R session disconnected from the data viewer.'; @@ -235,9 +230,6 @@ async function openJdvView( return; } // The same view with a newer generation (View() again, or a refresh) updates the panel. - if (existing.hintScope !== hintScope || existing.descriptor.generation !== descriptor.generation) { - rowCountHints.close(existing.hintScope, existing.descriptor.view, existing.descriptor.generation); - } existing.descriptor = descriptor; existing.peer = peer; existing.title = title; @@ -286,7 +278,6 @@ async function openJdvView( panels.delete(key); } state.stream?.dispose(); - rowCountHints.close(state.hintScope, state.descriptor.view, state.descriptor.generation); if (!state.closed) { const { view, generation } = state.descriptor; void state.peer.request('view/close', { view, generation }).catch(() => undefined); diff --git a/src/session.ts b/src/session.ts index 02b4e2d75..6b78d313a 100644 --- a/src/session.ts +++ b/src/session.ts @@ -2207,41 +2207,23 @@ export async function getTableHtml( let filteredRows = 0; let totalRows = 0; let isFiltered = false; - let isSequential = false; let sequentialKnownTotal; - let sequentialLoadedRows = 0; - let sequentialComplete = false; - let sequentialNotice; - let sequentialInfo; + let sequentialProgress; let verticalScrollbar; let scrollbarPositionAttached = false; let scrollbarPressed = false; const LONG_FETCH_DELAY_MS = 2000; const rowNumberFormatter = new Intl.NumberFormat(); - function isValidKnownTotal(value) { - return Number.isSafeInteger(value) && value >= 0; - } - function renderSequentialRowCount() { - if (!isSequential) { - return; - } - const rowCount = document.querySelector('#viewerRowCount'); - rowCount.textContent = formatSequentialRowCount( - sequentialLoadedRows, sequentialKnownTotal, sequentialComplete - ); - rowCount.title = sequentialRowCountTitle(sequentialKnownTotal, sequentialNotice, sequentialComplete); - if (sequentialInfo) { - sequentialInfo.title = sequentialRowCountTitle(sequentialKnownTotal, sequentialNotice, sequentialComplete); - } - } - - function receiveSequentialKnownTotal(value) { - if (!isValidKnownTotal(value)) { - return; + if (sequentialProgress) { + const { loadedRows, totalRows, notice } = sequentialProgress; + const complete = Number.isFinite(totalRows); + const rowCount = document.querySelector('#viewerRowCount'); + rowCount.textContent = formatSequentialRowCount( + complete ? totalRows : loadedRows, sequentialKnownTotal, complete + ); + rowCount.title = notice || ''; } - sequentialKnownTotal = value; - renderSequentialRowCount(); } function clearLongFetchTimer() { if (longFetchTimer) { @@ -2424,6 +2406,11 @@ export async function getTableHtml( window.addEventListener('message', (event) => { const data = event.data; + if (data?.message === 'dataview/rowCount' && data.documentGeneration === documentGeneration) { + sequentialKnownTotal = data.knownTotal; + renderSequentialRowCount(); + return; + } if (!data || data.message !== 'dataview/response' || data.documentGeneration !== documentGeneration) { return; @@ -2440,14 +2427,6 @@ export async function getTableHtml( } }); - window.addEventListener('message', (event) => { - const data = event.data; - if (data && data.message === 'dataview/rowCount' && - data.documentGeneration === documentGeneration) { - receiveSequentialKnownTotal(data.knownTotal); - } - }); - function updateTheme() { if (gridApi) { gridApi.setGridOption('theme', getAgTheme()); @@ -2470,10 +2449,6 @@ export async function getTableHtml( const columns = Array.isArray(init.columns) ? init.columns : []; filteredRows = init.totalRows; totalRows = init.totalRows; - isSequential = init.sequential === true; - if (isSequential && sequentialKnownTotal === undefined && isValidKnownTotal(init.knownTotal)) { - sequentialKnownTotal = init.knownTotal; - } if (init.live) { const info = document.createElement('span'); info.textContent = 'Full data'; @@ -2481,9 +2456,12 @@ export async function getTableHtml( document.querySelector('#viewerControls').append(info); } if (init.sequential) { - sequentialInfo = document.createElement('span'); - sequentialInfo.textContent = 'Arrow stream'; - document.querySelector('#viewerControls').append(sequentialInfo); + sequentialProgress = { loadedRows: 0, totalRows: null }; + sequentialKnownTotal ??= init.knownTotal; + const info = document.createElement('span'); + info.textContent = 'Arrow stream'; + info.title = 'Rows are read from the Arrow stream as you scroll. Sorting and filtering are not available.'; + document.querySelector('#viewerControls').append(info); } const bigintFields = prepareViewerColumns(columns); updateViewerRowCount(filteredRows, totalRows); @@ -2512,9 +2490,6 @@ export async function getTableHtml( const nextFilteredRows = Number.isFinite(result.totalRows) ? result.totalRows : result.loadedRows; const nextTotalRows = Number.isFinite(result.totalUnfiltered) ? result.totalUnfiltered : result.loadedRows; const nextIsFiltered = Object.keys(params.filterModel || {}).length > 0; - if (init.sequential && sequentialKnownTotal === undefined && isValidKnownTotal(result.knownTotal)) { - sequentialKnownTotal = result.knownTotal; - } const headerNeedsRefresh = filteredRows !== nextFilteredRows || totalRows !== nextTotalRows || isFiltered !== nextIsFiltered; filteredRows = nextFilteredRows; @@ -2522,11 +2497,8 @@ export async function getTableHtml( isFiltered = nextIsFiltered; updateViewerRowCount(filteredRows, totalRows); if (init.sequential) { - sequentialComplete = Number.isFinite(result.totalRows); - sequentialLoadedRows = sequentialComplete - ? result.totalRows - : (Number.isFinite(result.loadedRows) ? result.loadedRows : 0); - sequentialNotice = result.notice; + sequentialKnownTotal ??= result.knownTotal; + sequentialProgress = result; renderSequentialRowCount(); } if (headerNeedsRefresh) { diff --git a/src/test/node/jdvRowCount.test.ts b/src/test/node/jdvRowCount.test.ts index e015d823e..a555ff8e6 100644 --- a/src/test/node/jdvRowCount.test.ts +++ b/src/test/node/jdvRowCount.test.ts @@ -1,101 +1,61 @@ import * as assert from 'node:assert/strict'; -import { JdvRowCountHints } from '../../jdvRowCount'; +import * as fs from 'node:fs'; +import * as vm from 'node:vm'; +import { getDataViewerScript } from '../../dataViewer'; + +interface Viewer { + applyJdvRowCount(session: string, value: typeof hint): void; + bindRowCount(state: ReturnType): void; + panels: Map>; +} + +// Load the actual viewer module with its VS Code dependencies stubbed out. +function viewer(): Viewer { + return vm.runInNewContext(fs.readFileSync(require.resolve('../../jdvViewer'), 'utf8') + + '\n;({ applyJdvRowCount, bindRowCount, panels })', { + exports: {}, require: (name: string) => name === './session' ? { getDocumentGeneration: () => 1 } : {}, + }) as Viewer; +} +const hint = { address: 'socket-a', view: 'v-1', generation: 1, nrow: 1000 }; +function panel() { + const messages: { knownTotal: number }[] = []; + return { hintScope: JSON.stringify(['session-a', 'socket-a']), stream: {}, + descriptor: { view: 'v-1', generation: 1 }, knownTotal: undefined, + panel: { webview: { postMessage: (message: { knownTotal: number }) => messages.push(message) } }, messages }; +} suite('jdv row-count hints', () => { - test('applies a hint received before view/open', () => { - const hints = new JdvRowCountHints(); - hints.hint('session-a', 'view-1', 3, 42); - const applied: number[] = []; - - assert.equal(hints.open('session-a', 'view-1', 3, nrow => applied.push(nrow)), 42); - hints.hint('session-a', 'view-1', 3, 43); - assert.deepEqual(applied, [43]); - }); - - test('applies a hint received after view/open', () => { - const hints = new JdvRowCountHints(); - const applied: number[] = []; - - assert.equal(hints.open('session-a', 'view-1', 1, nrow => applied.push(nrow)), undefined); - hints.hint('session-a', 'view-1', 1, 9); - assert.deepEqual(applied, [9]); - assert.equal(hints.open('session-a', 'view-1', 1, nrow => applied.push(nrow)), 9); - }); - - test('replaces the active generation and ignores stale hints and closes', () => { - const hints = new JdvRowCountHints(); - const oldGeneration: number[] = []; - const newGeneration: number[] = []; - - hints.open('session-a', 'view-1', 1, nrow => oldGeneration.push(nrow)); - hints.open('session-a', 'view-1', 2, nrow => newGeneration.push(nrow)); - hints.hint('session-a', 'view-1', 1, 100); - hints.close('session-a', 'view-1', 1); - hints.hint('session-a', 'view-1', 2, 200); - - assert.deepEqual(oldGeneration, []); - assert.deepEqual(newGeneration, [200]); - }); - - test('closing an older active generation releases its callback while retaining newer pending data', () => { - const hints = new JdvRowCountHints(); - const applied: number[] = []; - hints.open('session-a', 'view-1', 1, nrow => applied.push(nrow)); - hints.hint('session-a', 'view-1', 2, 20); - hints.close('session-a', 'view-1', 1); - hints.hint('session-a', 'view-1', 1, 10); - - assert.equal(applied.length, 0); - assert.equal(hints.open('session-a', 'view-1', 2, nrow => applied.push(nrow)), 20); - }); - - test('a close tombstone blocks same-generation delayed open and hints', () => { - const hints = new JdvRowCountHints(); - const applied: number[] = []; - - hints.close('session-a', 'view-1', 7); - hints.hint('session-a', 'view-1', 7, 70); - assert.equal(hints.open('session-a', 'view-1', 7, nrow => applied.push(nrow)), undefined); - hints.hint('session-a', 'view-1', 7, 71); - assert.equal(applied.length, 0); - - hints.hint('session-a', 'view-1', 8, 80); - assert.equal(hints.open('session-a', 'view-1', 8, nrow => applied.push(nrow)), 80); - }); - - test('isolates scopes even when view and generation match', () => { - const hints = new JdvRowCountHints(); - hints.hint('session-a/socket-a', 'view-1', 1, 11); - hints.hint('session-b/socket-b', 'view-1', 1, 22); - hints.clear('session-a/socket-a'); - - assert.equal(hints.open('session-a/socket-a', 'view-1', 1, () => undefined), undefined); - assert.equal(hints.open('session-b/socket-b', 'view-1', 1, () => undefined), 22); - }); - - test('bounds pending hints and preserves live subscriptions', () => { - const hints = new JdvRowCountHints(2, 8); - const applied: number[] = []; - hints.open('session-a', 'live', 1, nrow => applied.push(nrow)); - hints.hint('session-a', 'one', 1, 1); - hints.hint('session-a', 'two', 1, 2); - hints.hint('session-a', 'three', 1, 3); - - assert.equal(hints.open('session-a', 'one', 1, () => undefined), undefined); - assert.equal(hints.open('session-a', 'two', 1, () => undefined), 2); - assert.equal(hints.open('session-a', 'three', 1, () => undefined), 3); - hints.hint('session-a', 'live', 1, 99); - assert.deepEqual(applied, [99]); - }); - - test('bounds inactive metadata without evicting open panel callbacks', () => { - const hints = new JdvRowCountHints(4, 1); - const applied: number[] = []; - hints.open('session-a', 'live', 1, nrow => applied.push(nrow)); - hints.hint('session-a', 'other', 1, 2); - hints.hint('session-a', 'another', 1, 3); - hints.hint('session-a', 'live', 1, 10); - - assert.deepEqual(applied, [10]); + test('takes a hint received before view/open', () => { + const host = viewer(), state = panel(); + host.applyJdvRowCount('session-a', hint); + host.bindRowCount(state); + assert.equal(state.knownTotal, 1000); + }); + + test('updates an open panel when its hint arrives later', () => { + const host = viewer(), state = panel(); + host.panels.set(JSON.stringify(['session-a', 'v-1']), state); + host.applyJdvRowCount('session-a', hint); + assert.equal(state.knownTotal, 1000); + assert.equal(state.messages[0].knownTotal, 1000); + }); + + test('does not attach another scope or generation to the panel', () => { + const host = viewer(), state = panel(); + host.panels.set(JSON.stringify(['session-a', 'v-1']), state); + host.applyJdvRowCount('session-b', hint); + host.applyJdvRowCount('session-a', { ...hint, address: 'socket-b' }); + host.applyJdvRowCount('session-a', { ...hint, generation: 2 }); + host.bindRowCount(state); + assert.equal(state.knownTotal, undefined); + assert.equal(state.messages.length, 0); + }); + + test('formats unknown, known and complete stream counts', () => { + const format = vm.runInNewContext(getDataViewerScript() + ';formatSequentialRowCount') as + (loaded: number, knownTotal: number | undefined, done: boolean) => string; + assert.equal(format(2000, undefined, false), '2,000 / ??? rows'); + assert.equal(format(2000, 1000000, false), '2,000 / 1,000,000 rows'); + assert.equal(format(1000000, 1000000, true), '1,000,000 rows'); }); }); diff --git a/src/test/suite/dataViewer.test.ts b/src/test/suite/dataViewer.test.ts index fa2e914a2..6d93fb305 100644 --- a/src/test/suite/dataViewer.test.ts +++ b/src/test/suite/dataViewer.test.ts @@ -18,8 +18,6 @@ interface ViewerScript { saveViewerState(): void; getAgTheme(): unknown; setViewerSizingMode(mode: 'fit' | 'content'): void; - formatSequentialRowCount(loaded: number, knownTotal: number | undefined, complete: boolean): string; - sequentialRowCountTitle(knownTotal: number | undefined, notice?: string, complete?: boolean): string; } function viewerScript(saved?: SavedViewer, gridState: GridState = {}, light = false): { @@ -32,8 +30,7 @@ function viewerScript(saved?: SavedViewer, gridState: GridState = {}, light = fa let appliedColumns: ColumnState[] = []; let contentSizingCalls = 0; const script = vm.runInNewContext(`${getDataViewerScript()} - ({ prepareViewerColumns, prepareViewerRows, getViewerGridOptions, saveViewerState, getAgTheme, setViewerSizingMode, - formatSequentialRowCount, sequentialRowCountTitle })`, { + ({ prepareViewerColumns, prepareViewerRows, getViewerGridOptions, saveViewerState, getAgTheme, setViewerSizingMode })`, { vscode: { getState: () => saved, setState: (value: SavedViewer) => { stored = value; }, @@ -181,19 +178,6 @@ suite('Data viewer', () => { assert.strictEqual(state?.rowSelection, undefined); }); - test('formats sequential row counts while the total is unknown, known, and finished', () => { - const script = viewerScript().script; - assert.strictEqual(script.formatSequentialRowCount(2000, undefined, false), '2,000 / ??? rows'); - assert.strictEqual(script.formatSequentialRowCount(2000, 1000000, false), '2,000 / 1,000,000 rows'); - assert.strictEqual(script.formatSequentialRowCount(1000000, 1000000, true), '1,000,000 rows'); - assert.match(script.sequentialRowCountTitle(undefined), /total is unknown until the stream ends/); - assert.match(script.sequentialRowCountTitle(undefined, undefined, true), /All rows have been read/); - assert.match(script.sequentialRowCountTitle(1000000), /total is known from the R object/); - assert.match(script.sequentialRowCountTitle(1000000, - 'Rows are read from the Arrow stream as you scroll. The total is unknown until the stream ends.'), - /total is known from the R object/); - }); - test('uses native date, datetime, boolean and bigint types in both viewer modes', () => { const defs: ColDef[] = [ ...columns(), From 39cb0f40eb4afe93e53580817fb03128e4375557 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 00:47:52 +0000 Subject: [PATCH 32/38] fix: bound inline previews by the encoded event size --- sess/R/interactive.R | 77 ++++++++++--------- sess/inst/tinytest/test-interactive-handles.R | 57 ++++++++++++-- src/interactive/agent.ts | 3 + src/interactive/inlineTable.ts | 9 ++- src/interactive/jdvTable.ts | 4 +- src/test/node/interactiveRuntime.test.ts | 21 +++++ src/test/node/jdvTable.test.ts | 19 +++++ 7 files changed, 143 insertions(+), 47 deletions(-) diff --git a/sess/R/interactive.R b/sess/R/interactive.R index 213af5d91..8e4b3be12 100644 --- a/sess/R/interactive.R +++ b/sess/R/interactive.R @@ -1,15 +1,21 @@ # Rich events use pure R IPC; small stdout markers preserve console ordering. +.interactive_event_json <- function(type, data) { + id <- .sess_env$interactive_id + as.character(jsonlite::toJSON( + c(list(type = type, executionId = if (is.null(id)) "" else id), data), + auto_unbox = TRUE, null = "null", na = "null", digits = NA + )) +} + .interactive_event <- function(type, data = list()) { if (!identical(.sess_env$interactive_pid, Sys.getpid())) return(invisible(NULL)) id <- .sess_env$interactive_id - event <- c(list(type = type, executionId = if (is.null(id)) "" else id), data) - json <- as.character(jsonlite::toJSON( - event, auto_unbox = TRUE, null = "null", na = "null", digits = NA - )) + json <- .interactive_event_json(type, data) # Bound encoded events before writing to the agent transport. # Agent-side display limits run only after parsing and cannot protect it from # oversized HTML, wide tables, or strings enlarged by JSON escaping. if (nchar(json, type = "bytes") > 2 * 1024 * 1024) { + if (!is.null(data$handleId)) .interactive_handle_close(data$handleId) json <- as.character(jsonlite::toJSON(list( type = "truncated", executionId = if (is.null(id)) "" else id, message = "Rich output exceeds 2 MiB; display a smaller preview or save it to a file." @@ -381,6 +387,8 @@ interactive_execute <- function(id, code, source = NULL) { .interactive_handle_limit <- 64L .interactive_page_bytes <- 256L * 1024L +# Leave 512 KiB for adapter normalization and agent event metadata. +.interactive_preview_bytes <- 1536L * 1024L .interactive_jdv_usable <- function() { !identical(getOption("sess.dataviewer", "auto"), "sess") && @@ -423,27 +431,35 @@ interactive_execute <- function(id, code, source = NULL) { # Capture `value` and read its descriptor and preview pages. Returns NULL, leaving no # handle behind, when jdv cannot represent the value; the caller then uses the # dataview registry. -.interactive_handle_preview <- function(value, kind) { +.interactive_handle_preview <- function(value, kind, data = list()) { if (!.interactive_jdv_usable()) return(NULL) handle <- NULL tryCatch({ handle <- jdv::jdv_handle(value, mode = "paged") descriptor <- jdv::jdv_handle_descriptor(handle, max_bytes = .interactive_page_bytes) + .interactive_handle_store() + id <- paste0("h", .sess_env$interactive_handle_seq + 1L) page <- if (kind == "table") { - pages <- jdv::jdv_handle_request(handle, "grid/rows", list(offset = 0L, limit = 20L), - max_bytes = .interactive_page_bytes) - parsed <- jsonlite::fromJSON(pages, simplifyVector = FALSE) - offset <- length(parsed$data$source_row) - while (offset > 0L && offset < parsed$total && offset < 20L) { + fits <- function(pages) { + event <- c(list(kind = kind, handleId = id, descriptor = descriptor, page = pages), data) + bytes <- nchar(.interactive_event_json("display", event), type = "bytes") + bytes <= .interactive_preview_bytes + } + # Preserve the table and viewer control when the printed fallback is too large. + if (!fits(character())) data$printedText <- NULL + pages <- character() + offset <- 0L + repeat { next_page <- jdv::jdv_handle_request( handle, "grid/rows", list(offset = offset, limit = 20L - offset), max_bytes = .interactive_page_bytes ) - next_parsed <- jsonlite::fromJSON(next_page, simplifyVector = FALSE) - returned <- length(next_parsed$data$source_row) - if (returned == 0L) break + if (!fits(c(pages, next_page))) break pages <- c(pages, next_page) + parsed <- jsonlite::fromJSON(next_page, simplifyVector = FALSE) + returned <- length(parsed$data$source_row) offset <- offset + returned + if (returned == 0L || offset >= parsed$total || offset >= 20L) break } pages } else { @@ -461,7 +477,7 @@ interactive_execute <- function(id, code, source = NULL) { while (length(.sess_env$interactive_handle_order) > .interactive_handle_limit) { .interactive_handle_close(.sess_env$interactive_handle_order[[1L]]) } - list(handleId = id, descriptor = descriptor, page = page) + c(list(handleId = id, descriptor = descriptor, page = page), data) }, error = function(e) { if (!is.null(handle)) try(jdv::jdv_handle_close(handle), silent = TRUE) NULL @@ -516,22 +532,21 @@ handle_jdv_open_viewer <- function(params) { .interactive_rich_value <- function(value) { if (dataview_is_table(value)) { snapshot <- .interactive_table_snapshot(value) + printed <- .interactive_table_text(snapshot$value) + if (snapshot$truncated && !is.null(printed$printedText)) { + note <- sprintf( + "\n[Snapshot: first %s of %s rows. Open Data viewer for the full table.]\n", + format(nrow(snapshot$value), big.mark = ",", scientific = FALSE, trim = TRUE), + format(snapshot$source_rows, big.mark = ",", scientific = FALSE, trim = TRUE) + ) + printed$printedText <- paste0(printed$printedText, note) + } preview <- if (is.data.frame(value) || is.matrix(value)) { - .interactive_handle_preview(value, "table") + .interactive_handle_preview(value, "table", + c(list(sourceRows = snapshot$source_rows), printed)) } if (!is.null(preview)) { - printed <- .interactive_table_text(snapshot$value) - if (snapshot$truncated && !is.null(printed$printedText)) { - note <- sprintf( - "\n[Snapshot: first %s of %s rows. Open Data viewer for the full table.]\n", - format(nrow(snapshot$value), big.mark = ",", scientific = FALSE, trim = TRUE), - format(snapshot$source_rows, big.mark = ",", scientific = FALSE, trim = TRUE) - ) - printed$printedText <- paste0(printed$printedText, note) - } - .interactive_event("display", c(list(kind = "table", handleId = preview$handleId, - descriptor = preview$descriptor, page = preview$page, - sourceRows = snapshot$source_rows), printed)) + .interactive_event("display", c(list(kind = "table"), preview)) return(TRUE) } registration <- dataview_register(snapshot$value) @@ -540,19 +555,11 @@ handle_jdv_open_viewer <- function(params) { startRow = 0L, endRow = 20L, formatNumbers = TRUE, sortModel = list(), filterModel = list())) - printed <- .interactive_table_text(snapshot$value) full_view <- NULL if (snapshot$truncated) { # Retain the full object without copying it. The cell remains a snapshot; # the expanded viewer can reflect later reference edits to this object. full_view <- dataview_register(value, live = TRUE)$view_id - if (!is.null(printed$printedText)) { - note <- - sprintf("\n[Snapshot: first %s of %s rows. Open Data viewer for the full table.]\n", - format(metadata$totalRows, big.mark = ",", scientific = FALSE, trim = TRUE), - format(snapshot$source_rows, big.mark = ",", scientific = FALSE, trim = TRUE)) - printed$printedText <- paste0(printed$printedText, note) - } } .interactive_event("display", c(list(kind = "table", viewId = registration$view_id, fullViewId = full_view, diff --git a/sess/inst/tinytest/test-interactive-handles.R b/sess/inst/tinytest/test-interactive-handles.R index ac1a9c761..83a949c01 100644 --- a/sess/inst/tinytest/test-interactive-handles.R +++ b/sess/inst/tinytest/test-interactive-handles.R @@ -37,8 +37,7 @@ single_event <- jsonlite::fromJSON(jsonlite::toJSON( ), simplifyVector = FALSE) expect_identical(single_event$page, table$page) -# A byte-limited page is refilled so the renderer can advance by its page size -# without skipping rows. +# Byte-limited pages stay unchanged and stop before the complete event budget. numeric <- rep(pi, 150) numeric[[3L]] <- Inf wide <- data.frame( @@ -51,7 +50,7 @@ expect_true(length(wide_preview$page) > 1L) wide_entry <- ns$.interactive_handle_entry(wide_preview$handleId) expected_pages <- character() offset <- 0L -while (offset < 20L) { +while (length(expected_pages) < length(wide_preview$page)) { raw_page <- jdv::jdv_handle_request( wide_entry$handle, "grid/rows", list(offset = offset, limit = 20L - offset), max_bytes = ns$.interactive_page_bytes @@ -69,7 +68,8 @@ multi_event <- jsonlite::fromJSON(jsonlite::toJSON( expect_identical(unlist(multi_event$page), expected_pages) wide_pages <- lapply(wide_preview$page, jsonlite::fromJSON, simplifyVector = FALSE) wide_rows <- unlist(lapply(wide_pages, function(page) page$data$source_row)) -expect_identical(wide_rows, 0:19) +expect_true(length(wide_rows) < 20L) +expect_identical(wide_rows, seq_len(length(wide_rows)) - 1L) factor_values <- sum(vapply(wide_pages, function(page) { length(page$data$columns[[3L]]$data$values) }, 0L)) @@ -78,11 +78,52 @@ row_labels <- unlist(lapply(wide_pages, function(page) page$data$row_labels)) empty_errors <- vapply(wide_pages, function(page) { identical(names(page$data$columns[[1L]]$errors), character()) }, logical(1L)) -expect_identical(factor_values, 20L) -expect_identical(missing_values, rep(c(TRUE, FALSE), 10L)) -expect_identical(row_labels, paste0("row-", 0:19)) +expect_identical(factor_values, length(wide_rows)) +expect_identical(missing_values, rep(c(TRUE, FALSE), length.out = length(wide_rows))) +expect_identical(row_labels, paste0("row-", wide_rows)) expect_true(all(empty_errors)) +# Capture the actual display event, including printed text and JSON escaping. +local({ + original_send <- ns$rpc_send + fields <- c("interactive_pid", "interactive_id", "interactive_managed") + saved <- mget(fields, env, ifnotfound = list(NULL)) + captured <- NULL + unlockBinding("rpc_send", ns) + assign("rpc_send", function(method, params) captured <<- params, envir = ns) + lockBinding("rpc_send", ns) + on.exit({ + unlockBinding("rpc_send", ns) + assign("rpc_send", original_send, envir = ns) + lockBinding("rpc_send", ns) + list2env(saved, env) + }, add = TRUE) + env$interactive_pid <- Sys.getpid() + env$interactive_id <- "preview-budget" + env$interactive_managed <- FALSE + expect_true(ns$.interactive_rich_value(wide)) + expect_identical(captured$type, "display") + expect_identical(captured$kind, "table") + encoded <- jsonlite::toJSON(captured, auto_unbox = TRUE, null = "null", digits = NA) + expect_true(nchar(encoded, type = "bytes") <= ns$.interactive_preview_bytes) + expect_true(is.character(captured$handleId)) + + # The printed fallback can be omitted; a valid table with no inline rows is OK. + oversized <- preview(wide, "table", list(printedText = strrep("x", 2L * 1024L * 1024L))) + expect_null(oversized$printedText) + empty <- preview(wide, "table", list(printedText = strrep("x", 1530L * 1024L))) + expect_identical(length(empty$page), 0L) + ns$.interactive_event("display", c(list(kind = "table"), empty)) + expect_identical(captured$type, "display") + expect_true(nchar(ns$.interactive_event_json("display", empty), type = "bytes") < 2L * 1024L * 1024L) + + # The last-resort truncation path releases the captured handle. + ns$.interactive_event("display", c(list(kind = "table"), empty, + list(extra = strrep("x", 2L * 1024L * 1024L)))) + expect_identical(captured$type, "truncated") + expect_error(ns$.interactive_handle_entry(empty$handleId), "no longer available") +}) + # Paging, sorting and filtering run in R through the same handle; ids are zero-based rows. sorted <- request(table$handleId, "grid/rows", list( offset = 0L, limit = 3L, sort = list(list(col = 0L, desc = TRUE)), @@ -103,6 +144,7 @@ expect_identical(request(tree$handleId, "object/children", list( # Opening an Interactive handle registers the selected value through jdv's handle API. # The custom [[ method must not run while opening a nested table. local({ + started_session <- identical(jdv::jdv_connection()$state, "disconnected") original_connect <- ns$.jdv_connect review_handle_id <- NULL registered_views <- list() @@ -120,6 +162,7 @@ local({ ), auto_unbox = TRUE)), silent = TRUE) } jdv::jdv_notifications() + if (started_session) jdv::jdv_disconnect() unlockBinding(".jdv_connect", ns) assign(".jdv_connect", original_connect, envir = ns) lockBinding(".jdv_connect", ns) diff --git a/src/interactive/agent.ts b/src/interactive/agent.ts index 8642f17b9..ea4a6471d 100644 --- a/src/interactive/agent.ts +++ b/src/interactive/agent.ts @@ -464,6 +464,9 @@ export class SessionAgent { delete display.data; } if (Buffer.byteLength(JSON.stringify(display)) > 2 * 1024 * 1024) { + if (typeof data.handleId === 'string') { + void this.backend.inspect({ method: 'jdv_handle_close', params: { handle_id: data.handleId }, timeout: 5000 }).catch(() => undefined); + } this.event('truncated', { message: 'Rich output exceeds 2 MiB; display a smaller preview or save it to a file.' }, executionId); return; } diff --git a/src/interactive/inlineTable.ts b/src/interactive/inlineTable.ts index 8197861a4..d17604a10 100644 --- a/src/interactive/inlineTable.ts +++ b/src/interactive/inlineTable.ts @@ -79,9 +79,9 @@ export function createInlineTable(parent: HTMLElement, toolbar: HTMLElement, sta const filtersButton = button('Filter columns', 'filter', () => { state.filterOpen = !state.filterOpen; remember(state); update(); }, 'Filters'); const reset = button('Reset table view', 'reset', () => { if (!active) { return; } - // Restoring saved output is local, and invalidates even an in-flight reply. + // Reset invalidates in-flight replies; a partial saved preview is refilled. pending = undefined; state = initial(); error = ''; remember(state); - draw(); buildFilters(true); update(); tableHost.scrollLeft = 0; tableHost.scrollTop = 0; + draw(); buildFilters(true); update(); completePreview(); tableHost.scrollLeft = 0; tableHost.scrollTop = 0; }, 'Reset'); reset.title = 'Restore the saved preview, first page, original column order, and clear sorting and filters'; toolbar.append(filtersButton, reset, pager); @@ -217,7 +217,10 @@ export function createInlineTable(parent: HTMLElement, toolbar: HTMLElement, sta status.title = state.live && !textMode ? 'Rows are fetched on demand from the original object. Reference edits may appear. Reset restores the saved preview.' : hasFull ? `${tableSnapshotSummary(original)}. Browse beyond these rows, sort or filter to retrieve live data.` : ''; } - draw(); buildFilters(); update(); + function completePreview(): void { + if (available && !state.live && state.data.previewPartial === true) { request(0); } + } + draw(); buildFilters(); update(); completePreview(); return { setTextMode(text) { textMode = text; update(); }, reply(message) { diff --git a/src/interactive/jdvTable.ts b/src/interactive/jdvTable.ts index 97f088e35..ffd0a3425 100644 --- a/src/interactive/jdvTable.ts +++ b/src/interactive/jdvTable.ts @@ -45,14 +45,14 @@ export function normalizeJdvDisplay(data: Record): Record parseJson(page, 'page')); const rows: Record[] = []; - let totalRows = 0; + let totalRows = descriptor.nrow ?? 0; for (const raw of pages) { const next = gridPageFromRows(raw, rows.length, TABLE_PAGE_SIZE - rows.length); rows.push(...next.rows); totalRows = next.totalRows; if (rows.length >= TABLE_PAGE_SIZE) { break; } } - return { ...rest, descriptor, columns: gridColumnDefs(descriptor), rows, totalRows }; + return { ...rest, descriptor, columns: gridColumnDefs(descriptor), rows, totalRows, previewPartial: rows.length < Math.min(TABLE_PAGE_SIZE, totalRows) }; } if (data.kind === 'list' && descriptor.kind === 'tree') { const adapter = new TreeAdapter(descriptor, 'List'); diff --git a/src/test/node/interactiveRuntime.test.ts b/src/test/node/interactiveRuntime.test.ts index 98f0d7016..e273d909f 100644 --- a/src/test/node/interactiveRuntime.test.ts +++ b/src/test/node/interactiveRuntime.test.ts @@ -407,6 +407,27 @@ sess::display("display-${cell}", "text/plain"); cat("stderr-${cell}\\n", file=st await assert.rejects(queryJdvTablePage(table.data, { start: 0, size: 20 }, inspect), /no longer available/); }); + test('inline jdv wide tables survive the event limit and page without gaps', async function () { + if (!(await jdvAvailable())) { this.skip(); return; } + const id = await submit('wide <- data.frame(id = 1:150, text = rep(strrep("a", 100000), 150)); wide'); + assert.strictEqual((await finished(id)).state, 'success'); + const table = events.find(event => event.executionId === id && event.data.kind === 'table'); assert.ok(table); + assert.ok(isJdvOutput(table.data)); + assert.equal(events.some(event => event.executionId === id && event.type === 'truncated'), false); + assert.ok(Buffer.byteLength(JSON.stringify(table.data)) < 2 * 1024 * 1024); + assert.equal(table.data.previewPartial, true); + assert.ok((table.data.rows as unknown[]).length < 20); + for (const size of [20, 100]) { + const ids: string[] = []; + for (let start = 0; start < 150; start += size) { + const page = await queryJdvTablePage(table.data, { start, size }, inspect); + assert.equal(page.startRow, start); + ids.push(...(page.rows as Record[]).map(row => row['1'])); + } + assert.deepEqual(ids, Array.from({ length: 150 }, (_, i) => String(i + 1))); + } + }); + test('inline jdv tables bound a huge table to its first page', async function () { if (!(await jdvAvailable())) { this.skip(); return; } const id = await submit(`n <- 832976871L diff --git a/src/test/node/jdvTable.test.ts b/src/test/node/jdvTable.test.ts index d6e410f02..26edea4b7 100644 --- a/src/test/node/jdvTable.test.ts +++ b/src/test/node/jdvTable.test.ts @@ -53,6 +53,25 @@ suite('Interactive jdv tables', () => { assert.equal(rows[3]['1'], '4'); }); + test('marks short and empty previews for completion from page zero', async () => { + for (const page of [[], [rowsPage(0, 2)]]) { + const data = normalizeJdvDisplay(display({ page })); + assert.equal(data.totalRows, 50); + assert.equal(data.previewPartial, true); + const offsets: number[] = []; + const result = await queryJdvTablePage(data, { start: 0, size: 20 }, request => { + const params = request.params.params as { offset: number; limit: number }; + offsets.push(params.offset); + return Promise.resolve({ json: rowsPage(params.offset, params.limit) }); + }); + assert.deepEqual(offsets, [0]); + assert.deepEqual((result.rows as Record[]).map(row => row.__sourceRow), + Array.from({ length: 20 }, (_, i) => i)); + } + assert.equal(normalizeJdvDisplay(display({ page: rowsPage(0, 20) })).previewPartial, false); + assert.equal(normalizeJdvDisplay(display({ page: rowsPage(0, 2, 2) })).previewPartial, false); + }); + test('offers the operators of the column type', () => { const columns = normalizeJdvDisplay(display()).columns as TableColumn[]; assert.deepEqual(columnOperators(columns[1]).slice(-2), ['blank', 'notBlank']); From de07a676c7af729ba8e30ea99a3091b34d885048 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 00:52:42 +0000 Subject: [PATCH 33/38] style: fix lintr warnings in interactive handle tests --- sess/inst/tinytest/test-interactive-handles.R | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sess/inst/tinytest/test-interactive-handles.R b/sess/inst/tinytest/test-interactive-handles.R index 83a949c01..6cada9e74 100644 --- a/sess/inst/tinytest/test-interactive-handles.R +++ b/sess/inst/tinytest/test-interactive-handles.R @@ -69,7 +69,7 @@ expect_identical(unlist(multi_event$page), expected_pages) wide_pages <- lapply(wide_preview$page, jsonlite::fromJSON, simplifyVector = FALSE) wide_rows <- unlist(lapply(wide_pages, function(page) page$data$source_row)) expect_true(length(wide_rows) < 20L) -expect_identical(wide_rows, seq_len(length(wide_rows)) - 1L) +expect_identical(wide_rows, seq_along(wide_rows) - 1L) factor_values <- sum(vapply(wide_pages, function(page) { length(page$data$columns[[3L]]$data$values) }, 0L)) @@ -115,11 +115,12 @@ local({ expect_identical(length(empty$page), 0L) ns$.interactive_event("display", c(list(kind = "table"), empty)) expect_identical(captured$type, "display") - expect_true(nchar(ns$.interactive_event_json("display", empty), type = "bytes") < 2L * 1024L * 1024L) + empty_json <- ns$.interactive_event_json("display", empty) + expect_true(nchar(empty_json, type = "bytes") < 2L * 1024L * 1024L) # The last-resort truncation path releases the captured handle. ns$.interactive_event("display", c(list(kind = "table"), empty, - list(extra = strrep("x", 2L * 1024L * 1024L)))) + list(extra = strrep("x", 2L * 1024L * 1024L)))) expect_identical(captured$type, "truncated") expect_error(ns$.interactive_handle_entry(empty$handleId), "no longer available") }) From cbf78d5655e317c43aca58731a435fc5f1913488 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 11:15:55 +0000 Subject: [PATCH 34/38] perf: avoid repeated buffer copies when receiving jdv frames Large jdv responses arrive in many socket chunks. Concatenating the whole receive buffer on every chunk made reception quadratic in the frame size. Hold chunks until a JSON line or Arrow body can complete and concatenate them once. --- src/jdvTransport.ts | 40 ++++++++--- src/test/node/jdvTransport.test.ts | 105 +++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 9 deletions(-) create mode 100644 src/test/node/jdvTransport.test.ts diff --git a/src/jdvTransport.ts b/src/jdvTransport.ts index ca978b447..e834005de 100644 --- a/src/jdvTransport.ts +++ b/src/jdvTransport.ts @@ -72,7 +72,8 @@ function validIdentity(value: unknown): boolean { /** One producer connection. Implements the host obligations of the jdv transport. */ export class JdvPeer { - private buffer: Buffer = Buffer.alloc(0); + private chunks: Buffer[] = []; + private buffered = 0; private body?: { header: JsonObject; length: number; timer: NodeJS.Timeout }; private limits?: ProducerLimits; private readonly outbound: OutboundFrame[] = []; @@ -144,36 +145,54 @@ export class JdvPeer { pending.reject(new JdvRequestError('view_not_found', reason)); } this.pending.clear(); + this.chunks = []; + this.buffered = 0; this.socket.destroy(); this.handlers.closed(this, reason); } private receive(chunk: Buffer): void { - this.buffer = this.buffer.length ? Buffer.concat([this.buffer, chunk]) : chunk; + if (this.closed) { + return; + } + // Hold chunks until a frame can complete, so a large frame split across + // many chunks is copied once instead of once per chunk. + this.chunks.push(chunk); + this.buffered += chunk.length; + if (this.body ? this.buffered < this.body.length : chunk.indexOf(0x0a) < 0) { + if (!this.body && this.buffered > MAX_JSON_LINE_BYTES) { + this.close('A jdv message exceeds the size limit.'); + } + return; + } + let buffer = this.chunks.length === 1 ? this.chunks[0] : Buffer.concat(this.chunks, this.buffered); try { while (!this.closed) { if (this.body) { - if (this.buffer.length < this.body.length) { + if (buffer.length < this.body.length) { return; } const { header, length, timer } = this.body; clearTimeout(timer); this.body = undefined; // Copy so a retained page does not keep the receive buffer alive. - const body = new Uint8Array(this.buffer.subarray(0, length)); - this.buffer = this.buffer.subarray(length); + const body = new Uint8Array(buffer.subarray(0, length)); + buffer = buffer.subarray(length); this.dispatch(header, body); continue; } - const newline = this.buffer.indexOf(0x0a); + const newline = buffer.indexOf(0x0a); if (newline < 0) { - if (this.buffer.length > MAX_JSON_LINE_BYTES) { + if (buffer.length > MAX_JSON_LINE_BYTES) { throw new Error('A jdv message exceeds the size limit.'); } return; } - const line = this.buffer.subarray(0, newline); - this.buffer = this.buffer.subarray(newline + 1); + if (newline > MAX_JSON_LINE_BYTES) { + throw new Error('A jdv message exceeds the size limit.'); + } + const line = buffer.subarray(0, newline); + buffer = buffer.subarray(newline + 1); const message = JSON.parse(line.toString('utf8')) as unknown; if (!isObject(message)) { throw new Error('Invalid jdv message.'); @@ -195,6 +214,9 @@ export class JdvPeer { } } catch (e) { this.close(e instanceof Error ? e.message : String(e)); + } finally { + this.chunks = !this.closed && buffer.length ? [buffer] : []; + this.buffered = this.closed ? 0 : buffer.length; } } diff --git a/src/test/node/jdvTransport.test.ts b/src/test/node/jdvTransport.test.ts new file mode 100644 index 000000000..d2f5cb81b --- /dev/null +++ b/src/test/node/jdvTransport.test.ts @@ -0,0 +1,105 @@ +import * as assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { Socket } from 'node:net'; +import { JdvPeer } from '../../jdvTransport'; +import { JsonObject } from '../../jdvAdapter'; + +class FakeSocket extends EventEmitter { + destroyed = false; + write(_bytes: Buffer): boolean { return true; } + destroy(): void { this.destroyed = true; } + setNoDelay(): this { return this; } +} + +function connection() { + const socket = new FakeSocket(); + const reasons: string[] = []; + const notifications: JsonObject[] = []; + const peer = new JdvPeer(socket as unknown as Socket, { + notification: (_peer, _method, params) => notifications.push(params), + closed: (_peer, reason) => reasons.push(reason), + }); + socket.emit('data', Buffer.from(JSON.stringify({ method: 'hello', params: { + protocol: 1, session: 'test', producer: { name: 'test', version: '1' }, + limits: { max_unacked_frames: 32, max_unacked_bytes: 10485760, queue_messages: 32, + queue_bytes: 10485760, max_request_bytes: 10485760, max_message_bytes: 10485760 }, + } }) + '\n')); + return { socket, peer, reasons, notifications }; +} + +suite('jdv transport framing', () => { + test('receives a large UTF-8 JSON line in tiny chunks', async () => { + const { socket, peer } = connection(); + try { + const response = peer.request('grid/rows', {}); + const value = 'λ🙂'.repeat(350000); + const frame = Buffer.from(JSON.stringify({ id: 'vscode-r-1', result: { value } }) + '\n'); + for (let offset = 0; offset < frame.length; offset += 17) { + socket.emit('data', frame.subarray(offset, offset + 17)); + } + assert.deepEqual(await response, { value }); + assert.equal(peer.isClosed, false); + } finally { peer.close('test complete'); } + }); + + test('dispatches several messages and retains a partial next line', () => { + const { socket, peer, notifications } = connection(); + try { + const line = (value: number) => JSON.stringify({ method: 'view/open', params: { value } }) + '\n'; + const third = line(3); + socket.emit('data', Buffer.from(line(1) + line(2) + third.slice(0, 10))); + assert.deepEqual(notifications, [{ value: 1 }, { value: 2 }]); + socket.emit('data', Buffer.from(third.slice(10))); + assert.deepEqual(notifications, [{ value: 1 }, { value: 2 }, { value: 3 }]); + } finally { peer.close('test complete'); } + }); + + test('receives Arrow headers and bodies across every split and in one-byte chunks', async () => { + const body = Buffer.from([0, 10, 255, 13, 123, 34, 0, 10]); + const result = { view: 'v', generation: 1, transfer: { encoding: 'arrow-ipc', byte_length: body.length } }; + const header = Buffer.from(JSON.stringify({ id: 'vscode-r-1', result }) + '\n'); + const following = Buffer.from(JSON.stringify({ id: 'vscode-r-2', result: 'next' }) + '\n'); + const frame = Buffer.concat([header, body, following]); + const deliveries: Buffer[][] = []; + for (let split = 1; split < frame.length; split++) { + deliveries.push([frame.subarray(0, split), frame.subarray(split)]); + } + deliveries.push(Array.from(frame, (_value, index) => frame.subarray(index, index + 1))); + for (const chunks of deliveries) { + const { socket, peer } = connection(); + try { + const response = peer.requestWithBody('stream/next', { view: 'v', generation: 1 }); + const next = peer.request('grid/rows', {}); + for (const chunk of chunks) { socket.emit('data', chunk); } + assert.deepEqual(await response, { result, body: new Uint8Array(body) }); + assert.equal(await next, 'next'); + assert.equal(peer.isClosed, false); + } finally { peer.close('test complete'); } + } + }); + + test('closes on an oversized unterminated line or a complete oversized line', () => { + for (const newline of ['', '\n']) { + const { socket, peer, reasons } = connection(); + const line = Buffer.from(JSON.stringify({ result: 'x'.repeat(10 * 1024 * 1024) }) + newline); + for (let offset = 0; offset < line.length; offset += 16384) { + socket.emit('data', line.subarray(offset, offset + 16384)); + } + assert.equal(peer.isClosed, true); + assert.equal(socket.destroyed, true); + assert.match(reasons[0], /exceeds the size limit/); + } + }); + + test('closes on invalid binary frame lengths and missing response ids', () => { + for (const [id, length] of [['x', -1], ['x', 1.5], ['x', 10 * 1024 * 1024], [null, 1]]) { + const { socket, peer, reasons } = connection(); + socket.emit('data', Buffer.from(JSON.stringify({ id, + result: { transfer: { encoding: 'arrow-ipc', byte_length: length } }, + }) + '\n')); + assert.equal(peer.isClosed, true); + assert.equal(socket.destroyed, true); + assert.match(reasons[0], /Invalid jdv binary frame/); + } + }); +}); From 50626660614142787b972df45ed75d38aaf17471 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 11:15:55 +0000 Subject: [PATCH 35/38] perf: fetch only displayed columns in the data viewer jdv builds every selected column for each grid/rows page, so wide data frames were slow to scroll even with most columns hidden. Send the displayed column positions as cols and reload the infinite cache when a hidden column is shown again. Fall back to all columns when an explicit selection exceeds jdv's page budget, since jdv only shortens pages that select every column. --- src/jdvAdapter.ts | 32 ++++++++++ src/jdvGrid.ts | 42 +++++++++++++ src/jdvViewer.ts | 31 ++-------- src/session.ts | 10 +++ src/test/node/interactiveRuntime.test.ts | 7 +++ src/test/node/jdvAdapter.test.ts | 15 +++++ src/test/node/jdvGrid.test.ts | 79 ++++++++++++++++++++++++ 7 files changed, 191 insertions(+), 25 deletions(-) create mode 100644 src/jdvGrid.ts create mode 100644 src/test/node/jdvGrid.test.ts diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index ae1da919f..b32252abc 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -567,10 +567,38 @@ export interface TableRowsParams extends JsonObject { limit: number; sort: JdvSort[]; filter?: FilterNode; + cols?: number[]; +} + +/** jdv's default `page_columns` limit for an explicit column selection. */ +const MAX_PAGE_COLUMNS = 256; + +/** + * Zero-based positions for the grid fields the webview displays, or undefined to + * request every column. Field ids are one-based positions; "0" is the row label. + */ +function pageColumns(descriptor: JdvDescriptor, fields: unknown): number[] | undefined { + const ncol = descriptor.columns?.length ?? 0; + if (!Array.isArray(fields)) { + return undefined; + } + const cols = new Set(); + for (const field of fields) { + if (field === '0') { + continue; + } + const position = typeof field === 'string' && /^[1-9][0-9]*$/.test(field) ? Number(field) - 1 : -1; + if (position < 0 || position >= ncol) { + return undefined; + } + cols.add(position); + } + return cols.size >= ncol || cols.size > MAX_PAGE_COLUMNS ? undefined : [...cols]; } export function tableRowsParams( descriptor: JdvDescriptor, startRow: number, endRow: number, sortModel: unknown, filterModel: unknown, + columns?: unknown, ): TableRowsParams { const offset = Math.max(0, Math.floor(startRow)); const limit = Math.max(1, Math.floor(endRow) - offset); @@ -585,6 +613,10 @@ export function tableRowsParams( if (filter) { params.filter = filter; } + const cols = pageColumns(descriptor, columns); + if (cols) { + params.cols = cols; + } return params; } diff --git a/src/jdvGrid.ts b/src/jdvGrid.ts new file mode 100644 index 000000000..02d985778 --- /dev/null +++ b/src/jdvGrid.ts @@ -0,0 +1,42 @@ +import { GridPage, JdvDescriptor, JdvRequestError, JsonObject, gridPageFromRows, tableRowsParams } from './jdvAdapter'; + +/** Fill an AG Grid block, retrying oversized column selections with adaptive all-column pages. */ +export async function fetchGridPage( + descriptor: JdvDescriptor, message: JsonObject, + request: (method: string, params: JsonObject) => Promise, +): Promise> { + const startRow = Number(message.startRow ?? 0); + const endRow = Number(message.endRow ?? 0); + // Only the displayed columns; the webview reloads its cache when a hidden column is shown. + let columns = message.columns; + const rows: Record[] = []; + let page: GridPage | undefined; + let offset = startRow; + while (offset < endRow) { + const params = tableRowsParams(descriptor, offset, endRow, message.sortModel, message.filterModel, columns); + let result: unknown; + try { + result = await request('grid/rows', params); + } catch (e) { + // jdv shortens pages only when every column is selected; an explicit + // selection that does not fit fails, so fall back to all columns. + if (params.cols && e instanceof JdvRequestError && e.code === 'limit_exceeded') { + columns = undefined; + continue; + } + throw e; + } + page = gridPageFromRows(result, offset, params.limit); + for (const row of page.rows) { + rows.push(row); + } + offset += page.returned; + if (page.returned === 0 || offset >= page.totalRows) { + break; + } + } + if (!page) { + throw new JdvRequestError('invalid_request', 'Empty page request'); + } + return { rows, totalRows: page.totalRows, totalUnfiltered: page.totalUnfiltered, lastRow: page.totalRows }; +} diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts index adf1fbc7f..1f4e6936c 100644 --- a/src/jdvViewer.ts +++ b/src/jdvViewer.ts @@ -9,9 +9,10 @@ import * as path from 'path'; import * as vscode from 'vscode'; import { Uri, ViewColumn, window } from 'vscode'; import { - GridPage, JdvDescriptor, cellOpenParams, JdvRequestError, JdvTracking, JsonObject, TreeAdapter, canRefresh, gridColumnDefs, - gridPageFromRows, parseDescriptor, parseTracking, revealsPanel, tableRowsParams, + JdvDescriptor, cellOpenParams, JdvRequestError, JdvTracking, JsonObject, TreeAdapter, canRefresh, gridColumnDefs, + parseDescriptor, parseTracking, revealsPanel, } from './jdvAdapter'; +import { fetchGridPage } from './jdvGrid'; import { SequentialStream } from './jdvArrow'; import { JdvHost, JdvPeer } from './jdvTransport'; import { disposeJdvHostIfCurrentAddress } from './jdvHostLifecycle'; @@ -345,7 +346,9 @@ async function handleMessage(state: JdvPanel, webview: vscode.Webview, message: if (message.action === 'init') { reply(true, { columns: gridColumnDefs(state.descriptor), totalRows: state.descriptor.nrow ?? 0 }); } else if (message.action === 'page') { - reply(true, await fetchGridPage(state, message)); + reply(true, await fetchGridPage( + state.descriptor, message, (method, params) => state.peer.request(method, params), + )); } else { reply(false, undefined, `Unsupported dataview action: ${String(message.action)}`); } @@ -490,28 +493,6 @@ async function trackView( } } -/** Fetch one AG Grid block, issuing further pages if jdv clamps the page size. */ -async function fetchGridPage(state: JdvPanel, message: JsonObject): Promise> { - const startRow = Number(message.startRow ?? 0); - const endRow = Number(message.endRow ?? 0); - const descriptor = state.descriptor; - let page: GridPage | undefined; - let offset = startRow; - while (offset < endRow) { - const params = tableRowsParams(descriptor, offset, endRow, message.sortModel, message.filterModel); - const next = gridPageFromRows(await state.peer.request('grid/rows', params), offset, params.limit); - page = page ? { ...next, rows: [...page.rows, ...next.rows] } : next; - offset += next.returned; - if (next.returned === 0 || offset >= next.totalRows) { - break; - } - } - if (!page) { - throw new JdvRequestError('invalid_request', 'Empty page request'); - } - return { rows: page.rows, totalRows: page.totalRows, totalUnfiltered: page.totalUnfiltered, lastRow: page.totalRows }; -} - /** Serve an AG Grid block from received Arrow pages, reading the stream forward as needed. */ async function fetchSequentialPage( state: JdvPanel, stream: SequentialStream, message: JsonObject, diff --git a/src/session.ts b/src/session.ts index 6b78d313a..42f2e225e 100644 --- a/src/session.ts +++ b/src/session.ts @@ -2486,6 +2486,10 @@ export async function getTableHtml( endRow: params.endRow, sortModel: params.sortModel, filterModel: params.filterModel, + // Fetch only displayed columns; wide tables otherwise cost R a lot per block. + columns: params.api.getAllDisplayedColumns() + .map(column => column.getColDef().field) + .filter(field => field !== '0'), }); const nextFilteredRows = Number.isFinite(result.totalRows) ? result.totalRows : result.loadedRows; const nextTotalRows = Number.isFinite(result.totalUnfiltered) ? result.totalUnfiltered : result.loadedRows; @@ -2527,6 +2531,12 @@ export async function getTableHtml( onPaginationChanged: updateScrollPosition, onGridSizeChanged: updateFetchStatusPosition, onDisplayedColumnsChanged: updateFetchStatusPosition, + onColumnVisible: function(event) { + // Cached rows lack columns that were hidden when they were fetched. + if (!init.sequential && (event.columns || []).some(column => column.isVisible())) { + event.api.purgeInfiniteCache(); + } + }, onFirstDataRendered: function() { updateFetchStatusPosition(); attachScrollbarPositionIndicator(); diff --git a/src/test/node/interactiveRuntime.test.ts b/src/test/node/interactiveRuntime.test.ts index e273d909f..b4366cd47 100644 --- a/src/test/node/interactiveRuntime.test.ts +++ b/src/test/node/interactiveRuntime.test.ts @@ -134,6 +134,13 @@ const delay = (ms: number): Promise => new Promise(resolve => setTimeout(r const data = result.data as JsonObject; assert.deepStrictEqual(data.source_row, [0, 1]); assert.strictEqual(result.total, 3); + const selected = await peer.request('grid/rows', { + view: params.view, generation: params.generation, offset: 0, limit: 2, cols: [1], + }) as JsonObject; + const selectedData = selected.data as { source_row: number[]; columns: { schema: { position: number }; data: { values: unknown[] } }[] }; + assert.deepStrictEqual(selectedData.source_row, [0, 1]); + assert.deepStrictEqual(selectedData.columns.map(column => column.schema.position), [1]); + assert.deepStrictEqual(selectedData.columns[0].data.values, ['a', 'b']); } finally { client.off('event', onEvent); host.dispose(); diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts index fd73db679..05636026a 100644 --- a/src/test/node/jdvAdapter.test.ts +++ b/src/test/node/jdvAdapter.test.ts @@ -166,6 +166,21 @@ suite('jdv adapter', () => { }); }); + test('selects displayed columns by field, excluding the row label', () => { + const select = (fields: unknown) => tableRowsParams(grid, 0, 10, [], {}, fields); + assert.deepEqual(select(['0', '3', '1', '3']).cols, [2, 0]); + assert.deepEqual(select(['0']).cols, []); + assert.deepEqual(select([]).cols, []); + for (const fields of [undefined, null, '1', ['bad'], [1], ['01'], ['-1'], ['7'], ['1.5'], + ['999999999999999999999'], ['1', '2', '3', '4', '5', '6'], ['0', '1', '2', '3', '4', '5', '6']]) { + assert.equal('cols' in select(fields), false, JSON.stringify(fields)); + } + const wide = { ...grid, columns: Array.from({ length: 300 }, (_, position) => ({ ...grid.columns![0], position })) }; + const fields = Array.from({ length: 257 }, (_, index) => String(index + 1)); + assert.equal('cols' in tableRowsParams(wide, 0, 10, [], {}, fields), false); + assert.equal(tableRowsParams(wide, 0, 10, [], {}, ['0', ...fields.slice(0, 256)]).cols?.length, 256); + }); + test('converts boolean filter models', () => { const booleanGrid = parseDescriptor({ ...grid, ncol: 1, columns: [ { position: 0, name: 'flag', type: 'boolean', native_type: ['logical'], sortable: true, filterable: true }, diff --git a/src/test/node/jdvGrid.test.ts b/src/test/node/jdvGrid.test.ts new file mode 100644 index 000000000..942a7a8ad --- /dev/null +++ b/src/test/node/jdvGrid.test.ts @@ -0,0 +1,79 @@ +import * as assert from 'node:assert/strict'; +import { JdvDescriptor, JdvRequestError, JsonObject, parseDescriptor } from '../../jdvAdapter'; +import { fetchGridPage } from '../../jdvGrid'; + +const descriptor: JdvDescriptor = parseDescriptor({ + view: 'v', generation: 1, kind: 'grid', access: 'random', encoding: 'jdv-json', + capabilities: { seek: true, known_total: true }, collation: 'C', nrow: 5, ncol: 2, + columns: [{ position: 0, name: 'a', type: 'string' }, { position: 1, name: 'b', type: 'string' }], +}); +const message = { startRow: 0, endRow: 5, columns: ['1'] }; +const page = (offset: number, count: number) => ({ + total: 5, total_unfiltered: 7, offset, limit: count, + data: { source_row: Array.from({ length: count }, (_, index) => offset + index), row_labels: null, columns: [] }, +}); + +suite('jdv grid block fetching', () => { + test('keeps selected columns across short pages', async () => { + const requests: JsonObject[] = []; + const result = await fetchGridPage(descriptor, message, (method, params) => { + assert.equal(method, 'grid/rows'); + requests.push(params); + return Promise.resolve(page(Number(params.offset), Math.min(2, Number(params.limit)))); + }); + assert.deepEqual(requests.map(params => [params.offset, params.limit, params.cols]), + [[0, 5, [0]], [2, 3, [0]], [4, 1, [0]]]); + assert.deepEqual(result.rows.map(row => row.__sourceRow), [0, 1, 2, 3, 4]); + assert.equal(result.lastRow, 5); + assert.equal(result.totalUnfiltered, 7); + }); + + test('retries at the same offset without cols for the rest of an oversized block', async () => { + const requests: JsonObject[] = []; + const result = await fetchGridPage(descriptor, message, (_method, params) => { + requests.push(params); + if (params.offset === 2 && params.cols) { + return Promise.reject(new JdvRequestError('limit_exceeded', 'page too large')); + } + return Promise.resolve(page(Number(params.offset), Math.min(2, Number(params.limit)))); + }); + assert.deepEqual(requests.map(params => [params.offset, params.cols]), + [[0, [0]], [2, [0]], [2, undefined], [4, undefined]]); + assert.deepEqual(result.rows.map(row => row.__sourceRow), [0, 1, 2, 3, 4]); + }); + + test('retries empty selections once and propagates all-column failures', async () => { + let calls = 0; + await assert.rejects(fetchGridPage(descriptor, { ...message, columns: [] }, (_method, params) => { + assert.deepEqual(params.cols, calls++ === 0 ? [] : undefined); + return Promise.reject(new JdvRequestError('limit_exceeded', 'row too large')); + }), /row too large/); + assert.equal(calls, 2); + }); + + test('propagates other errors without retrying', async () => { + let calls = 0; + await assert.rejects(fetchGridPage(descriptor, message, () => { + calls++; + return Promise.reject(new JdvRequestError('expired', 'expired')); + }), /expired/); + assert.equal(calls, 1); + }); + + test('stops on an empty page or the total row count', async () => { + let calls = 0; + const result = await fetchGridPage(descriptor, message, () => { + calls++; + return Promise.resolve(page(0, 0)); + }); + assert.equal(calls, 1); + assert.deepEqual(result.rows, []); + calls = 0; + const full = await fetchGridPage(descriptor, { ...message, endRow: 10 }, () => { + calls++; + return Promise.resolve(page(0, 5)); + }); + assert.equal(calls, 1); + assert.equal(full.rows.length, 5); + }); +}); From 6c73a816b2cf9a6a85b393d52a3808070faee9d9 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 12:50:24 +0000 Subject: [PATCH 36/38] refactor: drop the explicit column fallback jdv now shortens pages for explicit column selections as well, so a limit_exceeded response is a genuine failure. Cover the shortened selected-column page in the real R runtime test. --- src/jdvGrid.ts | 20 +++------------ src/test/node/interactiveRuntime.test.ts | 18 +++++++++++-- src/test/node/jdvGrid.test.ts | 32 ++++++++---------------- 3 files changed, 31 insertions(+), 39 deletions(-) diff --git a/src/jdvGrid.ts b/src/jdvGrid.ts index 02d985778..ef713722a 100644 --- a/src/jdvGrid.ts +++ b/src/jdvGrid.ts @@ -1,31 +1,19 @@ import { GridPage, JdvDescriptor, JdvRequestError, JsonObject, gridPageFromRows, tableRowsParams } from './jdvAdapter'; -/** Fill an AG Grid block, retrying oversized column selections with adaptive all-column pages. */ +/** Fill an AG Grid block, issuing further pages when jdv shortens a selected-column page. */ export async function fetchGridPage( descriptor: JdvDescriptor, message: JsonObject, request: (method: string, params: JsonObject) => Promise, ): Promise> { const startRow = Number(message.startRow ?? 0); const endRow = Number(message.endRow ?? 0); - // Only the displayed columns; the webview reloads its cache when a hidden column is shown. - let columns = message.columns; const rows: Record[] = []; let page: GridPage | undefined; let offset = startRow; while (offset < endRow) { - const params = tableRowsParams(descriptor, offset, endRow, message.sortModel, message.filterModel, columns); - let result: unknown; - try { - result = await request('grid/rows', params); - } catch (e) { - // jdv shortens pages only when every column is selected; an explicit - // selection that does not fit fails, so fall back to all columns. - if (params.cols && e instanceof JdvRequestError && e.code === 'limit_exceeded') { - columns = undefined; - continue; - } - throw e; - } + // Keep the displayed-column selection for every short page in the block. + const params = tableRowsParams(descriptor, offset, endRow, message.sortModel, message.filterModel, message.columns); + const result = await request('grid/rows', params); page = gridPageFromRows(result, offset, params.limit); for (const row of page.rows) { rows.push(row); diff --git a/src/test/node/interactiveRuntime.test.ts b/src/test/node/interactiveRuntime.test.ts index b4366cd47..96b7fb2b9 100644 --- a/src/test/node/interactiveRuntime.test.ts +++ b/src/test/node/interactiveRuntime.test.ts @@ -120,7 +120,8 @@ const delay = (ms: number): Promise => new Promise(resolve => setTimeout(r }; client.on('event', onEvent); try { - const id = await submit('utils::View(data.frame(a = 1:3, b = letters[1:3]), "interactive jdv test")'); + // Repeated strings keep the R fixture small while its encoded page exceeds the byte budget. + const id = await submit('utils::View(data.frame(a = 1:300, b = rep(letters, length.out = 300), c = rep(strrep("x", 12000), 300), d = rep(strrep("y", 12000), 300)), "interactive jdv test")'); assert.strictEqual((await finished(id)).state, 'success'); const { peer, params } = await Promise.race([ opened, @@ -133,7 +134,7 @@ const delay = (ms: number): Promise => new Promise(resolve => setTimeout(r }) as JsonObject; const data = result.data as JsonObject; assert.deepStrictEqual(data.source_row, [0, 1]); - assert.strictEqual(result.total, 3); + assert.strictEqual(result.total, 300); const selected = await peer.request('grid/rows', { view: params.view, generation: params.generation, offset: 0, limit: 2, cols: [1], }) as JsonObject; @@ -141,6 +142,19 @@ const delay = (ms: number): Promise => new Promise(resolve => setTimeout(r assert.deepStrictEqual(selectedData.source_row, [0, 1]); assert.deepStrictEqual(selectedData.columns.map(column => column.schema.position), [1]); assert.deepStrictEqual(selectedData.columns[0].data.values, ['a', 'b']); + const shortened = await peer.request('grid/rows', { + view: params.view, generation: params.generation, offset: 0, limit: 300, cols: [2, 3], + }) as JsonObject; + const shortenedData = shortened.data as typeof selectedData; + assert.strictEqual(shortened.total, 300); + assert.ok(shortenedData.source_row.length > 0 && shortenedData.source_row.length < 300); + assert.deepStrictEqual(shortenedData.source_row, + Array.from({ length: shortenedData.source_row.length }, (_, index) => index)); + assert.deepStrictEqual(shortenedData.columns.map(column => column.schema.position), [2, 3]); + for (const [index, column] of shortenedData.columns.entries()) { + assert.strictEqual(column.data.values.length, shortenedData.source_row.length); + assert.strictEqual(column.data.values[0], (index === 0 ? 'x' : 'y').repeat(12000)); + } } finally { client.off('event', onEvent); host.dispose(); diff --git a/src/test/node/jdvGrid.test.ts b/src/test/node/jdvGrid.test.ts index 942a7a8ad..b290f2716 100644 --- a/src/test/node/jdvGrid.test.ts +++ b/src/test/node/jdvGrid.test.ts @@ -28,27 +28,17 @@ suite('jdv grid block fetching', () => { assert.equal(result.totalUnfiltered, 7); }); - test('retries at the same offset without cols for the rest of an oversized block', async () => { - const requests: JsonObject[] = []; - const result = await fetchGridPage(descriptor, message, (_method, params) => { - requests.push(params); - if (params.offset === 2 && params.cols) { - return Promise.reject(new JdvRequestError('limit_exceeded', 'page too large')); - } - return Promise.resolve(page(Number(params.offset), Math.min(2, Number(params.limit)))); - }); - assert.deepEqual(requests.map(params => [params.offset, params.cols]), - [[0, [0]], [2, [0]], [2, undefined], [4, undefined]]); - assert.deepEqual(result.rows.map(row => row.__sourceRow), [0, 1, 2, 3, 4]); - }); - - test('retries empty selections once and propagates all-column failures', async () => { - let calls = 0; - await assert.rejects(fetchGridPage(descriptor, { ...message, columns: [] }, (_method, params) => { - assert.deepEqual(params.cols, calls++ === 0 ? [] : undefined); - return Promise.reject(new JdvRequestError('limit_exceeded', 'row too large')); - }), /row too large/); - assert.equal(calls, 2); + test('propagates limit_exceeded for selected, empty and all columns without retrying', async () => { + for (const columns of [['1'], [], undefined]) { + let calls = 0; + const error = new JdvRequestError('limit_exceeded', 'row too large'); + await assert.rejects(fetchGridPage(descriptor, { ...message, columns }, (_method, params) => { + calls++; + assert.deepEqual(params.cols, columns?.map(field => Number(field) - 1)); + return Promise.reject(error); + }), (received: unknown) => received === error); + assert.equal(calls, 1); + } }); test('propagates other errors without retrying', async () => { From 0cf8698bfb6604e1539d6c1200c552be838112f1 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 14:17:49 +0000 Subject: [PATCH 37/38] feat: read sparse jdv missing-row positions jdv now sends missingness as `na_rows`, a list of zero-based positions within the page, instead of a dense boolean `na` array. Build a set per column from it when converting grid pages. The `special` channel became a position object as well, but the viewer only shows R's display text for non-finite values, so it needs no change. --- src/jdvAdapter.ts | 12 +++++++----- src/test/node/jdvAdapter.test.ts | 6 +++--- src/test/node/jdvTable.test.ts | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index b32252abc..4e7d36baf 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -623,7 +623,8 @@ export function tableRowsParams( interface ColumnChannels { values?: unknown[]; display?: (string | null)[]; - na?: boolean[]; + /** Zero-based positions of missing rows within this page. */ + na_rows?: number[]; } interface PageColumn { @@ -641,8 +642,8 @@ export interface GridPage { returned: number; } -function cellValue(column: JdvColumnSchema, channels: ColumnChannels, row: number): unknown { - if (channels.na?.[row]) { +function cellValue(column: JdvColumnSchema, channels: ColumnChannels, naRows: Set, row: number): unknown { + if (naRows.has(row)) { // Missing cells use the grid's NA renderer, unless R labels them differently // (NaN, or a class-specific missing label). const label = channels.display?.[row]; @@ -693,9 +694,10 @@ export function gridPageFromRows(result: unknown, startRow: number, requested: n const field = String(column.schema.position + 1); const channels = column.data ?? {}; const error = columnErrorText(column.errors); + const naRows = new Set(channels.na_rows); for (let row = 0; row < rows.length; row++) { - rows[row][field] = channels.display || channels.values || channels.na - ? cellValue(column.schema, channels, row) + rows[row][field] = channels.display || channels.values || channels.na_rows + ? cellValue(column.schema, channels, naRows, row) : error ? `` : null; } } diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts index 05636026a..9678078dd 100644 --- a/src/test/node/jdvAdapter.test.ts +++ b/src/test/node/jdvAdapter.test.ts @@ -242,7 +242,7 @@ suite('jdv adapter', () => { (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported'); }); - test('converts pages to rows using display text and the NA mask', () => { + test('converts pages to rows using display text and the NA rows', () => { const page = gridPageFromRows({ view: 'v-1', generation: 2, total: 3, total_unfiltered: 5, offset: 0, limit: 3, data: { @@ -251,9 +251,9 @@ suite('jdv adapter', () => { columns: [ { schema: grid.columns![0], data: { values: [null, 1, null], display: ['Inf', 'NaN', 'NA'], - special: ['pos_inf', 'nan', 'na'], na: [false, true, true], + special: { pos_inf: [0], nan: [1] }, na_rows: [1, 2], }, errors: [] }, - { schema: grid.columns![1], data: { values: [null, 'x', 'y'], na: [true, false, false] }, errors: [] }, + { schema: grid.columns![1], data: { values: [null, 'x', 'y'], na_rows: [0] }, errors: [] }, ], }, }, 0, 3); diff --git a/src/test/node/jdvTable.test.ts b/src/test/node/jdvTable.test.ts index 26edea4b7..ed2798f02 100644 --- a/src/test/node/jdvTable.test.ts +++ b/src/test/node/jdvTable.test.ts @@ -16,7 +16,7 @@ const rowsPage = (offset: number, count: number, total = 50) => JSON.stringify({ data: { source_row: Array.from({ length: count }, (_, i) => offset + i), row_labels: null, columns: [{ schema: gridDescriptor.columns[0], data: { values: Array.from({ length: count }, (_, i) => offset + i + 1), - display: Array.from({ length: count }, (_, i) => String(offset + i + 1)), na: Array(count).fill(false) }, errors: {} }], + display: Array.from({ length: count }, (_, i) => String(offset + i + 1)), na_rows: [] }, errors: {} }], }, }); const display = (extra: Record = {}) => ({ From 470c448583ad345d6eb0722a3359adeda75681b7 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Mon, 19 Oct 2026 15:08:59 +0000 Subject: [PATCH 38/38] feat: add empty-string filters for jdv text columns String and categorical columns offer "Is empty" and "Is not empty" next to "Is NA" and "Is not NA", so empty strings and missing values can be filtered separately. Is empty maps to jdv's text_equals with an empty value; Is not empty also excludes NA, like the other negated text conditions. Combine Is NA and Is empty with OR to match either. Inline tables label the NA options as such instead of reusing blank, and jdv columns no longer accept blank/notBlank. --- src/dataViewer.ts | 6 ++++- src/interactive/inlineTable.ts | 4 +-- src/interactive/tableQuery.ts | 10 +++----- src/jdvAdapter.ts | 12 ++++++--- src/test/node/interactiveRuntime.test.ts | 9 +++++++ src/test/node/jdvAdapter.test.ts | 32 ++++++++++++++++++++++++ src/test/node/jdvTable.test.ts | 24 ++++++++++++------ src/test/suite/dataViewer.test.ts | 2 ++ 8 files changed, 80 insertions(+), 19 deletions(-) diff --git a/src/dataViewer.ts b/src/dataViewer.ts index 4a637fbe5..09b5589d3 100644 --- a/src/dataViewer.ts +++ b/src/dataViewer.ts @@ -186,6 +186,10 @@ export function getDataViewerScript(): string { if (column.jdv) { // jdv cells hold R-formatted display text, and R evaluates filters. // Offer only the operators jdv implements for this column. + const customOption = (displayKey, displayName) => ({ displayKey, displayName, predicate: () => true, numberOfInputs: 0 }); + const textOptions = column.jdv.filterOptions.map(option => + option === 'jdvIsEmpty' ? customOption(option, 'Is empty') : + option === 'jdvNotEmpty' ? customOption(option, 'Is not empty') : option); const naOptions = column.jdv.naOptions ? [ { displayKey: 'jdvIsNa', displayName: 'Is NA', predicate: () => true, numberOfInputs: 0 }, { displayKey: 'jdvNotNa', displayName: 'Is not NA', predicate: () => true, numberOfInputs: 0 } @@ -197,7 +201,7 @@ export function getDataViewerScript(): string { column.filterParams = { ...column.filterParams, buttons: ['reset', 'apply'], - filterOptions: [...column.jdv.filterOptions, ...naOptions], + filterOptions: [...textOptions, ...naOptions], inRangeInclusive: true, ...(column.jdv.units ? { filterPlaceholder: 'Value (' + column.jdv.units + ')' } : {}) }; diff --git a/src/interactive/inlineTable.ts b/src/interactive/inlineTable.ts index d17604a10..c555a60f4 100644 --- a/src/interactive/inlineTable.ts +++ b/src/interactive/inlineTable.ts @@ -157,7 +157,7 @@ export function createInlineTable(parent: HTMLElement, toolbar: HTMLElement, sta const column = columns().find(column => column.field === filterColumn.value); operator.replaceChildren(); if (column) { - for (const op of columnOperators(column)) { operator.add(new Option(filterOperatorLabel(op, column), op)); } + for (const op of columnOperators(column)) { operator.add(new Option(filterOperatorLabel(op), op)); } } const filter = state.filters[filterColumn.value]; value.value = ''; @@ -188,7 +188,7 @@ export function createInlineTable(parent: HTMLElement, toolbar: HTMLElement, sta const column = columns().find(column => column.field === field); const name = column?.headerName ?? field; const remove = document.createElement('button'); remove.type = 'button'; remove.className = 'r-interactive-button'; - remove.textContent = `${name} ${filterOperatorLabel(filter.type, column)} ${filter.filter ?? ''} ×`; + remove.textContent = `${name} ${filterOperatorLabel(filter.type)} ${filter.filter ?? ''} ×`; remove.setAttribute('aria-label', `Remove filter for ${name}`); remove.onclick = () => { const filters = { ...state.filters }; delete filters[field]; request(0, state.sort, filters); }; chips.append(remove); diff --git a/src/interactive/tableQuery.ts b/src/interactive/tableQuery.ts index 45cd35cc9..3b77a1b50 100644 --- a/src/interactive/tableQuery.ts +++ b/src/interactive/tableQuery.ts @@ -11,22 +11,20 @@ export type TableFilters = Record; export const filterOperators = { equals: '=', notEqual: '≠', lessThan: '<', lessThanOrEqual: '≤', greaterThan: '>', greaterThanOrEqual: '≥', contains: 'contains', notContains: 'does not contain', startsWith: 'starts with', endsWith: 'ends with', - blank: 'is blank', notBlank: 'is not blank', true: 'is true', false: 'is false', + blank: 'is blank', notBlank: 'is not blank', jdvIsEmpty: 'is empty', jdvNotEmpty: 'is not empty', jdvIsNa: 'is NA', jdvNotNa: 'is not NA', true: 'is true', false: 'is false', }; -export function filterOperatorLabel(type: string, column?: TableColumn): string { - if (column?.jdv && type === 'blank') { return 'is NA'; } - if (column?.jdv && type === 'notBlank') { return 'is not NA'; } +export function filterOperatorLabel(type: string): string { return filterOperators[type as keyof typeof filterOperators] ?? type; } export function columnOperators(column: TableColumn): string[] { // An inline filter is one condition, so the range operator is not offered. - if (column.jdv) { return [...column.jdv.filterOptions.filter(option => option !== 'inRange'), 'blank', 'notBlank']; } + if (column.jdv) { return [...column.jdv.filterOptions.filter(option => option !== 'inRange'), 'jdvIsNa', 'jdvNotNa']; } if (column.type === 'booleanColumn') { return ['true', 'false', 'blank', 'notBlank']; } return ['numericColumn', 'bigintColumn', 'dateColumn', 'datetimeColumn'].includes(String(column.type)) ? ['equals', 'notEqual', 'lessThan', 'lessThanOrEqual', 'greaterThan', 'greaterThanOrEqual', 'blank', 'notBlank'] : ['contains', 'notContains', 'equals', 'notEqual', 'startsWith', 'endsWith', 'blank', 'notBlank']; } -export const filterNeedsValue = (type: string): boolean => !['blank', 'notBlank', 'true', 'false'].includes(type); +export const filterNeedsValue = (type: string): boolean => !['blank', 'notBlank', 'jdvIsEmpty', 'jdvNotEmpty', 'jdvIsNa', 'jdvNotNa', 'true', 'false'].includes(type); // Slicing can turn automatic row numbers into explicit character row names. // The synthetic index is never sorted/filtered and is not part of the data schema. export const tableSchema = (columns: TableColumn[]): unknown[][] => columns.filter(column => column.field !== '0') diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts index 4e7d36baf..a2f7d1472 100644 --- a/src/jdvAdapter.ts +++ b/src/jdvAdapter.ts @@ -198,7 +198,11 @@ export interface GridColumnDef { jdv?: { filterOptions: string[]; naOptions: boolean; units?: string }; } -const TEXT_FILTER_OPTIONS = ['contains', 'notContains', 'equals', 'notEqual', 'startsWith', 'endsWith']; +export const EMPTY_FILTER_OPTION = 'jdvIsEmpty'; +export const NOT_EMPTY_FILTER_OPTION = 'jdvNotEmpty'; +const TEXT_FILTER_OPTIONS = [ + 'contains', 'notContains', 'equals', 'notEqual', 'startsWith', 'endsWith', EMPTY_FILTER_OPTION, NOT_EMPTY_FILTER_OPTION, +]; const ORDER_FILTER_OPTIONS = [ 'equals', 'notEqual', 'lessThan', 'lessThanOrEqual', 'greaterThan', 'greaterThanOrEqual', 'inRange', ]; @@ -438,10 +442,10 @@ const ORDER_OPS: Record = { function conditionNode(condition: JsonObject, col: number, column: JdvColumnSchema): FilterNode { const type = condition.type; const isNa = { op: 'is_na', col }; - if (type === NA_FILTER_OPTION || type === 'blank') { + if (type === NA_FILTER_OPTION) { return isNa; } - if (type === NOT_NA_FILTER_OPTION || type === 'notBlank') { + if (type === NOT_NA_FILTER_OPTION) { return { op: 'not', arg: isNa }; } if (!column.filterable) { @@ -495,6 +499,8 @@ function conditionNode(condition: JsonObject, col: number, column: JdvColumnSche op: 'and', args: [{ op: 'not', arg: node }, { op: 'not', arg: isNa }], }); switch (type) { + case EMPTY_FILTER_OPTION: return { op: 'text_equals', col, value: '' }; + case NOT_EMPTY_FILTER_OPTION: return negate({ op: 'text_equals', col, value: '' }); case 'contains': return { op: 'contains', col, value }; case 'notContains': return negate({ op: 'contains', col, value }); case 'equals': return { op: 'text_equals', col, value }; diff --git a/src/test/node/interactiveRuntime.test.ts b/src/test/node/interactiveRuntime.test.ts index 96b7fb2b9..b1b8ac0cb 100644 --- a/src/test/node/interactiveRuntime.test.ts +++ b/src/test/node/interactiveRuntime.test.ts @@ -416,6 +416,15 @@ sess::display("display-${cell}", "text/plain"); cat("stderr-${cell}\\n", file=st // Row labels show the original row after sorting. assert.strictEqual((filtered.rows as Record[])[0]['0'], '99'); + // Empty strings and NA are separate filters. + const blankId = await submit('blanks <- data.frame(s=c("a", "", NA, "b"), stringsAsFactors=FALSE); blanks'); + assert.strictEqual((await finished(blankId)).state, 'success'); + const blanks = events.find(event => event.executionId === blankId && event.data.kind === 'table'); assert.ok(blanks); + for (const [type, expected] of [['jdvIsEmpty', 1], ['jdvNotEmpty', 2], ['jdvIsNa', 1], ['jdvNotNa', 3]] as const) { + const result = await queryJdvTablePage(blanks.data, { start: 0, size: 20, filterModel: { '1': { type } } }, inspect); + assert.strictEqual(result.totalRows, expected, type); + } + // A page past the end is clamped to the last page. const clamped = await queryJdvTablePage(table.data, { start: 400, size: 20 }, inspect); assert.strictEqual(clamped.startRow, 80); diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts index 9678078dd..5210f4cfb 100644 --- a/src/test/node/jdvAdapter.test.ts +++ b/src/test/node/jdvAdapter.test.ts @@ -166,6 +166,38 @@ suite('jdv adapter', () => { }); }); + test('converts empty-string filters for text columns only', () => { + const textGrid = parseDescriptor({ + ...grid, columns: [ + ...grid.columns!.slice(0, 2), + { position: 2, name: 's', type: 'string', native_type: ['character'], sortable: true, filterable: true }, + ], + }); + const filter = (field: string, type: string) => toJdvFilter({ [field]: { filterType: 'text', type } }, textGrid); + const isNa = (col: number) => ({ op: 'is_na', col }); + const empty = (col: number) => ({ op: 'text_equals', col, value: '' }); + // Categorical and string columns; NA stays a separate condition. + for (const col of [1, 2]) { + const field = String(col + 1); + assert.ok(gridColumnDefs(textGrid)[col + 1].jdv?.filterOptions.includes('jdvIsEmpty')); + assert.deepEqual(filter(field, 'jdvIsEmpty'), empty(col)); + assert.deepEqual(filter(field, 'jdvNotEmpty'), + { op: 'and', args: [{ op: 'not', arg: empty(col) }, { op: 'not', arg: isNa(col) }] }); + assert.deepEqual(filter(field, 'jdvIsNa'), isNa(col)); + // blank and notBlank are no longer mapped to is_na. + for (const type of ['blank', 'notBlank']) { + assert.throws(() => filter(field, type), (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported'); + } + } + // Is NA OR Is empty in one AG Grid model. + assert.deepEqual(toJdvFilter({ '3': { filterType: 'text', operator: 'OR', conditions: [ + { filterType: 'text', type: 'jdvIsNa' }, { filterType: 'text', type: 'jdvIsEmpty' }, + ] } }, textGrid), { op: 'or', args: [isNa(2), empty(2)] }); + // A number column has no empty-string filters. + assert.ok(!gridColumnDefs(textGrid)[1].jdv?.filterOptions.includes('jdvIsEmpty')); + assert.throws(() => filter('1', 'jdvIsEmpty'), (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported'); + }); + test('selects displayed columns by field, excluding the row label', () => { const select = (fields: unknown) => tableRowsParams(grid, 0, 10, [], {}, fields); assert.deepEqual(select(['0', '3', '1', '3']).cols, [2, 0]); diff --git a/src/test/node/jdvTable.test.ts b/src/test/node/jdvTable.test.ts index ed2798f02..21af8b658 100644 --- a/src/test/node/jdvTable.test.ts +++ b/src/test/node/jdvTable.test.ts @@ -1,6 +1,6 @@ import * as assert from 'node:assert/strict'; import { createListAdapter, isJdvOutput, listRequest, normalizeJdvDisplay, queryJdvTablePage } from '../../interactive/jdvTable'; -import { columnOperators, filterOperatorLabel, TableColumn } from '../../interactive/tableQuery'; +import { columnOperators, filterNeedsValue, filterOperatorLabel, TableColumn } from '../../interactive/tableQuery'; const capabilities = { seek: true, sort: true, filter: true, known_total: true, next: false, children: false, refresh: false, track: false }; const gridDescriptor = { @@ -74,13 +74,23 @@ suite('Interactive jdv tables', () => { test('offers the operators of the column type', () => { const columns = normalizeJdvDisplay(display()).columns as TableColumn[]; - assert.deepEqual(columnOperators(columns[1]).slice(-2), ['blank', 'notBlank']); - assert.equal(filterOperatorLabel('blank', columns[1]), 'is NA'); - assert.equal(filterOperatorLabel('notBlank', columns[1]), 'is not NA'); + assert.deepEqual(columnOperators(columns[1]).slice(-2), ['jdvIsNa', 'jdvNotNa']); + assert.equal(filterOperatorLabel('jdvIsNa'), 'is NA'); + assert.equal(filterOperatorLabel('jdvNotNa'), 'is not NA'); assert.ok(columnOperators(columns[1]).includes('greaterThan')); - assert.deepEqual(columnOperators(columns[3]), ['equals', 'blank', 'notBlank']); - assert.equal(filterOperatorLabel('blank', { field: 'legacy', headerName: 'legacy' }), 'is blank'); - assert.equal(filterOperatorLabel('notBlank', { field: 'legacy', headerName: 'legacy' }), 'is not blank'); + assert.ok(!columnOperators(columns[1]).includes('blank')); + assert.deepEqual(columnOperators(columns[3]), ['equals', 'jdvIsNa', 'jdvNotNa']); + assert.equal(filterOperatorLabel('blank'), 'is blank'); + assert.equal(filterOperatorLabel('notBlank'), 'is not blank'); + assert.equal(filterNeedsValue('jdvIsNa'), false); + // Text columns additionally offer empty-string filters, separate from NA. + const text = { field: '1', headerName: 's', jdv: { filterOptions: ['contains', 'jdvIsEmpty', 'jdvNotEmpty'] } }; + assert.deepEqual(columnOperators(text), ['contains', 'jdvIsEmpty', 'jdvNotEmpty', 'jdvIsNa', 'jdvNotNa']); + assert.equal(filterOperatorLabel('jdvIsEmpty'), 'is empty'); + assert.equal(filterOperatorLabel('jdvNotEmpty'), 'is not empty'); + assert.equal(filterNeedsValue('jdvIsEmpty'), false); + assert.equal(filterNeedsValue('jdvNotEmpty'), false); + assert.equal(filterNeedsValue('contains'), true); assert.ok(!columnOperators(columns[1]).includes('inRange')); }); diff --git a/src/test/suite/dataViewer.test.ts b/src/test/suite/dataViewer.test.ts index 6d93fb305..f24b77669 100644 --- a/src/test/suite/dataViewer.test.ts +++ b/src/test/suite/dataViewer.test.ts @@ -260,6 +260,7 @@ suite('Data viewer', () => { { field: '2', headerName: 'i', type: 'bigintColumn', filter: 'agBigIntColumnFilter', jdv: { filterOptions: ['equals'], naOptions: false } }, { field: '3', headerName: 'd', type: 'dateColumn', filter: 'agDateColumnFilter', jdv: { filterOptions: [], naOptions: true } }, { field: '4', headerName: 'dt', type: 'datetimeColumn', filter: 'agDateColumnFilter', jdv: { filterOptions: ['equals', 'inRange'], naOptions: true } }, + { field: '5', headerName: 's', filter: 'agTextColumnFilter', jdv: { filterOptions: ['contains', 'jdvIsEmpty', 'jdvNotEmpty'], naOptions: true } }, ] as (ColDef & { jdv?: unknown })[]; const bigintFields = viewer.script.prepareViewerColumns(jdvColumns); assert.strictEqual(bigintFields.length, 0); @@ -270,6 +271,7 @@ suite('Data viewer', () => { assert.deepStrictEqual(options(jdvColumns[0]), ['equals', 'inRange', 'jdvIsNa', 'jdvNotNa']); assert.deepStrictEqual(options(jdvColumns[1]), ['equals']); assert.deepStrictEqual(options(jdvColumns[2]), ['jdvIsNa', 'jdvNotNa']); + assert.deepStrictEqual(options(jdvColumns[4]), ['contains', 'jdvIsEmpty', 'jdvNotEmpty', 'jdvIsNa', 'jdvNotNa']); assert.strictEqual((jdvColumns[0].filterParams as { inRangeInclusive: boolean }).inRangeInclusive, true); assert.strictEqual((jdvColumns[0].filterParams as { filterPlaceholder: string }).filterPlaceholder, 'Value (hours)'); assert.strictEqual((jdvColumns[2].filterParams as { browserDatePicker: boolean }).browserDatePicker, true);