diff --git a/R/attach_sess.R b/R/attach_sess.R index 3b2849a84..9c448d109 100644 --- a/R/attach_sess.R +++ b/R/attach_sess.R @@ -184,6 +184,7 @@ vscode_r_attach_sess <- function(endpoint, pkg_path, managed_root, consent_dir, timeout_seconds = 180, setup_timeout_seconds = 300, use_rstudioapi = TRUE, + track_variables = TRUE, attachment_id = NULL, auto_activate = TRUE) { if (!is.null(attachment_id) && @@ -235,6 +236,11 @@ vscode_r_attach_sess <- function(endpoint, pkg_path, managed_root, consent_dir, connect <- get("connect", envir = ns, inherits = FALSE) args <- list(endpoint = endpoint, plot_backend = plot_backend, use_rstudioapi = use_rstudioapi) + # Tracking is on by default, so only a disabled setting is passed. This + # keeps older sess namespaces working. + if (!isTRUE(track_variables)) { + args$track_variables <- FALSE + } # Preserve compatibility with older sess namespaces and helper stubs # when the caller uses the legacy, untagged attach path. if (!is.null(attachment_id)) { diff --git a/R/profile.R b/R/profile.R index 9ca4c2c3b..c76e168bd 100644 --- a/R/profile.R +++ b/R/profile.R @@ -52,7 +52,9 @@ invisible(local({ plot_backend <- Sys.getenv("SESS_PLOT_BACKEND", "auto") connect <- get("connect", envir = ns, inherits = FALSE) use_rstudioapi <- as.logical(Sys.getenv("SESS_RSTUDIOAPI", "TRUE")) - result <- connect(use_rstudioapi = use_rstudioapi, plot_backend = plot_backend) + track_variables <- as.logical(Sys.getenv("SESS_TRACK_VARIABLES", "TRUE")) + result <- connect(use_rstudioapi = use_rstudioapi, plot_backend = plot_backend, + track_variables = track_variables) isTRUE(result) } diff --git a/package.json b/package.json index ffdcad25b..f07023363 100644 --- a/package.json +++ b/package.json @@ -2108,6 +2108,11 @@ "default": 500, "markdownDescription": "The maximum number of rows per page in the data viewer. `0` means to disable pagination." }, + "r.session.viewers.trackVariables": { + "type": "boolean", + "default": true, + "markdownDescription": "Update the data viewer when the variable passed to `View()` is changed in R, for example `View(df)` followed by `df <- transform(df, ...)`. Requires R >= 4.6 and the [jdv](https://github.com/eitsupi/jdv) package; otherwise the data viewer shows a fixed snapshot. Applies to R sessions started or attached after the setting is changed. `options(sess.dataviewer.track = TRUE/FALSE)` in R takes precedence over this setting. Requires `#r.sessionWatcher#` to be set to `true`." + }, "r.session.viewers.viewColumn": { "type": "object", "markdownDescription": "Which view column should R-related webviews be displayed? Requires `#r.sessionWatcher#` to be set to `true`.", @@ -2638,6 +2643,7 @@ "dependencies": { "@vscode/codicons": "0.0.46-24", "ag-grid-community": "^36.2.0", + "apache-arrow": "21.2.0", "cheerio": "1.0.0-rc.12", "ejs": "^3.1.10", "highlight.js": "^11.11.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d42aaac3..015dfabd4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: ag-grid-community: specifier: ^36.2.0 version: 36.2.0 + apache-arrow: + specifier: 21.2.0 + version: 21.2.0 cheerio: specifier: 1.0.0-rc.12 version: 1.0.0-rc.12 @@ -823,6 +826,9 @@ packages: '@types/node@22.20.5': resolution: {integrity: sha512-U2+DNr+wSjpsTS/wZGYHq7GcwfuSmKiKvoPvK22zwTlRhU91yOniN4qRR5KhIjvif7ysw/dz/hKmfDH0Ris4aA==} + '@types/node@25.9.9': + resolution: {integrity: sha512-b4e2xxj/yMeT2hNlD6x7zNFiQK2Dlmkb9CQ6v3/1G2RezKN4YS2lcEv8lzGm1mhs4f2lM4yNAMI/zUDBZBBt7w==} + '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} @@ -1070,6 +1076,10 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + apache-arrow@21.2.0: + resolution: {integrity: sha512-Hxe6Agq26gQOM954qpzYSllJBPJl+e16U5CkfuMUhLrNba+5nKkttIVlflaovN6oaTratqMGAO8H5u/aNhmHWQ==} + hasBin: true + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -1429,6 +1439,9 @@ packages: resolution: {integrity: sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==} hasBin: true + flatbuffers@25.9.23: + resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} + follow-redirects@1.16.0: resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} engines: {node: '>=4.0'} @@ -1738,6 +1751,9 @@ packages: json-schema-traverse@1.0.0: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + json-with-bigint@3.5.12: + resolution: {integrity: sha512-uwbF/wSSuOgC7qqlq27Xp5B6a2MHVug3t0idZdTqu0JnlFvgJuH7ju+KAk/J06C7GfhoYy2gnb9wz2INqcne7w==} + json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} @@ -2391,6 +2407,9 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici-types@7.24.6: + resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + unicorn-magic@0.1.0: resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==} engines: {node: '>=18'} @@ -3053,6 +3072,10 @@ snapshots: dependencies: undici-types: 6.21.0 + '@types/node@25.9.9': + dependencies: + undici-types: 7.24.6 + '@types/normalize-package-data@2.4.4': {} '@types/sarif@2.1.7': {} @@ -3273,6 +3296,13 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.2 + apache-arrow@21.2.0: + dependencies: + '@types/node': 25.9.9 + flatbuffers: 25.9.23 + json-with-bigint: 3.5.12 + tslib: 2.8.1 + argparse@2.0.1: {} astral-regex@2.0.0: {} @@ -3666,6 +3696,8 @@ snapshots: flat@5.0.2: {} + flatbuffers@25.9.23: {} + follow-redirects@1.16.0(debug@4.4.3(supports-color@8.1.1)): optionalDependencies: debug: 4.4.3(supports-color@8.1.1) @@ -3955,6 +3987,8 @@ snapshots: json-schema-traverse@1.0.0: {} + json-with-bigint@3.5.12: {} + json5@2.2.3: {} jsonc-parser@3.3.1: {} @@ -4718,6 +4752,8 @@ snapshots: undici-types@6.21.0: {} + undici-types@7.24.6: {} + unicorn-magic@0.1.0: {} unicorn-magic@0.3.0: {} diff --git a/sess/DESCRIPTION b/sess/DESCRIPTION index 77a5b5f43..67a923b78 100644 --- a/sess/DESCRIPTION +++ b/sess/DESCRIPTION @@ -1,7 +1,7 @@ Package: sess Type: Package Title: High-Performance IPC Bridge for R Sessions -Version: 3.2.0 +Version: 3.2.9000.9000 Authors@R: c( person(given = "Randy", family = "Lai", @@ -37,6 +37,7 @@ Suggests: htmlwidgets, systemfonts, httpgd, + jdv, jgd, svglite, tinytest diff --git a/sess/R/handlers.R b/sess/R/handlers.R index ad0f10f50..9c8bb48a2 100644 --- a/sess/R/handlers.R +++ b/sess/R/handlers.R @@ -1291,3 +1291,155 @@ handle_dataview_dispose <- function(params) { } TRUE } + +# jdv data viewer backend (experimental) ----------------------------------- +# +# When the jdv package is available, View() registers data frames, matrices, +# lists and Arrow data with jdv instead of the sess data viewer. jdv talks to the +# viewer over its own local transport, whose address VS Code provides per +# session; sess only registers the views. Set `options(sess.dataviewer = "sess")` +# to keep using the sess data viewer. + +.jdv_arrow_classes <- c("ArrowTabular", "RecordBatchReader", "nanoarrow_array_stream", + "polars_data_frame") + +.jdv_is_arrow <- function(x) { + inherits(x, .jdv_arrow_classes) +} + +# Return a row-count hint when an Arrow source exposes dimensions without reading +# its data. Forward-only readers and streams return NULL from nrow(). +.jdv_arrow_nrow <- function(x) { + count <- tryCatch(nrow(x), error = function(e) NULL) + if (is.numeric(count) && !is.complex(count) && length(count) == 1L && + is.finite(count) && count >= 0 && count == floor(count)) { + unname(as.numeric(count)) + } +} + +# Make sure jdv has a persistent connection to this session's viewer host. +# Returns FALSE, leaving the user's own jdv state untouched, when the sess data +# viewer should be used instead: jdv is missing or opted out, VS Code offers no +# transport, or jdv is already connected elsewhere or owned by an embedding host. +# Every failure, whether a jdv_error (jdv_error_in_use, jdv_error_connect_failed) +# or the transport request itself failing, means the same fallback, so errors +# are not told apart. +.jdv_connect <- function(address = NULL) { + if (identical(getOption("sess.dataviewer", "auto"), "sess") || + !requireNamespace("jdv", quietly = TRUE)) { + return(FALSE) + } + tryCatch({ + connection <- jdv::jdv_connection() + if (identical(connection$state, "connected")) { + return(is.character(.sess_env$jdv_address) && + identical(connection$address, .sess_env$jdv_address)) + } + if (!identical(connection$state, "disconnected")) { + return(FALSE) + } + # Callers that cannot wait for a reply (an inspection being answered) pass it in. + if (is.null(address)) { + address <- request_client("jdv/address")$address + } + if (!is.character(address) || length(address) != 1L || is.na(address) || + !nzchar(address)) { + return(FALSE) + } + # An explicit address leaves discovery.json and the `jdv.address` option of + # other jdv hosts the user may run alone. + jdv::jdv_connect(address) + .sess_env$jdv_address <- address + TRUE + }, error = function(e) FALSE) +} + +# Close the connection that .jdv_connect() opened, with all of its views. +.jdv_release <- function() { + if (is.null(.sess_env$jdv_address)) { + return(invisible(NULL)) + } + address <- .sess_env$jdv_address + .sess_env$jdv_address <- NULL + if (isNamespaceLoaded("jdv")) { + try({ + if (identical(jdv::jdv_connection()$address, address)) { + jdv::jdv_disconnect() + } + }, silent = TRUE) + } + invisible(NULL) +} + +# The variable that View() was called with, for jdv to follow. +# jdv::jdv_binding() does the lookup without evaluating anything: it returns +# list(name, env) for a symbol bound in `env` or one of its parents, and NULL +# for other expressions, unreadable bindings (promises, active bindings), unbound +# names and R older than 4.6.0. +# sess only adds the gates: the VS Code setting `r.session.viewers.trackVariables` +# (passed to connect() as `track_variables`, default on), which +# `options(sess.dataviewer.track = TRUE/FALSE)` overrides in both directions. +.jdv_track_binding <- function(expression, env) { + track <- getOption("sess.dataviewer.track", .sess_env$track_variables %||% TRUE) + if (!is.symbol(expression) || !nzchar(as.character(expression)) || !isTRUE(track) || + !requireNamespace("jdv", quietly = TRUE)) { + return(NULL) + } + tryCatch(jdv::jdv_binding(expression, env), error = function(e) NULL) +} + +# Show `x` with jdv. Returns FALSE when jdv cannot be used, so the caller falls +# back to the sess data viewer. +# +# The title is jdv's display slot: View() with the same title replaces the view +# and increments its generation. Grids are paged so that sorting and filtering +# stay in R. When `binding` names the variable that holds `x`, jdv follows it and +# rebuilds the view as the next generation after the variable is reassigned. jdv +# rejects the binding with a jdv_error if it does not hold `x` (for example +# View(x) called through do.call()); then the view is registered without +# tracking, since View() must not fail because of it. Arrow sources are +# sequential and cannot be tracked, so they never get a binding. +# +# jdv checks tracked variables from its own task callback and sends the update +# over the transport itself, so sess needs no task callback of its own for it. +# Only an embedding host that collects jdv_notifications() must call +# jdv_rpc(NULL) first. +.jdv_show <- function(x, title_key, binding = NULL, address = NULL) { + if (.jdv_is_arrow(x) && !requireNamespace("nanoarrow", quietly = TRUE)) { + return(FALSE) + } + row_count <- if (.jdv_is_arrow(x)) .jdv_arrow_nrow(x) else NULL + title <- if (nzchar(title_key)) title_key else NULL + # Only jdv errors about the binding are retried without it; anything else + # ends in FALSE so that View() falls back to the sess data viewer. + .jdv_show_registered(function() { + if (!is.null(binding) && !.jdv_is_arrow(x)) { + registered <- tryCatch({ + jdv::jdv_register(x, title = title, mode = "paged", binding = binding) + }, jdv_error = function(e) FALSE) + if (!identical(registered, FALSE)) { + return(registered) + } + } + jdv::jdv_register(x, title = title, mode = "paged") + }, address, row_count) +} + +# Connect and register a view, sharing the same failure contract for values and +# Interactive handles. A row-count notification is best-effort display metadata. +.jdv_show_registered <- function(register, address = NULL, row_count = NULL) { + if (!.jdv_connect(address)) return(FALSE) + tryCatch({ + registration <- register() + if (!is.null(row_count)) { + # A notification is only a hint; it must not make a successful View fail. + try(notify_client("jdv/rowCount", list( + view = registration$id, + generation = registration$generation, + nrow = row_count, + address = .sess_env$jdv_address + )), silent = TRUE) + } + TRUE + }, error = function(e) FALSE) +} diff --git a/sess/R/hooks.R b/sess/R/hooks.R index 0d207ea44..3dbfc7c6d 100644 --- a/sess/R/hooks.R +++ b/sess/R/hooks.R @@ -74,11 +74,9 @@ runtime_start <- function(use_rstudioapi = TRUE, owner <- as.character(expression) } force(title) - - if (is.null(.sess_env$view_owner) && isTRUE(.sess_env$interactive_connected) && - .interactive_rich_value(x)) { - return(invisible(NULL)) - } + # Resolve the variable behind View() before x is forced, so jdv can + # follow it. The lookup starts at the caller, never at this function's frame. + tracked_binding <- .jdv_track_binding(original_expression, parent.frame()) view_type <- if (dataview_is_table(x)) { "table" @@ -88,6 +86,16 @@ runtime_start <- function(use_rstudioapi = TRUE, "object" } title_key <- paste(as.character(title), collapse = "\n") + if ((view_type %in% c("table", "list") || .jdv_is_arrow(x)) && + .jdv_show(x, title_key, tracked_binding)) { + return(invisible(NULL)) + } + # Explicit View() in Interactive should open the normal data viewer when + # jdv is available. Inline rich output remains the fallback when it is not. + if (is.null(.sess_env$view_owner) && isTRUE(.sess_env$interactive_connected) && + .interactive_rich_value(x)) { + return(invisible(NULL)) + } owner <- owner %||% title_key registry_key <- paste0(view_type, ":", owner) dataview_registry <- .sess_env$dataview_registry diff --git a/sess/R/interactive.R b/sess/R/interactive.R index 03e935025..8e4b3be12 100644 --- a/sess/R/interactive.R +++ b/sess/R/interactive.R @@ -1,15 +1,21 @@ # Rich events use pure R IPC; small stdout markers preserve console ordering. +.interactive_event_json <- function(type, data) { + id <- .sess_env$interactive_id + as.character(jsonlite::toJSON( + c(list(type = type, executionId = if (is.null(id)) "" else id), data), + auto_unbox = TRUE, null = "null", na = "null", digits = NA + )) +} + .interactive_event <- function(type, data = list()) { if (!identical(.sess_env$interactive_pid, Sys.getpid())) return(invisible(NULL)) id <- .sess_env$interactive_id - event <- c(list(type = type, executionId = if (is.null(id)) "" else id), data) - json <- as.character(jsonlite::toJSON( - event, auto_unbox = TRUE, null = "null", na = "null", digits = NA - )) + json <- .interactive_event_json(type, data) # Bound encoded events before writing to the agent transport. # Agent-side display limits run only after parsing and cannot protect it from # oversized HTML, wide tables, or strings enlarged by JSON escaping. if (nchar(json, type = "bytes") > 2 * 1024 * 1024) { + if (!is.null(data$handleId)) .interactive_handle_close(data$handleId) json <- as.character(jsonlite::toJSON(list( type = "truncated", executionId = if (is.null(id)) "" else id, message = "Rich output exceeds 2 MiB; display a smaller preview or save it to a file." @@ -369,28 +375,191 @@ interactive_execute <- function(id, code, source = NULL) { list(value = snapshot, source_rows = source_rows, truncated = truncated) } +# jdv handles back the inline table and list previews ------------------------ +# +# Each table or list output holds one registry-free jdv handle (no viewer slot, +# no notifications). The agent sends the descriptor and the first page to the +# extension, which pages, sorts, filters and explores through `jdv_handle_request` +# inspections. Handles are closed when the output is cleared, the registry +# exceeds its bound, or the Interactive bridge stops; jdv also closes +# unreachable handles. Without jdv (or with options(sess.dataviewer = "sess")) +# the previous dataview registry serves the same outputs. + +.interactive_handle_limit <- 64L +.interactive_page_bytes <- 256L * 1024L +# Leave 512 KiB for adapter normalization and agent event metadata. +.interactive_preview_bytes <- 1536L * 1024L + +.interactive_jdv_usable <- function() { + !identical(getOption("sess.dataviewer", "auto"), "sess") && + requireNamespace("jdv", quietly = TRUE) +} + +.interactive_handle_store <- function() { + if (is.null(.sess_env$interactive_handles)) { + .sess_env$interactive_handles <- new.env(parent = emptyenv()) + .sess_env$interactive_handle_order <- character() + .sess_env$interactive_handle_seq <- 0L + } + .sess_env$interactive_handles +} + +.interactive_handle_close <- function(id) { + store <- .interactive_handle_store() + if (is.character(id) && length(id) == 1L && exists(id, envir = store, inherits = FALSE)) { + entry <- get(id, envir = store, inherits = FALSE) + rm(list = id, envir = store) + .sess_env$interactive_handle_order <- setdiff(.sess_env$interactive_handle_order, id) + try(jdv::jdv_handle_close(entry$handle), silent = TRUE) + } + invisible(NULL) +} + +.interactive_handle_close_all <- function() { + for (id in .sess_env$interactive_handle_order) .interactive_handle_close(id) + invisible(NULL) +} + +.interactive_handle_entry <- function(id) { + store <- .interactive_handle_store() + if (!is.character(id) || length(id) != 1L || !exists(id, envir = store, inherits = FALSE)) { + stop("This output's data is no longer available. Run its code again.") + } + get(id, envir = store, inherits = FALSE) +} + +# Capture `value` and read its descriptor and preview pages. Returns NULL, leaving no +# handle behind, when jdv cannot represent the value; the caller then uses the +# dataview registry. +.interactive_handle_preview <- function(value, kind, data = list()) { + if (!.interactive_jdv_usable()) return(NULL) + handle <- NULL + tryCatch({ + handle <- jdv::jdv_handle(value, mode = "paged") + descriptor <- jdv::jdv_handle_descriptor(handle, max_bytes = .interactive_page_bytes) + .interactive_handle_store() + id <- paste0("h", .sess_env$interactive_handle_seq + 1L) + page <- if (kind == "table") { + fits <- function(pages) { + event <- c(list(kind = kind, handleId = id, descriptor = descriptor, page = pages), data) + bytes <- nchar(.interactive_event_json("display", event), type = "bytes") + bytes <= .interactive_preview_bytes + } + # Preserve the table and viewer control when the printed fallback is too large. + if (!fits(character())) data$printedText <- NULL + pages <- character() + offset <- 0L + repeat { + next_page <- jdv::jdv_handle_request( + handle, "grid/rows", list(offset = offset, limit = 20L - offset), + max_bytes = .interactive_page_bytes + ) + if (!fits(c(pages, next_page))) break + pages <- c(pages, next_page) + parsed <- jsonlite::fromJSON(next_page, simplifyVector = FALSE) + returned <- length(parsed$data$source_row) + offset <- offset + returned + if (returned == 0L || offset >= parsed$total || offset >= 20L) break + } + pages + } else { + jdv::jdv_handle_request( + handle, "object/children", + list(path = list(), offset = 0L, limit = 200L, attributes = FALSE), + max_bytes = .interactive_page_bytes + ) + } + store <- .interactive_handle_store() + .sess_env$interactive_handle_seq <- .sess_env$interactive_handle_seq + 1L + id <- paste0("h", .sess_env$interactive_handle_seq) + assign(id, list(handle = handle), envir = store) + .sess_env$interactive_handle_order <- c(.sess_env$interactive_handle_order, id) + while (length(.sess_env$interactive_handle_order) > .interactive_handle_limit) { + .interactive_handle_close(.sess_env$interactive_handle_order[[1L]]) + } + c(list(handleId = id, descriptor = descriptor, page = page), data) + }, error = function(e) { + if (!is.null(handle)) try(jdv::jdv_handle_close(handle), silent = TRUE) + NULL + }) +} + +handle_jdv_request <- function(params) { + entry <- .interactive_handle_entry(params$handle_id) + method <- params$method + if (!identical(method, "grid/rows") && !identical(method, "object/children")) { + stop("Unsupported jdv handle method") + } + max_bytes <- suppressWarnings(as.integer(params$max_bytes %||% .interactive_page_bytes)) + if (is.na(max_bytes) || max_bytes < 1L) max_bytes <- .interactive_page_bytes + list(json = jdv::jdv_handle_request(entry$handle, method, params$params %||% list(), + max_bytes = max_bytes)) +} + +handle_jdv_handle_close <- function(params) { + .interactive_handle_close(params$handle_id) + TRUE +} + +# Register the captured value without invoking R subsetting methods or forcing +# bindings. A temporary child is released even when registration fails; the view +# itself has an independent lifetime. +.jdv_show_handle <- function(handle, path, title_key, address = NULL) { + .jdv_show_registered(function() { + if (length(path)) { + handle <- jdv::jdv_handle_open(handle, path, mode = "paged") + on.exit(jdv::jdv_handle_close(handle), add = TRUE) + } + # NULL would reuse the handle's title, hence refresh an unrelated view. The + # handle API requires an explicit unique title for an untitled independent view. + title <- if (nzchar(title_key)) title_key else basename(tempfile("Interactive output ")) + jdv::jdv_handle_register(handle, title = title, mode = "paged") + }, address) +} + +# Show the output's value, or the element at `path`, in the Data Viewer panel by +# registering it through the stage A jdv connection. Returns FALSE when the panel +# cannot be opened. +handle_jdv_open_viewer <- function(params) { + entry <- .interactive_handle_entry(params$handle_id) + title <- params$title + address <- params$address + .jdv_show_handle(entry$handle, params$path %||% list(), + if (is.character(title) && length(title) == 1L && !is.na(title)) title else "", + address = if (is.character(address) && length(address) == 1L) address) +} + .interactive_rich_value <- function(value) { if (dataview_is_table(value)) { snapshot <- .interactive_table_snapshot(value) + printed <- .interactive_table_text(snapshot$value) + if (snapshot$truncated && !is.null(printed$printedText)) { + note <- sprintf( + "\n[Snapshot: first %s of %s rows. Open Data viewer for the full table.]\n", + format(nrow(snapshot$value), big.mark = ",", scientific = FALSE, trim = TRUE), + format(snapshot$source_rows, big.mark = ",", scientific = FALSE, trim = TRUE) + ) + printed$printedText <- paste0(printed$printedText, note) + } + preview <- if (is.data.frame(value) || is.matrix(value)) { + .interactive_handle_preview(value, "table", + c(list(sourceRows = snapshot$source_rows), printed)) + } + if (!is.null(preview)) { + .interactive_event("display", c(list(kind = "table"), preview)) + return(TRUE) + } registration <- dataview_register(snapshot$value) metadata <- handle_dataview_init(list(view_id = registration$view_id)) preview <- handle_dataview_page(list(view_id = registration$view_id, startRow = 0L, endRow = 20L, formatNumbers = TRUE, sortModel = list(), filterModel = list())) - printed <- .interactive_table_text(snapshot$value) full_view <- NULL if (snapshot$truncated) { # Retain the full object without copying it. The cell remains a snapshot; # the expanded viewer can reflect later reference edits to this object. full_view <- dataview_register(value, live = TRUE)$view_id - if (!is.null(printed$printedText)) { - note <- - sprintf("\n[Snapshot: first %s of %s rows. Open Data viewer for the full table.]\n", - format(metadata$totalRows, big.mark = ",", scientific = FALSE, trim = TRUE), - format(snapshot$source_rows, big.mark = ",", scientific = FALSE, trim = TRUE)) - printed$printedText <- paste0(printed$printedText, note) - } } .interactive_event("display", c(list(kind = "table", viewId = registration$view_id, fullViewId = full_view, @@ -402,6 +571,13 @@ interactive_execute <- function(id, code, source = NULL) { return(TRUE) } if (is.list(value) && !is.object(value)) { + preview <- .interactive_handle_preview(value, "list") + if (!is.null(preview)) { + .interactive_event("display", c(list(kind = "list", handleId = preview$handleId, + descriptor = preview$descriptor, page = preview$page), + .interactive_table_text(value))) + return(TRUE) + } view_id <- dataview_new_id() # Nested panels reuse the root's owner; separate cell results must stay independent. root <- listview_state(value, "List", paste0("interactive:", view_id)) @@ -455,6 +631,7 @@ display <- function(x, mime = NULL) { #' @export interactive_stop <- function() { .sess_env$interactive_stop <- TRUE + try(.interactive_handle_close_all(), silent = TRUE) if (!is.null(.sess_env$interactive_task)) { removeTaskCallback(.sess_env$interactive_task) .sess_env$interactive_task <- NULL diff --git a/sess/R/runtime.R b/sess/R/runtime.R index c62eb22fd..69bbdb380 100644 --- a/sess/R/runtime.R +++ b/sess/R/runtime.R @@ -16,6 +16,8 @@ } .runtime_clear_viewer_state <- function() { + try(.interactive_handle_close_all(), silent = TRUE) + .jdv_release() # Open dataviews and title-to-id mappings belong to one runtime connection. # Keep an empty registry available, but never restore data from a prior run. .sess_env$dataviews <- list() diff --git a/sess/R/server.R b/sess/R/server.R index 0fb37b410..038593f40 100644 --- a/sess/R/server.R +++ b/sess/R/server.R @@ -11,6 +11,11 @@ #' It is not an authentication token or an exclusive ownership claim. #' @param auto_activate Logical. Whether vscode-R should select this attachment #' automatically. `FALSE` requires `attachment_id`. +#' @param track_variables Logical or NULL. Whether the data viewer follows the +#' variable passed to `View()` (R >= 4.6 and jdv). The VS Code extension passes +#' its `r.session.viewers.trackVariables` setting. `NULL` and `TRUE` both +#' enable tracking. `options(sess.dataviewer.track)`, when set, takes +#' precedence over this argument. #' @return Invisibly returns `TRUE` when the IPC connection and runtime startup #' succeed, or `FALSE` when the IPC endpoint is unavailable. #' @details When SESS_DISCOVERY_FILE describes the connected endpoint, an @@ -23,7 +28,8 @@ #' @export connect <- function(endpoint = NULL, use_rstudioapi = TRUE, plot_backend = c("auto", "jgd", "httpgd", "standard", "native"), - attachment_id = NULL, auto_activate = TRUE) { + attachment_id = NULL, auto_activate = TRUE, + track_variables = NULL) { if (!is.null(attachment_id) && !(is.character(attachment_id) && length(attachment_id) == 1L && !is.na(attachment_id) && @@ -43,6 +49,7 @@ connect <- function(endpoint = NULL, use_rstudioapi = TRUE, # Metadata belongs to this connect call, not to the R process. In particular, # a later native attach must not inherit an external terminal's identity. .sess_env$attachment_id <- attachment_id + .sess_env$track_variables <- !isFALSE(track_variables) .sess_env$auto_activate <- auto_activate .sess_env$con <- NULL .sess_env$pending_responses <- list() @@ -72,7 +79,8 @@ connect <- function(endpoint = NULL, use_rstudioapi = TRUE, .sess_env$reconnect <- list( path = discovery_file, endpoint = endpoint, options = list(use_rstudioapi = use_rstudioapi, plot_backend = plot_backend, - attachment_id = attachment_id, auto_activate = auto_activate) + attachment_id = attachment_id, auto_activate = auto_activate, + track_variables = track_variables) ) } @@ -488,7 +496,10 @@ dispatch_message <- function(line) { "listview_view" = function(p) handle_listview_view(p$view_id, p$index, p$path), "dataview_init" = function(p) handle_dataview_init(p), "dataview_page" = function(p) handle_dataview_page(p), - "dataview_dispose" = function(p) handle_dataview_dispose(p) + "dataview_dispose" = function(p) handle_dataview_dispose(p), + "jdv_handle_request" = function(p) handle_jdv_request(p), + "jdv_handle_close" = function(p) handle_jdv_handle_close(p), + "jdv_open_viewer" = function(p) handle_jdv_open_viewer(p) ) if (payload$method %in% names(handlers)) { diff --git a/sess/README.md b/sess/README.md index 441bbbb12..43023f909 100644 --- a/sess/README.md +++ b/sess/README.md @@ -87,7 +87,8 @@ When you start an R terminal from VS Code, the extension's R profile calls sess::connect( endpoint = NULL, # socket/pipe endpoint; see below use_rstudioapi = TRUE, # emulate rstudioapi functions - plot_backend = "auto" # or "standard", "httpgd", "jgd", "native" + plot_backend = "auto", # or "standard", "httpgd", "jgd", "native" + track_variables = NULL # follow the variable passed to View() (R >= 4.6 with jdv); NULL means TRUE ) ``` @@ -187,6 +188,8 @@ is used. | `SESS_ENDPOINT` | env var | Socket/pipe path used by `connect()`. | | `SESS_RSTUDIOAPI` | env var | `TRUE`/`FALSE`; passed as `use_rstudioapi` by the extension's R profile. | | `SESS_PLOT_BACKEND` | env var | `auto`, `standard`, `httpgd`, `jgd` or `native`; passed as `plot_backend` by the extension's R profile. | +| `SESS_TRACK_VARIABLES` | env var | `TRUE`/`FALSE`; passed as `track_variables` by the extension's R profile (setting `r.session.viewers.trackVariables`). | +| `sess.dataviewer.track` | R option | `TRUE`/`FALSE`; whether `View()` follows later changes of the variable (R >= 4.6 with jdv). When set, it takes precedence over `track_variables`. | | `JGD_SOCKET` | env var | Socket used by the jgd device; set by the extension. | | `sess.quiet` | R option | Set to `TRUE` to suppress the successful connection message. Connection failures remain visible. | diff --git a/sess/inst/tinytest/test-interactive-handles.R b/sess/inst/tinytest/test-interactive-handles.R new file mode 100644 index 000000000..a96cc61c4 --- /dev/null +++ b/sess/inst/tinytest/test-interactive-handles.R @@ -0,0 +1,295 @@ +# Inline table and list previews use registry-free jdv handles when jdv is installed. +if (!requireNamespace("jdv", quietly = TRUE)) { + exit_file("jdv is not installed") +} + +ns <- asNamespace("sess") +env <- sess:::.sess_env +previous <- list(handles = env$interactive_handles, order = env$interactive_handle_order, + seq = env$interactive_handle_seq) +on.exit({ + ns$.interactive_handle_close_all() + env$interactive_handles <- previous$handles + env$interactive_handle_order <- previous$order + env$interactive_handle_seq <- previous$seq + options(sess.dataviewer = NULL) +}) +options(sess.dataviewer = NULL) +preview <- ns$.interactive_handle_preview +request <- function(id, method, params = list()) { + jsonlite::fromJSON(ns$handle_jdv_request(list(handle_id = id, method = method, + params = params))$json, + simplifyVector = FALSE) +} + +# A table yields a descriptor and a bounded first page without using the viewer registry. +table <- preview(data.frame(id = 1:100, name = rep(c("a", "b"), 50)), "table") +expect_true(is.character(table$handleId)) +descriptor <- jsonlite::fromJSON(table$descriptor, simplifyVector = FALSE) +expect_identical(descriptor$kind, "grid") +expect_identical(descriptor$access, "random") +expect_identical(descriptor$nrow, 100L) +first <- jsonlite::fromJSON(table$page, simplifyVector = FALSE) +expect_identical(length(first$data$source_row), 20L) +expect_identical(first$total, 100L) +single_event <- jsonlite::fromJSON(jsonlite::toJSON( + list(page = table$page), auto_unbox = TRUE, null = "null", digits = NA +), simplifyVector = FALSE) +expect_identical(single_event$page, table$page) + +# Byte-limited pages stay unchanged and stop before the complete event budget. +numeric <- rep(pi, 150) +numeric[[3L]] <- Inf +wide <- data.frame( + id = 1:150, text = rep(strrep("a", 100000), 150), + category = factor(rep(c("a", "b"), 75)), missing = rep(c(NA, "x"), 75), + numeric = numeric, row.names = paste0("row-", 0:149) +) +wide_preview <- preview(wide, "table") +expect_true(length(wide_preview$page) > 1L) +wide_entry <- ns$.interactive_handle_entry(wide_preview$handleId) +expected_pages <- character() +offset <- 0L +while (length(expected_pages) < length(wide_preview$page)) { + raw_page <- jdv::jdv_handle_request( + wide_entry$handle, "grid/rows", list(offset = offset, limit = 20L - offset), + max_bytes = ns$.interactive_page_bytes + ) + expected_pages <- c(expected_pages, raw_page) + parsed_page <- jsonlite::fromJSON(raw_page, simplifyVector = FALSE) + returned <- length(parsed_page$data$source_row) + if (returned == 0L) break + offset <- offset + returned +} +expect_identical(wide_preview$page, expected_pages) +multi_event <- jsonlite::fromJSON(jsonlite::toJSON( + list(page = wide_preview$page), auto_unbox = TRUE, null = "null", digits = NA +), simplifyVector = FALSE) +expect_identical(unlist(multi_event$page), expected_pages) +wide_pages <- lapply(wide_preview$page, jsonlite::fromJSON, simplifyVector = FALSE) +wide_rows <- unlist(lapply(wide_pages, function(page) page$data$source_row)) +expect_true(length(wide_rows) < 20L) +expect_identical(wide_rows, seq_along(wide_rows) - 1L) +factor_values <- sum(vapply(wide_pages, function(page) { + length(page$data$columns[[3L]]$data$values) +}, 0L)) +missing_values <- unlist(lapply(wide_pages, function(page) { + (seq_along(page$data$source_row) - 1L) %in% page$data$columns[[4L]]$data[["na_rows"]] +})) +row_labels <- unlist(lapply(wide_pages, function(page) page$data$row_labels)) +empty_errors <- vapply(wide_pages, function(page) { + identical(names(page$data$columns[[1L]]$errors), character()) +}, logical(1L)) +expect_identical(factor_values, length(wide_rows)) +expect_identical(missing_values, rep(c(TRUE, FALSE), length.out = length(wide_rows))) +expect_identical(row_labels, paste0("row-", wide_rows)) +expect_true(all(empty_errors)) + +# Capture the actual display event, including printed text and JSON escaping. +local({ + original_send <- ns$rpc_send + fields <- c("interactive_pid", "interactive_id", "interactive_managed") + saved <- mget(fields, env, ifnotfound = list(NULL)) + captured <- NULL + unlockBinding("rpc_send", ns) + assign("rpc_send", function(method, params) captured <<- params, envir = ns) + lockBinding("rpc_send", ns) + on.exit({ + unlockBinding("rpc_send", ns) + assign("rpc_send", original_send, envir = ns) + lockBinding("rpc_send", ns) + list2env(saved, env) + }, add = TRUE) + env$interactive_pid <- Sys.getpid() + env$interactive_id <- "preview-budget" + env$interactive_managed <- FALSE + expect_true(ns$.interactive_rich_value(wide)) + expect_identical(captured$type, "display") + expect_identical(captured$kind, "table") + encoded <- jsonlite::toJSON(captured, auto_unbox = TRUE, null = "null", digits = NA) + expect_true(nchar(encoded, type = "bytes") <= ns$.interactive_preview_bytes) + expect_true(is.character(captured$handleId)) + + # The printed fallback can be omitted; a valid table with no inline rows is OK. + oversized <- preview(wide, "table", list(printedText = strrep("x", 2L * 1024L * 1024L))) + expect_null(oversized$printedText) + empty <- preview(wide, "table", list(printedText = strrep("x", 1530L * 1024L))) + expect_identical(length(empty$page), 0L) + ns$.interactive_event("display", c(list(kind = "table"), empty)) + expect_identical(captured$type, "display") + empty_json <- ns$.interactive_event_json("display", empty) + expect_true(nchar(empty_json, type = "bytes") < 2L * 1024L * 1024L) + + # The last-resort truncation path releases the captured handle. + ns$.interactive_event("display", c(list(kind = "table"), empty, + list(extra = strrep("x", 2L * 1024L * 1024L)))) + expect_identical(captured$type, "truncated") + expect_error(ns$.interactive_handle_entry(empty$handleId), "no longer available") +}) + +# Paging, sorting and filtering run in R through the same handle; ids are zero-based rows. +sorted <- request(table$handleId, "grid/rows", list( + offset = 0L, limit = 3L, sort = list(list(col = 0L, desc = TRUE)), + filter = list(op = "text_equals", col = 1L, value = "a") +)) +expect_identical(sorted$total, 50L) +expect_identical(unlist(sorted$data$source_row), c(98L, 96L, 94L)) + +# A list yields a tree descriptor and its first children page. +tree <- preview(list(a = 1, b = data.frame(z = 1:2), 3), "list") +expect_identical(jsonlite::fromJSON(tree$descriptor, simplifyVector = FALSE)$kind, "tree") +children <- jsonlite::fromJSON(tree$page, simplifyVector = FALSE) +expect_identical(vapply(children$children, function(child) child$label, ""), c("a", "b", "")) +expect_identical(request(tree$handleId, "object/children", list( + path = list(list(type = "index", index = 2L)), offset = 0L, limit = 10L, attributes = FALSE +))$total_children, 1L) + +# Opening an Interactive handle registers the selected value through jdv's handle API. +# The custom [[ method must not run while opening a nested table. +local({ + started_session <- identical(jdv::jdv_connection()$state, "disconnected") + original_connect <- ns$.jdv_connect + review_handle_id <- NULL + registered_views <- list() + unlockBinding(".jdv_connect", ns) + assign(".jdv_connect", function(address = NULL) TRUE, envir = ns) + lockBinding(".jdv_connect", ns) + on.exit({ + if (!is.null(review_handle_id)) { + ns$handle_jdv_handle_close(list(handle_id = review_handle_id)) + } + for (view in registered_views) { + try(jdv::jdv_rpc(jsonlite::toJSON(list( + id = 1L, method = "view/close", + params = list(view = view$id, generation = view$generation) + ), auto_unbox = TRUE)), silent = TRUE) + } + jdv::jdv_notifications() + if (started_session) jdv::jdv_disconnect() + unlockBinding(".jdv_connect", ns) + assign(".jdv_connect", original_connect, envir = ns) + lockBinding(".jdv_connect", ns) + }, add = TRUE) + + anchor <- jdv::jdv_register(data.frame(anchor = 1L), title = "interactive handles test anchor") + registered_views <- list(anchor) + jdv::jdv_notifications() + method_calls <- 0L + previous_method <- if (exists("[[.review_list", envir = .GlobalEnv, inherits = FALSE)) { + get("[[.review_list", envir = .GlobalEnv, inherits = FALSE) + } + assign("[[.review_list", function(x, i, ...) { + method_calls <<- method_calls + 1L + stop("review_list [[ method must not be called") + }, envir = .GlobalEnv) + on.exit({ + if (is.null(previous_method)) { + if (exists("[[.review_list", envir = .GlobalEnv, inherits = FALSE)) { + rm("[[.review_list", envir = .GlobalEnv) + } + } else { + assign("[[.review_list", previous_method, envir = .GlobalEnv) + } + }, add = TRUE) + review <- structure(list(table = data.frame(z = 1:2)), class = "review_list") + review_preview <- preview(review, "list") + review_handle_id <- review_preview$handleId + expect_true(!is.null(review_preview)) + + opens <- function() { + messages <- lapply(jdv::jdv_notifications(), jsonlite::fromJSON, + simplifyVector = FALSE) + result <- Filter(function(message) identical(message$method, "view/open"), messages) + registered_views <<- c(registered_views, lapply(result, function(message) { + list(id = message$params$view, generation = message$params$generation) + })) + result + } + opened <- function() lapply(opens(), `[[`, "params") + request <- function(method, params) { + parsed <- jsonlite::fromJSON(jdv::jdv_rpc(jsonlite::toJSON( + list(id = 1L, method = method, params = params), auto_unbox = TRUE + )), simplifyVector = FALSE) + parsed$result + } + + expect_true(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, + path = list(list(type = "index", index = 1L)), + title = "Cell: List$table" + ))) + child <- opened()[[1L]] + expect_identical(child$kind, "grid") + expect_identical(child$title, "Cell: List$table") + rows <- request("grid/rows", list(view = child$view, generation = child$generation, + offset = 0L, limit = 2L)) + expect_identical(length(rows$data$source_row), 2L) + expect_identical(method_calls, 0L) + + # Empty paths register the root handle itself; named titles retain refresh behavior. + expect_true(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, path = list(), title = "Interactive root" + ))) + root_first <- opened()[[1L]] + expect_identical(root_first$kind, "tree") + expect_identical(root_first$title, "Interactive root") + expect_true(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, path = list(), title = "Interactive root" + ))) + root_refresh <- opened()[[1L]] + expect_identical(root_refresh$view, root_first$view) + expect_identical(root_refresh$generation, root_first$generation + 1L) + + # Empty titles remain independent even though jdv handles carry their own fallback title. + expect_true(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, path = list(), title = "" + ))) + expect_true(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, path = list(), title = "" + ))) + untitled <- opened() + expect_identical(length(untitled), 2L) + expect_true(all(vapply(untitled, function(view) startsWith(view$title, "Interactive output "), + logical(1L)))) + expect_false(identical(untitled[[1L]]$view, untitled[[2L]]$view)) + + expect_false(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, + path = list(list(type = "index", index = 99L)), title = "invalid path" + ))) + unlockBinding(".jdv_connect", ns) + assign(".jdv_connect", function(address = NULL) FALSE, envir = ns) + lockBinding(".jdv_connect", ns) + expect_false(ns$handle_jdv_open_viewer(list( + handle_id = review_preview$handleId, path = list(), title = "disconnected" + ))) + expect_true(ns$handle_jdv_handle_close(list(handle_id = review_preview$handleId))) + rows_after_handle_close <- request( + "grid/rows", list(view = child$view, generation = child$generation, offset = 0L, limit = 2L) + ) + expect_identical(length(rows_after_handle_close$data$source_row), 2L) +}) + +# Only grid rows and object children can be requested. +expect_error(ns$handle_jdv_request(list(handle_id = table$handleId, method = "view/close")), + "Unsupported") + +# Closing releases the handle, idempotently; unknown ids report that the data is gone. +expect_true(ns$handle_jdv_handle_close(list(handle_id = table$handleId))) +expect_true(ns$handle_jdv_handle_close(list(handle_id = table$handleId))) +expect_error(request(table$handleId, "grid/rows"), "no longer available") + +# The number of live handles is bounded; the oldest are closed first. +ns$.interactive_handle_close_all() +ids <- vapply(seq_len(ns$.interactive_handle_limit + 3L), function(i) { + preview(list(i), "list")$handleId +}, "") +expect_identical(length(env$interactive_handle_order), ns$.interactive_handle_limit) +expect_error(request(ids[[1L]], "object/children"), "no longer available") +expect_silent(request(ids[[length(ids)]], "object/children", list(path = list()))) + +# Values jdv cannot represent, or an opt-out, leave the dataview registry in charge. +expect_null(preview(quote(x), "list")) +expect_null(preview(data.frame(text = strrep("a", 300000)), "table")) +options(sess.dataviewer = "sess") +expect_null(preview(data.frame(a = 1), "table")) diff --git a/sess/inst/tinytest/test-ipc.R b/sess/inst/tinytest/test-ipc.R index fa7722b2e..1be545483 100644 --- a/sess/inst/tinytest/test-ipc.R +++ b/sess/inst/tinytest/test-ipc.R @@ -1,3 +1,6 @@ +# These tests cover the sess data viewer; keep it when jdv is installed. +options(sess.dataviewer = "sess") + # dispatch_message routes responses to pending_responses local({ .sess_env <- sess:::.sess_env diff --git a/sess/inst/tinytest/test-jdv.R b/sess/inst/tinytest/test-jdv.R new file mode 100644 index 000000000..43c198603 --- /dev/null +++ b/sess/inst/tinytest/test-jdv.R @@ -0,0 +1,308 @@ +# View() uses jdv over a persistent connection to the viewer host when jdv is installed. +if (!requireNamespace("jdv", quietly = TRUE)) { + exit_file("jdv is not installed") +} + +local({ + ns <- asNamespace("sess") + runtime <- sess:::.sess_env + previous_con <- runtime$con + previous_plot_path <- runtime$latest_plot_path + previous_interactive_connected <- runtime$interactive_connected + original_request_client <- ns$request_client + pipe <- processx::conn_create_pipepair() + host <- NULL + on.exit({ + sess:::runtime_stop() + jdv::jdv_disconnect() + unlockBinding("request_client", ns) + assign("request_client", original_request_client, envir = ns) + lockBinding("request_client", ns) + runtime$con <- previous_con + runtime$latest_plot_path <- previous_plot_path + runtime$interactive_connected <- previous_interactive_connected + if (!is.null(host)) host$kill() + lapply(pipe, close) + }, add = TRUE) + runtime$con <- pipe[[2L]] + # The standard plot backend needs the path that connect() normally sets. + if (is.null(runtime$latest_plot_path)) runtime$latest_plot_path <- tempfile(fileext = ".png") + options(sess.dataviewer = NULL) + sess:::runtime_start(use_rstudioapi = FALSE, plot_backend = "standard") + + # VS Code answers `jdv/address` with the address of its per-session listener. + stub_address <- function(address) { + unlockBinding("request_client", ns) + assign("request_client", function(action, args = list()) { + if (is.null(address)) stop("no jdv transport") else list(address = address) + }, envir = ns) + lockBinding("request_client", ns) + } + read_messages <- function() { + messages <- strsplit(processx::conn_read_chars(pipe[[1L]]), "\n", fixed = TRUE)[[1L]] + lapply(messages[nzchar(messages)], jsonlite::fromJSON, simplifyVector = FALSE) + } + used_sess_viewer <- function() { + any(vapply(read_messages(), function(message) identical(message$method, "dataview"), + logical(1L))) + } + + sample <- data.frame(x = c(2, 1, NA), y = c("b", "a", "c")) + + # Only sources with an inexpensive, known row count get a hint. Readers and + # malformed counts remain unknown without consuming a stream. + table <- structure(sample, class = c("Table", class(sample))) + batch <- structure(sample, class = c("RecordBatch", class(sample))) + method_name <- "dim.polars_data_frame" + had_method <- exists(method_name, envir = .GlobalEnv, inherits = FALSE) + old_method <- if (had_method) get(method_name, envir = .GlobalEnv) else NULL + assign(method_name, function(x) x$height, envir = .GlobalEnv) + on.exit({ + if (had_method) { + assign(method_name, old_method, envir = .GlobalEnv) + } else if (exists(method_name, envir = .GlobalEnv, inherits = FALSE)) { + rm(list = method_name, envir = .GlobalEnv) + } + }, add = TRUE) + expect_identical(sess:::.jdv_arrow_nrow(table), 3) + expect_identical(sess:::.jdv_arrow_nrow(batch), 3) + polars <- structure(list(height = 4), class = "polars_data_frame") + expect_identical(sess:::.jdv_arrow_nrow(polars), 4) + for (invalid in list(NA_real_, Inf, -Inf, -1, 1.5, "3", 1 + 1i)) { + polars <- structure(list(height = invalid), class = "polars_data_frame") + expect_null(sess:::.jdv_arrow_nrow(polars)) + } + expect_null(sess:::.jdv_arrow_nrow(structure(list(), class = "RecordBatchReader"))) + if (requireNamespace("arrow", quietly = TRUE)) { + expect_identical(sess:::.jdv_arrow_nrow(arrow::arrow_table(x = 1:3)), 3) + } + if (requireNamespace("polars", quietly = TRUE)) { + expect_identical(sess:::.jdv_arrow_nrow(polars::pl$DataFrame(x = 1:3)), 3) + } + if (requireNamespace("nanoarrow", quietly = TRUE)) { + expect_null(sess:::.jdv_arrow_nrow(nanoarrow::as_nanoarrow_array_stream(sample))) + } + + # The opt-out keeps the sess data viewer. + options(sess.dataviewer = "sess") + stub_address("tcp://127.0.0.1:1") + expect_false(sess:::.jdv_connect()) + utils::View(sample) + expect_true(used_sess_viewer()) + options(sess.dataviewer = NULL) + + # Without a transport, or with an unreachable one, View() falls back and leaves jdv alone. + stub_address(NULL) + expect_false(sess:::.jdv_connect()) + stub_address("tcp://127.0.0.1:1") + expect_false(sess:::.jdv_connect()) + expect_identical(jdv::jdv_connection()$state, "disconnected") + expect_null(runtime$jdv_address) + utils::View(sample) + expect_true(used_sess_viewer()) + + # An embedded session of another jdv user is never taken over. + jdv::jdv_register(sample) + expect_identical(jdv::jdv_connection()$state, "embedded") + stub_address("tcp://127.0.0.1:1") + expect_false(sess:::.jdv_connect()) + utils::View(sample) + expect_true(used_sess_viewer()) + expect_identical(jdv::jdv_connection()$state, "embedded") + jdv::jdv_disconnect() + + # A minimal viewer host: it records every line that jdv sends. + received <- tempfile(fileext = ".jsonl") + port <- NULL + for (candidate in sample(20000:60000, 20L)) { + host <- processx::process$new(file.path(R.home("bin"), "Rscript"), c("--vanilla", "-e", " + args <- commandArgs(TRUE) + server <- serverSocket(as.integer(args[1L])) + connection <- socketAccept(server, blocking = TRUE, open = 'r+') + repeat { + line <- readLines(connection, n = 1L) + if (!length(line)) break + cat(line, '\\n', file = args[2L], append = TRUE, sep = '') + } + ", candidate, received), stderr = "|") + Sys.sleep(1) + if (host$is_alive()) { + port <- candidate + break + } + } + if (is.null(port)) exit_file("no free TCP port for the viewer host") + address <- sprintf("tcp://127.0.0.1:%d", port) + stub_address(address) + host_messages <- function(n, timeout = 10) { + deadline <- Sys.time() + timeout + repeat { + lines <- if (file.exists(received)) readLines(received, warn = FALSE) else character() + if (length(lines) >= n || Sys.time() > deadline) break + Sys.sleep(0.05) + } + lapply(lines, jsonlite::fromJSON, simplifyVector = FALSE) + } + opens <- function() { + Filter(function(message) identical(message$method, "view/open"), host_messages(0L)) + } + + utils::View(sample) + expect_false(used_sess_viewer()) + messages <- host_messages(2L) + expect_identical(messages[[1L]]$method, "hello") + expect_identical(messages[[1L]]$params$producer$name, "jdv") + expect_identical(runtime$jdv_address, address) + connection <- jdv::jdv_connection() + expect_identical(connection$state, "connected") + expect_true(connection$persistent) + expect_identical(connection$address, address) + + # A successful registration publishes its row count to the session client. + expect_true(sess:::.jdv_show_registered(function() { + list(id = "row-count-view", generation = 7L) + }, address, 3)) + row_count_messages <- read_messages() + row_count_message <- Filter(function(message) identical(message$method, "jdv/rowCount"), + row_count_messages) + expect_length(row_count_message, 1L) + expect_identical(row_count_message[[1L]]$params$view, "row-count-view") + expect_equal(row_count_message[[1L]]$params$generation, 7) + expect_equal(row_count_message[[1L]]$params$nrow, 3) + expect_identical(row_count_message[[1L]]$params$address, address) + + # A client notification failure does not undo a successful registration. + original_notify_client <- ns$notify_client + unlockBinding("notify_client", ns) + assign("notify_client", function(...) stop("client is unavailable"), envir = ns) + lockBinding("notify_client", ns) + expect_true(sess:::.jdv_show_registered(function() { + list(id = "row-count-view", generation = 7L) + }, address, 3)) + unlockBinding("notify_client", ns) + assign("notify_client", original_notify_client, envir = ns) + lockBinding("notify_client", ns) + + # Data frames are paged and titled by their expression. + open <- opens()[[1L]]$params + expect_identical(open$kind, "grid") + expect_identical(open$access, "random") + expect_identical(open$encoding, "jdv-json") + expect_identical(open$collation, "C") + expect_identical(open$title, "sample") + expect_identical(open$generation, 1L) + + # The same title replaces the view and increments its generation. + utils::View(sample) + host_messages(3L) + reopened <- opens()[[2L]]$params + expect_identical(reopened$view, open$view) + expect_identical(reopened$generation, 2L) + + # Lists open as trees on the same connection. + nested <- list(a = 1:3, df = sample) + utils::View(nested) + host_messages(4L) + tree <- opens()[[3L]]$params + expect_identical(tree$kind, "tree") + expect_identical(tree$title, "nested") + expect_false(used_sess_viewer()) + + # Arrow data is a forward-only stream on the same connection. + if (requireNamespace("nanoarrow", quietly = TRUE)) { + utils::View(nanoarrow::as_nanoarrow_array_stream(sample), "stream") + host_messages(5L) + stream <- opens()[[4L]]$params + expect_identical(stream$access, "sequential") + expect_identical(stream$title, "stream") + stream_notifications <- read_messages() + expect_false(any(vapply(stream_notifications, function(message) { + identical(message$method, "jdv/rowCount") + }, logical(1L)))) + } + + # View() follows the variable on R 4.6.0 or later, and never on older R. + generations <- function(title) { + vapply(Filter(function(open) identical(open$title, title), lapply(opens(), `[[`, "params")), + function(open) open$generation, integer(1L)) + } + wait_generations <- function(title, n, timeout = 5) { + deadline <- Sys.time() + timeout + repeat { + jdv::jdv_rpc(NULL) + found <- generations(title) + if (length(found) >= n || Sys.time() > deadline) break + Sys.sleep(0.05) + } + found + } + tracking <- data.frame(a = 1:3) + expect_identical(sess:::.jdv_track_binding(quote(tracking), environment()), + if (getRversion() >= "4.6.0") list(name = "tracking", env = environment())) + expect_null(sess:::.jdv_track_binding(quote(head(tracking)), environment())) + expect_null(sess:::.jdv_track_binding(quote(no_such_variable_for_sess), environment())) + options(sess.dataviewer.track = FALSE) + expect_null(sess:::.jdv_track_binding(quote(tracking), environment())) + options(sess.dataviewer.track = NULL) + + # The connect() setting turns tracking off; the R option overrides it. + sess_env <- sess:::.sess_env + setting <- sess_env$track_variables + sess_env$track_variables <- FALSE + expect_null(sess:::.jdv_track_binding(quote(tracking), environment())) + options(sess.dataviewer.track = TRUE) + expect_identical(sess:::.jdv_track_binding(quote(tracking), environment()), + if (getRversion() >= "4.6.0") list(name = "tracking", env = environment())) + options(sess.dataviewer.track = NULL) + sess_env$track_variables <- TRUE + expect_identical(sess:::.jdv_track_binding(quote(tracking), environment()), + if (getRversion() >= "4.6.0") list(name = "tracking", env = environment())) + sess_env$track_variables <- setting + + utils::View(tracking) + expect_identical(wait_generations("tracking", 1L), 1L) + tracking <- transform(tracking, b = a * 2L) + if (getRversion() >= "4.6.0") { + expect_identical(wait_generations("tracking", 2L), 1:2) + # Assigning the same object again changes nothing. + tracking <- tracking + expect_identical(wait_generations("tracking", 3L, timeout = 0.5), 1:2) + } else { + expect_identical(wait_generations("tracking", 2L, timeout = 0.5), 1L) + } + + # The opt-out and expressions that are not a single variable stay fixed. + options(sess.dataviewer.track = FALSE) + fixed <- data.frame(a = 1:3) + utils::View(fixed) + options(sess.dataviewer.track = NULL) + utils::View(head(fixed), "head") + fixed <- transform(fixed, b = a * 2L) + expect_identical(wait_generations("fixed", 2L, timeout = 0.5), 1L) + expect_identical(wait_generations("head", 2L, timeout = 0.5), 1L) + + # A binding that does not hold the object is dropped instead of failing View(). + expect_true(sess:::.jdv_show(fixed, "mismatch", list(name = "tracking", env = environment()))) + expect_identical(wait_generations("mismatch", 1L), 1L) + + # Sequential sources cannot be tracked, so no binding is passed for them. + if (requireNamespace("nanoarrow", quietly = TRUE)) { + stream <- nanoarrow::as_nanoarrow_array_stream(data.frame(a = 1:3)) + expect_true(sess:::.jdv_show(stream, "arrow", list(name = "stream", env = environment()))) + expect_identical(wait_generations("arrow", 1L), 1L) + } + + # Explicit View() in Interactive opens the regular jdv panel before trying + # the inline rich-output path used by ordinary cell results. + runtime$interactive_connected <- TRUE + utils::View(sample, "interactive explicit") + runtime$interactive_connected <- FALSE + host_messages(1L) + expect_identical(tail(vapply(opens(), function(message) message$params$title, ""), 1L), + "interactive explicit") + + # Releasing the runtime closes the connection that sess opened. + sess:::.jdv_release() + expect_identical(jdv::jdv_connection()$state, "disconnected") + expect_null(runtime$jdv_address) +}) diff --git a/sess/inst/tinytest/test-listview.R b/sess/inst/tinytest/test-listview.R index abb071211..a206e2a4b 100644 --- a/sess/inst/tinytest/test-listview.R +++ b/sess/inst/tinytest/test-listview.R @@ -1,3 +1,6 @@ +# These tests cover the sess data viewer; keep it when jdv is installed. +options(sess.dataviewer = "sess") + # Expanding and opening descendants keeps one list/table viewer per root object. local({ runtime <- sess:::.sess_env diff --git a/sess/man/connect.Rd b/sess/man/connect.Rd index deb4bfee2..247b437db 100644 --- a/sess/man/connect.Rd +++ b/sess/man/connect.Rd @@ -9,7 +9,8 @@ connect( use_rstudioapi = TRUE, plot_backend = c("auto", "jgd", "httpgd", "standard", "native"), attachment_id = NULL, - auto_activate = TRUE + auto_activate = TRUE, + track_variables = NULL ) } \arguments{ @@ -29,6 +30,12 @@ ownership claim.} \item{auto_activate}{Logical. Whether vscode-R should select this attachment automatically. \code{FALSE} requires \code{attachment_id}.} + +\item{track_variables}{Logical or NULL. Whether the data viewer follows the +variable passed to \code{View()} (R >= 4.6 and jdv). The VS Code extension passes +its \code{r.session.viewers.trackVariables} setting. \code{NULL} and \code{TRUE} both +enable tracking. \code{options(sess.dataviewer.track)}, when set, takes +precedence over this argument.} } \value{ Invisibly returns \code{TRUE} when the IPC connection and runtime startup diff --git a/src/dataViewer.ts b/src/dataViewer.ts index 525e9a811..cd6498367 100644 --- a/src/dataViewer.ts +++ b/src/dataViewer.ts @@ -36,11 +36,23 @@ export function getDataViewerStyle(): string { font-size: 12px; color: var(--vscode-descriptionForeground); } ${viewerSessionStyle} + .dataview-drill { display: flex; align-items: center; gap: 4px; } + .dataview-drill-text { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; } + .dataview-drill-open { + flex-shrink: 0; padding: 0 4px; border: 1px solid transparent; border-radius: 3px; + background: transparent; color: var(--vscode-textLink-foreground); font: inherit; cursor: pointer; + line-height: 1.2; + } + .dataview-drill-open:hover { background: var(--vscode-toolbar-hoverBackground); } .dataview-na { color: var(--vscode-descriptionForeground); font-style: italic; opacity: 0.75; } `; } -export function getDataViewerToolbarHtml(sessionHtml = ''): string { +export function getDataViewerToolbarHtml( + sessionHtml = '', + refreshable = false, + followable = false, +): string { return `
@@ -50,6 +62,8 @@ export function getDataViewerToolbarHtml(sessionHtml = ''): string { + ${refreshable ? '' : ''} + ${followable ? '' : ''}
@@ -75,6 +89,30 @@ export function getDataViewerScript(): string { element.textContent = 'NA'; return element; }; + // A drillable jdv cell shows its text and a button that opens the cell as a child view. + // The cell is addressed by its original source row, so it works after sorting and filtering. + const drillCellRenderer = params => { + const element = document.createElement('span'); + element.className = 'dataview-drill'; + const text = document.createElement('span'); + text.className = 'dataview-drill-text'; + text.textContent = params.value; + const button = document.createElement('button'); + button.type = 'button'; + button.className = 'dataview-drill-open'; + button.title = 'Open this cell in a new view'; + button.setAttribute('aria-label', 'Open this cell in a new view'); + button.textContent = '\u2197'; + button.addEventListener('click', event => { + event.stopPropagation(); + vscode.postMessage({ + message: 'dataview/openCell', documentGeneration, + sourceRow: params.data.__sourceRow, column: params.colDef.context.drillColumn + }); + }); + element.append(text, button); + return element; + }; function getAgTheme() { const light = document.body.classList.contains('vscode-light') || @@ -102,23 +140,40 @@ export function getDataViewerScript(): string { } const bigintFields = []; for (const column of columns) { + const drillColumn = column.jdvDrill; + delete column.jdvDrill; + if (Number.isInteger(drillColumn)) { + column.context = { drillColumn }; + } column.cellRendererSelector = params => { if (params.data == null) { return { component: emptyCellRenderer }; } - return params.value == null ? { component: naCellRenderer } : undefined; + if (params.value == null) { + return { component: naCellRenderer }; + } + return Number.isInteger(drillColumn) && params.data.__sourceRow !== undefined + ? { component: drillCellRenderer } : undefined; }; if (column.type === 'dateColumn' || column.type === 'datetimeColumn') { column.cellDataType = column.type === 'dateColumn' ? 'dateString' : 'dateTimeString'; column.filter = 'agDateColumnFilter'; - column.filterParams = { browserDatePicker: true }; + column.filterParams = { + browserDatePicker: true, + includeTime: column.type === 'datetimeColumn' + }; column.initialWidth = 200; } else if (column.type === 'bigintColumn') { column.cellDataType = 'bigint'; column.filter = 'agBigIntColumnFilter'; bigintFields.push(column.field); } else if (column.type === 'numericColumn') { - column.cellDataType = 'number'; + // Sequential Arrow values are already formatted display strings. + // Keep them as strings to preserve integer/decimal precision while + // retaining numericColumn's right alignment. + if (column.cellDataType !== false) { + column.cellDataType = 'number'; + } } else if (column.type === 'booleanColumn') { column.cellDataType = 'boolean'; column.filterParams = { buttons: ['reset'] }; @@ -132,6 +187,30 @@ export function getDataViewerScript(): string { column.suppressSizeToFit = true; column.initialFlex = 0; } + if (column.jdv) { + // jdv cells hold R-formatted display text, and R evaluates filters. + // Offer only the operators jdv implements for this column. + const customOption = (displayKey, displayName) => ({ displayKey, displayName, predicate: () => true, numberOfInputs: 0 }); + const textOptions = column.jdv.filterOptions.map(option => + option === 'jdvIsEmpty' ? customOption(option, 'Is empty') : + option === 'jdvNotEmpty' ? customOption(option, 'Is not empty') : option); + const naOptions = column.jdv.naOptions ? [ + { displayKey: 'jdvIsNa', displayName: 'Is NA', predicate: () => true, numberOfInputs: 0 }, + { displayKey: 'jdvNotNa', displayName: 'Is not NA', predicate: () => true, numberOfInputs: 0 } + ] : []; + delete column.cellDataType; + if (column.type === 'bigintColumn') { + bigintFields.pop(); + } + column.filterParams = { + ...column.filterParams, + buttons: ['reset', 'apply'], + filterOptions: [...textOptions, ...naOptions], + inRangeInclusive: true, + ...(column.jdv.units ? { filterPlaceholder: 'Value (' + column.jdv.units + ')' } : {}) + }; + delete column.jdv; + } if (column.type !== 'numericColumn') { delete column.type; } @@ -213,6 +292,15 @@ export function getDataViewerScript(): string { ? format(total) + ' rows' : format(filtered) + ' of ' + format(total) + ' rows'; } + function formatSequentialRowCount(loaded, knownTotal, complete) { + const format = value => new Intl.NumberFormat().format(value); + if (complete) { + return format(loaded) + ' rows'; + } + return format(loaded) + ' / ' + (Number.isFinite(knownTotal) + ? format(knownTotal) : '???') + ' rows'; + } + function updateViewerSizingButtons() { document.querySelector('#viewerFit').setAttribute('aria-pressed', String(viewerSizingMode === 'fit')); document.querySelector('#viewerAutoSize').setAttribute('aria-pressed', String(viewerSizingMode === 'content')); diff --git a/src/interactive/agent.ts b/src/interactive/agent.ts index 91898a6a1..db20a03fe 100644 --- a/src/interactive/agent.ts +++ b/src/interactive/agent.ts @@ -21,6 +21,7 @@ import { sessionLabel, } from './protocol'; import { searchHistory } from './history'; +import { normalizeJdvDisplay } from './jdvTable'; interface Client { socket: net.Socket; @@ -776,6 +777,8 @@ export class SessionAgent { } private display(data: Record, executionId?: string): void { + // A jdv handle's descriptor and first page become the renderer's table or list data. + data = normalizeJdvDisplay(data); const display: Record = { ...data, displayId: typeof data.displayId === 'string' ? data.displayId : randomUUID(), @@ -795,6 +798,15 @@ export class SessionAgent { delete display.data; } if (Buffer.byteLength(JSON.stringify(display)) > 2 * 1024 * 1024) { + if (typeof data.handleId === 'string') { + void this.backend + .inspect({ + method: 'jdv_handle_close', + params: { handle_id: data.handleId }, + timeout: 5000, + }) + .catch(() => undefined); + } this.event( 'truncated', { diff --git a/src/interactive/backend.ts b/src/interactive/backend.ts index 3192841da..3167ca5a7 100644 --- a/src/interactive/backend.ts +++ b/src/interactive/backend.ts @@ -19,7 +19,10 @@ export type InspectionMethod = | 'completion' | 'dataview_init' | 'dataview_page' - | 'dataview_dispose'; + | 'dataview_dispose' + | 'jdv_handle_request' + | 'jdv_handle_close' + | 'jdv_open_viewer'; export interface InspectionRequest { method: InspectionMethod; params: Record; @@ -91,6 +94,9 @@ export function inspectionMethod(value: unknown): InspectionMethod { 'dataview_init', 'dataview_page', 'dataview_dispose', + 'jdv_handle_request', + 'jdv_handle_close', + 'jdv_open_viewer', ].includes(String(value)) ) { throw new Error('Unsupported inspection method'); diff --git a/src/interactive/inlineTable.ts b/src/interactive/inlineTable.ts index af5b83522..31529f44d 100644 --- a/src/interactive/inlineTable.ts +++ b/src/interactive/inlineTable.ts @@ -4,7 +4,7 @@ import { tableColumnAlignment, tableDisplayValue, tableSnapshotSummary } from '. import { columnOperators, filterNeedsValue, - filterOperators, + filterOperatorLabel, TableColumn, TableFilters, tableSchema, @@ -65,7 +65,7 @@ export function createInlineTable( let error = ''; const available = original.connected !== false && original.archived !== true && original.running !== false; - const hasFull = !!original.fullViewId; + const hasFull = !!original.fullViewId || !!original.handleId; const columns = (): TableColumn[] => (state.data.columns ?? []) as TableColumn[]; const count = (): number => !state.live && hasFull && !state.sort.length && !Object.keys(state.filters).length @@ -200,7 +200,7 @@ export function createInlineTable( if (!active) { return; } - // Restoring saved output is local, and invalidates even an in-flight reply. + // Reset invalidates in-flight replies; a partial saved preview is refilled. pending = undefined; state = initial(); error = ''; @@ -208,6 +208,7 @@ export function createInlineTable( draw(); buildFilters(true); update(); + completePreview(); tableHost.scrollLeft = 0; tableHost.scrollTop = 0; }, @@ -361,7 +362,7 @@ export function createInlineTable( operator.replaceChildren(); if (column) { for (const op of columnOperators(column)) { - operator.add(new Option(filterOperators[op as keyof typeof filterOperators], op)); + operator.add(new Option(filterOperatorLabel(op), op)); } } const filter = state.filters[filterColumn.value]; @@ -428,11 +429,12 @@ export function createInlineTable( updateOperators(!resetDraft && filterColumn.value === selected); chips.replaceChildren(); for (const [field, filter] of Object.entries(state.filters)) { - const name = columns().find((column) => column.field === field)?.headerName ?? field; + const column = columns().find((column) => column.field === field); + const name = column?.headerName ?? field; const remove = document.createElement('button'); remove.type = 'button'; remove.className = 'r-interactive-button'; - remove.textContent = `${name} ${filterOperators[filter.type as keyof typeof filterOperators]} ${filter.filter ?? ''} ×`; + remove.textContent = `${name} ${filterOperatorLabel(filter.type)} ${filter.filter ?? ''} ×`; remove.setAttribute('aria-label', `Remove filter for ${name}`); remove.onclick = () => { const filters = { ...state.filters }; @@ -499,9 +501,15 @@ export function createInlineTable( ? `${tableSnapshotSummary(original)}. Browse beyond these rows, sort or filter to retrieve live data.` : ''; } + function completePreview(): void { + if (available && !state.live && state.data.previewPartial === true) { + request(0); + } + } draw(); buildFilters(); update(); + completePreview(); return { setTextMode(text) { textMode = text; diff --git a/src/interactive/jdvAddress.ts b/src/interactive/jdvAddress.ts new file mode 100644 index 000000000..8402c37b0 --- /dev/null +++ b/src/interactive/jdvAddress.ts @@ -0,0 +1,36 @@ +export interface InteractiveJdvAddressOptions { + sessionId: string; + hostKey: string; + canConnect(): boolean; + isCurrent(): boolean; + getAddress(sessionId: string, hostKey: string): Promise; + disposeHost(hostKey: string, expectedAddress: string): void | Promise; +} + +/** Acquire a session-scoped listener without returning an address to a stale client. */ +export async function acquireInteractiveJdvAddress( + options: InteractiveJdvAddressOptions, +): Promise { + if (!options.canConnect()) { + throw new Error('Only the controlling Interactive client can connect the data viewer.'); + } + const address = await options.getAddress(options.sessionId, options.hostKey); + if (!options.canConnect() || !options.isCurrent()) { + await options.disposeHost(options.hostKey, address); + throw new Error('The Interactive session changed while opening its data viewer.'); + } + return address; +} + +/** Drop a listener when another client takes control of the same R process. */ +export function disposeInteractiveJdvHostOnControlLoss( + previouslyControlled: boolean, + currentlyControlled: boolean, + disposeHost: () => void, +): boolean { + if (!previouslyControlled || currentlyControlled) { + return false; + } + disposeHost(); + return true; +} diff --git a/src/interactive/jdvTable.ts b/src/interactive/jdvTable.ts new file mode 100644 index 000000000..a10ed2c0e --- /dev/null +++ b/src/interactive/jdvTable.ts @@ -0,0 +1,314 @@ +'use strict'; + +// Inline table and list previews backed by registry-free jdv handles. The R side +// owns one handle per output and answers `jdv_handle_request` inspections; this +// module converts between jdv's protocol shapes and the renderer's data. It has no +// VS Code dependency, so the agent, the extension host and tests can all use it. + +import { + JdvDescriptor, + JdvPathStep, + JdvRequestError, + JsonObject, + ListViewChild, + TreeAdapter, + gridColumnDefs, + gridPageFromRows, + parseDescriptor, + tableRowsParams, +} from '../jdvAdapter'; +import { TABLE_PAGE_SIZE, tablePage } from './tablePaging'; +import { + columnOperators, + filterNeedsValue, + TableColumn, + TableFilters, + TableSort, +} from './tableQuery'; + +export type Inspect = (request: { + method: string; + params: Record; +}) => Promise>; + +/** Largest JSON result requested for one page, matching the R side's preview bound. */ +export const PAGE_BYTES = 256 * 1024; + +/** Whether a display event came from a jdv handle (as opposed to the dataview registry). */ +export function isJdvOutput(data: Record): boolean { + return typeof data.handleId === 'string' && data.handleId !== ''; +} + +function parseJson(text: unknown, what: string): unknown { + if (typeof text !== 'string') { + throw new Error(`Invalid jdv ${what}`); + } + return JSON.parse(text); +} + +function descriptorOf(data: Record): JdvDescriptor { + return parseDescriptor( + typeof data.descriptor === 'string' ? JSON.parse(data.descriptor) : data.descriptor, + ); +} + +/** + * Agent side: turn the raw descriptor and preview pages that R sent into the data the + * renderer, exports and transcript already understand (columns and rows for a + * table; navigation and children for a list). The parsed descriptor stays so the + * host can query the same handle later. + */ +export function normalizeJdvDisplay(data: Record): Record { + if (!isJdvOutput(data)) { + return data; + } + const descriptor = parseDescriptor(parseJson(data.descriptor, 'descriptor')); + const rawPage = data.page; + const { page: _page, ...rest } = data; + if (data.kind === 'table' && descriptor.kind === 'grid') { + const pages = (Array.isArray(rawPage) ? rawPage : [rawPage]).map((page) => + parseJson(page, 'page'), + ); + const rows: Record[] = []; + let totalRows = descriptor.nrow ?? 0; + for (const raw of pages) { + const next = gridPageFromRows(raw, rows.length, TABLE_PAGE_SIZE - rows.length); + rows.push(...next.rows); + totalRows = next.totalRows; + if (rows.length >= TABLE_PAGE_SIZE) { + break; + } + } + return { + ...rest, + descriptor, + // AG Grid supplies boolean filters itself; inline tables need explicit operators. + columns: gridColumnDefs(descriptor).map((column) => + column.type === 'booleanColumn' && column.filter !== false + ? { ...column, jdv: { filterOptions: ['true', 'false'] } } + : column, + ), + rows, + totalRows, + previewPartial: rows.length < Math.min(TABLE_PAGE_SIZE, totalRows), + }; + } + if (data.kind === 'list' && descriptor.kind === 'tree') { + const adapter = new TreeAdapter(descriptor, 'List'); + return { + ...rest, + descriptor, + navigation: adapter.navigation([]), + ...adapter.children([], parseJson(rawPage, 'page')), + }; + } + throw new Error('The jdv descriptor does not match the output kind'); +} + +// --------------------------------------------------------------------------- +// Tables + +/** AG Grid names the date operand `dateFrom`; the inline filter UI writes `filter`. */ +function filterModel(filters: TableFilters, columns: TableColumn[]): Record { + const model: Record = {}; + for (const [field, filter] of Object.entries(filters)) { + const type = columns.find((column) => column.field === field)?.type; + model[field] = + type === 'dateColumn' || type === 'datetimeColumn' + ? { ...filter, dateFrom: filter.filter } + : filter; + } + return model; +} + +/** Validate a renderer table query, as the dataview path does. */ +function validateQuery(columns: TableColumn[], sort: TableSort[], filters: TableFilters): void { + for (const item of sort) { + if ( + !item || + !['asc', 'desc'].includes(item.sort) || + !columns.some( + (column) => + column.field === item.colId && + column.field !== '0' && + column.sortable !== false, + ) + ) { + throw new Error('This column cannot be sorted.'); + } + } + for (const [field, filter] of Object.entries(filters)) { + const column = columns.find( + (column) => column.field === field && column.field !== '0' && column.filter !== false, + ); + if ( + !column || + !filter || + !columnOperators(column).includes(filter.type) || + (filterNeedsValue(filter.type) && + (typeof filter.filter !== 'string' || filter.filter.length > 1000)) + ) { + throw new Error('Invalid column filter.'); + } + } +} + +/** Fetch one inline table page from the output's handle, sorted and filtered in R. */ +export async function queryJdvTablePage( + data: Record, + message: Record, + inspect: Inspect, +): Promise> { + const size = message.size === undefined ? TABLE_PAGE_SIZE : Number(message.size); + if (![20, 50, 100].includes(size)) { + throw new Error('Choose 20, 50 or 100 rows per page.'); + } + const offset = Number(message.start ?? 0); + if (!Number.isSafeInteger(offset) || offset < 0 || offset > 2147483646) { + throw new Error('Invalid table page.'); + } + const sort = (message.sortModel ?? []) as TableSort[]; + const filters = (message.filterModel ?? {}) as TableFilters; + if ( + !Array.isArray(sort) || + sort.length > 1 || + !filters || + typeof filters !== 'object' || + Array.isArray(filters) || + Object.keys(filters).length > 100 + ) { + throw new Error('Invalid table query.'); + } + const columns = data.columns as TableColumn[]; + if (!Array.isArray(columns) || columns.length > 100000) { + throw new Error('Invalid table columns.'); + } + validateQuery(columns, sort, filters); + const descriptor = descriptorOf(data); + const model = filterModel(filters, columns); + + const fetch = async (from: number) => { + const rows: Record[] = []; + let total = 0; + let unfiltered = 0; + let cursor = from; + while (rows.length < size) { + const wanted = size - rows.length; + const { + view: _view, + generation: _generation, + ...params + } = tableRowsParams(descriptor, cursor, cursor + wanted, sort, model); + if (!params.sort.length) { + delete (params as { sort?: unknown }).sort; + } + const reply = await inspect({ + method: 'jdv_handle_request', + params: { + handle_id: data.handleId, + method: 'grid/rows', + params, + max_bytes: PAGE_BYTES, + }, + }); + const page = gridPageFromRows(parseJson(reply.json, 'page'), cursor, wanted); + rows.push(...page.rows); + total = page.totalRows; + unfiltered = page.totalUnfiltered; + cursor += page.returned; + // jdv may return fewer rows than requested to fit the byte budget. + if (page.returned === 0 || cursor >= total) { + break; + } + } + return { rows, total, unfiltered }; + }; + let start = Math.floor(offset / size) * size; + let result = await fetch(start); + if (result.total > 0 && start >= result.total) { + start = tablePage(result.total, start, size).start; + result = await fetch(start); + } + return { + rows: result.rows, + totalRows: result.total, + totalUnfiltered: result.unfiltered, + startRow: result.total === 0 ? 0 : start, + columns, + live: true, + queryReset: false, + }; +} + +// --------------------------------------------------------------------------- +// Lists + +/** What a list request needs the host to do besides replying. */ +export interface ListOutcome { + result?: unknown; + /** Open this item (typed path and title) in the Data Viewer panel. */ + open?: { path: JdvPathStep[]; title: string }; +} + +/** A fresh adapter for an output, primed with the root page the renderer already shows. */ +export async function createListAdapter( + data: Record, + inspect: Inspect, +): Promise { + const adapter = new TreeAdapter(descriptorOf(data), 'List'); + const reply = await inspect({ + method: 'jdv_handle_request', + params: { + handle_id: data.handleId, + method: 'object/children', + params: stripOwner(adapter.childrenParams([], 1)), + max_bytes: PAGE_BYTES, + }, + }); + adapter.children([], parseJson(reply.json, 'children')); + return adapter; +} + +function stripOwner(params: JsonObject): JsonObject { + const { view: _view, generation: _generation, ...rest } = params; + return rest; +} + +/** Serve a renderer list action (`list`, `listPage`, `listNavigate`, `listItem`) from the handle. */ +export async function listRequest( + adapter: TreeAdapter, + data: Record, + message: Record, + inspect: Inspect, +): Promise { + const path = message.path as number[]; + switch (message.action) { + case 'listPage': { + const reply = await inspect({ + method: 'jdv_handle_request', + params: { + handle_id: data.handleId, + method: 'object/children', + params: stripOwner(adapter.childrenParams(path, Number(message.start))), + max_bytes: PAGE_BYTES, + }, + }); + return { result: adapter.children(path, parseJson(reply.json, 'children')) }; + } + case 'listNavigate': + return { result: { navigation: adapter.navigation(path) } }; + case 'listItem': { + const action = adapter.open(path, Number(message.index)); + if (action.kind === 'navigate') { + return { result: { navigation: action.navigation } }; + } + return { open: adapter.locate([...path, Number(message.index)]) }; + } + case 'list': + return { open: adapter.locate(path) }; + default: + throw new JdvRequestError('invalid_request', 'Unsupported list action'); + } +} + +export type { ListViewChild }; diff --git a/src/interactive/manager.ts b/src/interactive/manager.ts index 7902c5075..7cdc3b409 100644 --- a/src/interactive/manager.ts +++ b/src/interactive/manager.ts @@ -38,6 +38,7 @@ import { } from './notebook'; import { setInteractiveExecutor } from './executionTarget'; import * as session from '../session'; +import { applyJdvRowCount, disposeJdvHost, getJdvAddress } from '../jdvViewer'; import * as util from '../util'; import type { ListViewNavigation } from '../listViewer'; import { escapeXml } from './plotSvg'; @@ -49,6 +50,9 @@ import { readExecutionRecords, readPreviousJournal } from './journal'; import { sessionAge, sessionPresentation } from './sessionPresentation'; import { runningSessions, stopSession } from './sessionLifecycle'; import { InteractiveInputFeatures } from './inputFeatures'; +import { createListAdapter, isJdvOutput, listRequest, queryJdvTablePage } from './jdvTable'; +import type { TreeAdapter } from '../jdvAdapter'; +import { acquireInteractiveJdvAddress, disposeInteractiveJdvHostOnControlLoss } from './jdvAddress'; interface InteractiveView { client: AgentClient; @@ -82,6 +86,10 @@ interface SavedConnection { notebookUri?: string; } +function interactiveJdvHostKey(id: string, generation: string): string { + return `interactive:${id}:${generation}`; +} + export class InteractiveManager implements vscode.Disposable, vscode.TreeDataProvider { @@ -687,11 +695,22 @@ export class InteractiveManager return; } const edits: vscode.NotebookEdit[] = []; + const handles: string[] = []; for (const cell of view.notebook.getCells().slice().reverse()) { const id = cell.metadata.rExecutionId as string | undefined; - const record = - id && this.transcriptCells(view).find((cell) => cell.record.id === id)?.record; + const transcript = id + ? this.transcriptCells(view).find((cell) => cell.record.id === id) + : undefined; + const record = transcript?.record; if (record && !['queued', 'running', 'unknown'].includes(record.state)) { + // Cleared outputs release their jdv handles in R. + if (transcript?.generation === view.model.generation) { + for (const output of transcript.outputs) { + if (output.type === 'display' && isJdvOutput(output.data)) { + handles.push(String(output.data.handleId)); + } + } + } view.hidden.add(record.id); const adjacent = edits.at(-1); if (adjacent?.range.start === cell.index + 1) { @@ -710,6 +729,16 @@ export class InteractiveManager const edit = new vscode.WorkspaceEdit(); edit.set(view.notebook.uri, edits); await vscode.workspace.applyEdit(edit); + if (!['exited', 'stopping'].includes(view.client.manifest.status)) { + for (const handle of handles) { + await view.client + .request('inspect', { + method: 'jdv_handle_close', + params: { handle_id: handle }, + }) + .catch(() => undefined); + } + } }); await view.chain; } @@ -1575,6 +1604,8 @@ export class InteractiveManager } private listen(view: InteractiveView, client: AgentClient): void { + const hostKey = interactiveJdvHostKey(client.manifest.id, client.manifest.generation); + let hadControl = client.control; client.on('event', (event: SessionEvent) => { if (view.client !== client) { return; @@ -1584,12 +1615,20 @@ export class InteractiveManager .catch((error) => this.report(error)); }); client.on('activity', () => { - if (view.client === client) { - this.updateStatus(); - this.changes.fire(undefined); + if (view.client !== client) { + return; } + const hasControl = client.control; + disposeInteractiveJdvHostOnControlLoss(hadControl, hasControl, () => + disposeJdvHost(hostKey), + ); + hadControl = hasControl; + this.updateStatus(); + this.changes.fire(undefined); }); client.on('disconnect', () => { + disposeJdvHost(hostKey); + hadControl = false; if (view.client !== client || view.restarting) { return; } @@ -1900,6 +1939,11 @@ export class InteractiveManager view.target.pid = typeof event.data.rPid === 'number' ? String(event.data.rPid) : ''; view.target.rVer = typeof event.data.rVersion === 'string' ? event.data.rVersion : ''; this.refresh(); + if (event.data.status === 'exited') { + disposeJdvHost( + interactiveJdvHostKey(view.client.manifest.id, view.client.manifest.generation), + ); + } if (event.data.status === 'exited' && !view.restarting) { await this.sessionNotice(view, 'stopped'); for (const cell of this.transcriptCells(view)) { @@ -1962,6 +2006,9 @@ export class InteractiveManager } } else if (event.type === 'notification') { const params = object(event.data.params ?? {}); + if (event.data.method === 'jdv/rowCount') { + applyJdvRowCount(view.target.sessionId, params); + } if (event.data.method === 'help') { await session.showHelpNotification(params); } @@ -2341,7 +2388,12 @@ export class InteractiveManager const client = view.client; const params = object(data.params ?? {}); try { - const result = await session.handleEditorRequest(String(data.method), params); + let result: unknown; + if (data.method === 'jdv/address') { + result = { address: await this.jdvAddress(view) }; + } else { + result = await session.handleEditorRequest(String(data.method), params); + } await client.request('clientReply', { id: data.id, result }); } catch (error) { await client @@ -2350,6 +2402,71 @@ export class InteractiveManager } } + /** The session-scoped jdv listener address, created on first use. */ + private jdvAddress(view: InteractiveView): Promise { + const client = view.client; + const sessionId = view.target.sessionId; + const generation = view.model.generation; + const manifestGeneration = client.manifest.generation; + return acquireInteractiveJdvAddress({ + sessionId, + hostKey: interactiveJdvHostKey(client.manifest.id, manifestGeneration), + canConnect: () => + client.control && + !view.target.processExited && + !['stopping', 'exited'].includes(client.manifest.status), + isCurrent: () => + !view.disposed && + view.client === client && + !view.restarting && + view.target.sessionId === sessionId && + view.model.generation === generation && + client.manifest.generation === manifestGeneration, + getAddress: getJdvAddress, + disposeHost: disposeJdvHost, + }); + } + + /** Positional-path bookkeeping of an inline list, rebuilt from its handle when missing. */ + private readonly listAdapters = new Map(); + + private async listAdapter( + view: InteractiveView, + data: Record, + ): Promise { + const key = `${view.client.manifest.id}:${view.model.generation}:${String(data.handleId)}`; + let adapter = this.listAdapters.get(key); + if (!adapter) { + adapter = await createListAdapter(data, (request) => + view.client.request('inspect', request), + ); + this.listAdapters.set(key, adapter); + while (this.listAdapters.size > 64) { + this.listAdapters.delete(this.listAdapters.keys().next().value as string); + } + } + return adapter; + } + + /** Register the output's value (or one item of it) with jdv so the Data Viewer panel shows it. */ + private async openHandleViewer( + view: InteractiveView, + data: Record, + path: unknown[], + title: string, + ): Promise { + // R cannot wait for a reply to its own request while answering this inspection, + // so the listener address is handed over instead of being requested. + const address = await this.jdvAddress(view); + const opened = await view.client.request('inspect', { + method: 'jdv_open_viewer', + params: { handle_id: data.handleId, path, title, address }, + }); + if (opened !== true) { + throw new Error('Unable to open the Data Viewer. Check the R session and try again.'); + } + } + private async rendererMessage( editor: vscode.NotebookEditor, message: Record, @@ -2447,6 +2564,15 @@ export class InteractiveManager if (data.kind !== 'table') { return; } + if (isJdvOutput(data)) { + await this.openHandleViewer( + view, + data, + [], + `Cell ${owner.record.order}: table`, + ); + break; + } await session.showDataView( 'table', 'json', @@ -2462,8 +2588,10 @@ export class InteractiveManager if (data.kind !== 'table') { return; } - result = await queryTablePage(data, message, (request) => - view.client.request('inspect', request), + result = await (isJdvOutput(data) ? queryJdvTablePage : queryTablePage)( + data, + message, + (request) => view.client.request('inspect', request), ); break; } @@ -2481,6 +2609,25 @@ export class InteractiveManager ) { throw new Error('Invalid list path'); } + if (isJdvOutput(data)) { + const adapter = await this.listAdapter(view, data); + const outcome = await listRequest( + adapter, + data, + { ...message, path: listPath }, + (request) => view.client.request('inspect', request), + ); + if (outcome.open) { + await this.openHandleViewer( + view, + data, + outcome.open.path, + `Cell ${owner.record.order}: ${outcome.open.title}`, + ); + } + result = outcome.result; + break; + } const params = { view_id: data.viewId, path: listPath }; if (message.action === 'listPage') { if (!Number.isSafeInteger(message.start) || Number(message.start) < 1) { @@ -2952,6 +3099,9 @@ export class InteractiveManager if (answer !== 'Restart Session' || view.disposed) { return; } + disposeJdvHost( + interactiveJdvHostKey(view.client.manifest.id, view.client.manifest.generation), + ); const storage = path.join(this.root, view.client.manifest.id); let config = JSON.parse( fs.readFileSync(path.join(storage, 'config.json'), 'utf8'), @@ -3303,6 +3453,9 @@ export class InteractiveManager view.disposed = true; clearTimeout(view.timer); clearTimeout(view.reconnectTimer); + disposeJdvHost( + interactiveJdvHostKey(view.client.manifest.id, view.client.manifest.generation), + ); view.client.close(); for (const { task } of view.executions.values()) { task.end(undefined); diff --git a/src/interactive/renderer.ts b/src/interactive/renderer.ts index 07ab973dc..4c7edb3f8 100644 --- a/src/interactive/renderer.ts +++ b/src/interactive/renderer.ts @@ -185,7 +185,7 @@ export function activate(context: RendererContext): { () => send(item, data, 'table'), 'Data viewer', ); - if (data.fullViewId) { + if (data.fullViewId || data.handleId) { open.title = 'Open the full table without copying it. Reference edits may appear here; reopen to refresh. Sorting and filtering scan the full data.'; } diff --git a/src/interactive/tableQuery.ts b/src/interactive/tableQuery.ts index 7366da120..73935d3f6 100644 --- a/src/interactive/tableQuery.ts +++ b/src/interactive/tableQuery.ts @@ -7,6 +7,8 @@ export interface TableColumn { type?: unknown; sortable?: boolean; filter?: unknown; + /** jdv columns list the operators their type supports. */ + jdv?: { filterOptions: string[] }; } export interface TableSort { colId: string; @@ -30,10 +32,25 @@ export const filterOperators = { endsWith: 'ends with', blank: 'is blank', notBlank: 'is not blank', - true: 'is true', - false: 'is false', + jdvIsEmpty: 'is empty', + jdvNotEmpty: 'is not empty', + jdvIsNa: 'is NA', + jdvNotNa: 'is not NA', + true: 'is TRUE', + false: 'is FALSE', }; +export function filterOperatorLabel(type: string): string { + return filterOperators[type as keyof typeof filterOperators] ?? type; +} export function columnOperators(column: TableColumn): string[] { + // An inline filter is one condition, so the range operator is not offered. + if (column.jdv) { + return [ + ...column.jdv.filterOptions.filter((option) => option !== 'inRange'), + 'jdvIsNa', + 'jdvNotNa', + ]; + } if (column.type === 'booleanColumn') { return ['true', 'false', 'blank', 'notBlank']; } @@ -62,7 +79,16 @@ export function columnOperators(column: TableColumn): string[] { ]; } export const filterNeedsValue = (type: string): boolean => - !['blank', 'notBlank', 'true', 'false'].includes(type); + ![ + 'blank', + 'notBlank', + 'jdvIsEmpty', + 'jdvNotEmpty', + 'jdvIsNa', + 'jdvNotNa', + 'true', + 'false', + ].includes(type); // Slicing can turn automatic row numbers into explicit character row names. // The synthetic index is never sorted/filtered and is not part of the data schema. export const tableSchema = (columns: TableColumn[]): unknown[][] => diff --git a/src/jdvAdapter.ts b/src/jdvAdapter.ts new file mode 100644 index 000000000..3208ee3c4 --- /dev/null +++ b/src/jdvAdapter.ts @@ -0,0 +1,1097 @@ +'use strict'; + +// Adapter between the jdv protocol and the existing Data Viewer and List Viewer +// webviews. It is independent of VS Code so the conversions can be tested in Node. + +export type JsonObject = Record; + +/** Capability flags of a descriptor; parseDescriptor() fills omitted flags with false. */ +export interface JdvCapabilities { + seek: boolean; + sort: boolean; + filter: boolean; + known_total: boolean; + next: boolean; + children: boolean; + refresh: boolean; + /** True only when the view tracks a variable and has not lost it. */ + track: boolean; +} + +const CAPABILITY_NAMES = [ + 'seek', + 'sort', + 'filter', + 'known_total', + 'next', + 'children', + 'refresh', + 'track', +] as const; + +export interface JdvColumnSchema { + position: number; + name: string; + type: string; + /** Producer-defined display hint (R: `class()`); never interpreted. */ + native_type?: string[]; + sortable: boolean; + filterable: boolean; + drillable?: boolean; + levels?: string[]; + ordered?: boolean; + tz?: string; + /** `duration` columns: the unit of the values (secs, mins, hours, days or weeks). */ + units?: string; +} + +export interface JdvDescriptor { + view: string; + generation: number; + kind: 'grid' | 'tree'; + access: string; + encoding: string; + capabilities: JdvCapabilities; + title?: string | null; + collation?: 'C' | 'locale'; + /** Sequential views: the row cap captured when the view was opened. */ + max_page_rows?: number; + nrow?: number | null; + ncol?: number; + columns?: JdvColumnSchema[]; + path?: JdvPathStep[]; + /** Tree views: `class()` and `typeof()` of the root object. */ + class?: string[]; + type?: string; + parent?: string; + /** + * Why jdv sent this `view/open`: "register", "child", "tracked" or "refresh". + * Missing or unknown values mean "register". + */ + reason?: string; + /** Present only on views that can follow a variable. */ + tracking?: JdvTracking; +} + +export interface JdvTracking { + name: string; + state: 'active' | 'paused' | 'lost'; + /** Fixed token when lost: "removed", "collected" or an unknown value. */ + reason?: string; + /** English display text when lost; not to be parsed. */ + message?: string; +} + +export type JdvPathStep = + | { type: 'index'; index: number } + | { type: 'binding' | 'slot' | 'attr'; name: string }; + +export class JdvRequestError extends Error { + constructor( + readonly code: string, + message: string, + ) { + super(message); + } +} + +function isObject(value: unknown): value is JsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Text of a scalar filter operand or channel value; other values become ''. */ +function scalarText(value: unknown): string { + return typeof value === 'string' || + typeof value === 'number' || + typeof value === 'bigint' || + typeof value === 'boolean' + ? String(value) + : ''; +} + +/** Validate the params of a `view/open` notification. */ +export function parseDescriptor(value: unknown): JdvDescriptor { + if ( + !isObject(value) || + typeof value.view !== 'string' || + value.view === '' || + typeof value.generation !== 'number' || + !Number.isSafeInteger(value.generation) || + value.generation < 1 || + (value.kind !== 'grid' && value.kind !== 'tree') || + typeof value.access !== 'string' || + typeof value.encoding !== 'string' + ) { + throw new Error('Invalid jdv view descriptor'); + } + // An omitted capabilities object or flag means false. + const supplied = value.capabilities === undefined ? {} : value.capabilities; + if ( + !isObject(supplied) || + CAPABILITY_NAMES.some((name) => name in supplied && typeof supplied[name] !== 'boolean') + ) { + throw new Error('Invalid jdv view descriptor capabilities'); + } + const capabilities = Object.fromEntries( + CAPABILITY_NAMES.map((name) => [name, supplied[name] === true]), + ); + value.capabilities = capabilities; + if (value.kind === 'grid') { + validateGridDescriptor(value, capabilities as unknown as JdvCapabilities); + } + const descriptor = value as unknown as JdvDescriptor; + // `reason` is optional and may gain values; keep only a string. + if (typeof descriptor.reason !== 'string') { + delete descriptor.reason; + } + const tracking = parseTracking(descriptor.tracking); + if (tracking) { + descriptor.tracking = tracking; + } else { + delete descriptor.tracking; + } + return descriptor; +} + +function validateGridDescriptor(value: JsonObject, capabilities: JdvCapabilities): void { + const invalid = (reason: string) => new Error(`Invalid jdv grid descriptor: ${reason}`); + if (!Array.isArray(value.columns)) { + throw invalid('columns'); + } + if (value.access === 'sequential') { + const cap = value.max_page_rows; + if ( + value.encoding !== 'arrow-ipc' || + typeof cap !== 'number' || + !Number.isSafeInteger(cap) || + cap < 1 || + !capabilities.next + ) { + throw invalid('sequential access'); + } + return; + } + if ( + (value.access !== 'snapshot' && value.access !== 'random') || + value.encoding !== 'jdv-json' + ) { + throw invalid('access or encoding'); + } + if (value.collation !== 'C' && value.collation !== 'locale') { + throw invalid('collation'); + } + if ( + value.access === 'random' && + (typeof value.nrow !== 'number' || + !Number.isSafeInteger(value.nrow) || + value.nrow < 0 || + !capabilities.seek || + !capabilities.known_total) + ) { + throw invalid('random access'); + } +} + +/** Validate a `tracking` object; a malformed one is dropped (`undefined`). */ +export function parseTracking(value: unknown): JdvTracking | undefined { + if ( + !isObject(value) || + typeof value.name !== 'string' || + (value.state !== 'active' && value.state !== 'paused' && value.state !== 'lost') + ) { + return undefined; + } + const tracking: JdvTracking = { name: value.name, state: value.state }; + if (typeof value.reason === 'string') { + tracking.reason = value.reason; + } + if (typeof value.message === 'string') { + tracking.message = value.message; + } + return tracking; +} + +/** + * Whether a newer generation of an open view should bring its panel to the front. + * Updates that R made on its own (a tracked variable, a Refresh click) must not + * switch tabs; an explicit View(), an unknown reason or none still reveal. + */ +export function revealsPanel(descriptor: JdvDescriptor): boolean { + return descriptor.reason !== 'tracked' && descriptor.reason !== 'refresh'; +} + +/** Whether the view accepts `view/refresh`. Sequential streams cannot be rewound. */ +export function canRefresh(descriptor: JdvDescriptor): boolean { + return descriptor.capabilities.refresh === true && descriptor.access !== 'sequential'; +} + +// --------------------------------------------------------------------------- +// Grid + +/** Row label column. Data columns use their one-based position as field id. */ +export const ROW_LABEL_FIELD = '0'; +/** Row data key holding the zero-based source row; not a column field. */ +export const SOURCE_ROW_FIELD = '__sourceRow'; + +export interface GridColumnDef { + headerName: string; + headerTooltip?: string; + field: string; + type: string; + filter: string | boolean; + sortable: boolean; + cellDataType?: string | false; + suppressHeaderMenuButton?: boolean; + lockPosition?: string; + /** Drillable grid cells: the zero-based schema position that `object/open` addresses. */ + jdvDrill?: number; + /** Webview-only options; see prepareViewerColumns(). */ + jdv?: { filterOptions: string[]; naOptions: boolean; units?: string }; +} + +export const EMPTY_FILTER_OPTION = 'jdvIsEmpty'; +export const NOT_EMPTY_FILTER_OPTION = 'jdvNotEmpty'; +const TEXT_FILTER_OPTIONS = [ + 'contains', + 'notContains', + 'equals', + 'notEqual', + 'startsWith', + 'endsWith', + EMPTY_FILTER_OPTION, + NOT_EMPTY_FILTER_OPTION, +]; +const ORDER_FILTER_OPTIONS = [ + 'equals', + 'notEqual', + 'lessThan', + 'lessThanOrEqual', + 'greaterThan', + 'greaterThanOrEqual', + 'inRange', +]; +// jdv implements only exact equality and inclusive ranges for integer64. +const INT64_FILTER_OPTIONS = ['equals', 'inRange']; +export const NA_FILTER_OPTION = 'jdvIsNa'; +export const NOT_NA_FILTER_OPTION = 'jdvNotNa'; + +function columnTooltip(column: JdvColumnSchema): string { + let tooltip = `${column.name}, class: [${(column.native_type ?? []).join(', ')}], type: ${column.type}`; + if (column.type === 'categorical' && Array.isArray(column.levels)) { + const levels = column.levels.slice(0, 20).join(', '); + tooltip += `, levels: ${levels}${column.levels.length > 20 ? ', ...' : ''}`; + } + if (column.type === 'datetime' && column.tz !== undefined) { + tooltip += `, tz: ${column.tz || '(local)'}`; + } + if (column.type === 'duration' && typeof column.units === 'string') { + tooltip += `, units: ${column.units}`; + } + return tooltip; +} + +const ARROW_NUMERIC_FORMATS = new Set(['c', 'C', 's', 'S', 'i', 'I', 'l', 'L', 'e', 'f', 'g']); + +function rowLabelColumn(): GridColumnDef { + return { + headerName: '', + field: ROW_LABEL_FIELD, + type: 'textColumn', + filter: false, + sortable: false, + lockPosition: 'left', + suppressHeaderMenuButton: true, + }; +} + +/** Columns of a sequential Arrow view; they can be neither sorted nor filtered. */ +function sequentialColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { + const columns = [rowLabelColumn()]; + for (const column of (descriptor.columns ?? []) as unknown as { + position: number; + name: string; + type: string; + unsupported: string | null; + }[]) { + // The descriptor no longer carries the Arrow schema, so a dictionary column + // (whose format is its index type) is aligned like a number. + const numeric = ARROW_NUMERIC_FORMATS.has(column.type) || column.type.startsWith('d:'); + columns.push({ + headerName: column.name, + headerTooltip: column.unsupported + ? `${column.name}, unsupported: ${column.unsupported}` + : `${column.name}, Arrow format: ${column.type}`, + field: String(column.position + 1), + type: numeric ? 'numericColumn' : 'textColumn', + ...(numeric ? { cellDataType: false } : {}), + filter: false, + sortable: false, + }); + } + return columns; +} + +export function gridColumnDefs(descriptor: JdvDescriptor): GridColumnDef[] { + if (descriptor.access === 'sequential') { + return sequentialColumnDefs(descriptor); + } + const columns: GridColumnDef[] = [rowLabelColumn()]; + for (const column of descriptor.columns ?? []) { + const sortable = descriptor.capabilities.sort && column.sortable; + const filterEnabled = descriptor.capabilities.filter; + let type = 'textColumn'; + let filter: string | boolean = false; + let filterOptions: string[] = []; + switch (column.type) { + case 'integer': + case 'number': + case 'duration': + type = 'numericColumn'; + filter = 'agNumberColumnFilter'; + filterOptions = ORDER_FILTER_OPTIONS; + break; + case 'int64': + type = 'bigintColumn'; + filter = 'agBigIntColumnFilter'; + filterOptions = INT64_FILTER_OPTIONS; + break; + case 'date': + type = 'dateColumn'; + filter = 'agDateColumnFilter'; + filterOptions = ORDER_FILTER_OPTIONS; + break; + case 'datetime': + type = 'datetimeColumn'; + filter = 'agDateColumnFilter'; + filterOptions = ORDER_FILTER_OPTIONS; + break; + case 'boolean': + // AG Grid's boolean data type supplies its own True/False filter. + type = 'booleanColumn'; + filter = true; + break; + case 'string': + case 'categorical': + filter = 'agTextColumnFilter'; + filterOptions = TEXT_FILTER_OPTIONS; + break; + default: + // `list`, `other` and unknown types are display-only: only the NA + // options apply. Never infer semantics from `class`. + filter = 'agTextColumnFilter'; + } + // Value operators require `filterable`; `is_na` only needs a missingness mask, + // which every column has unless its channel failed. + if (!column.filterable) { + filterOptions = []; + } + const def: GridColumnDef = { + headerName: column.name, + headerTooltip: columnTooltip(column), + field: String(column.position + 1), + type, + filter: filterEnabled ? filter : false, + sortable, + jdv: { + filterOptions, + naOptions: true, + ...(column.type === 'duration' && typeof column.units === 'string' + ? { units: column.units } + : {}), + }, + }; + if (column.drillable === true && descriptor.capabilities.children) { + def.jdvDrill = column.position; + } + if (column.type === 'boolean') { + def.cellDataType = 'boolean'; + delete def.jdv; + if (!column.filterable) { + def.filter = false; + } + } + if (!filterEnabled) { + delete def.jdv; + } + columns.push(def); + } + return columns; +} + +export interface AgSortModelItem { + colId: string; + sort: 'asc' | 'desc'; +} + +export interface JdvSort { + col: number; + desc: boolean; +} + +function fieldPosition(field: string, descriptor: JdvDescriptor): number { + const position = Number(field) - 1; + if ( + !Number.isSafeInteger(position) || + position < 0 || + position >= (descriptor.columns?.length ?? 0) + ) { + throw new JdvRequestError('invalid_request', `Unknown column: ${field}`); + } + return position; +} + +export function toJdvSort(sortModel: unknown, descriptor: JdvDescriptor): JdvSort[] { + if (!Array.isArray(sortModel)) { + return []; + } + return sortModel.map((item) => { + const sortItem = item as Partial; + const col = fieldPosition(String(sortItem.colId), descriptor); + if (!descriptor.columns![col].sortable) { + throw new JdvRequestError( + 'unsupported', + `Column "${descriptor.columns![col].name}" cannot be sorted`, + ); + } + return { col, desc: sortItem.sort === 'desc' }; + }); +} + +type FilterNode = JsonObject; + +function unsupportedFilter(column: JdvColumnSchema, type: unknown): never { + throw new JdvRequestError( + 'unsupported', + `Filter "${String(type)}" is not supported for column "${column.name}" (${column.type})`, + ); +} + +function canonicalInt64(value: unknown, column: JdvColumnSchema): string { + const text = scalarText(value).trim(); + if (!/^[+-]?\d+$/.test(text)) { + throw new JdvRequestError( + 'invalid_request', + `Invalid integer64 value for column "${column.name}"`, + ); + } + return BigInt(text).toString(); +} + +function numberOperand(value: unknown, column: JdvColumnSchema): unknown { + const number = typeof value === 'number' ? value : Number(value); + if (Number.isNaN(number) || value === null || value === undefined || value === '') { + throw new JdvRequestError('invalid_request', `Invalid number for column "${column.name}"`); + } + if (number === Infinity) { + return { special: 'pos_inf' }; + } + if (number === -Infinity) { + return { special: 'neg_inf' }; + } + return number; +} + +/** Duration operand: a number in the column's units, or a typed `[-]H:MM:SS[.fff]` string. */ +function durationOperand(value: unknown, column: JdvColumnSchema): unknown { + if (typeof value === 'string' && /^\s*-?\d+:\d{2}:\d{2}(\.\d+)?\s*$/.test(value)) { + return value.trim(); + } + return numberOperand(value, column); +} + +function dateOperand(value: unknown, column: JdvColumnSchema): string { + // AG Grid's date filter model uses "YYYY-MM-DD HH:mm:ss". Keep the + // datetime text intact (including fractional seconds and explicit offsets) + // so jdv can interpret it without a JavaScript Date round trip. + const text = scalarText(value); + const invalid = () => + new JdvRequestError('invalid_request', `Invalid date for column "${column.name}"`); + if (column.type === 'date') { + const date = /^(\d{4}-\d{2}-\d{2})$(?![\s\S])/.exec(text); + if (date) { + return date[1]; + } + // AG Grid's filter model writes a date as "YYYY-MM-DD 00:00:00"; an actual time + // or timezone is never discarded. + const midnight = /^(\d{4}-\d{2}-\d{2})[ T]00:00:00$(?![\s\S])/.exec(text); + if (midnight) { + return midnight[1]; + } + throw invalid(); + } + const datetime = + /^(\d{4}-\d{2}-\d{2})(?:[ T](\d{2}:\d{2}:\d{2})(\.\d+)?(Z|[+-]\d{2}:\d{2})?)?$(?![\s\S])/.exec( + text, + ); + if (!datetime) { + throw invalid(); + } + // An operand without offset uses the column's time zone in jdv. + return `${datetime[1]}T${datetime[2] ?? '00:00:00'}${datetime[3] ?? ''}${datetime[4] ?? ''}`; +} + +const ORDER_OPS: Record = { + equals: 'eq', + notEqual: 'ne', + lessThan: 'lt', + lessThanOrEqual: 'le', + greaterThan: 'gt', + greaterThanOrEqual: 'ge', +}; + +function conditionNode(condition: JsonObject, col: number, column: JdvColumnSchema): FilterNode { + const type = condition.type; + const isNa = { op: 'is_na', col }; + if (type === NA_FILTER_OPTION) { + return isNa; + } + if (type === NOT_NA_FILTER_OPTION) { + return { op: 'not', arg: isNa }; + } + if (!column.filterable) { + unsupportedFilter(column, type); + } + switch (column.type) { + case 'boolean': + if (type === 'true') { + return { op: 'is_true', col }; + } + if (type === 'false') { + return { op: 'is_false', col }; + } + return unsupportedFilter(column, type); + case 'integer': + case 'number': + case 'duration': { + const operand = column.type === 'duration' ? durationOperand : numberOperand; + if (type === 'inRange') { + return { + op: 'between', + col, + lower: operand(condition.filter, column), + upper: operand(condition.filterTo, column), + }; + } + const op = ORDER_OPS[String(type)]; + return op + ? { op, col, value: operand(condition.filter, column) } + : unsupportedFilter(column, type); + } + case 'int64': + if (type === 'equals') { + return { op: 'eq', col, value: canonicalInt64(condition.filter, column) }; + } + if (type === 'inRange') { + return { + op: 'between', + col, + lower: canonicalInt64(condition.filter, column), + upper: canonicalInt64(condition.filterTo, column), + }; + } + return unsupportedFilter(column, type); + case 'date': + case 'datetime': { + if (type === 'inRange') { + return { + op: 'between', + col, + lower: dateOperand(condition.dateFrom, column), + upper: dateOperand(condition.dateTo, column), + }; + } + const op = ORDER_OPS[String(type)]; + return op + ? { op, col, value: dateOperand(condition.dateFrom, column) } + : unsupportedFilter(column, type); + } + case 'string': + case 'categorical': { + const value = scalarText(condition.filter); + // As in the previous viewer, negated text conditions exclude missing values. + const negate = (node: FilterNode): FilterNode => ({ + op: 'and', + args: [ + { op: 'not', arg: node }, + { op: 'not', arg: isNa }, + ], + }); + switch (type) { + case EMPTY_FILTER_OPTION: + return { op: 'text_equals', col, value: '' }; + case NOT_EMPTY_FILTER_OPTION: + return negate({ op: 'text_equals', col, value: '' }); + case 'contains': + return { op: 'contains', col, value }; + case 'notContains': + return negate({ op: 'contains', col, value }); + case 'equals': + return { op: 'text_equals', col, value }; + case 'notEqual': + return negate({ op: 'text_equals', col, value }); + case 'startsWith': + return { op: 'starts_with', col, value }; + case 'endsWith': + return { op: 'ends_with', col, value }; + default: + return unsupportedFilter(column, type); + } + } + default: + // Display-only types (`list`, `other`, unknown) accept only the NA options above. + return unsupportedFilter(column, type); + } +} + +/** Convert an AG Grid filter model to the jdv filter AST, or `undefined` for no filter. */ +export function toJdvFilter( + filterModel: unknown, + descriptor: JdvDescriptor, +): FilterNode | undefined { + if (!isObject(filterModel)) { + return undefined; + } + const nodes: FilterNode[] = []; + for (const [field, model] of Object.entries(filterModel)) { + if (!isObject(model)) { + continue; + } + const col = fieldPosition(field, descriptor); + const column = descriptor.columns![col]; + const conditions = Array.isArray(model.conditions) + ? model.conditions.filter(isObject) + : undefined; + if (conditions && conditions.length > 0) { + const args = conditions.map((condition) => conditionNode(condition, col, column)); + nodes.push( + args.length === 1 + ? args[0] + : { + op: String(model.operator).toUpperCase() === 'OR' ? 'or' : 'and', + args, + }, + ); + } else { + nodes.push(conditionNode(model, col, column)); + } + } + if (nodes.length === 0) { + return undefined; + } + return nodes.length === 1 ? nodes[0] : { op: 'and', args: nodes }; +} + +/** `object/open` params that open one drillable grid cell as a child view. */ +export function cellOpenParams( + descriptor: JdvDescriptor, + sourceRow: unknown, + column: unknown, +): JsonObject { + if (descriptor.kind !== 'grid' || !descriptor.capabilities.children) { + throw new JdvRequestError('unsupported', 'This view has no cells that can be opened.'); + } + if ( + typeof sourceRow !== 'number' || + !Number.isSafeInteger(sourceRow) || + sourceRow < 0 || + typeof column !== 'number' || + !Number.isSafeInteger(column) || + column < 0 + ) { + throw new JdvRequestError('invalid_request', 'Invalid cell address.'); + } + const schema = descriptor.columns?.find((candidate) => candidate.position === column); + if (!schema?.drillable) { + throw new JdvRequestError('invalid_request', 'This column cannot be opened.'); + } + return { + view: descriptor.view, + generation: descriptor.generation, + path: [{ kind: 'cell', source_row: sourceRow, column }], + // Paged keeps sorting and filtering in R, as for top-level grids. + mode: 'paged', + }; +} + +export interface TableRowsParams extends JsonObject { + view: string; + generation: number; + offset: number; + limit: number; + sort: JdvSort[]; + filter?: FilterNode; + cols?: number[]; +} + +/** jdv's default `page_columns` limit for an explicit column selection. */ +const MAX_PAGE_COLUMNS = 256; + +/** + * Zero-based positions for the grid fields the webview displays, or undefined to + * request every column. Field ids are one-based positions; "0" is the row label. + */ +function pageColumns(descriptor: JdvDescriptor, fields: unknown): number[] | undefined { + const ncol = descriptor.columns?.length ?? 0; + if (!Array.isArray(fields)) { + return undefined; + } + const cols = new Set(); + for (const field of fields) { + if (field === '0') { + continue; + } + const position = + typeof field === 'string' && /^[1-9][0-9]*$/.test(field) ? Number(field) - 1 : -1; + if (position < 0 || position >= ncol) { + return undefined; + } + cols.add(position); + } + return cols.size >= ncol || cols.size > MAX_PAGE_COLUMNS ? undefined : [...cols]; +} + +export function tableRowsParams( + descriptor: JdvDescriptor, + startRow: number, + endRow: number, + sortModel: unknown, + filterModel: unknown, + columns?: unknown, +): TableRowsParams { + const offset = Math.max(0, Math.floor(startRow)); + const limit = Math.max(1, Math.floor(endRow) - offset); + const params: TableRowsParams = { + view: descriptor.view, + generation: descriptor.generation, + offset, + limit, + sort: toJdvSort(sortModel, descriptor), + }; + const filter = toJdvFilter(filterModel, descriptor); + if (filter) { + params.filter = filter; + } + const cols = pageColumns(descriptor, columns); + if (cols) { + params.cols = cols; + } + return params; +} + +interface ColumnChannels { + values?: unknown[]; + display?: (string | null)[]; + /** Zero-based positions of missing rows within this page. */ + na_rows?: number[]; +} + +interface PageColumn { + schema?: JdvColumnSchema; + data?: ColumnChannels; + errors?: unknown; +} + +export interface GridPage { + rows: Record[]; + totalRows: number; + totalUnfiltered: number; + lastRow: number; + /** Rows returned; may be fewer than requested when jdv clamps the page. */ + returned: number; +} + +function cellValue( + column: JdvColumnSchema, + channels: ColumnChannels, + naRows: Set, + row: number, +): unknown { + if (naRows.has(row)) { + // Missing cells use the grid's NA renderer, unless R labels them differently + // (NaN, or a class-specific missing label). + const label = channels.display?.[row]; + return typeof label === 'string' && label !== 'NA' ? label : null; + } + // Booleans keep their value for the grid's checkbox renderer. Everything else + // shows the R-formatted display text; comparisons happen in R. + if (column.type === 'boolean' && channels.values) { + return channels.values[row] ?? null; + } + const display = channels.display?.[row]; + if (display !== undefined && display !== null) { + return display; + } + // jdv omits display for strings, categoricals and int64 when it equals the value. + const value = channels.values?.[row]; + return value === undefined || value === null ? null : scalarText(value); +} + +function columnErrorText(errors: unknown): string | undefined { + if (!isObject(errors)) { + return undefined; + } + const messages = Object.entries(errors).map( + ([channel, message]) => `${channel}: ${String(message)}`, + ); + return messages.length > 0 ? messages.join('; ') : undefined; +} + +/** Convert a `grid/rows` result to row objects keyed by the grid field ids. */ +export function gridPageFromRows(result: unknown, startRow: number, requested: number): GridPage { + if ( + !isObject(result) || + !isObject(result.data) || + typeof result.total !== 'number' || + typeof result.total_unfiltered !== 'number' || + !Array.isArray(result.data.source_row) + ) { + throw new Error('Invalid jdv grid/rows result'); + } + const data = result.data; + const sourceRow = data.source_row as number[]; + const rowLabels = Array.isArray(data.row_labels) + ? (data.row_labels as (string | null)[]) + : undefined; + const columns = Array.isArray(data.columns) ? (data.columns as PageColumn[]) : []; + // `source_row` is zero-based; the browser shows it plus one. + const rows: Record[] = sourceRow.map((index, row) => ({ + [ROW_LABEL_FIELD]: rowLabels?.[row] ?? String(index + 1), + // Original zero-based row, for opening a drillable cell after sorting or filtering. + [SOURCE_ROW_FIELD]: index, + })); + for (const column of columns) { + if (!column.schema) { + continue; + } + const field = String(column.schema.position + 1); + const channels = column.data ?? {}; + const error = columnErrorText(column.errors); + const naRows = new Set(channels.na_rows); + for (let row = 0; row < rows.length; row++) { + rows[row][field] = + channels.display || channels.values || channels.na_rows + ? cellValue(column.schema, channels, naRows, row) + : error + ? `` + : null; + } + } + const total = result.total; + const returned = rows.length; + // When jdv returns fewer rows than requested before the end, report an unknown + // last row so the grid asks again for the remainder of the block. + const lastRow = returned < requested && startRow + returned < total ? -1 : total; + return { rows, totalRows: total, totalUnfiltered: result.total_unfiltered, lastRow, returned }; +} + +// --------------------------------------------------------------------------- +// Tree + +export interface ListViewNavigation { + title: string; + path: number[]; + breadcrumbs: { label: string; path: number[] }[]; + vector?: boolean; +} + +export interface ListViewChild { + label: string; + str: string; + index: number; + viewable?: boolean; + has_children?: boolean; +} + +interface TreeNode { + path: JdvPathStep[]; + label: string; + title: string; + class: string[]; + type?: string; + hasChildren: boolean; + viewable: boolean; +} + +interface TreeChild { + index: number; + label: string; + class?: string[]; + type?: string; + summary?: string; + has_children?: boolean; + viewable?: boolean; + unavailable?: string; + path?: JdvPathStep[]; +} + +export const TREE_PAGE_SIZE = 200; + +const TABLE_CLASSES = new Set(['data.frame', 'matrix']); +const ATOMIC_TYPES = new Set(['logical', 'integer', 'double', 'complex', 'character', 'raw']); + +/** Atomic vectors are shown as value rows, as in the previous List Viewer. */ +export function isVectorType(type: string | undefined, classes: string[]): boolean { + return type !== undefined && ATOMIC_TYPES.has(type) && !classes.includes('matrix'); +} + +export function isTableClass(classes: string[]): boolean { + return classes.some((cls) => TABLE_CLASSES.has(cls)); +} + +function childTitle(parent: TreeNode, child: TreeChild): string { + const step = child.path?.[child.path.length - 1]; + if (step?.type === 'slot') { + return `${parent.title}@${step.name}`; + } + if (step?.type === 'attr') { + return `attr(${parent.title}, "${step.name}")`; + } + if (step?.type === 'binding') { + return `${parent.title}$${step.name}`; + } + const label = child.label; + return /^\[\[?\d+\]\]?$/.test(label) || label === '' + ? `${parent.title}[[${child.index}]]` + : `${parent.title}$${label}`; +} + +/** + * Maps the List Viewer's positional paths (child `index` values) to jdv typed + * paths. Typed paths are learned from `object/children` results. + */ +export class TreeAdapter { + private readonly nodes = new Map(); + + constructor( + public descriptor: JdvDescriptor, + private readonly title: string, + ) { + this.nodes.set('[]', { + path: [], + label: title, + title, + class: descriptor.class ?? [], + type: descriptor.type, + hasChildren: true, + viewable: true, + }); + } + + private node(path: number[]): TreeNode { + const node = this.nodes.get(JSON.stringify(path)); + if (!node) { + throw new JdvRequestError( + 'unavailable', + 'This item is no longer available. Reload the parent list.', + ); + } + return node; + } + + /** The typed jdv path and display title of the item at a positional path. */ + locate(path: number[]): { path: JdvPathStep[]; title: string } { + const node = this.node(path); + return { path: node.path, title: node.title }; + } + + navigation(path: number[] = []): ListViewNavigation { + const node = this.node(path); + const breadcrumbs = []; + for (let depth = 0; depth <= path.length; depth++) { + const prefix = path.slice(0, depth); + breadcrumbs.push({ label: this.node(prefix).label, path: prefix }); + } + return { + title: node.title, + path, + breadcrumbs, + vector: isVectorType(node.type, node.class), + }; + } + + childrenParams(path: number[], start: number): JsonObject { + const node = this.node(path); + return { + view: this.descriptor.view, + generation: this.descriptor.generation, + path: node.path, + offset: Math.max(0, start - 1), + limit: TREE_PAGE_SIZE, + // Like the previous List Viewer, show elements without attributes such as `names`. + attributes: false, + }; + } + + /** Record the children of `path` and convert them to List Viewer rows. */ + children( + path: number[], + result: unknown, + ): { children: ListViewChild[]; next_start: number | null } { + if ( + !isObject(result) || + !Array.isArray(result.children) || + typeof result.offset !== 'number' || + typeof result.total_children !== 'number' + ) { + throw new Error('Invalid jdv object/children result'); + } + const parent = this.node(path); + const vector = isVectorType(parent.type, parent.class); + const children = (result.children as TreeChild[]).map((child) => { + const addressable = Array.isArray(child.path) && child.viewable !== false; + if (addressable) { + this.nodes.set(JSON.stringify([...path, child.index]), { + path: child.path!, + label: child.label, + title: childTitle(parent, child), + class: child.class ?? [], + type: child.type, + hasChildren: child.has_children === true, + viewable: child.viewable !== false, + }); + } + const hasChildren = child.has_children === true; + const summary = child.unavailable ? `(${child.unavailable})` : (child.summary ?? ''); + return { + // Unnamed elements are labeled by position, as in the previous List Viewer. + label: child.label === '' ? `[[${child.index}]]` : child.label, + str: summary, + index: child.index, + // Vector elements are terminal rows; other leaves have nothing to open. + viewable: + addressable && !vector && (hasChildren || isTableClass(child.class ?? [])), + has_children: hasChildren, + }; + }); + const end = result.offset + children.length; + return { children, next_start: end < result.total_children ? end + 1 : null }; + } + + /** What opening the child at `path`/`index` should do. */ + open( + path: number[], + index: number, + ): + | { kind: 'navigate'; navigation: ListViewNavigation } + | { kind: 'object'; params: JsonObject; title: string } { + const childPath = [...path, index]; + const node = this.node(childPath); + if (!node.viewable) { + throw new JdvRequestError('unavailable', 'This item cannot be opened.'); + } + if (isTableClass(node.class)) { + return { + kind: 'object', + title: node.title, + // Paged keeps sorting and filtering in R, as for top-level grids. + params: { + view: this.descriptor.view, + generation: this.descriptor.generation, + path: node.path, + mode: 'paged', + }, + }; + } + return { kind: 'navigate', navigation: this.navigation(childPath) }; + } +} diff --git a/src/jdvArrow.ts b/src/jdvArrow.ts new file mode 100644 index 000000000..aef155a27 --- /dev/null +++ b/src/jdvArrow.ts @@ -0,0 +1,501 @@ +'use strict'; + +// Sequential (forward-only) jdv Arrow views. The extension host keeps the +// received Arrow IPC pages, decodes them with apache-arrow, and serves the +// existing AG Grid infinite row model with formatted rows. + +import { tableFromIPC } from 'apache-arrow'; +import { JdvDescriptor, JdvRequestError, ROW_LABEL_FIELD } from './jdvAdapter'; + +// --------------------------------------------------------------------------- +// Formatting. Values are formatted from physical buffers where JavaScript +// number getters lose precision. Ported from jdv's reference viewer +// (server/web/sequential.js, MIT). + +export const CELL_TEXT_LIMIT = 4096; +const NANOSECONDS = [1_000_000_000n, 1_000_000n, 1_000n, 1n]; +const UNIT_NAMES = ['s', 'ms', 'us', 'ns']; + +// apache-arrow Type ids used below. +const TYPE_DICTIONARY = -1; +const TYPE_NULL = 1; +const TYPE_INT = 2; +const TYPE_DECIMAL = 7; +const TYPE_DATE = 8; +const TYPE_TIME = 9; +const TYPE_TIMESTAMP = 10; +const TYPE_LIST = 12; +const TYPE_STRUCT = 13; +const TYPE_FIXED_SIZE_LIST = 16; +const TYPE_MAP = 17; +const TYPE_DURATION = 18; +const TYPE_LARGE_LIST = 21; + +interface ArrowType { + typeId: number; + bitWidth?: number; + scale?: number; + unit?: number; + timezone?: string | null; + listSize?: number; + children?: unknown[]; +} + +interface ArrowData { + length: number; + stride: number; + values: ArrayLike; + valueOffsets?: ArrayLike; + dictionary?: ArrowVector; +} + +interface ArrowVector { + length: number; + type: ArrowType; + data: ArrowData[]; + isValid(index: number): boolean; + get(index: number): unknown; +} + +interface ArrowTable { + numRows: number; + numCols: number; + getChildAt(index: number): unknown; +} + +export function formatScaled(coefficient: bigint, scale: number): string { + if (!Number.isSafeInteger(scale)) { + throw new Error('Invalid Arrow decimal scale: expected a safe integer.'); + } + const negative = coefficient < 0n; + const digits = (negative ? -coefficient : coefficient).toString(); + const sign = negative ? '-' : ''; + const fixedLength = + sign.length + (scale <= 0 ? digits.length - scale : Math.max(digits.length, scale + 1) + 1); + // Arrow decimal coefficients have at most 76 digits; never expand a huge scale. + if (fixedLength > CELL_TEXT_LIMIT) { + return `${sign}${digits}e${-scale}`; + } + if (scale <= 0) { + return sign + digits + '0'.repeat(-scale); + } + const padded = digits.padStart(scale + 1, '0'); + return `${sign}${padded.slice(0, -scale)}.${padded.slice(-scale)}`; +} + +function signedWords(values: ArrayLike, start: number, width: number): bigint { + let result = 0n; + for (let word = 0; word < width; word++) { + result |= BigInt(Number(values[start + word]) >>> 0) << BigInt(word * 32); + } + return BigInt.asIntN(width * 32, result); +} + +function floorDiv(value: bigint, divisor: bigint): bigint { + const quotient = value / divisor; + return value % divisor < 0n ? quotient - 1n : quotient; +} + +export function formatTimestamp( + ticks: bigint, + unit: number, + timezone: string | null = null, +): string { + const nanoseconds = ticks * NANOSECONDS[unit]; + const seconds = floorDiv(nanoseconds, 1_000_000_000n); + const fraction = nanoseconds - seconds * 1_000_000_000n; + const date = new Date(Number(seconds) * 1000); + if (!Number.isSafeInteger(Number(seconds)) || !Number.isFinite(date.getTime())) { + return `${ticks} ${UNIT_NAMES[unit]}${timezone ? ` [${timezone}]` : ''}`; + } + const digits = [0, 3, 6, 9][unit]; + const suffix = digits ? `.${fraction.toString().padStart(9, '0').slice(0, digits)}` : ''; + if (!timezone || timezone === 'UTC' || timezone === 'Etc/UTC') { + return date.toISOString().replace(/\.\d{3}Z$/, suffix + (timezone ? 'Z' : '')); + } + try { + const formatter = new Intl.DateTimeFormat('en-US-u-ca-iso8601-nu-latn', { + timeZone: timezone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hourCycle: 'h23', + timeZoneName: 'longOffset', + }); + const parts = Object.fromEntries( + formatter.formatToParts(date).map(({ type, value }) => [type, value]), + ); + const offset = (parts.timeZoneName ?? '').replace(/^GMT/, '') || '+00:00'; + return `${parts.year}-${parts.month}-${parts.day}T${parts.hour}:${parts.minute}:${parts.second}${suffix}${offset} [${timezone}]`; + } catch { + return `${date.toISOString().replace(/\.\d{3}Z$/, suffix + 'Z')} [${timezone}]`; + } +} + +function chunkAt(vector: ArrowVector, row: number): { data: ArrowData; row: number } { + for (const data of vector.data) { + if (row < data.length) { + return { data, row }; + } + row -= data.length; + } + throw new Error('Arrow row is outside its buffers.'); +} + +function boundedText(value: string): string { + return value.length > CELL_TEXT_LIMIT + ? value.slice(0, CELL_TEXT_LIMIT) + '… [truncated]' + : value; +} + +/** Display text of one Arrow value, or `null` for a missing value. */ +export function formatArrowValue( + vector: ArrowVector, + row: number, + dictionaries = new Set(), +): string | null { + if (!Number.isSafeInteger(row) || row < 0 || row >= vector.length) { + throw new Error('Arrow row is outside its buffers.'); + } + const type = vector.type; + if (type.typeId === TYPE_NULL || !vector.isValid(row)) { + return null; + } + const { data, row: local } = chunkAt(vector, row); + // Arrow Data slices already adjust values; offset belongs to the validity bitmap. + const start = local * data.stride; + if (type.typeId === TYPE_DICTIONARY) { + const index = Number(data.values[start]); + if ( + !data.dictionary || + !Number.isSafeInteger(index) || + index < 0 || + index >= data.dictionary.length + ) { + throw new Error('Invalid Arrow dictionary index.'); + } + if (dictionaries.has(data) || dictionaries.size >= 32) { + throw new Error('Arrow dictionary nesting is cyclic or exceeds 32 levels.'); + } + dictionaries.add(data); + return formatArrowValue(data.dictionary, index, dictionaries); + } + if (type.typeId === TYPE_INT && type.bitWidth === 64) { + return BigInt(data.values[start]).toString(); + } + if (type.typeId === TYPE_DECIMAL) { + return formatScaled( + signedWords(data.values, start, (type.bitWidth ?? 128) / 32), + type.scale ?? 0, + ); + } + if (type.typeId === TYPE_TIMESTAMP) { + return formatTimestamp(BigInt(data.values[start]), type.unit ?? 0, type.timezone ?? null); + } + if (type.typeId === TYPE_TIME || type.typeId === TYPE_DURATION) { + return `${String(data.values[start])} ${UNIT_NAMES[type.unit ?? 0]}`; + } + if ( + type.typeId === TYPE_LIST || + type.typeId === TYPE_FIXED_SIZE_LIST || + type.typeId === TYPE_LARGE_LIST + ) { + const size = + type.listSize ?? + Number(data.valueOffsets![local + 1]) - Number(data.valueOffsets![local]); + return ``; + } + if (type.typeId === TYPE_STRUCT) { + return ``; + } + if (type.typeId === TYPE_MAP) { + return ''; + } + const value = vector.get(row); + if (value instanceof Uint8Array) { + const shown = value.subarray(0, CELL_TEXT_LIMIT / 2); + return ( + Array.from(shown, (byte) => byte.toString(16).padStart(2, '0')).join('') + + (shown.length < value.length ? '… [truncated]' : '') + ); + } + if (type.typeId === TYPE_DATE) { + return new Date(Number(value)).toISOString().slice(0, 10); + } + return boundedText( + typeof value === 'string' ? value : String(value as number | boolean | bigint), + ); +} + +// --------------------------------------------------------------------------- +// Page cache + +interface SequentialColumn { + position: number; + name: string; + type: string; + ipc_index: number | null; + unsupported: string | null; +} + +export const VIEW_CACHE_BYTES = 64 * 1024 * 1024; +export const TOTAL_CACHE_BYTES = 256 * 1024 * 1024; +const PAGE_OVERHEAD_BYTES = 1024; +const DECODED_PAGES = 4; +/** Empty non-EOF pages are legal; bound how many one row request may consume. */ +const MAX_EMPTY_PAGES = 64; +export const UNAVAILABLE_TEXT = '(no longer retained)'; + +interface StoredPage { + firstRow: number; + rows: number; + ipc: Uint8Array; + charge: number; + sequence: number; +} + +export interface StreamPageResult { + first_row: number; + rows_fetched: number; + rows_seen: number; + done: boolean; +} + +export type FetchNext = (maxRows: number) => Promise<{ result: unknown; body?: Uint8Array }>; + +let pageSequence = 0; +const streams = new Set(); + +function totalBytes(): number { + let total = 0; + for (const stream of streams) { + total += stream.bytes; + } + return total; +} + +function parseProgress(result: unknown, requested: number, rowsSeen: number): StreamPageResult { + const value = result as Partial | null; + const integer = (field: unknown) => + typeof field === 'number' && Number.isSafeInteger(field) && field >= 0; + if ( + !value || + !integer(value.first_row) || + !integer(value.rows_fetched) || + !integer(value.rows_seen) || + typeof value.done !== 'boolean' || + value.rows_seen !== value.first_row! + value.rows_fetched! || + value.rows_fetched! > requested || + value.first_row! < rowsSeen + ) { + throw new JdvRequestError('invalid_request', 'Invalid jdv stream page metadata.'); + } + return value as StreamPageResult; +} + +/** + * Received pages of one view generation. R cannot rewind the stream, so pages + * are kept within byte budgets; evicted rows are reported as unavailable. + */ +export class SequentialStream { + private pages: StoredPage[] = []; + private readonly decoded = new Map(); + private fetching?: Promise; + bytes = 0; + rowsSeen = 0; + done = false; + /** Rows before this index are no longer held. */ + unavailableBefore = 0; + private failed?: Error; + readonly columns: SequentialColumn[]; + readonly pageRows: number; + + constructor( + readonly descriptor: JdvDescriptor, + private readonly next: FetchNext, + preferredRows = 1000, + private readonly viewCacheBytes = VIEW_CACHE_BYTES, + ) { + this.columns = (descriptor.columns ?? []) as unknown as SequentialColumn[]; + // parseDescriptor() guarantees a positive safe-integer cap for sequential views. + this.pageRows = Math.min(preferredRows, descriptor.max_page_rows!); + streams.add(this); + } + + dispose(): void { + streams.delete(this); + this.pages = []; + this.decoded.clear(); + this.bytes = 0; + } + + /** Fetch pages until `endRow` rows have been seen or the stream ends. */ + async ensure(endRow: number): Promise { + while (!this.done && this.rowsSeen < endRow) { + if (this.failed) { + throw this.failed; + } + // One stream/next per view generation at a time. + this.fetching ??= this.fetchUntil(endRow).finally(() => { + this.fetching = undefined; + }); + const rowsBeforeFetch = this.rowsSeen; + await this.fetching; + // fetchUntil() bounds consecutive empty pages. Do not immediately + // start another round when that bound was reached without progress. + // A gap page advances rowsSeen and therefore remains progress. + if (this.rowsSeen === rowsBeforeFetch && !this.done) { + return; + } + } + } + + private async fetchUntil(endRow: number): Promise { + let empty = 0; + while (!this.done && this.rowsSeen < endRow) { + let response; + try { + response = await this.next(this.pageRows); + } catch (e) { + // A producer failure is terminal; R releases the stream. + if (e instanceof JdvRequestError && e.code === 'stream_failed') { + this.failed = e; + } + throw e; + } + const progress = parseProgress(response.result, this.pageRows, this.rowsSeen); + // A gap (first_row > rowsSeen) has no page; rows() reports it as unavailable. + if (progress.rows_fetched > 0) { + if (!response.body) { + throw new JdvRequestError( + 'invalid_request', + 'The jdv stream page has no Arrow body.', + ); + } + this.store(progress, response.body); + empty = 0; + } else if (progress.rows_seen > this.rowsSeen) { + // A gap has no body, but it advances the stream and is progress. + empty = 0; + } else { + empty++; + } + this.rowsSeen = progress.rows_seen; + this.done = progress.done; + if (empty >= MAX_EMPTY_PAGES && !this.done) { + return; + } + } + } + + private store(progress: StreamPageResult, ipc: Uint8Array): void { + const page: StoredPage = { + firstRow: progress.first_row, + rows: progress.rows_fetched, + ipc, + charge: ipc.byteLength + PAGE_OVERHEAD_BYTES, + sequence: ++pageSequence, + }; + this.pages.push(page); + this.bytes += page.charge; + while (this.bytes > this.viewCacheBytes && this.pages.length > 0) { + this.evictOldest(); + } + // Evict across views, oldest page first. + while (totalBytes() > TOTAL_CACHE_BYTES) { + let victim: SequentialStream | undefined; + for (const stream of streams) { + if ( + stream.pages.length > 0 && + (!victim || stream.pages[0].sequence < victim.pages[0].sequence) + ) { + victim = stream; + } + } + if (!victim) { + break; + } + victim.evictOldest(); + } + } + + private evictOldest(): void { + const page = this.pages.shift(); + if (page) { + this.bytes -= page.charge; + this.decoded.delete(page); + this.unavailableBefore = Math.max(this.unavailableBefore, page.firstRow + page.rows); + } + } + + private pageFor(row: number): StoredPage | undefined { + let low = 0; + let high = this.pages.length - 1; + while (low <= high) { + const middle = (low + high) >> 1; + const page = this.pages[middle]; + if (row < page.firstRow) { + high = middle - 1; + } else if (row >= page.firstRow + page.rows) { + low = middle + 1; + } else { + return page; + } + } + return undefined; + } + + private table(page: StoredPage): ArrowTable { + let table = this.decoded.get(page); + if (table) { + // Refresh the small decoded LRU. + this.decoded.delete(page); + } else { + table = tableFromIPC(page.ipc) as unknown as ArrowTable; + if (table.numRows !== page.rows && table.numCols !== 0) { + throw new JdvRequestError( + 'invalid_request', + 'Arrow rows do not match the stream page metadata.', + ); + } + } + this.decoded.set(page, table); + while (this.decoded.size > DECODED_PAGES) { + this.decoded.delete(this.decoded.keys().next().value!); + } + return table; + } + + /** Formatted rows in `[startRow, endRow)` that have been seen. */ + rows(startRow: number, endRow: number): Record[] { + const rows: Record[] = []; + const end = Math.min(endRow, this.rowsSeen); + for (let row = Math.max(0, startRow); row < end; row++) { + const output: Record = { [ROW_LABEL_FIELD]: String(row + 1) }; + const page = this.pageFor(row); + const table = page ? this.table(page) : undefined; + for (const column of this.columns) { + const field = String(column.position + 1); + if (column.unsupported || column.ipc_index === null) { + output[field] = ``; + } else if (!page || !table) { + output[field] = UNAVAILABLE_TEXT; + } else { + const vector = table.getChildAt(column.ipc_index) as ArrowVector | null; + if (!vector) { + throw new JdvRequestError( + 'invalid_request', + 'Arrow column mapping is outside the IPC schema.', + ); + } + output[field] = formatArrowValue(vector, row - page.firstRow); + } + } + rows.push(output); + } + return rows; + } +} diff --git a/src/jdvGrid.ts b/src/jdvGrid.ts new file mode 100644 index 000000000..c9de8ff91 --- /dev/null +++ b/src/jdvGrid.ts @@ -0,0 +1,50 @@ +import { + GridPage, + JdvDescriptor, + JdvRequestError, + JsonObject, + gridPageFromRows, + tableRowsParams, +} from './jdvAdapter'; + +/** Fill an AG Grid block, issuing further pages when jdv shortens a selected-column page. */ +export async function fetchGridPage( + descriptor: JdvDescriptor, + message: JsonObject, + request: (method: string, params: JsonObject) => Promise, +): Promise> { + const startRow = Number(message.startRow ?? 0); + const endRow = Number(message.endRow ?? 0); + const rows: Record[] = []; + let page: GridPage | undefined; + let offset = startRow; + while (offset < endRow) { + // Keep the displayed-column selection for every short page in the block. + const params = tableRowsParams( + descriptor, + offset, + endRow, + message.sortModel, + message.filterModel, + message.columns, + ); + const result = await request('grid/rows', params); + page = gridPageFromRows(result, offset, params.limit); + for (const row of page.rows) { + rows.push(row); + } + offset += page.returned; + if (page.returned === 0 || offset >= page.totalRows) { + break; + } + } + if (!page) { + throw new JdvRequestError('invalid_request', 'Empty page request'); + } + return { + rows, + totalRows: page.totalRows, + totalUnfiltered: page.totalUnfiltered, + lastRow: page.totalRows, + }; +} diff --git a/src/jdvHostLifecycle.ts b/src/jdvHostLifecycle.ts new file mode 100644 index 000000000..c250bdf33 --- /dev/null +++ b/src/jdvHostLifecycle.ts @@ -0,0 +1,28 @@ +/** Dispose only the exact host currently registered at a key and address. */ +export async function disposeJdvHostIfCurrentAddress( + hosts: Map>, + hostKey: string, + expectedAddress: string, + addressOf: (host: T) => string, + dispose: (host: T) => void, +): Promise { + const pending = hosts.get(hostKey); + if (!pending) { + return false; + } + let host: T; + try { + host = await pending; + } catch { + if (hosts.get(hostKey) === pending) { + hosts.delete(hostKey); + } + return false; + } + if (hosts.get(hostKey) !== pending || addressOf(host) !== expectedAddress) { + return false; + } + hosts.delete(hostKey); + dispose(host); + return true; +} diff --git a/src/jdvTransport.ts b/src/jdvTransport.ts new file mode 100644 index 000000000..bd8a34296 --- /dev/null +++ b/src/jdvTransport.ts @@ -0,0 +1,535 @@ +'use strict'; + +// Extension-host side of the jdv local transport (see `?jdv_spec`). R connects +// to a listener owned by one vscode-R session and exchanges JSON Lines, with +// Arrow IPC bodies following `stream/next` response headers. + +import * as crypto from 'crypto'; +import * as fs from 'fs'; +import * as net from 'net'; +import * as os from 'os'; +import * as path from 'path'; +import { JdvRequestError, JsonObject } from './jdvAdapter'; + +const PROTOCOL_MAJOR = 1; +const MAX_LIVE_REQUESTS = 32; +const WAKE_INTERVAL_MS = 50; +const MAX_JSON_LINE_BYTES = 10 * 1024 * 1024; +const MAX_BINARY_HEADER_BYTES = 64 * 1024; +const MAX_BINARY_FRAME_BYTES = 10 * 1024 * 1024; +const BINARY_BODY_TIMEOUT_MS = 30_000; + +export interface JdvResponse { + result: unknown; + /** Arrow IPC body for responses carrying `result.transfer`. */ + body?: Uint8Array; +} + +interface ProducerLimits { + maxUnackedFrames: number; + maxUnackedBytes: number; + maxRequestBytes: number; +} + +/** Error codes of the protocol; any other code is a general producer failure. */ +const ERROR_CODES = new Set([ + 'invalid_request', + 'unsupported', + 'limit_exceeded', + 'schema_changed', + 'producer_error', + 'interrupted', + 'cancelled', + 'expired', + 'unavailable', + 'stale_generation', + 'view_not_found', + 'cache_changed', + 'in_use', + 'stream_failed', + 'method_not_found', +]); +const SESSION_ID = /^[A-Za-z0-9._:-]{1,128}$/; +const LIMIT_NAMES = [ + 'max_unacked_frames', + 'max_unacked_bytes', + 'queue_messages', + 'queue_bytes', + 'max_request_bytes', + 'max_message_bytes', +] as const; + +interface OutboundFrame { + bytes: Buffer; + wake?: boolean; + requestId?: string; +} + +interface PendingRequest { + resolve(response: JdvResponse): void; + reject(error: Error): void; + view?: unknown; + generation?: unknown; +} + +function isObject(value: unknown): value is JsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function positiveInteger(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0; +} + +/** Producer or runtime metadata: string `name` and `version` of 1 to 256 UTF-8 bytes. */ +function validIdentity(value: unknown): boolean { + return ( + isObject(value) && + [value.name, value.version].every( + (field) => + typeof field === 'string' && + Buffer.byteLength(field) >= 1 && + Buffer.byteLength(field) <= 256, + ) + ); +} + +/** One producer connection. Implements the host obligations of the jdv transport. */ +export class JdvPeer { + private chunks: Buffer[] = []; + private buffered = 0; + private body?: { header: JsonObject; length: number; timer: NodeJS.Timeout }; + private limits?: ProducerLimits; + private readonly outbound: OutboundFrame[] = []; + /** Bytes of frames written but not yet acknowledged, in send order. */ + private readonly unconfirmed: number[] = []; + private unconfirmedBytes = 0; + private sentFrames = 0; + private ackedFrames = 0; + private readonly pending = new Map(); + private wakeTimer?: NodeJS.Timeout; + private sequence = 0; + private closed = false; + /** Whether the producer asked for periodic `wake` notifications. */ + private wanted = false; + /** Opaque session id from `hello`; it carries no process id. */ + session?: string; + + constructor( + private readonly socket: net.Socket, + private readonly handlers: { + notification(peer: JdvPeer, method: string, params: JsonObject): void; + closed(peer: JdvPeer, reason: string): void; + }, + ) { + socket.on('data', (chunk) => this.receive(chunk)); + socket.on('error', (error) => this.close(`jdv connection error: ${error.message}`)); + socket.on('close', () => this.close('The R session closed the jdv connection.')); + } + + get isClosed(): boolean { + return this.closed; + } + + async request(method: string, params: JsonObject): Promise { + return (await this.requestWithBody(method, params)).result; + } + + requestWithBody(method: string, params: JsonObject): Promise { + if (this.closed) { + return Promise.reject( + new JdvRequestError('view_not_found', 'The jdv connection is closed.'), + ); + } + if (this.pending.size >= MAX_LIVE_REQUESTS) { + return Promise.reject( + new JdvRequestError('limit_exceeded', 'Too many pending jdv requests.'), + ); + } + const id = `vscode-r-${++this.sequence}`; + const bytes = Buffer.from(JSON.stringify({ id, method, params }) + '\n', 'utf8'); + if (this.limits && bytes.length > this.requestByteLimit()) { + return Promise.reject( + new JdvRequestError('limit_exceeded', 'The jdv request is too large.'), + ); + } + return new Promise((resolve, reject) => { + this.pending.set(id, { + resolve, + reject, + view: params.view, + generation: params.generation, + }); + this.outbound.push({ bytes, requestId: id }); + this.flush(); + this.updateWake(); + }); + } + + close(reason: string): void { + if (this.closed) { + return; + } + this.closed = true; + if (this.body) { + clearTimeout(this.body.timer); + this.body = undefined; + } + this.stopWake(); + for (const pending of this.pending.values()) { + pending.reject(new JdvRequestError('view_not_found', reason)); + } + this.pending.clear(); + this.chunks = []; + this.buffered = 0; + this.socket.destroy(); + this.handlers.closed(this, reason); + } + + private receive(chunk: Buffer): void { + if (this.closed) { + return; + } + // Hold chunks until a frame can complete, so a large frame split across + // many chunks is copied once instead of once per chunk. + this.chunks.push(chunk); + this.buffered += chunk.length; + if (this.body ? this.buffered < this.body.length : chunk.indexOf(0x0a) < 0) { + if (!this.body && this.buffered > MAX_JSON_LINE_BYTES) { + this.close('A jdv message exceeds the size limit.'); + } + return; + } + let buffer = + this.chunks.length === 1 ? this.chunks[0] : Buffer.concat(this.chunks, this.buffered); + try { + while (!this.closed) { + if (this.body) { + if (buffer.length < this.body.length) { + return; + } + const { header, length, timer } = this.body; + clearTimeout(timer); + this.body = undefined; + // Copy so a retained page does not keep the receive buffer alive. + const body = new Uint8Array(buffer.subarray(0, length)); + buffer = buffer.subarray(length); + this.dispatch(header, body); + continue; + } + const newline = buffer.indexOf(0x0a); + if (newline < 0) { + if (buffer.length > MAX_JSON_LINE_BYTES) { + throw new Error('A jdv message exceeds the size limit.'); + } + return; + } + if (newline > MAX_JSON_LINE_BYTES) { + throw new Error('A jdv message exceeds the size limit.'); + } + const line = buffer.subarray(0, newline); + buffer = buffer.subarray(newline + 1); + const message = JSON.parse(line.toString('utf8')) as unknown; + if (!isObject(message)) { + throw new Error('Invalid jdv message.'); + } + const transfer = isObject(message.result) ? message.result.transfer : undefined; + if (isObject(transfer) && transfer.encoding === 'arrow-ipc') { + const length = transfer.byte_length; + if ( + typeof length !== 'number' || + !Number.isSafeInteger(length) || + length < 0 || + typeof message.id !== 'string' || + line.length > MAX_BINARY_HEADER_BYTES || + line.length + 1 + length > MAX_BINARY_FRAME_BYTES + ) { + throw new Error('Invalid jdv binary frame.'); + } + const timer = setTimeout( + () => this.close('Timed out receiving a jdv Arrow page.'), + BINARY_BODY_TIMEOUT_MS, + ); + this.body = { header: message, length, timer }; + continue; + } + this.dispatch(message); + } + } catch (e) { + this.close(e instanceof Error ? e.message : String(e)); + } finally { + this.chunks = !this.closed && buffer.length ? [buffer] : []; + this.buffered = this.closed ? 0 : buffer.length; + } + } + + private dispatch(message: JsonObject, body?: Uint8Array): void { + if (typeof message.method === 'string') { + const params = isObject(message.params) ? message.params : {}; + if (message.method === 'hello') { + this.acceptHello(params); + } else if (message.method === 'ack') { + this.acknowledge(params); + } else if (this.limits) { + this.handlers.notification(this, message.method, params); + } + return; + } + if (typeof message.id !== 'string') { + // Best-effort errors for malformed requests carry a null id. + return; + } + const pending = this.pending.get(message.id); + if (!pending) { + return; + } + this.pending.delete(message.id); + this.updateWake(); + if (isObject(message.error)) { + const { code, message: text } = message.error; + const known = typeof code === 'string' && ERROR_CODES.has(code); + const detail = typeof text === 'string' ? text : 'jdv request failed'; + // An unrecognized code is a general producer failure; keep it visible in the text. + pending.reject( + known + ? new JdvRequestError(code, detail) + : new JdvRequestError( + 'producer_error', + typeof code === 'string' ? `${detail} (${code})` : detail, + ), + ); + return; + } + const result = message.result; + if ( + body && + isObject(result) && + (result.view !== pending.view || result.generation !== pending.generation) + ) { + pending.reject( + new JdvRequestError( + 'stale_generation', + 'The Arrow page belongs to another view generation.', + ), + ); + return; + } + pending.resolve({ result, body }); + } + + private acceptHello(params: JsonObject): void { + if (this.limits) { + throw new Error('Duplicate jdv hello.'); + } + const version = typeof params.protocol === 'number' ? Math.trunc(params.protocol) : NaN; + if (version !== PROTOCOL_MAJOR) { + throw new Error(`Unsupported jdv protocol: ${String(params.protocol)}`); + } + if ( + typeof params.session !== 'string' || + !SESSION_ID.test(params.session) || + !validIdentity(params.producer) || + (isObject(params.producer) && + params.producer.runtime !== undefined && + !validIdentity(params.producer.runtime)) || + (params.wake !== undefined && typeof params.wake !== 'boolean') + ) { + throw new Error('Invalid jdv hello.'); + } + const limits = params.limits; + if (!isObject(limits) || !LIMIT_NAMES.every((name) => positiveInteger(limits[name]))) { + throw new Error('Invalid jdv hello limits.'); + } + this.limits = { + maxUnackedFrames: limits.max_unacked_frames as number, + maxUnackedBytes: limits.max_unacked_bytes as number, + maxRequestBytes: limits.max_request_bytes as number, + }; + this.rejectOversizedQueuedRequests(); + this.session = params.session; + // The host sends periodic wakes only when the producer asks for them. + this.wanted = params.wake === true; + this.flush(); + this.updateWake(); + } + + private requestByteLimit(): number { + return Math.min(this.limits!.maxRequestBytes, this.limits!.maxUnackedBytes); + } + + /** Reject requests that were queued before hello supplied the byte limits. */ + private rejectOversizedQueuedRequests(): void { + const limit = this.requestByteLimit(); + for (let index = this.outbound.length - 1; index >= 0; index--) { + const frame = this.outbound[index]; + if (frame.requestId && frame.bytes.length > limit) { + this.outbound.splice(index, 1); + const pending = this.pending.get(frame.requestId); + if (pending) { + this.pending.delete(frame.requestId); + pending.reject( + new JdvRequestError('limit_exceeded', 'The jdv request is too large.'), + ); + } + } + } + } + + private acknowledge(params: JsonObject): void { + const frames = params.frames; + if ( + typeof frames !== 'number' || + !Number.isSafeInteger(frames) || + frames < this.ackedFrames || + frames > this.sentFrames + ) { + throw new Error('Invalid jdv acknowledgement.'); + } + while (this.ackedFrames < frames) { + this.unconfirmedBytes -= this.unconfirmed.shift() ?? 0; + this.ackedFrames++; + } + this.flush(); + } + + /** Write queued frames while the cumulative unconfirmed output fits R's receive quota. */ + private flush(): void { + const limits = this.limits; + if (!limits || this.closed) { + return; + } + while (this.outbound.length > 0) { + const frame = this.outbound[0]; + if ( + this.unconfirmed.length + 1 > limits.maxUnackedFrames || + this.unconfirmedBytes + frame.bytes.length > limits.maxUnackedBytes + ) { + return; + } + this.outbound.shift(); + this.unconfirmed.push(frame.bytes.length); + this.unconfirmedBytes += frame.bytes.length; + this.sentFrames++; + this.socket.write(frame.bytes); + } + } + + /** Wake the producer repeatedly while a request is pending, if it asked for wakes. */ + private updateWake(): void { + if (!this.wanted) { + return; + } + if (this.pending.size === 0) { + this.stopWake(); + } else if (!this.wakeTimer) { + this.wakeTimer = setInterval(() => { + // At most one queued wake; skip while frames are waiting for credit. + if (this.outbound.length === 0) { + this.outbound.push({ bytes: Buffer.from('{"method":"wake"}\n'), wake: true }); + this.flush(); + } + }, WAKE_INTERVAL_MS); + } + } + + private stopWake(): void { + if (this.wakeTimer) { + clearInterval(this.wakeTimer); + this.wakeTimer = undefined; + } + for (let i = this.outbound.length - 1; i >= 0; i--) { + if (this.outbound[i].wake) { + this.outbound.splice(i, 1); + } + } + } +} + +export interface JdvHostEvents { + /** `view/open` received from R. */ + viewOpen(peer: JdvPeer, params: JsonObject): void; + /** `view/closed` received from R. */ + viewClosed(peer: JdvPeer, view: string, generation: number): void; + /** `view/update_failed` received from R: a tracked variable changed but the view could not be rebuilt. */ + viewUpdateFailed(peer: JdvPeer, params: JsonObject): void; + /** `view/tracking` received from R: the tracking state changed without a rebuild (lost, paused or resumed). */ + viewTracking(peer: JdvPeer, params: JsonObject): void; + /** The R connection closed; its views are gone. */ + disconnected(peer: JdvPeer, reason: string): void; +} + +/** + * A listener owned by one vscode-R session. Its address is handed to that R + * session only, so jdv never connects through a shared discovery file. + */ +export class JdvHost { + private peer?: JdvPeer; + + private constructor( + private readonly server: net.Server, + /** Address in jdv's syntax (`npipe:////./pipe/...` on Windows), for R. */ + readonly address: string, + /** The same endpoint in the form Node's `net` module accepts. */ + readonly socketPath: string, + private readonly events: JdvHostEvents, + ) { + server.on('connection', (socket) => this.accept(socket)); + } + + static async create(events: JdvHostEvents): Promise { + const id = crypto.randomBytes(8).toString('hex'); + // Unix socket paths are limited to about 104-108 bytes; keep the name short. + const socketPath = + process.platform === 'win32' + ? `\\\\.\\pipe\\vscode-r-jdv-${id}` + : path.join(os.tmpdir(), `vscr-jdv-${id}.sock`); + const address = + process.platform === 'win32' ? `npipe:////./pipe/vscode-r-jdv-${id}` : socketPath; + const server = net.createServer(); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(socketPath, () => { + server.off('error', reject); + resolve(); + }); + }); + if (process.platform !== 'win32') { + fs.chmodSync(socketPath, 0o600); + } + return new JdvHost(server, address, socketPath, events); + } + + get connection(): JdvPeer | undefined { + return this.peer && !this.peer.isClosed ? this.peer : undefined; + } + + dispose(): void { + this.peer?.close('The R session ended.'); + this.server.close(); + if (process.platform !== 'win32') { + fs.rmSync(this.socketPath, { force: true }); + } + } + + private accept(socket: net.Socket): void { + // jdv opens a new connection after a disconnect; only one is active. + this.peer?.close('The R session opened a new jdv connection.'); + this.peer = new JdvPeer(socket, { + notification: (peer, method, params) => { + if (method === 'view/open') { + this.events.viewOpen(peer, params); + } else if ( + method === 'view/closed' && + typeof params.view === 'string' && + typeof params.generation === 'number' + ) { + this.events.viewClosed(peer, params.view, params.generation); + } else if (method === 'view/update_failed') { + this.events.viewUpdateFailed(peer, params); + } else if (method === 'view/tracking') { + this.events.viewTracking(peer, params); + } + }, + closed: (peer, reason) => this.events.disconnected(peer, reason), + }); + } +} diff --git a/src/jdvViewer.ts b/src/jdvViewer.ts new file mode 100644 index 000000000..27aa3aff1 --- /dev/null +++ b/src/jdvViewer.ts @@ -0,0 +1,671 @@ +'use strict'; + +// Data Viewer panels backed by jdv. The webviews are the existing AG Grid and List +// Viewer documents; this module translates their requests to the jdv protocol. +// R connects to a listener owned by the session (see JdvHost) and every view, +// including child views opened from trees, arrives there as a `view/open`. + +import * as path from 'path'; +import * as vscode from 'vscode'; +import { Uri, ViewColumn, window } from 'vscode'; +import { + JdvDescriptor, + cellOpenParams, + JdvRequestError, + JdvTracking, + JsonObject, + TreeAdapter, + canRefresh, + gridColumnDefs, + parseDescriptor, + parseTracking, + revealsPanel, +} from './jdvAdapter'; +import { fetchGridPage } from './jdvGrid'; +import { SequentialStream } from './jdvArrow'; +import { JdvHost, JdvPeer } from './jdvTransport'; +import { disposeJdvHostIfCurrentAddress } from './jdvHostLifecycle'; +import { extensionContext } from './extension'; +import * as session from './session'; +import { config, UriIcon } from './util'; + +interface JdvPanel { + readonly key: string; + readonly panel: vscode.WebviewPanel; + readonly sessionId: string | null; + /** The R connection that owns the view. */ + peer: JdvPeer; + descriptor: JdvDescriptor; + title: string; + hintScope: string; + knownTotal?: number; + tree?: TreeAdapter; + /** Received Arrow pages of a sequential view. */ + stream?: SequentialStream; + /** Title for the child view that a pending `object/open` is about to open. */ + childTitle?: string; + /** Set when R closed or evicted this view generation. */ + closed?: string; +} + +const panels = new Map(); +const hosts = new Map>(); +const pendingRowCounts = new Map(); + +function rowCountScope(sessionId: string | null, address: string): string { + return JSON.stringify([sessionId, address]); +} + +/** A display-only hint from the authenticated sess notification channel. */ +export function applyJdvRowCount(sessionId: string | null, params: Record): void { + if ( + !sessionId || + typeof params.address !== 'string' || + !params.address || + typeof params.view !== 'string' || + !params.view || + !Number.isSafeInteger(params.generation) || + Number(params.generation) < 1 || + !Number.isSafeInteger(params.nrow) || + Number(params.nrow) < 0 + ) { + return; + } + const scope = rowCountScope(sessionId, params.address); + const state = panels.get(panelKey(sessionId, params.view)); + if ( + state?.stream && + !state.closed && + state.hintScope === scope && + state.descriptor.generation === params.generation + ) { + state.knownTotal = Number(params.nrow); + const webview = state.panel.webview; + void webview.postMessage({ + message: 'dataview/rowCount', + documentGeneration: session.getDocumentGeneration(webview), + knownTotal: state.knownTotal, + }); + } else { + pendingRowCounts.set( + JSON.stringify([scope, params.view, params.generation]), + Number(params.nrow), + ); + if (pendingRowCounts.size > 64) { + pendingRowCounts.delete(pendingRowCounts.keys().next().value!); + } + } +} + +function bindRowCount(state: JdvPanel): void { + const key = JSON.stringify([ + state.hintScope, + state.descriptor.view, + state.descriptor.generation, + ]); + state.knownTotal = state.stream ? pendingRowCounts.get(key) : undefined; + pendingRowCounts.delete(key); +} + +/** The jdv transport address for a session, creating its listener on first use. */ +export function getJdvAddress(sessionId: string, hostKey = sessionId): Promise { + let host = hosts.get(hostKey); + if (!host) { + let address = ''; + host = JdvHost.create({ + viewOpen: (peer, params) => { + let descriptor: JdvDescriptor; + try { + descriptor = parseDescriptor(params); + } catch (e) { + void window.showErrorMessage( + `Unable to open the data viewer: ${errorMessage(e)}`, + ); + return; + } + const viewer = viewerColumn(); + if (viewer) { + void openJdvView( + descriptor, + sessionId, + viewer, + peer, + rowCountScope(sessionId, address), + ); + } else { + void peer + .request('view/close', { + view: descriptor.view, + generation: descriptor.generation, + }) + .catch(() => undefined); + } + }, + viewClosed: (_peer, view, generation) => markClosed(sessionId, view, generation), + viewUpdateFailed: (_peer, params) => warnUpdateFailed(sessionId, params), + viewTracking: (_peer, params) => applyTracking(sessionId, params), + disconnected: (peer) => { + for (const state of panels.values()) { + if (state.peer === peer && !state.closed) { + state.closed = 'The R session disconnected from the data viewer.'; + } + } + }, + }).then((value) => { + address = value.address; + return value; + }); + hosts.set(hostKey, host); + void host.catch(() => { + if (hosts.get(hostKey) === host) { + hosts.delete(hostKey); + } + }); + } + return host.then((value) => value.address); +} + +/** Close the session's jdv listener when the session ends. */ +export function disposeJdvHost(hostKey: string): void; +export function disposeJdvHost(hostKey: string, expectedAddress: string): Promise; +export function disposeJdvHost(hostKey: string, expectedAddress?: string): void | Promise { + if (expectedAddress !== undefined) { + return disposeJdvHostIfCurrentAddress( + hosts, + hostKey, + expectedAddress, + (host) => host.address, + (host) => host.dispose(), + ).then(() => undefined); + } + const host = hosts.get(hostKey); + hosts.delete(hostKey); + void host?.then( + (value) => value.dispose(), + () => undefined, + ); +} + +function markClosed(sessionId: string | null, view: string, generation: number): void { + const state = panels.get(panelKey(sessionId, view)); + if (state && state.descriptor.generation === generation) { + state.closed = 'This view was closed in R. Run View() again to refresh it.'; + } +} + +/** A tracked variable changed but jdv could not rebuild the view; the previous generation stays. */ +function warnUpdateFailed(sessionId: string | null, params: JsonObject): void { + if (typeof params.view !== 'string') { + return; + } + const state = panels.get(panelKey(sessionId, params.view)); + // Ignore a failure reported for a generation that is no longer shown. + if (!state || state.closed || params.generation !== state.descriptor.generation) { + return; + } + const reason = typeof params.message === 'string' ? params.message : 'unknown error'; + void window.showWarningMessage(`The data viewer could not update "${state.title}": ${reason}`); +} + +/** + * `view/tracking` changes the state of the shown generation without a rebuild. + * The generation is accepted as is: a pause or loss concerns the view, and a + * newer generation arrives as a `view/open` that carries its own state. + */ +function applyTracking(sessionId: string | null, params: JsonObject): void { + const tracking = parseTracking(params.tracking); + if (typeof params.view !== 'string' || !tracking) { + return; + } + const state = panels.get(panelKey(sessionId, params.view)); + if (!state || !state.descriptor.tracking) { + return; + } + setTracking(state, tracking); +} + +function setTracking(state: JdvPanel, tracking: JdvTracking): void { + state.descriptor.tracking = tracking; + state.descriptor.capabilities.track = tracking.state !== 'lost'; + // The webview is gone once the panel is disposed. + try { + const webview = state.panel.webview; + void webview.postMessage({ + message: 'viewer/tracking', + documentGeneration: session.getDocumentGeneration(webview), + tracking, + }); + } catch { + // Disposed panel: nothing to update. + } +} + +function createStream(descriptor: JdvDescriptor, peer: JdvPeer): SequentialStream | undefined { + if (descriptor.access !== 'sequential') { + return undefined; + } + const { view, generation } = descriptor; + return new SequentialStream(descriptor, (maxRows) => + peer.requestWithBody('stream/next', { view, generation, max_rows: maxRows }), + ); +} + +function panelKey(sessionId: string | null, view: string): string { + return JSON.stringify([sessionId, view]); +} + +function viewerColumn(): string | undefined { + const viewer = + config().get>('session.viewers.viewColumn')?.view ?? 'Two'; + return viewer === 'Disable' ? undefined : viewer; +} + +function errorMessage(error: unknown): string { + if (error instanceof JdvRequestError) { + return `${error.message} (${error.code})`; + } + return error instanceof Error ? error.message : String(error); +} + +async function openJdvView( + descriptor: JdvDescriptor, + sessionId: string | null, + viewer: string, + peer: JdvPeer, + hintScope: string, +): Promise { + const key = panelKey(sessionId, descriptor.view); + const existing = panels.get(key); + // jdv titles a child view when its parent has a title; an untitled parent leaves it + // to the host-side bookkeeping of the item that was opened. + const parent = + descriptor.parent === undefined + ? undefined + : panels.get(panelKey(sessionId, descriptor.parent)); + const title = descriptor.title ?? parent?.childTitle ?? existing?.title ?? 'View'; + if ( + descriptor.kind === 'grid' && + descriptor.access !== 'random' && + descriptor.access !== 'sequential' + ) { + // Snapshot grids are not supported; release the view. + void peer + .request('view/close', { view: descriptor.view, generation: descriptor.generation }) + .catch(() => undefined); + void window.showWarningMessage( + `The data viewer cannot display "${title}" yet (${descriptor.access} view).`, + ); + return; + } + if (existing) { + if ( + existing.hintScope === hintScope && + descriptor.generation < existing.descriptor.generation + ) { + return; + } + // The same view with a newer generation (View() again, or a refresh) updates the panel. + existing.descriptor = descriptor; + existing.peer = peer; + existing.title = title; + existing.closed = undefined; + existing.tree = descriptor.kind === 'tree' ? new TreeAdapter(descriptor, title) : undefined; + existing.stream?.dispose(); + existing.stream = createStream(descriptor, peer); + existing.hintScope = hintScope; + bindRowCount(existing); + existing.panel.title = title; + // Updates that R made on its own (a tracked variable, a Refresh click) keep + // the panel where it is. Otherwise, including when jdv sends no reason, + // reveal without taking focus so that an explicit View() surfaces the panel. + if (revealsPanel(descriptor)) { + existing.panel.reveal(existing.panel.viewColumn, true); + } + existing.panel.webview.html = await renderPanel(existing); + return; + } + + const resDir = path.join(extensionContext.extensionPath, 'dist', 'resources'); + const panel = window.createWebviewPanel( + 'dataview', + title, + { + preserveFocus: true, + viewColumn: ViewColumn[viewer as keyof typeof ViewColumn], + }, + { + enableScripts: true, + enableFindWidget: true, + retainContextWhenHidden: true, + localResourceRoots: [Uri.file(resDir)], + }, + ); + panel.iconPath = new UriIcon(descriptor.kind === 'grid' ? 'open-preview' : 'preview'); + session.attachViewerSessionBridge(panel, sessionId); + const state: JdvPanel = { + key, + panel, + sessionId, + peer, + descriptor, + title, + hintScope, + tree: descriptor.kind === 'tree' ? new TreeAdapter(descriptor, title) : undefined, + stream: createStream(descriptor, peer), + }; + panels.set(key, state); + bindRowCount(state); + // The panel's webview getter throws once onDidDispose fires. + const webview = panel.webview; + panel.onDidDispose(() => { + if (panels.get(key) === state) { + panels.delete(key); + } + state.stream?.dispose(); + if (!state.closed) { + const { view, generation } = state.descriptor; + void state.peer.request('view/close', { view, generation }).catch(() => undefined); + } + }); + webview.onDidReceiveMessage((message: unknown) => { + void handleMessage(state, webview, message as JsonObject); + }); + webview.html = await renderPanel(state); +} + +async function renderPanel(state: JdvPanel): Promise { + const webview = state.panel.webview; + const refreshable = canRefresh(state.descriptor); + const tracking = state.descriptor.tracking; + if (state.descriptor.kind === 'grid') { + return session.getTableHtml( + webview, + undefined, + state.title, + state.sessionId, + refreshable, + tracking, + ); + } + return session.getListHtml( + webview, + state.title, + state.tree!.navigation([]), + state.sessionId, + refreshable, + tracking, + ); +} + +function isPositivePath(value: unknown): value is number[] { + return Array.isArray(value) && value.every((index) => Number.isSafeInteger(index) && index > 0); +} + +async function handleMessage( + state: JdvPanel, + webview: vscode.Webview, + message: JsonObject, +): Promise { + const documentGeneration = message.documentGeneration; + if ( + typeof documentGeneration !== 'number' || + documentGeneration !== session.getDocumentGeneration(webview) + ) { + return; + } + const current = () => documentGeneration === session.getDocumentGeneration(webview); + if (message.message === 'dataview/refresh' || message.message === 'listview/refresh') { + await refreshView(state, webview, documentGeneration); + return; + } + if (message.message === 'dataview/openCell') { + await openCell(state, webview, documentGeneration, message); + return; + } + if (message.message === 'viewer/track' && typeof message.enabled === 'boolean') { + await trackView(state, webview, documentGeneration, message.enabled); + return; + } + if (message.message === 'dataview/request' && typeof message.requestId === 'number') { + const reply = (ok: boolean, result?: unknown, error?: string) => { + if (current()) { + void webview.postMessage({ + message: 'dataview/response', + documentGeneration, + requestId: message.requestId, + ok, + result, + error, + }); + } + }; + try { + if (state.stream) { + // Rows already received stay viewable after R closes the view. + if (message.action === 'init') { + reply(true, { + columns: gridColumnDefs(state.descriptor), + totalRows: 0, + sequential: true, + knownTotal: state.knownTotal, + }); + } else if (message.action === 'page') { + reply(true, await fetchSequentialPage(state, state.stream, message)); + } + return; + } + if (state.closed) { + throw new Error(state.closed); + } + if (message.action === 'init') { + reply(true, { + columns: gridColumnDefs(state.descriptor), + totalRows: state.descriptor.nrow ?? 0, + }); + } else if (message.action === 'page') { + reply( + true, + await fetchGridPage(state.descriptor, message, (method, params) => + state.peer.request(method, params), + ), + ); + } else { + reply(false, undefined, `Unsupported dataview action: ${String(message.action)}`); + } + } catch (e) { + reply(false, undefined, errorMessage(e)); + } + return; + } + + const tree = state.tree; + if (!tree || !isPositivePath(message.path)) { + return; + } + const treePath = message.path; + if (message.message === 'listview/navigate' || message.message === 'listview/view') { + let navigation; + let error: string | undefined; + try { + if (state.closed) { + throw new Error(state.closed); + } + if (message.message === 'listview/navigate') { + navigation = tree.navigation(treePath); + } else if (Number.isSafeInteger(message.index)) { + const action = tree.open(treePath, message.index as number); + if (action.kind === 'navigate') { + navigation = action.navigation; + } else { + // The child arrives as a `view/open` notification before the response. + state.childTitle = action.title; + try { + await state.peer.request('object/open', action.params); + } finally { + state.childTitle = undefined; + } + } + } + } catch (e) { + error = `Unable to open this item: ${errorMessage(e)}`; + } + if (navigation) { + state.panel.title = navigation.title; + } + if (current()) { + void webview.postMessage({ + message: 'listview/navigation', + documentGeneration, + requestId: message.requestId, + navigation, + error, + }); + } + } else if ( + message.message === 'listview/page' && + Number.isSafeInteger(message.start) && + typeof message.requestId === 'number' + ) { + let page; + let error: string | undefined; + try { + if (state.closed) { + throw new Error(state.closed); + } + const result = await state.peer.request( + 'object/children', + tree.childrenParams(treePath, message.start as number), + ); + page = tree.children(treePath, result); + } catch (e) { + error = `Unable to load items: ${errorMessage(e)}`; + } + if (current()) { + void webview.postMessage({ + message: 'listview/page', + documentGeneration, + requestId: message.requestId, + ...page, + error, + }); + } + } +} + +/** + * Open a drillable grid cell as a child view. The child arrives as a `view/open` + * notification (with jdv's title) before the response, so only a failure is reported. + */ +async function openCell( + state: JdvPanel, + webview: vscode.Webview, + documentGeneration: number, + message: JsonObject, +): Promise { + let error: string | undefined; + try { + if (state.closed) { + throw new Error(state.closed); + } + await state.peer.request( + 'object/open', + cellOpenParams(state.descriptor, message.sourceRow, message.column), + ); + } catch (e) { + error = `Unable to open this cell: ${errorMessage(e)}`; + } + if (error !== undefined && documentGeneration === session.getDocumentGeneration(webview)) { + void webview.postMessage({ message: 'viewer/openFailed', documentGeneration, error }); + } +} + +/** + * Ask R to rebuild the view. On success the next generation arrives as a `view/open` + * that re-renders the panel, so only a failure is reported to the webview. + */ +async function refreshView( + state: JdvPanel, + webview: vscode.Webview, + documentGeneration: number, +): Promise { + const { view, generation } = state.descriptor; + let error: string | undefined; + try { + if (state.closed) { + throw new Error(state.closed); + } + if (!canRefresh(state.descriptor)) { + throw new Error('This view cannot be refreshed.'); + } + await state.peer.request('view/refresh', { view, generation }); + } catch (e) { + error = `Unable to refresh: ${errorMessage(e)}`; + } + if (error !== undefined && documentGeneration === session.getDocumentGeneration(webview)) { + void webview.postMessage({ message: 'viewer/refreshFailed', documentGeneration, error }); + } +} + +/** + * Ask R to pause or resume following the variable. Resuming may rebuild the view, + * in which case the new generation arrives as a `view/open` that re-renders the panel; + * otherwise the toggle is updated from the result. + */ +async function trackView( + state: JdvPanel, + webview: vscode.Webview, + documentGeneration: number, + enabled: boolean, +): Promise { + const { view, generation } = state.descriptor; + let error: string | undefined; + try { + if (state.closed) { + throw new Error(state.closed); + } + if (!state.descriptor.tracking) { + throw new Error('This view does not follow a variable.'); + } + const result = await state.peer.request('view/track', { view, generation, enabled }); + const tracking = parseTracking((result as JsonObject | undefined)?.tracking); + // A rebuilt view was already applied by its `view/open`. + if ( + tracking && + state.descriptor.tracking && + (result as JsonObject).generation === state.descriptor.generation + ) { + setTracking(state, tracking); + } + } catch (e) { + error = `Unable to ${enabled ? 'follow' : 'pause following'} the variable: ${errorMessage(e)}`; + } + if (error !== undefined && documentGeneration === session.getDocumentGeneration(webview)) { + void webview.postMessage({ message: 'viewer/trackFailed', documentGeneration, error }); + } +} + +/** Serve an AG Grid block from received Arrow pages, reading the stream forward as needed. */ +async function fetchSequentialPage( + state: JdvPanel, + stream: SequentialStream, + message: JsonObject, +): Promise { + const startRow = Math.max(0, Number(message.startRow ?? 0)); + const endRow = Math.max(startRow, Number(message.endRow ?? 0)); + if (!state.closed) { + await stream.ensure(endRow); + } + // After the stream ends, or once R has closed the view, the received rows are final. + const known = stream.done || state.closed !== undefined; + return { + rows: stream.rows(startRow, endRow), + totalRows: known ? stream.rowsSeen : null, + totalUnfiltered: known ? stream.rowsSeen : null, + lastRow: known ? stream.rowsSeen : -1, + loadedRows: stream.rowsSeen, + knownTotal: state.knownTotal, + done: stream.done, + unavailableBefore: stream.unavailableBefore, + notice: state.closed, + }; +} diff --git a/src/listViewer.ts b/src/listViewer.ts index f0e2088e1..31e9cf755 100644 --- a/src/listViewer.ts +++ b/src/listViewer.ts @@ -1,6 +1,8 @@ /// import { viewerSessionStyle } from './viewerSession'; +import { getFollowScript } from './viewerFollow'; +import type { JdvTracking } from './jdvAdapter'; export interface ListViewNavigation { title: string; @@ -302,6 +304,8 @@ export function getListViewerScript( path: [], breadcrumbs: [{ label: '', path: [] }], }, + refreshable = false, + tracking?: JdvTracking, ): string { return ` const vscode = acquireVsCodeApi(); @@ -319,6 +323,35 @@ export function getListViewerScript( const message = event.data; if (['listview/page', 'listview/navigation'].includes(message.message) && message.documentGeneration === documentGeneration) viewer.reply(message); }); + ${ + refreshable + ? ` + const refreshButton = document.getElementById('refresh'); + refreshButton.addEventListener('click', () => { + refreshButton.disabled = true; + document.getElementById('navigation-status').textContent = ''; + vscode.postMessage({ message: 'listview/refresh', documentGeneration }); + }); + // A successful refresh replaces this document, so only failures arrive. + window.addEventListener('message', event => { + const message = event.data; + if (message && message.message === 'viewer/refreshFailed' && message.documentGeneration === documentGeneration) { + refreshButton.disabled = false; + document.getElementById('navigation-status').textContent = message.error; + } + });` + : '' + } + ${ + tracking + ? getFollowScript( + tracking, + 'follow', + "document.getElementById('navigation-status').textContent = '';", + "document.getElementById('navigation-status').textContent = message.error;", + ) + : '' + } `; } @@ -335,7 +368,11 @@ export const listViewerStyle = ` } .r-list-viewer .back { gap: 4px; padding: 4px 6px; flex-shrink: 0; border-radius: 3px; } .r-list-viewer .back:disabled { opacity: 0.4; cursor: default; background: transparent; } - .r-list-viewer .reset { gap: 4px; padding: 4px 6px; flex-shrink: 0; border-radius: 3px; } + .r-list-viewer .reset, .r-list-viewer .refresh { gap: 4px; padding: 4px 6px; flex-shrink: 0; border-radius: 3px; } + .r-list-viewer .follow { gap: 4px; padding: 4px 6px; flex-shrink: 0; border-radius: 3px; } + .r-list-viewer .follow[aria-pressed="true"] { background: var(--vscode-toolbar-activeBackground, var(--vscode-button-secondaryBackground)); } + .r-list-viewer .follow:disabled { opacity: 0.4; cursor: default; background: transparent; } + .r-list-viewer .refresh:disabled { opacity: 0.4; cursor: default; background: transparent; } ${viewerSessionStyle} .r-list-viewer .viewer-session { margin-left: auto; } .r-list-viewer .codicon { flex-shrink: 0; } diff --git a/src/rTerminal.ts b/src/rTerminal.ts index 3d4cd02e7..e382fc5cf 100644 --- a/src/rTerminal.ts +++ b/src/rTerminal.ts @@ -346,6 +346,10 @@ export async function makeTerminalOptions(resource?: vscode.Uri): Promise('session.emulateRStudioAPI') ? 'TRUE' : 'FALSE', + SESS_TRACK_VARIABLES: + config(configResource).get('session.viewers.trackVariables') === false + ? 'FALSE' + : 'TRUE', SESS_PLOT_BACKEND: backend, }; Object.assign(env, jgdVars); diff --git a/src/session.ts b/src/session.ts index 0305f652c..a51248b8f 100644 --- a/src/session.ts +++ b/src/session.ts @@ -42,6 +42,9 @@ import type { import { showWebView } from './webViewer'; import { getListViewerScript, listViewerStyle, ListViewNavigation } from './listViewer'; import { getDataViewerScript, getDataViewerStyle, getDataViewerToolbarHtml } from './dataViewer'; +import { applyJdvRowCount, disposeJdvHost, getJdvAddress } from './jdvViewer'; +import { getFollowScript } from './viewerFollow'; +import type { JdvTracking } from './jdvAdapter'; import { getDataViewerColumnPanelHtml, getDataViewerColumnPanelScript, @@ -712,6 +715,11 @@ const dynamicDataViewStateGenerations = new WeakMap const documentGenerations = new WeakMap(); let documentGenerationRevision = 0; +/** The generation of the HTML document currently loaded in a viewer webview. */ +export function getDocumentGeneration(webview: Webview): number | undefined { + return documentGenerations.get(webview); +} + function escapeHtml(text: string): string { const map: Record = { '&': '&', @@ -1563,6 +1571,11 @@ function asRStringLiteral(value: string): string { return `"${escaped}"`; } +/** Whether the data viewer should follow the variable passed to View(); on unless disabled. */ +export function trackVariablesSetting(resource?: vscode.Uri): boolean { + return config(resource).get('session.viewers.trackVariables') !== false; +} + function getAttachSessionScriptPath(pipePath: string): string { const scriptBase = path.basename(pipePath).replace(/[^a-zA-Z0-9_.-]/g, '_') || 'attach_session'; return path.join(extensionContext.globalStorageUri.fsPath, 'tmp', 'attach', `${scriptBase}.R`); @@ -1597,7 +1610,7 @@ function buildAttachSessionScript( ]), ' source(startup_helper, local = TRUE)', ' source(attach_helper, local = TRUE)', - ` vscode_r_attach_sess(endpoint, sess_src, managed_root, consent_dir, source_helper, installer_helper, ${asRStringLiteral(backend)}, use_rstudioapi = ${config().get('session.emulateRStudioAPI') ? 'TRUE' : 'FALSE'}${attachment ? `, attachment_id = ${asRStringLiteral(attachment.attachmentId)}, auto_activate = ${attachment.autoActivate ? 'TRUE' : 'FALSE'}` : ''})`, + ` vscode_r_attach_sess(endpoint, sess_src, managed_root, consent_dir, source_helper, installer_helper, ${asRStringLiteral(backend)}, use_rstudioapi = ${config().get('session.emulateRStudioAPI') ? 'TRUE' : 'FALSE'}, track_variables = ${trackVariablesSetting() ? 'TRUE' : 'FALSE'}${attachment ? `, attachment_id = ${asRStringLiteral(attachment.attachmentId)}, auto_activate = ${attachment.autoActivate ? 'TRUE' : 'FALSE'}` : ''})`, ' })', ' invisible(isTRUE(attached))', ' }, error = function(error) {', @@ -2587,7 +2600,10 @@ function getViewerSessionHtml(sessionId: string | null): string { return getViewerSessionContext(sessionId)?.getHtml() ?? ''; } -function attachViewerSessionBridge(panel: vscode.WebviewPanel, sessionId: string | null): void { +export function attachViewerSessionBridge( + panel: vscode.WebviewPanel, + sessionId: string | null, +): void { getViewerSessionContext(sessionId)?.attach(panel); } @@ -2612,6 +2628,8 @@ export async function getTableHtml( file: string | undefined, title: string, sessionId: string | null = null, + refreshable = false, + tracking?: JdvTracking, ): Promise { const sessionHtml = getViewerSessionHtml(sessionId); const codicons = webview.asWebviewUri(Uri.file(path.join(resDir, 'codicon.css'))).toString(); @@ -2808,11 +2826,24 @@ export async function getTableHtml( let filteredRows = 0; let totalRows = 0; let isFiltered = false; + let sequentialKnownTotal; + let sequentialProgress; let verticalScrollbar; let scrollbarPositionAttached = false; let scrollbarPressed = false; const LONG_FETCH_DELAY_MS = 2000; const rowNumberFormatter = new Intl.NumberFormat(); + function renderSequentialRowCount() { + if (sequentialProgress) { + const { loadedRows, totalRows, notice } = sequentialProgress; + const complete = Number.isFinite(totalRows); + const rowCount = document.querySelector('#viewerRowCount'); + rowCount.textContent = formatSequentialRowCount( + complete ? totalRows : loadedRows, sequentialKnownTotal, complete + ); + rowCount.title = notice || ''; + } + } function clearLongFetchTimer() { if (longFetchTimer) { clearTimeout(longFetchTimer); @@ -2994,6 +3025,11 @@ export async function getTableHtml( window.addEventListener('message', (event) => { const data = event.data; + if (data?.message === 'dataview/rowCount' && data.documentGeneration === documentGeneration) { + sequentialKnownTotal = data.knownTotal; + renderSequentialRowCount(); + return; + } if (!data || data.message !== 'dataview/response' || data.documentGeneration !== documentGeneration) { return; @@ -3038,8 +3074,19 @@ export async function getTableHtml( info.title = 'This view retains the full object without a copy. Reference edits may appear here. Reopen from the cell to refresh after edits. Sorting and filtering scan the full data.'; document.querySelector('#viewerControls').append(info); } + if (init.sequential) { + sequentialProgress = { loadedRows: 0, totalRows: null }; + sequentialKnownTotal ??= init.knownTotal; + const info = document.createElement('span'); + info.textContent = 'Arrow stream'; + info.title = 'Rows are read from the Arrow stream as you scroll. Sorting and filtering are not available.'; + document.querySelector('#viewerControls').append(info); + } const bigintFields = prepareViewerColumns(columns); updateViewerRowCount(filteredRows, totalRows); + if (init.sequential) { + renderSequentialRowCount(); + } const rowIndexColumn = columns.find(column => column.field === '0'); if (rowIndexColumn) { rowIndexColumn.headerValueGetter = () => isFiltered @@ -3058,12 +3105,28 @@ export async function getTableHtml( endRow: params.endRow, sortModel: params.sortModel, filterModel: params.filterModel, + // Fetch only displayed columns; wide tables otherwise cost R a lot per block. + columns: params.api.getAllDisplayedColumns() + .map(column => column.getColDef().field) + .filter(field => field !== '0'), }); - filteredRows = result.totalRows; - totalRows = result.totalUnfiltered; - isFiltered = Object.keys(params.filterModel || {}).length > 0; + const nextFilteredRows = Number.isFinite(result.totalRows) ? result.totalRows : result.loadedRows; + const nextTotalRows = Number.isFinite(result.totalUnfiltered) ? result.totalUnfiltered : result.loadedRows; + const nextIsFiltered = Object.keys(params.filterModel || {}).length > 0; + const headerNeedsRefresh = filteredRows !== nextFilteredRows || + totalRows !== nextTotalRows || isFiltered !== nextIsFiltered; + filteredRows = nextFilteredRows; + totalRows = nextTotalRows; + isFiltered = nextIsFiltered; updateViewerRowCount(filteredRows, totalRows); - gridApi?.refreshHeader(); + if (init.sequential) { + sequentialKnownTotal ??= result.knownTotal; + sequentialProgress = result; + renderSequentialRowCount(); + } + if (headerNeedsRefresh) { + gridApi?.refreshHeader(); + } updateScrollPosition(); const resolvedLastRow = Number.isFinite(result.totalRows) ? result.totalRows : result.lastRow; const rows = prepareViewerRows(result.rows || [], bigintFields); @@ -3087,6 +3150,12 @@ export async function getTableHtml( onPaginationChanged: updateScrollPosition, onGridSizeChanged: updateFetchStatusPosition, onDisplayedColumnsChanged: updateFetchStatusPosition, + onColumnVisible: function(event) { + // Cached rows lack columns that were hidden when they were fetched. + if (!init.sequential && (event.columns || []).some(column => column.isVisible())) { + event.api.purgeInfiniteCache(); + } + }, onFirstDataRendered: function() { updateFetchStatusPosition(); attachScrollbarPositionIndicator(); @@ -3098,6 +3167,10 @@ export async function getTableHtml( console.log('[dataview] Creating grid with options:', gridOptions); gridApi = window.agGrid.createGrid(gridDiv, gridOptions); initializeViewerToolbar(); + if (init.sequential) { + // Arrow streams are read forward only; jdv offers no sorting or filtering. + document.querySelector('#viewerFilters').disabled = true; + } console.log('[dataview] Grid created successfully'); updateFetchStatusPosition(); } catch (e) { @@ -3113,6 +3186,43 @@ export async function getTableHtml( retryBtn.addEventListener('click', retryCurrentPage); } + window.addEventListener('message', (event) => { + const data = event.data; + if (data && data.message === 'viewer/openFailed' && + data.documentGeneration === documentGeneration) { + setFetchStatus('error', data.error || 'Unable to open this cell.', false); + } + }); + + const refreshBtn = document.querySelector('#viewerRefresh'); + if (refreshBtn) { + refreshBtn.addEventListener('click', () => { + refreshBtn.disabled = true; + setFetchStatus('loading', 'Refreshing...', false); + vscode.postMessage({ message: 'dataview/refresh', documentGeneration }); + }); + // A successful refresh replaces this document, so only failures arrive. + window.addEventListener('message', (event) => { + const data = event.data; + if (data && data.message === 'viewer/refreshFailed' && + data.documentGeneration === documentGeneration) { + refreshBtn.disabled = false; + setFetchStatus('error', data.error || 'Unable to refresh.', false); + } + }); + } + + ${ + tracking + ? getFollowScript( + tracking, + 'viewerFollow', + '', + "setFetchStatus('error', message.error || 'Unable to change following.', false);", + ) + : '' + } + updateTheme(); initialize().catch((e) => { console.error('[dataview] Initialization failed', e); @@ -3133,7 +3243,7 @@ export async function getTableHtml( - ${getDataViewerToolbarHtml(sessionHtml)} + ${getDataViewerToolbarHtml(sessionHtml, refreshable, tracking !== undefined)}
${getDataViewerColumnPanelHtml()} @@ -3320,6 +3430,8 @@ export function getListHtml( title: string, navigation?: ListViewNavigation, sessionId: string | null = null, + refreshable = false, + tracking?: JdvTracking, ): string { const documentGeneration = ++documentGenerationRevision; documentGenerations.set(webview, documentGeneration); @@ -3353,6 +3465,8 @@ export function getListHtml( + ${refreshable ? '' : ''} + ${tracking ? '' : ''} ${getViewerSessionHtml(sessionId)}
@@ -3365,6 +3479,8 @@ export function getListHtml( path: [], breadcrumbs: [{ label: title, path: [] }], }, + refreshable, + tracking, )} ${getViewerSessionScript()} @@ -3919,6 +4035,10 @@ async function handleNotification(message: Record, socket: IpcS } break; } + case 'jdv/rowCount': { + applyJdvRowCount(socket._sessionId ?? null, params); + break; + } case 'plot_updated': { void updatePlot(); break; @@ -3968,6 +4088,15 @@ async function handleRequest(message: Record, socket?: IpcSocke try { switch (method) { + case 'jdv/address': { + // The per-session jdv transport; sess connects jdv to it when View() is used. + const owner = sessions.get(socket?._sessionId ?? ''); + if (!owner) { + throw new Error('The jdv transport requires an attached session'); + } + result = { address: await getJdvAddress(owner.sessionId) }; + break; + } case 'rstudioapi/active_editor_context': result = rstudioapi.activeEditorContext(); break; @@ -4071,6 +4200,7 @@ export async function cleanupSession(sessionId: string, closingSocket?: IpcSocke return; } sessions.delete(sessionId); + disposeJdvHost(sessionId); terminalRegistry.releaseSession(session); const disconnected = invalidateSessionConnection(session); if (pipeClient === session.socket) { diff --git a/src/test/node/interactiveJdvAddress.test.ts b/src/test/node/interactiveJdvAddress.test.ts new file mode 100644 index 000000000..e70437556 --- /dev/null +++ b/src/test/node/interactiveJdvAddress.test.ts @@ -0,0 +1,184 @@ +import * as assert from 'assert'; +import { + acquireInteractiveJdvAddress, + disposeInteractiveJdvHostOnControlLoss, +} from '../../interactive/jdvAddress'; +import { disposeJdvHostIfCurrentAddress } from '../../jdvHostLifecycle'; + +suite('Interactive jdv address', () => { + test('returns a listener address scoped to the captured session and generation', async () => { + const address = await acquireInteractiveJdvAddress({ + sessionId: 'session:g1', + hostKey: 'interactive:session:g1', + canConnect: () => true, + isCurrent: () => true, + getAddress: (sessionId, hostKey) => { + assert.strictEqual(sessionId, 'session:g1'); + assert.strictEqual(hostKey, 'interactive:session:g1'); + return Promise.resolve('/tmp/jdv.sock'); + }, + disposeHost: () => assert.fail('A current session must keep its listener'), + }); + assert.strictEqual(address, '/tmp/jdv.sock'); + }); + + test('refuses observers without creating a listener', async () => { + let created = false; + await assert.rejects( + acquireInteractiveJdvAddress({ + sessionId: 'session:g1', + hostKey: 'interactive:session:g1', + canConnect: () => false, + isCurrent: () => true, + getAddress: () => { + created = true; + return Promise.resolve('/tmp/jdv.sock'); + }, + disposeHost: () => assert.fail('No listener was created'), + }), + /controlling Interactive client/, + ); + assert.strictEqual(created, false); + }); + + test('disposes an existing listener when control moves to another client', () => { + let disposed = 0; + assert.strictEqual( + disposeInteractiveJdvHostOnControlLoss(true, false, () => { + disposed++; + }), + true, + ); + assert.strictEqual(disposed, 1); + assert.strictEqual( + disposeInteractiveJdvHostOnControlLoss(false, false, () => { + disposed++; + }), + false, + ); + assert.strictEqual( + disposeInteractiveJdvHostOnControlLoss(true, true, () => { + disposed++; + }), + false, + ); + assert.strictEqual(disposed, 1); + }); + + test('closes a listener when a completed restart changes the captured generation', async () => { + let resolveAddress!: (address: string) => void; + const pending = new Promise((resolve) => { + resolveAddress = resolve; + }); + let generation = 'g1'; + const disposed: string[] = []; + const request = acquireInteractiveJdvAddress({ + sessionId: 'session:g1', + hostKey: 'interactive:session:g1', + canConnect: () => true, + isCurrent: () => generation === 'g1', + getAddress: () => pending, + disposeHost: (hostKey) => { + disposed.push(hostKey); + }, + }); + // A restart may finish before the asynchronous listener creation resumes. + generation = 'g2'; + resolveAddress('/tmp/old-jdv.sock'); + await assert.rejects(request, /session changed/); + assert.deepStrictEqual(disposed, ['interactive:session:g1']); + }); + + test('does not dispose a reattached host when an old address acquisition finishes late', async () => { + const hostKey = 'interactive:session:g1'; + let resolveAddress!: (address: string) => void; + const pendingAddress = new Promise((resolve) => { + resolveAddress = resolve; + }); + let oldHostDisposed = false; + let newHostDisposed = false; + const oldHostPromise = Promise.resolve({ + address: '/tmp/old.sock', + dispose: () => { + oldHostDisposed = true; + }, + }); + const newHost = { + address: '/tmp/new.sock', + dispose: () => { + newHostDisposed = true; + }, + }; + const hosts = new Map>([[hostKey, oldHostPromise]]); + const request = acquireInteractiveJdvAddress({ + sessionId: 'session:g1', + hostKey, + canConnect: () => true, + isCurrent: () => false, + getAddress: () => pendingAddress, + disposeHost: (key, address) => + disposeJdvHostIfCurrentAddress( + hosts, + key, + address, + (host) => host.address, + (host) => host.dispose(), + ).then(() => undefined), + }); + // Detach removed the old host and a same-generation reattach installed a new one. + hosts.delete(hostKey); + hosts.set(hostKey, Promise.resolve(newHost)); + resolveAddress('/tmp/old.sock'); + await assert.rejects(request, /session changed/); + assert.strictEqual(hosts.get(hostKey) !== undefined, true); + assert.strictEqual((await hosts.get(hostKey))?.address, '/tmp/new.sock'); + assert.strictEqual(oldHostDisposed, false); + assert.strictEqual(newHostDisposed, false); + }); + + test('closes a listener when detach wins while its address is being created', async () => { + let resolveAddress!: (address: string) => void; + const pending = new Promise((resolve) => { + resolveAddress = resolve; + }); + let attached = true; + let disposed = false; + const request = acquireInteractiveJdvAddress({ + sessionId: 'session:g1', + hostKey: 'interactive:session:g1', + canConnect: () => true, + isCurrent: () => attached, + getAddress: () => pending, + disposeHost: () => { + disposed = true; + }, + }); + attached = false; + resolveAddress('/tmp/detached-jdv.sock'); + await assert.rejects(request, /session changed/); + assert.strictEqual(disposed, true); + }); + + test('closes a listener if R exits while its address is being created', async () => { + let resolveAddress!: (address: string) => void; + const pending = new Promise((resolve) => { + resolveAddress = resolve; + }); + let running = true; + let disposed = false; + const request = acquireInteractiveJdvAddress({ + sessionId: 'session:g1', + hostKey: 'interactive:session:g1', + canConnect: () => running, + isCurrent: () => true, + getAddress: () => pending, + disposeHost: () => { + disposed = true; + }, + }); + running = false; + resolveAddress('/tmp/stopped-jdv.sock'); + await assert.rejects(request, /session changed/); + assert.strictEqual(disposed, true); + }); +}); diff --git a/src/test/node/interactiveRuntime.test.ts b/src/test/node/interactiveRuntime.test.ts index a7543c33d..0f1e42668 100644 --- a/src/test/node/interactiveRuntime.test.ts +++ b/src/test/node/interactiveRuntime.test.ts @@ -22,6 +22,14 @@ import { queryTablePage, TableColumn, tableSchema } from '../../interactive/tabl import { AssetStorageStats, readAsset } from '../../interactive/assets'; import { assertSvgTextVisible } from '../svgAssertions'; import { rString } from '../../interactive/backends/rCode'; +import { JdvHost, JdvPeer } from '../../jdvTransport'; +import { JsonObject } from '../../jdvAdapter'; +import { + createListAdapter, + isJdvOutput, + listRequest, + queryJdvTablePage, +} from '../../interactive/jdvTable'; const run = promisify(execFile); const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); @@ -137,6 +145,129 @@ const delay = (ms: number): Promise => new Promise((resolve) => setTimeout .join(''); } + /** The dataview registry serves inline previews when jdv is not used (or not installed). */ + const legacyViewerTest = (name: string, body: (this: Mocha.Context) => Promise): void => { + test(name, async function () { + await finished(await submit('options(sess.dataviewer = "sess")')); + try { + await body.call(this); + } finally { + await finished(await submit('options(sess.dataviewer = NULL)')).catch( + () => undefined, + ); + } + }); + }; + + test('opens jdv from an Interactive cell and serves a page while R is idle', async function () { + const availability = await run('Rscript', [ + '--vanilla', + '-e', + "cat(requireNamespace('jdv', quietly=TRUE))", + ]); + if (availability.stdout.trim() !== 'TRUE') { + this.skip(); + return; + } + + let resolveOpen!: (value: { peer: JdvPeer; params: JsonObject }) => void; + let rejectOpen!: (reason: Error) => void; + const opened = new Promise<{ peer: JdvPeer; params: JsonObject }>((resolve, reject) => { + resolveOpen = resolve; + rejectOpen = reject; + }); + const host = await JdvHost.create({ + viewOpen: (peer, params) => resolveOpen({ peer, params }), + viewClosed: () => undefined, + viewUpdateFailed: () => undefined, + viewTracking: () => undefined, + disconnected: (_peer, reason) => rejectOpen(new Error(reason)), + }); + const onEvent = (event: SessionEvent): void => { + if (event.type !== 'clientRequest' || event.data.method !== 'jdv/address') { + return; + } + void client + .request('clientReply', { + id: event.data.id, + result: { address: host.address }, + }) + .catch((error) => + rejectOpen(error instanceof Error ? error : new Error(String(error))), + ); + }; + client.on('event', onEvent); + try { + // Repeated strings keep the R fixture small while its encoded page exceeds the byte budget. + const id = await submit( + 'utils::View(data.frame(a = 1:300, b = rep(letters, length.out = 300), c = rep(strrep("x", 12000), 300), d = rep(strrep("y", 12000), 300)), "interactive jdv test")', + ); + assert.strictEqual((await finished(id)).state, 'success'); + const { peer, params } = await Promise.race([ + opened, + new Promise((_resolve, reject) => + setTimeout( + () => reject(new Error('Timed out waiting for jdv view/open')), + 10000, + ), + ), + ]); + assert.strictEqual(params.kind, 'grid'); + assert.strictEqual(params.access, 'random'); + const result = (await peer.request('grid/rows', { + view: params.view, + generation: params.generation, + offset: 0, + limit: 2, + })) as JsonObject; + const data = result.data as JsonObject; + assert.deepStrictEqual(data.source_row, [0, 1]); + assert.strictEqual(result.total, 300); + const selected = (await peer.request('grid/rows', { + view: params.view, + generation: params.generation, + offset: 0, + limit: 2, + cols: [1], + })) as JsonObject; + const selectedData = selected.data as { + source_row: number[]; + columns: { schema: { position: number }; data: { values: unknown[] } }[]; + }; + assert.deepStrictEqual(selectedData.source_row, [0, 1]); + assert.deepStrictEqual( + selectedData.columns.map((column) => column.schema.position), + [1], + ); + assert.deepStrictEqual(selectedData.columns[0].data.values, ['a', 'b']); + const shortened = (await peer.request('grid/rows', { + view: params.view, + generation: params.generation, + offset: 0, + limit: 300, + cols: [2, 3], + })) as JsonObject; + const shortenedData = shortened.data as typeof selectedData; + assert.strictEqual(shortened.total, 300); + assert.ok(shortenedData.source_row.length > 0 && shortenedData.source_row.length < 300); + assert.deepStrictEqual( + shortenedData.source_row, + Array.from({ length: shortenedData.source_row.length }, (_, index) => index), + ); + assert.deepStrictEqual( + shortenedData.columns.map((column) => column.schema.position), + [2, 3], + ); + for (const [index, column] of shortenedData.columns.entries()) { + assert.strictEqual(column.data.values.length, shortenedData.source_row.length); + assert.strictEqual(column.data.values[0], (index === 0 ? 'x' : 'y').repeat(12000)); + } + } finally { + client.off('event', onEvent); + host.dispose(); + } + }); + test('streams output before completion and keeps one R environment', async () => { const id = await submit( 'answer <- 42; cat("early λ🙂\\n"); Sys.sleep(0.5); cat("late\\n"); answer', @@ -441,135 +572,571 @@ sess::display("display-${cell}", "text/plain"); cat("stderr-${cell}\\n", file=st } }); - test('data inspection preserves random draws and reproducible resampling across cells', async () => { - const first = await submit(`set.seed(2026) -expected <- replicate(100, mean(sample(mtcars$mpg, replace=TRUE))) -set.seed(2026) -head(mtcars)`); - assert.strictEqual((await finished(first)).state, 'success'); - const second = await submit(`View(mtcars) -sess::display(head(iris)) -actual <- replicate(100, mean(sample(mtcars$mpg, replace=TRUE))) -stopifnot(identical(actual, expected))`); - assert.strictEqual((await finished(second)).state, 'success'); - assert.strictEqual( - events.filter((event) => event.type === 'display' && event.data.kind === 'table') - .length, - 3, - ); + const jdvAvailable = async (): Promise => + ( + await run('Rscript', ['--vanilla', '-e', "cat(requireNamespace('jdv', quietly=TRUE))"]) + ).stdout.trim() === 'TRUE'; + const inspect = (request: { + method: string; + params: Record; + }): Promise> => client.request('inspect', request); + + test('jdv Arrow views send display-only totals for sized sources', async function () { + if (!(await jdvAvailable())) { + this.skip(); + return; + } + const available = ( + await run('Rscript', [ + '--vanilla', + '-e', + "cat(requireNamespace('nanoarrow', quietly=TRUE), requireNamespace('arrow', quietly=TRUE))", + ]) + ).stdout + .trim() + .split(/\s+/); + if (available[0] !== 'TRUE') { + this.skip(); + return; + } + const sized = available[1] === 'TRUE'; + const opened: { peer: JdvPeer; params: JsonObject }[] = []; + const host = await JdvHost.create({ + viewOpen: (peer, params) => opened.push({ peer, params }), + viewClosed: () => undefined, + viewUpdateFailed: () => undefined, + viewTracking: () => undefined, + disconnected: () => undefined, + }); + const onEvent = (event: SessionEvent): void => { + if (event.type === 'clientRequest' && event.data.method === 'jdv/address') { + void client.request('clientReply', { + id: event.data.id, + result: { address: host.address }, + }); + } + }; + client.on('event', onEvent); + try { + const source = sized + ? 'arrow::Table$create(data.frame(a=1:3))' + : 'nanoarrow::as_nanoarrow_array_stream(data.frame(a=1:3))'; + const id = await submit(`utils::View(${source}, "Arrow count test")`); + assert.strictEqual((await finished(id)).state, 'success'); + await until(() => opened.length > 0); + const { peer, params } = opened[0]; + assert.strictEqual(params.access, 'sequential'); + assert.strictEqual(params.nrow, null); + const hints = events.filter( + (event) => event.type === 'notification' && event.data.method === 'jdv/rowCount', + ); + assert.strictEqual(hints.length, sized ? 1 : 0); + if (sized) { + assert.deepStrictEqual(hints[0].data.params, { + view: params.view, + generation: params.generation, + nrow: 3, + address: host.address, + }); + } + const page = await peer.requestWithBody('stream/next', { + view: params.view, + generation: params.generation, + max_rows: 2, + }); + assert.strictEqual((page.result as JsonObject).rows_seen, 2); + assert.strictEqual((page.result as JsonObject).done, false); + assert.ok(page.body); + } finally { + client.off('event', onEvent); + host.dispose(); + } }); - test('table pages retain their original data after reference-based cleaning and schema changes', async function () { - if ( - !( - await run('Rscript', ['-e', 'cat(requireNamespace("data.table", quietly=TRUE))']) - ).stdout.includes('TRUE') - ) { + test('inline tables page, sort and filter through one jdv handle per output', async function () { + if (!(await jdvAvailable())) { this.skip(); + return; } - const id = await submit('dt <- data.table::data.table(id=1:50, value=1:50); dt'); + const id = await submit( + 'tbl <- data.frame(id=1:100, name=rep(c("a", "b"), 50), x=seq(0.5, 50, by=0.5)); tbl', + ); assert.strictEqual((await finished(id)).state, 'success'); const table = events.find( (event) => event.executionId === id && event.data.kind === 'table', ); assert.ok(table); - await finished( - await submit( - 'dt[, value := value * 100]; data.table::setnames(dt, "value", "changed"); data.table::setorder(dt, -id)', - ), + assert.ok(isJdvOutput(table.data)); + assert.strictEqual(table.data.fullViewId, undefined); + assert.strictEqual(table.data.totalRows, 100); + assert.strictEqual(table.data.sourceRows, 100); + assert.strictEqual((table.data.rows as unknown[]).length, 20); + assert.deepStrictEqual( + (table.data.columns as { field: string }[]).map((column) => column.field), + ['0', '1', '2', '3'], ); - const page = await client.request<{ rows: Record[] }>('inspect', { - method: 'dataview_page', - params: { - view_id: table.data.viewId, - startRow: 20, - endRow: 40, - }, - }); + assert.match(String(table.data.printedText), /name/); + + const page = await queryJdvTablePage(table.data, { start: 40, size: 20 }, inspect); + assert.strictEqual(page.startRow, 40); assert.deepStrictEqual( - page.rows.map((row) => row['2']), - Array.from({ length: 20 }, (_, i) => i + 21), + (page.rows as Record[]).map((row) => row['1']).slice(0, 2), + ['41', '42'], ); - const filtered = await client.request<{ rows: Record[] }>('inspect', { - method: 'dataview_page', - params: { - view_id: table.data.viewId, - startRow: 0, - endRow: 20, - filterModel: { '2': { type: 'greaterThan', filter: 48 } }, + assert.strictEqual((page.rows as Record[])[0]['0'], '41'); + + // Sorting and filtering happen in R and report the filtered count. + const filtered = await queryJdvTablePage( + table.data, + { + start: 0, + size: 20, sortModel: [{ colId: '1', sort: 'desc' }], + filterModel: { + '2': { type: 'equals', filter: 'a' }, + '1': { type: 'greaterThan', filter: '90' }, + }, }, - }); + inspect, + ); + assert.strictEqual(filtered.totalRows, 5); assert.deepStrictEqual( - filtered.rows.map((row) => row['2']), - [50, 49], + (filtered.rows as Record[]).map((row) => row['1']), + ['99', '97', '95', '93', '91'], + ); + // Row labels show the original row after sorting. + assert.strictEqual((filtered.rows as Record[])[0]['0'], '99'); + + // Empty strings and NA are separate filters. + const blankId = await submit( + 'blanks <- data.frame(s=c("a", "", NA, "b"), stringsAsFactors=FALSE); blanks', + ); + assert.strictEqual((await finished(blankId)).state, 'success'); + const blanks = events.find( + (event) => event.executionId === blankId && event.data.kind === 'table', + ); + assert.ok(blanks); + for (const [type, expected] of [ + ['jdvIsEmpty', 1], + ['jdvNotEmpty', 2], + ['jdvIsNa', 1], + ['jdvNotNa', 3], + ] as const) { + const result = await queryJdvTablePage( + blanks.data, + { start: 0, size: 20, filterModel: { '1': { type } } }, + inspect, + ); + assert.strictEqual(result.totalRows, expected, type); + } + + const flagsId = await submit('flags <- data.frame(flag=c(TRUE, FALSE, NA)); flags'); + assert.strictEqual((await finished(flagsId)).state, 'success'); + const flags = events.find( + (event) => event.executionId === flagsId && event.data.kind === 'table', + ); + assert.ok(flags); + for (const [type, expected] of [ + ['jdvIsNa', 1], + ['jdvNotNa', 2], + ['true', 1], + ] as const) { + const result = await queryJdvTablePage( + flags.data, + { start: 0, size: 20, filterModel: { '1': { type } } }, + inspect, + ); + assert.strictEqual(result.totalRows, expected, type); + } + + // A page past the end is clamped to the last page. + const clamped = await queryJdvTablePage(table.data, { start: 400, size: 20 }, inspect); + assert.strictEqual(clamped.startRow, 80); + + // The handle outlives reconnects and is released on request. + client.close(); + await client.connect(); + await client.subscribe(0); + const restored = (await client.snapshot()).events.find((event) => event.seq === table.seq); + assert.strictEqual(restored?.data.handleId, table.data.handleId); + await inspect({ method: 'jdv_handle_close', params: { handle_id: table.data.handleId } }); + await assert.rejects( + queryJdvTablePage(table.data, { start: 0, size: 20 }, inspect), + /no longer available/, + ); + }); + + test('inline jdv wide tables survive the event limit and page without gaps', async function () { + if (!(await jdvAvailable())) { + this.skip(); + return; + } + const id = await submit( + 'wide <- data.frame(id = 1:150, text = rep(strrep("a", 100000), 150)); wide', + ); + assert.strictEqual((await finished(id)).state, 'success'); + const table = events.find( + (event) => event.executionId === id && event.data.kind === 'table', + ); + assert.ok(table); + assert.ok(isJdvOutput(table.data)); + assert.equal( + events.some((event) => event.executionId === id && event.type === 'truncated'), + false, + ); + assert.ok(Buffer.byteLength(JSON.stringify(table.data)) < 2 * 1024 * 1024); + assert.equal(table.data.previewPartial, true); + assert.ok((table.data.rows as unknown[]).length < 20); + for (const size of [20, 100]) { + const ids: string[] = []; + for (let start = 0; start < 150; start += size) { + const page = await queryJdvTablePage(table.data, { start, size }, inspect); + assert.equal(page.startRow, start); + ids.push(...(page.rows as Record[]).map((row) => row['1'])); + } + assert.deepEqual( + ids, + Array.from({ length: 150 }, (_, i) => String(i + 1)), + ); + } + }); + + test('inline jdv tables bound a huge table to its first page', async function () { + if (!(await jdvAvailable())) { + this.skip(); + return; + } + const id = await submit(`n <- 832976871L +huge <- structure(rep(list(seq_len(n)), 23L), names=paste0("x", 1:23), + class="data.frame", row.names=c(NA_integer_, -n)) +huge`); + assert.strictEqual((await finished(id)).state, 'success'); + const table = events.find( + (event) => event.executionId === id && event.data.kind === 'table', ); - const current = await submit('stopifnot(names(dt)[2] == "changed", dt$changed[1] == 5000)'); - assert.strictEqual((await finished(current)).state, 'success'); + assert.ok(table); + assert.ok(isJdvOutput(table.data)); + assert.strictEqual(table.data.totalRows, 832976871); + assert.strictEqual(table.data.sourceRows, 832976871); + assert.match(String(table.data.printedText), /first 1,000 of 832,976,871/); + const last = await queryJdvTablePage(table.data, { start: 832976860, size: 20 }, inspect); + assert.strictEqual(last.startRow, 832976860); + assert.strictEqual((last.rows as unknown[]).length, 11); + assert.strictEqual((last.rows as Record[])[10]['23'], '832976871'); }); - test('unclassed lists retain printable previews and lazy pages across reconnects', async () => { + test('inline lists explore typed paths through one jdv handle', async function () { + if (!(await jdvAvailable())) { + this.skip(); + return; + } const id = await submit( - 'x <- list(nested=list(value=c(1,2)), table=data.frame(id=1:2), "λ "); x', + 'lst <- list(nested=list(value=c(1,2)), table=data.frame(id=1:2), "λ "); lst', ); assert.strictEqual((await finished(id)).state, 'success'); const output = events.find( (event) => event.executionId === id && event.data.kind === 'list', ); assert.ok(output); + assert.ok(isJdvOutput(output.data)); const children = output.data.children as { label: string; has_children: boolean }[]; assert.deepStrictEqual( children.map((child) => child.label), - ['$ nested', '$ table', '[[3]]'], + ['nested', 'table', '[[3]]'], ); assert.strictEqual(children[0].has_children, true); assert.match(String(output.data.printedText), /\$nested\$value/); - assert.ok( - !text(id).includes('$nested'), - 'The rich list replaces automatic console printing', + + const adapter = await createListAdapter(output.data, inspect); + const nested = await listRequest( + adapter, + output.data, + { action: 'listItem', path: [], index: 1 }, + inspect, ); - const expected = await submit('print(x)'); - await finished(expected); - assert.strictEqual(output.data.printedText, text(expected)); - await finished(await submit('x$nested$value <- 99; invisible(NULL)')); - const page = await client.request<{ children: { str: string }[] }>('inspect', { - method: 'workspace_children', - params: { - view_id: output.data.viewId, - path: [1], - start: 1, - }, - }); - assert.match(page.children[0].str, /1/); - assert.ok(!page.children[0].str.includes('99')); - const explicit = await submit( - 'sess::display(list()); View(list(a=1)); structure(list(a=2), class="custom_list")', + assert.deepStrictEqual( + (nested.result as { navigation: { path: number[] } }).navigation.path, + [1], + ); + const page = await listRequest( + adapter, + output.data, + { action: 'listPage', path: [1], start: 1 }, + inspect, ); - assert.strictEqual((await finished(explicit)).state, 'success'); assert.strictEqual( - events.filter((event) => event.executionId === explicit && event.data.kind === 'list') - .length, - 2, - ); - assert.match(text(explicit), /custom_list/); - const large = await submit('as.list(seq_len(501))'); - await finished(large); - const largeList = events.find( - (event) => event.executionId === large && event.data.kind === 'list', - ); - assert.ok(largeList); - assert.strictEqual((largeList.data.children as unknown[]).length, 500); - assert.strictEqual(largeList.data.next_start, 501); - client.close(); - await client.connect(); - await client.subscribe(0); - assert.deepStrictEqual( - (await client.snapshot()).events.find((event) => event.seq === output.seq)?.data, + (page.result as { children: { label: string }[] }).children[0].label, + 'value', + ); + // A table inside a list is opened in the Data Viewer rather than navigated into. + const table = await listRequest( + adapter, output.data, + { action: 'listItem', path: [], index: 2 }, + inspect, ); + assert.deepStrictEqual(table.open, { + path: [{ type: 'index', index: 2 }], + title: 'List$table', + }); + }); + + test('opens an inline table and a list item in the Data Viewer through jdv', async function () { + if (!(await jdvAvailable())) { + this.skip(); + return; + } + const opened: JsonObject[] = []; + const host = await JdvHost.create({ + viewOpen: (_peer, params) => { + opened.push(params); + }, + viewClosed: () => undefined, + viewUpdateFailed: () => undefined, + viewTracking: () => undefined, + disconnected: () => undefined, + }); + try { + const id = await submit('opened <- list(a=1, df=data.frame(z=1:3)); opened'); + assert.strictEqual((await finished(id)).state, 'success'); + const output = events.find( + (event) => event.executionId === id && event.data.kind === 'list', + ); + assert.ok(output); + const viewer = await inspect({ + method: 'jdv_open_viewer', + params: { + handle_id: output.data.handleId, + path: [{ type: 'index', index: 2 }], + title: 'Cell: List$df', + address: host.address, + }, + }); + assert.strictEqual(viewer as unknown, true); + const deadline = Date.now() + 10000; + while (opened.length === 0 && Date.now() < deadline) { + await delay(25); + } + assert.strictEqual(opened[0]?.kind, 'grid'); + assert.strictEqual(opened[0]?.title, 'Cell: List$df'); + assert.strictEqual(opened[0]?.nrow, 3); + } finally { + host.dispose(); + } + }); + + test('opens nested and root jdv handles without dispatching list subsetting methods', async function () { + if (!(await jdvAvailable())) { + this.skip(); + return; + } + const opened: JsonObject[] = []; + const host = await JdvHost.create({ + viewOpen: (_peer, params) => { + opened.push(params); + }, + viewClosed: () => undefined, + viewUpdateFailed: () => undefined, + viewTracking: () => undefined, + disconnected: () => undefined, + }); + try { + const id = await submit( + '`[[.review_list` <- function(x, i, ...) stop("review_list [[ called"); review <- structure(list(table=data.frame(z=1:3)), class="review_list"); outer <- list(review=review); outer', + ); + assert.strictEqual((await finished(id)).state, 'success'); + const output = events.find( + (event) => event.executionId === id && event.data.kind === 'list', + ); + assert.ok(output); + assert.ok(isJdvOutput(output.data)); + + const nested = await inspect({ + method: 'jdv_open_viewer', + params: { + handle_id: output.data.handleId, + path: [ + { type: 'index', index: 1 }, + { type: 'index', index: 1 }, + ], + title: 'Cell: List$review$table', + address: host.address, + }, + }); + assert.strictEqual(nested as unknown, true); + const deadline = Date.now() + 10000; + while (opened.length === 0 && Date.now() < deadline) { + await delay(25); + } + assert.strictEqual(opened[0]?.kind, 'grid'); + assert.strictEqual(opened[0]?.title, 'Cell: List$review$table'); + assert.strictEqual(opened[0]?.nrow, 3); + + const root = await inspect({ + method: 'jdv_open_viewer', + params: { + handle_id: output.data.handleId, + path: [], + title: 'Interactive root', + address: host.address, + }, + }); + assert.strictEqual(root as unknown, true); + const rootDeadline = Date.now() + 10000; + while (opened.length < 2 && Date.now() < rootDeadline) { + await delay(25); + } + assert.strictEqual(opened[1]?.kind, 'tree'); + assert.strictEqual(opened[1]?.title, 'Interactive root'); + } finally { + host.dispose(); + } }); - test('nested table viewers from separate list cells stay independent', async () => { + legacyViewerTest( + 'data inspection preserves random draws and reproducible resampling across cells', + async () => { + const first = await submit(`set.seed(2026) +expected <- replicate(100, mean(sample(mtcars$mpg, replace=TRUE))) +set.seed(2026) +head(mtcars)`); + assert.strictEqual((await finished(first)).state, 'success'); + const second = await submit(`View(mtcars) +sess::display(head(iris)) +actual <- replicate(100, mean(sample(mtcars$mpg, replace=TRUE))) +stopifnot(identical(actual, expected))`); + assert.strictEqual((await finished(second)).state, 'success'); + assert.strictEqual( + events.filter((event) => event.type === 'display' && event.data.kind === 'table') + .length, + 3, + ); + }, + ); + + legacyViewerTest( + 'table pages retain their original data after reference-based cleaning and schema changes', + async function () { + if ( + !( + await run('Rscript', [ + '-e', + 'cat(requireNamespace("data.table", quietly=TRUE))', + ]) + ).stdout.includes('TRUE') + ) { + this.skip(); + } + const id = await submit('dt <- data.table::data.table(id=1:50, value=1:50); dt'); + assert.strictEqual((await finished(id)).state, 'success'); + const table = events.find( + (event) => event.executionId === id && event.data.kind === 'table', + ); + assert.ok(table); + await finished( + await submit( + 'dt[, value := value * 100]; data.table::setnames(dt, "value", "changed"); data.table::setorder(dt, -id)', + ), + ); + const page = await client.request<{ rows: Record[] }>('inspect', { + method: 'dataview_page', + params: { + view_id: table.data.viewId, + startRow: 20, + endRow: 40, + }, + }); + assert.deepStrictEqual( + page.rows.map((row) => row['2']), + Array.from({ length: 20 }, (_, i) => i + 21), + ); + const filtered = await client.request<{ rows: Record[] }>('inspect', { + method: 'dataview_page', + params: { + view_id: table.data.viewId, + startRow: 0, + endRow: 20, + filterModel: { '2': { type: 'greaterThan', filter: 48 } }, + sortModel: [{ colId: '1', sort: 'desc' }], + }, + }); + assert.deepStrictEqual( + filtered.rows.map((row) => row['2']), + [50, 49], + ); + const current = await submit( + 'stopifnot(names(dt)[2] == "changed", dt$changed[1] == 5000)', + ); + assert.strictEqual((await finished(current)).state, 'success'); + }, + ); + + legacyViewerTest( + 'unclassed lists retain printable previews and lazy pages across reconnects', + async () => { + const id = await submit( + 'x <- list(nested=list(value=c(1,2)), table=data.frame(id=1:2), "λ "); x', + ); + assert.strictEqual((await finished(id)).state, 'success'); + const output = events.find( + (event) => event.executionId === id && event.data.kind === 'list', + ); + assert.ok(output); + const children = output.data.children as { label: string; has_children: boolean }[]; + assert.deepStrictEqual( + children.map((child) => child.label), + ['$ nested', '$ table', '[[3]]'], + ); + assert.strictEqual(children[0].has_children, true); + assert.match(String(output.data.printedText), /\$nested\$value/); + assert.ok( + !text(id).includes('$nested'), + 'The rich list replaces automatic console printing', + ); + const expected = await submit('print(x)'); + await finished(expected); + assert.strictEqual(output.data.printedText, text(expected)); + await finished(await submit('x$nested$value <- 99; invisible(NULL)')); + const page = await client.request<{ children: { str: string }[] }>('inspect', { + method: 'workspace_children', + params: { + view_id: output.data.viewId, + path: [1], + start: 1, + }, + }); + assert.match(page.children[0].str, /1/); + assert.ok(!page.children[0].str.includes('99')); + const explicit = await submit( + 'sess::display(list()); View(list(a=1)); structure(list(a=2), class="custom_list")', + ); + assert.strictEqual((await finished(explicit)).state, 'success'); + assert.strictEqual( + events.filter( + (event) => event.executionId === explicit && event.data.kind === 'list', + ).length, + 2, + ); + assert.match(text(explicit), /custom_list/); + const large = await submit('as.list(seq_len(501))'); + await finished(large); + const largeList = events.find( + (event) => event.executionId === large && event.data.kind === 'list', + ); + assert.ok(largeList); + assert.strictEqual((largeList.data.children as unknown[]).length, 500); + assert.strictEqual(largeList.data.next_start, 501); + client.close(); + await client.connect(); + await client.subscribe(0); + assert.deepStrictEqual( + (await client.snapshot()).events.find((event) => event.seq === output.seq)?.data, + output.data, + ); + }, + ); + + legacyViewerTest('nested table viewers from separate list cells stay independent', async () => { const lists: SessionEvent[] = []; for (const value of [11, 21]) { const id = await submit( @@ -638,177 +1205,200 @@ stopifnot(identical(actual, expected))`); ); }); - test('huge tables keep bounded snapshots and page the full data without materializing it', async () => { - const id = await submit(`n <- 832976871L + legacyViewerTest( + 'huge tables keep bounded snapshots and page the full data without materializing it', + async () => { + const id = await submit(`n <- 832976871L huge <- structure(rep(list(seq_len(n)), 23L), names=paste0("x", 1:23), class="data.frame", row.names=c(NA_integer_, -n)) huge`); - assert.strictEqual((await finished(id)).state, 'success'); - const table = events.find( - (event) => event.executionId === id && event.data.kind === 'table', - ); - assert.ok(table); - assert.strictEqual(table.data.totalRows, 1000); - assert.strictEqual(table.data.sourceRows, 832976871); - assert.notStrictEqual(table.data.fullViewId, table.data.viewId); - assert.match(String(table.data.printedText), /first 1,000 of 832,976,871/); - const last = await client.request<{ rows: Record[]; totalRows: number }>( - 'inspect', - { + assert.strictEqual((await finished(id)).state, 'success'); + const table = events.find( + (event) => event.executionId === id && event.data.kind === 'table', + ); + assert.ok(table); + assert.strictEqual(table.data.totalRows, 1000); + assert.strictEqual(table.data.sourceRows, 832976871); + assert.notStrictEqual(table.data.fullViewId, table.data.viewId); + assert.match(String(table.data.printedText), /first 1,000 of 832,976,871/); + const last = await client.request<{ + rows: Record[]; + totalRows: number; + }>('inspect', { method: 'dataview_page', params: { view_id: table.data.fullViewId, startRow: 832976870, endRow: 832976871, }, - }, - ); - assert.strictEqual(last.totalRows, 832976871); - assert.strictEqual(last.rows[0]['23'], 832976871); - const inline = await queryTablePage( - table.data, - { start: 832976860, live: true, refresh: true }, - (request) => client.request('inspect', request), - ); - assert.strictEqual(inline.startRow, 832976860); - assert.strictEqual((inline.rows as unknown[]).length, 11); - assert.strictEqual((inline.rows as Record[])[10]['23'], 832976871); - client.close(); - await client.connect(); - await client.subscribe(0); - const restored = (await client.snapshot()).events.find((event) => event.seq === table.seq); - assert.strictEqual(restored?.data.fullViewId, table.data.fullViewId); - assert.strictEqual(restored?.data.sourceRows, table.data.sourceRows); - }); + }); + assert.strictEqual(last.totalRows, 832976871); + assert.strictEqual(last.rows[0]['23'], 832976871); + const inline = await queryTablePage( + table.data, + { start: 832976860, live: true, refresh: true }, + (request) => client.request('inspect', request), + ); + assert.strictEqual(inline.startRow, 832976860); + assert.strictEqual((inline.rows as unknown[]).length, 11); + assert.strictEqual((inline.rows as Record[])[10]['23'], 832976871); + client.close(); + await client.connect(); + await client.subscribe(0); + const restored = (await client.snapshot()).events.find( + (event) => event.seq === table.seq, + ); + assert.strictEqual(restored?.data.fullViewId, table.data.fullViewId); + assert.strictEqual(restored?.data.sourceRows, table.data.sourceRows); + }, + ); - test('large data.table snapshots stay stable while full-view queries refresh after reference edits', async function () { - if ( - !( - await run('Rscript', ['-e', 'cat(requireNamespace("data.table", quietly=TRUE))']) - ).stdout.includes('TRUE') - ) { - this.skip(); - } - const id = await submit('dt <- data.table::data.table(id=1:2000, value=1:2000); dt'); - assert.strictEqual((await finished(id)).state, 'success'); - const table = events.find( - (event) => event.executionId === id && event.data.kind === 'table', - ); - assert.ok(table); - const query = { - method: 'dataview_page', - params: { - view_id: table.data.fullViewId, - startRow: 0, - endRow: 2, - sortModel: [{ colId: '2', sort: 'desc' }], - }, - }; - const before = await client.request<{ rows: Record[] }>('inspect', query); - assert.deepStrictEqual( - before.rows.map((row) => row['2']), - [2000, 1999], - ); - await finished( - await submit( - 'data.table::set(dt, j="value", value=-seq_len(2000L)); stop("after mutation")', - ), - ); - const after = await client.request<{ rows: Record[] }>('inspect', query); - assert.deepStrictEqual( - after.rows.map((row) => row['2']), - [-1, -2], - ); - await finished(await submit('data.table::setnames(dt, "value", "changed")')); - await assert.rejects(client.request('inspect', query), /Reopen Data viewer/); - await client.request('inspect', { - method: 'dataview_init', - params: { view_id: table.data.fullViewId }, - }); - const refreshed = await client.request<{ rows: Record[] }>( - 'inspect', - query, - ); - assert.deepStrictEqual( - refreshed.rows.map((row) => row['2']), - [-1, -2], - ); - const saved = await client.request<{ rows: Record[]; totalRows: number }>( - 'inspect', - { + legacyViewerTest( + 'large data.table snapshots stay stable while full-view queries refresh after reference edits', + async function () { + if ( + !( + await run('Rscript', [ + '-e', + 'cat(requireNamespace("data.table", quietly=TRUE))', + ]) + ).stdout.includes('TRUE') + ) { + this.skip(); + } + const id = await submit('dt <- data.table::data.table(id=1:2000, value=1:2000); dt'); + assert.strictEqual((await finished(id)).state, 'success'); + const table = events.find( + (event) => event.executionId === id && event.data.kind === 'table', + ); + assert.ok(table); + const query = { + method: 'dataview_page', + params: { + view_id: table.data.fullViewId, + startRow: 0, + endRow: 2, + sortModel: [{ colId: '2', sort: 'desc' }], + }, + }; + const before = await client.request<{ rows: Record[] }>( + 'inspect', + query, + ); + assert.deepStrictEqual( + before.rows.map((row) => row['2']), + [2000, 1999], + ); + await finished( + await submit( + 'data.table::set(dt, j="value", value=-seq_len(2000L)); stop("after mutation")', + ), + ); + const after = await client.request<{ rows: Record[] }>( + 'inspect', + query, + ); + assert.deepStrictEqual( + after.rows.map((row) => row['2']), + [-1, -2], + ); + await finished(await submit('data.table::setnames(dt, "value", "changed")')); + await assert.rejects(client.request('inspect', query), /Reopen Data viewer/); + await client.request('inspect', { + method: 'dataview_init', + params: { view_id: table.data.fullViewId }, + }); + const refreshed = await client.request<{ rows: Record[] }>( + 'inspect', + query, + ); + assert.deepStrictEqual( + refreshed.rows.map((row) => row['2']), + [-1, -2], + ); + const saved = await client.request<{ + rows: Record[]; + totalRows: number; + }>('inspect', { method: 'dataview_page', params: { view_id: table.data.viewId, startRow: 980, endRow: 1020, }, - }, - ); - assert.strictEqual(saved.totalRows, 1000); - assert.strictEqual(saved.rows.length, 20); - assert.strictEqual(saved.rows[19]['2'], 1000); - }); + }); + assert.strictEqual(saved.totalRows, 1000); + assert.strictEqual(saved.rows.length, 20); + assert.strictEqual(saved.rows[19]['2'], 1000); + }, + ); - test('inline full-table queries sort, filter, widen pages and recover after schema edits', async function () { - if ( - !( - await run('Rscript', ['-e', 'cat(requireNamespace("data.table", quietly=TRUE))']) - ).stdout.includes('TRUE') - ) { - this.skip(); - } - const id = await submit( - 'dt <- data.table::data.table(id=1:2000, name=rep(c("a", "b"), 1000), date=as.Date("2026-01-01")+0:1999, flag=rep(c(TRUE, FALSE),1000)); dt', - ); - assert.strictEqual((await finished(id)).state, 'success'); - const table = events.find( - (event) => event.executionId === id && event.data.kind === 'table', - ); - assert.ok(table); - const schema = tableSchema(table.data.columns as TableColumn[]); - const inspect = (request: { - method: string; - params: Record; - }): Promise> => client.request('inspect', request); - const filters = { - '2': { type: 'equals', filter: 'a' }, - '4': { type: 'true' }, - '3': { type: 'greaterThan', filter: '2026-01-01' }, - }; - const query = { - start: 0, - size: 50, - refresh: true, - schema, - sortModel: [{ colId: '1', sort: 'desc' }], - filterModel: filters, - }; - const result = await queryTablePage(table.data, query, inspect); - assert.strictEqual( - result.queryReset, - false, - JSON.stringify({ schema, columns: result.columns }), - ); - assert.strictEqual(result.totalRows, 999); - assert.strictEqual((result.rows as unknown[]).length, 50); - assert.strictEqual((result.rows as Record[])[0]['1'], 1999); - const finalPage = await queryTablePage( - table.data, - { ...query, refresh: false, live: true, start: 950 }, - inspect, - ); - assert.strictEqual((finalPage.rows as unknown[]).length, 49); - assert.strictEqual((finalPage.rows as Record[])[48]['1'], 3); - await finished(await submit('data.table::setnames(dt, "id", "renamed")')); - const refreshed = await queryTablePage(table.data, query, inspect); - assert.strictEqual(refreshed.queryReset, true); - assert.strictEqual(refreshed.totalRows, 2000); - assert.strictEqual((refreshed.columns as TableColumn[])[1].headerName, 'renamed'); - const saved = await queryTablePage(table.data, { start: 0 }, inspect); - assert.strictEqual(saved.live, false); - assert.strictEqual((saved.columns as TableColumn[])[1].headerName, 'id'); - assert.strictEqual((saved.rows as Record[])[0]['1'], 1); - }); + legacyViewerTest( + 'inline full-table queries sort, filter, widen pages and recover after schema edits', + async function () { + if ( + !( + await run('Rscript', [ + '-e', + 'cat(requireNamespace("data.table", quietly=TRUE))', + ]) + ).stdout.includes('TRUE') + ) { + this.skip(); + } + const id = await submit( + 'dt <- data.table::data.table(id=1:2000, name=rep(c("a", "b"), 1000), date=as.Date("2026-01-01")+0:1999, flag=rep(c(TRUE, FALSE),1000)); dt', + ); + assert.strictEqual((await finished(id)).state, 'success'); + const table = events.find( + (event) => event.executionId === id && event.data.kind === 'table', + ); + assert.ok(table); + const schema = tableSchema(table.data.columns as TableColumn[]); + const inspect = (request: { + method: string; + params: Record; + }): Promise> => client.request('inspect', request); + const filters = { + '2': { type: 'equals', filter: 'a' }, + '4': { type: 'true' }, + '3': { type: 'greaterThan', filter: '2026-01-01' }, + }; + const query = { + start: 0, + size: 50, + refresh: true, + schema, + sortModel: [{ colId: '1', sort: 'desc' }], + filterModel: filters, + }; + const result = await queryTablePage(table.data, query, inspect); + assert.strictEqual( + result.queryReset, + false, + JSON.stringify({ schema, columns: result.columns }), + ); + assert.strictEqual(result.totalRows, 999); + assert.strictEqual((result.rows as unknown[]).length, 50); + assert.strictEqual((result.rows as Record[])[0]['1'], 1999); + const finalPage = await queryTablePage( + table.data, + { ...query, refresh: false, live: true, start: 950 }, + inspect, + ); + assert.strictEqual((finalPage.rows as unknown[]).length, 49); + assert.strictEqual((finalPage.rows as Record[])[48]['1'], 3); + await finished(await submit('data.table::setnames(dt, "id", "renamed")')); + const refreshed = await queryTablePage(table.data, query, inspect); + assert.strictEqual(refreshed.queryReset, true); + assert.strictEqual(refreshed.totalRows, 2000); + assert.strictEqual((refreshed.columns as TableColumn[])[1].headerName, 'renamed'); + const saved = await queryTablePage(table.data, { start: 0 }, inspect); + assert.strictEqual(saved.live, false); + assert.strictEqual((saved.columns as TableColumn[])[1].headerName, 'id'); + assert.strictEqual((saved.rows as Record[])[0]['1'], 1); + }, + ); test('data.table reference assignments stay quiet while explicit results and display remain visible', async function () { if ( @@ -917,7 +1507,7 @@ dt`); ); }); - test('retains rich tables, plot assets, and structured errors', async () => { + legacyViewerTest('retains rich tables, plot assets, and structured errors', async () => { const id = await submit( 'plot(1:3); data.frame(x=1:3, label=c("a", "b", "c")); stop("expected error")', ); @@ -998,54 +1588,69 @@ dt`); ); }); - test('retains R numeric formatting in previews and pages while preserving raw precision', async () => { - const id = await submit( - 'options(digits=7, scipen=0, OutDec="."); numeric_table <- data.frame(price_per_carat=rep(c(326/0.23, 326/0.21, 1400), 8)); numeric_table', - ); - assert.strictEqual((await finished(id)).state, 'success'); - const table = events.find( - (event) => - event.executionId === id && event.type === 'display' && event.data.kind === 'table', - ); - assert.ok(table); - const labels = table.data.formattedColumns as Record; - assert.deepStrictEqual(labels['1'].slice(0, 3), ['1417.391', '1552.381', '1400.000']); - assert.strictEqual(labels['1'].length, 20); - const first = (table.data.rows as Record[])[0]['1']; - assert.ok(Math.abs(first - 326 / 0.23) < 1e-10); - const page = await client.request<{ - rows: Record[]; - formattedColumns: Record; - }>('inspect', { - method: 'dataview_page', - params: { view_id: table.data.viewId, startRow: 20, endRow: 24, formatNumbers: true }, - }); - assert.deepStrictEqual(page.formattedColumns['1'], [ - '1400.000', - '1417.391', - '1552.381', - '1400.000', - ]); - assert.strictEqual(page.rows[1]['1'], first); - await finished(await submit('options(digits=4, scipen=999, OutDec=",")')); - const localized = await client.request<{ formattedColumns: Record }>( - 'inspect', - { + legacyViewerTest( + 'retains R numeric formatting in previews and pages while preserving raw precision', + async () => { + const id = await submit( + 'options(digits=7, scipen=0, OutDec="."); numeric_table <- data.frame(price_per_carat=rep(c(326/0.23, 326/0.21, 1400), 8)); numeric_table', + ); + assert.strictEqual((await finished(id)).state, 'success'); + const table = events.find( + (event) => + event.executionId === id && + event.type === 'display' && + event.data.kind === 'table', + ); + assert.ok(table); + const labels = table.data.formattedColumns as Record; + assert.deepStrictEqual(labels['1'].slice(0, 3), ['1417.391', '1552.381', '1400.000']); + assert.strictEqual(labels['1'].length, 20); + const first = (table.data.rows as Record[])[0]['1']; + assert.ok(Math.abs(first - 326 / 0.23) < 1e-10); + const page = await client.request<{ + rows: Record[]; + formattedColumns: Record; + }>('inspect', { method: 'dataview_page', - params: { view_id: table.data.viewId, startRow: 1, endRow: 2, formatNumbers: true }, - }, - ); - assert.deepStrictEqual(localized.formattedColumns['1'], ['1552']); - const formatted = await submit('data.frame(value = 1.23456789)'); - await finished(formatted); - const display = events.find( - (event) => event.executionId === formatted && event.data.kind === 'table', - ); - assert.ok(display); - assert.deepStrictEqual(display.data.formattedColumns, { '1': ['1,235'] }); - const saved = (await client.snapshot()).events.find((event) => event.seq === table.seq); - assert.deepStrictEqual(saved?.data.formattedColumns, labels); - }); + params: { + view_id: table.data.viewId, + startRow: 20, + endRow: 24, + formatNumbers: true, + }, + }); + assert.deepStrictEqual(page.formattedColumns['1'], [ + '1400.000', + '1417.391', + '1552.381', + '1400.000', + ]); + assert.strictEqual(page.rows[1]['1'], first); + await finished(await submit('options(digits=4, scipen=999, OutDec=",")')); + const localized = await client.request<{ formattedColumns: Record }>( + 'inspect', + { + method: 'dataview_page', + params: { + view_id: table.data.viewId, + startRow: 1, + endRow: 2, + formatNumbers: true, + }, + }, + ); + assert.deepStrictEqual(localized.formattedColumns['1'], ['1552']); + const formatted = await submit('data.frame(value = 1.23456789)'); + await finished(formatted); + const display = events.find( + (event) => event.executionId === formatted && event.data.kind === 'table', + ); + assert.ok(display); + assert.deepStrictEqual(display.data.formattedColumns, { '1': ['1,235'] }); + const saved = (await client.snapshot()).events.find((event) => event.seq === table.seq); + assert.deepStrictEqual(saved?.data.formattedColumns, labels); + }, + ); test('bounds large Unicode output without corrupting it or disconnecting R', async () => { const id = await submit('cat(strrep("λ🙂", 900000))'); diff --git a/src/test/node/jdvAdapter.test.ts b/src/test/node/jdvAdapter.test.ts new file mode 100644 index 000000000..bc43426bf --- /dev/null +++ b/src/test/node/jdvAdapter.test.ts @@ -0,0 +1,898 @@ +import * as assert from 'node:assert/strict'; +import { + JdvDescriptor, + JdvRequestError, + TreeAdapter, + gridColumnDefs, + gridPageFromRows, + isVectorType, + canRefresh, + cellOpenParams, + revealsPanel, + parseDescriptor, + parseTracking, + tableRowsParams, + toJdvFilter, +} from '../../jdvAdapter'; + +const capabilities = { + seek: true, + sort: true, + filter: true, + known_total: true, + next: false, + children: false, + refresh: true, + track: false, +}; + +const grid: JdvDescriptor = parseDescriptor({ + view: 'v-1', + generation: 2, + kind: 'grid', + access: 'random', + encoding: 'jdv-json', + capabilities, + title: 'df', + collation: 'C', + nrow: 3, + ncol: 6, + columns: [ + { + position: 0, + name: 'a', + type: 'number', + native_type: ['numeric'], + sortable: true, + filterable: true, + }, + { + position: 1, + name: 'b', + type: 'categorical', + native_type: ['factor'], + sortable: true, + filterable: true, + levels: ['x', 'y'], + }, + { + position: 2, + name: 'd', + type: 'date', + native_type: ['Date'], + sortable: true, + filterable: true, + }, + { + position: 3, + name: 't', + type: 'datetime', + native_type: ['POSIXct', 'POSIXt'], + sortable: true, + filterable: true, + tz: 'UTC', + }, + { + position: 4, + name: 'i', + type: 'int64', + native_type: ['integer64'], + sortable: true, + filterable: true, + }, + { + position: 5, + name: 'a', + type: 'other', + native_type: ['AsIs'], + sortable: false, + filterable: false, + drillable: true, + }, + ], +}); + +suite('jdv adapter', () => { + test('validates descriptors', () => { + assert.equal(grid.view, 'v-1'); + assert.equal(grid.generation, 2); + assert.throws(() => parseDescriptor({})); + assert.throws(() => parseDescriptor({ ...grid, columns: undefined })); + assert.throws(() => parseDescriptor({ ...grid, view: '' })); + assert.throws(() => parseDescriptor({ ...grid, generation: 0 })); + assert.throws(() => parseDescriptor({ ...grid, collation: 'locale-ish' })); + assert.throws(() => parseDescriptor({ ...grid, collation: undefined })); + assert.throws(() => parseDescriptor({ ...grid, encoding: 'unsupported-encoding' })); + assert.throws(() => parseDescriptor({ ...grid, nrow: null })); + assert.throws(() => + parseDescriptor({ ...grid, capabilities: { ...capabilities, seek: false } }), + ); + assert.throws(() => + parseDescriptor({ ...grid, capabilities: { ...capabilities, sort: 'yes' } }), + ); + }); + + test('treats omitted capabilities as false', () => { + const bare = parseDescriptor({ ...grid, access: 'snapshot', capabilities: undefined }); + assert.deepEqual(Object.values(bare.capabilities), Array(8).fill(false)); + }); + + test('requires max_page_rows for sequential views', () => { + const sequential = { + view: 'v-3', + generation: 1, + kind: 'grid', + access: 'sequential', + encoding: 'arrow-ipc', + capabilities: { next: true }, + nrow: null, + ncol: 0, + columns: [], + }; + assert.throws(() => parseDescriptor(sequential)); + assert.equal(parseDescriptor({ ...sequential, max_page_rows: 10 }).max_page_rows, 10); + }); + + test('reveals the panel unless the update is tracked or a refresh', () => { + assert.equal(grid.reason, undefined); + assert.equal(revealsPanel(grid), true); + for (const reason of ['register', 'child', 'something-new']) { + assert.equal(revealsPanel(parseDescriptor({ ...grid, reason })), true, reason); + } + for (const reason of ['tracked', 'refresh']) { + const descriptor = parseDescriptor({ ...grid, reason }); + assert.equal(descriptor.reason, reason); + assert.equal(revealsPanel(descriptor), false, reason); + } + // A non-string reason is ignored. + assert.equal(parseDescriptor({ ...grid, reason: 5 }).reason, undefined); + }); + + test('offers refresh only for refreshable, non-sequential views', () => { + assert.equal(canRefresh(grid), true); + assert.equal(canRefresh({ ...grid, access: 'sequential' }), false); + assert.equal( + canRefresh({ ...grid, capabilities: { ...capabilities, refresh: false } }), + false, + ); + assert.equal( + canRefresh( + parseDescriptor({ ...grid, capabilities: { seek: true, known_total: true } }), + ), + false, + ); + }); + + test('builds column definitions from the schema', () => { + const columns = gridColumnDefs(grid); + assert.deepEqual( + columns.map((column) => column.field), + ['0', '1', '2', '3', '4', '5', '6'], + ); + assert.equal(columns[0].sortable, false); + assert.equal(columns[1].type, 'numericColumn'); + assert.ok(columns[1].jdv?.filterOptions.includes('inRange')); + assert.deepEqual(columns[5].jdv?.filterOptions, ['equals', 'inRange']); + assert.equal(columns[6].sortable, false); + // Non-filterable columns still offer the NA operators. + assert.deepEqual(columns[6].jdv, { filterOptions: [], naOptions: true }); + assert.match(columns[2].headerTooltip ?? '', /levels: x, y/); + }); + + test('keeps text operators to strings and factors, and handles durations', () => { + const descriptor = parseDescriptor({ + ...grid, + columns: [ + { + position: 0, + name: 's', + type: 'string', + class: ['character'], + sortable: true, + filterable: true, + }, + { + position: 1, + name: 'l', + type: 'list', + class: ['list'], + sortable: false, + filterable: false, + }, + { + position: 2, + name: 'u', + type: 'future', + class: ['character'], + sortable: true, + filterable: true, + }, + { + position: 3, + name: 'o', + type: 'other', + class: ['character'], + sortable: false, + filterable: true, + }, + { + position: 4, + name: 'dt', + type: 'duration', + class: ['difftime'], + sortable: true, + filterable: true, + units: 'mins', + }, + ], + }); + const columns = gridColumnDefs(descriptor); + assert.ok(columns[1].jdv?.filterOptions.includes('contains')); + // List, other and unknown types are display-only: NA options but no value operators, + // whatever `class` suggests. They still follow `sortable`. + for (const index of [2, 3, 4]) { + assert.deepEqual( + columns[index].jdv, + { filterOptions: [], naOptions: true }, + String(index), + ); + } + assert.equal(columns[3].sortable, true); + assert.equal(columns[4].sortable, false); + for (const field of ['2', '3', '4']) { + assert.deepEqual( + toJdvFilter({ [field]: { filterType: 'text', type: 'jdvNotNa' } }, descriptor), + { op: 'not', arg: { op: 'is_na', col: Number(field) - 1 } }, + ); + assert.throws( + () => + toJdvFilter( + { [field]: { filterType: 'text', type: 'contains', filter: 'x' } }, + descriptor, + ), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported', + field, + ); + } + // The server may allow an unknown type to be filtered; the host still never guesses. + assert.throws( + () => + toJdvFilter( + { '3': { filterType: 'text', type: 'contains', filter: 'x' } }, + descriptor, + ), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported', + ); + + assert.equal(columns[5].type, 'numericColumn'); + assert.equal(columns[5].filter, 'agNumberColumnFilter'); + assert.ok(columns[5].jdv?.filterOptions.includes('inRange')); + assert.equal(columns[5].jdv?.units, 'mins'); + assert.match(columns[5].headerTooltip ?? '', /units: mins/); + assert.deepEqual( + toJdvFilter( + { '5': { filterType: 'number', type: 'lessThanOrEqual', filter: 90 } }, + descriptor, + ), + { op: 'le', col: 4, value: 90 }, + ); + assert.deepEqual( + toJdvFilter( + { + '5': { + filterType: 'number', + type: 'inRange', + filter: 1.5, + filterTo: '2:00:00', + }, + }, + descriptor, + ), + { op: 'between', col: 4, lower: 1.5, upper: '2:00:00' }, + ); + assert.throws( + () => + toJdvFilter( + { '5': { filterType: 'number', type: 'equals', filter: 'abc' } }, + descriptor, + ), + (e: unknown) => e instanceof JdvRequestError && e.code === 'invalid_request', + ); + }); + + test('parses tracking and drops a malformed one', () => { + assert.equal(grid.tracking, undefined); + const lost = { name: 'df', state: 'lost', reason: 'removed', message: 'gone' }; + assert.deepEqual(parseDescriptor({ ...grid, tracking: lost }).tracking, lost); + assert.deepEqual( + parseDescriptor({ ...grid, tracking: { name: 'df', state: 'active', extra: 1 } }) + .tracking, + { name: 'df', state: 'active' }, + ); + for (const tracking of [ + null, + 'df', + { name: 'df' }, + { name: 'df', state: 'weird' }, + { state: 'active' }, + ]) { + assert.equal( + parseDescriptor({ ...grid, tracking }).tracking, + undefined, + JSON.stringify(tracking), + ); + } + assert.equal(parseTracking([]), undefined); + }); + + test('converts sort and filter models', () => { + const params = tableRowsParams( + grid, + 100, + 600, + [ + { colId: '2', sort: 'desc' }, + { colId: '1', sort: 'asc' }, + ], + { + '1': { + filterType: 'number', + operator: 'OR', + conditions: [ + { filterType: 'number', type: 'inRange', filter: 1, filterTo: 3 }, + { filterType: 'number', type: 'jdvIsNa' }, + ], + }, + '2': { filterType: 'text', type: 'notContains', filter: 'X' }, + }, + ); + assert.equal(params.offset, 100); + assert.equal(params.limit, 500); + assert.deepEqual(params.sort, [ + { col: 1, desc: true }, + { col: 0, desc: false }, + ]); + assert.deepEqual(params.filter, { + op: 'and', + args: [ + { + op: 'or', + args: [ + { op: 'between', col: 0, lower: 1, upper: 3 }, + { op: 'is_na', col: 0 }, + ], + }, + { + op: 'and', + args: [ + { op: 'not', arg: { op: 'contains', col: 1, value: 'X' } }, + { op: 'not', arg: { op: 'is_na', col: 1 } }, + ], + }, + ], + }); + }); + + test('converts empty-string filters for text columns only', () => { + const textGrid = parseDescriptor({ + ...grid, + columns: [ + ...grid.columns!.slice(0, 2), + { + position: 2, + name: 's', + type: 'string', + native_type: ['character'], + sortable: true, + filterable: true, + }, + ], + }); + const filter = (field: string, type: string) => + toJdvFilter({ [field]: { filterType: 'text', type } }, textGrid); + const isNa = (col: number) => ({ op: 'is_na', col }); + const empty = (col: number) => ({ op: 'text_equals', col, value: '' }); + // Categorical and string columns; NA stays a separate condition. + for (const col of [1, 2]) { + const field = String(col + 1); + assert.ok(gridColumnDefs(textGrid)[col + 1].jdv?.filterOptions.includes('jdvIsEmpty')); + assert.deepEqual(filter(field, 'jdvIsEmpty'), empty(col)); + assert.deepEqual(filter(field, 'jdvNotEmpty'), { + op: 'and', + args: [ + { op: 'not', arg: empty(col) }, + { op: 'not', arg: isNa(col) }, + ], + }); + assert.deepEqual(filter(field, 'jdvIsNa'), isNa(col)); + // blank and notBlank are no longer mapped to is_na. + for (const type of ['blank', 'notBlank']) { + assert.throws( + () => filter(field, type), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported', + ); + } + } + // Is NA OR Is empty in one AG Grid model. + assert.deepEqual( + toJdvFilter( + { + '3': { + filterType: 'text', + operator: 'OR', + conditions: [ + { filterType: 'text', type: 'jdvIsNa' }, + { filterType: 'text', type: 'jdvIsEmpty' }, + ], + }, + }, + textGrid, + ), + { op: 'or', args: [isNa(2), empty(2)] }, + ); + // A number column has no empty-string filters. + assert.ok(!gridColumnDefs(textGrid)[1].jdv?.filterOptions.includes('jdvIsEmpty')); + assert.throws( + () => filter('1', 'jdvIsEmpty'), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported', + ); + }); + + test('selects displayed columns by field, excluding the row label', () => { + const select = (fields: unknown) => tableRowsParams(grid, 0, 10, [], {}, fields); + assert.deepEqual(select(['0', '3', '1', '3']).cols, [2, 0]); + assert.deepEqual(select(['0']).cols, []); + assert.deepEqual(select([]).cols, []); + for (const fields of [ + undefined, + null, + '1', + ['bad'], + [1], + ['01'], + ['-1'], + ['7'], + ['1.5'], + ['999999999999999999999'], + ['1', '2', '3', '4', '5', '6'], + ['0', '1', '2', '3', '4', '5', '6'], + ]) { + assert.equal('cols' in select(fields), false, JSON.stringify(fields)); + } + const wide = { + ...grid, + columns: Array.from({ length: 300 }, (_, position) => ({ + ...grid.columns![0], + position, + })), + }; + const fields = Array.from({ length: 257 }, (_, index) => String(index + 1)); + assert.equal('cols' in tableRowsParams(wide, 0, 10, [], {}, fields), false); + assert.equal( + tableRowsParams(wide, 0, 10, [], {}, ['0', ...fields.slice(0, 256)]).cols?.length, + 256, + ); + }); + + test('converts boolean filter models', () => { + const booleanGrid = parseDescriptor({ + ...grid, + ncol: 1, + columns: [ + { + position: 0, + name: 'flag', + type: 'boolean', + native_type: ['logical'], + sortable: true, + filterable: true, + }, + ], + }); + assert.deepEqual(toJdvFilter({ '1': { filterType: 'text', type: 'true' } }, booleanGrid), { + op: 'is_true', + col: 0, + }); + assert.deepEqual(toJdvFilter({ '1': { filterType: 'text', type: 'false' } }, booleanGrid), { + op: 'is_false', + col: 0, + }); + }); + + test('converts date, datetime and integer64 operands', () => { + assert.deepEqual( + toJdvFilter( + { + '3': { + filterType: 'date', + type: 'greaterThan', + dateFrom: '2020-01-02 00:00:00', + }, + }, + grid, + ), + { op: 'gt', col: 2, value: '2020-01-02' }, + ); + assert.deepEqual( + toJdvFilter( + { '4': { filterType: 'date', type: 'lessThan', dateFrom: '2020-01-02 10:30:00' } }, + grid, + ), + { op: 'lt', col: 3, value: '2020-01-02T10:30:00' }, + ); + for (const [input, expected] of [ + ['2020-01-02T10:30:00.125Z', '2020-01-02T10:30:00.125Z'], + ['2020-01-02 10:30:00.125+09:00', '2020-01-02T10:30:00.125+09:00'], + ['2020-01-02T10:30:00-05:30', '2020-01-02T10:30:00-05:30'], + ['2020-01-02', '2020-01-02T00:00:00'], + ]) { + assert.deepEqual( + toJdvFilter({ '4': { filterType: 'date', type: 'equals', dateFrom: input } }, grid), + { op: 'eq', col: 3, value: expected }, + ); + } + for (const input of [ + '2020-01-02T10:30:00garbage', + '2020-01-02T10:30:00.1Ztail', + '2020-01-02 10:30', + ]) { + assert.throws( + () => + toJdvFilter( + { '4': { filterType: 'date', type: 'equals', dateFrom: input } }, + grid, + ), + (e: unknown) => e instanceof JdvRequestError && e.code === 'invalid_request', + input, + ); + } + for (const input of ['2020-01-02T10:30:00\n', '2020-01-02T10:30:00\r\n']) { + assert.throws( + () => + toJdvFilter( + { '4': { filterType: 'date', type: 'equals', dateFrom: input } }, + grid, + ), + (e: unknown) => e instanceof JdvRequestError && e.code === 'invalid_request', + JSON.stringify(input), + ); + } + assert.deepEqual( + toJdvFilter( + { '3': { filterType: 'date', type: 'equals', dateFrom: '2020-01-02' } }, + grid, + ), + { op: 'eq', col: 2, value: '2020-01-02' }, + ); + assert.deepEqual( + toJdvFilter( + { '3': { filterType: 'date', type: 'equals', dateFrom: '2020-01-02 00:00:00' } }, + grid, + ), + { op: 'eq', col: 2, value: '2020-01-02' }, + ); + for (const input of ['2020-01-02 10:30:00', '2020-01-02T00:00:00Z', '2020-01-02garbage']) { + assert.throws( + () => + toJdvFilter( + { '3': { filterType: 'date', type: 'equals', dateFrom: input } }, + grid, + ), + (e: unknown) => e instanceof JdvRequestError && e.code === 'invalid_request', + input, + ); + } + for (const input of ['2020-01-02\n', '2020-01-02\r\n']) { + assert.throws( + () => + toJdvFilter( + { '3': { filterType: 'date', type: 'equals', dateFrom: input } }, + grid, + ), + (e: unknown) => e instanceof JdvRequestError && e.code === 'invalid_request', + JSON.stringify(input), + ); + } + assert.deepEqual( + toJdvFilter( + { + '4': { + filterType: 'date', + type: 'inRange', + dateFrom: '2020-01-02 10:30:00.125+09:00', + dateTo: '2020-01-03T11:45:00.875-05:30', + }, + }, + grid, + ), + { + op: 'between', + col: 3, + lower: '2020-01-02T10:30:00.125+09:00', + upper: '2020-01-03T11:45:00.875-05:30', + }, + ); + assert.deepEqual( + toJdvFilter({ '5': { filterType: 'bigint', type: 'equals', filter: '+0012' } }, grid), + { op: 'eq', col: 4, value: '12' }, + ); + assert.throws( + () => + toJdvFilter({ '5': { filterType: 'bigint', type: 'lessThan', filter: '1' } }, grid), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported', + ); + assert.throws( + () => toJdvFilter({ '6': { filterType: 'text', type: 'contains', filter: '1' } }, grid), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported', + ); + assert.throws( + () => tableRowsParams(grid, 0, 10, [{ colId: '6', sort: 'asc' }], undefined), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported', + ); + }); + + test('converts pages to rows using display text and the NA rows', () => { + const page = gridPageFromRows( + { + view: 'v-1', + generation: 2, + total: 3, + total_unfiltered: 5, + offset: 0, + limit: 3, + data: { + source_row: [2, 0, 1], + row_labels: ['r3', 'r1', 'r2'], + columns: [ + { + schema: grid.columns![0], + data: { + values: [null, 1, null], + display: ['Inf', 'NaN', 'NA'], + special: { pos_inf: [0], nan: [1] }, + na_rows: [1, 2], + }, + errors: [], + }, + { + schema: grid.columns![1], + data: { values: [null, 'x', 'y'], na_rows: [0] }, + errors: [], + }, + ], + }, + }, + 0, + 3, + ); + assert.deepEqual(page.rows, [ + { '0': 'r3', __sourceRow: 2, '1': 'Inf', '2': null }, + { '0': 'r1', __sourceRow: 0, '1': 'NaN', '2': 'x' }, + { '0': 'r2', __sourceRow: 1, '1': null, '2': 'y' }, + ]); + assert.equal(page.totalRows, 3); + assert.equal(page.totalUnfiltered, 5); + const unnamed = gridPageFromRows( + { + total: 1, + total_unfiltered: 1, + offset: 0, + limit: 1, + data: { source_row: [6], row_labels: null, columns: [] }, + }, + 0, + 1, + ); + assert.deepEqual(unnamed.rows, [{ '0': '7', __sourceRow: 6 }]); + }); + + test('opens drillable cells by original source row', () => { + // Column 5 is the only drillable column; the grid reports children. + const drillable = parseDescriptor({ + ...grid, + capabilities: { ...capabilities, children: true }, + }); + assert.equal( + gridColumnDefs(drillable).filter((column) => column.jdvDrill !== undefined).length, + 1, + ); + assert.equal(gridColumnDefs(drillable)[6].jdvDrill, 5); + assert.equal(gridColumnDefs(grid)[6].jdvDrill, undefined); + assert.deepEqual(cellOpenParams(drillable, 204, 5), { + view: 'v-1', + generation: 2, + path: [{ kind: 'cell', source_row: 204, column: 5 }], + mode: 'paged', + }); + const rejected = (code: string) => (e: unknown) => + e instanceof JdvRequestError && e.code === code; + assert.throws(() => cellOpenParams(grid, 0, 5), rejected('unsupported')); + assert.throws(() => cellOpenParams(drillable, 0, 0), rejected('invalid_request')); + assert.throws(() => cellOpenParams(drillable, -1, 5), rejected('invalid_request')); + assert.throws(() => cellOpenParams(drillable, 1.5, 5), rejected('invalid_request')); + assert.throws(() => cellOpenParams(drillable, 0, 9), rejected('invalid_request')); + }); + + test('maps List Viewer paths to typed tree paths', () => { + const tree = new TreeAdapter( + parseDescriptor({ + view: 'v-2', + generation: 1, + kind: 'tree', + access: 'random', + encoding: 'jdv-json', + capabilities: { ...capabilities, sort: false, filter: false, children: true }, + path: [], + class: ['list'], + type: 'list', + }), + 'x', + ); + assert.deepEqual(tree.childrenParams([], 1), { + view: 'v-2', + generation: 1, + path: [], + offset: 0, + limit: 200, + attributes: false, + }); + const page = tree.children([], { + path: [], + offset: 0, + total_children: 4, + children: [ + { + index: 1, + label: 'a', + class: ['integer'], + type: 'integer', + summary: 'integer (3): 1, 2, 3', + has_children: true, + viewable: true, + path: [{ type: 'index', index: 1 }], + }, + { + index: 2, + label: 'b', + class: ['list'], + type: 'list', + summary: 'list (1)', + has_children: true, + viewable: true, + path: [{ type: 'index', index: 2 }], + }, + { + index: 3, + label: 'df', + class: ['data.frame'], + type: 'list', + summary: 'data.frame', + has_children: true, + viewable: true, + path: [{ type: 'index', index: 3 }], + }, + ], + }); + assert.equal(page.next_start, 4); + assert.deepEqual(page.children[1], { + label: 'b', + str: 'list (1)', + index: 2, + viewable: true, + has_children: true, + }); + + const vector = tree.open([], 1); + assert.equal(vector.kind, 'navigate'); + assert.equal(vector.kind === 'navigate' && vector.navigation.vector, true); + const elements = tree.children([1], { + path: [{ type: 'index', index: 1 }], + offset: 0, + total_children: 1, + children: [ + { + index: 1, + label: '[1]', + class: ['integer'], + type: 'integer', + summary: 'integer (1): 1', + has_children: false, + viewable: true, + path: [ + { type: 'index', index: 1 }, + { type: 'index', index: 1 }, + ], + }, + ], + }); + assert.deepEqual(elements.children[0], { + label: '[1]', + str: 'integer (1): 1', + index: 1, + viewable: false, + has_children: false, + }); + + const nested = tree.open([], 2); + assert.deepEqual(nested.kind === 'navigate' && nested.navigation, { + title: 'x$b', + path: [2], + vector: false, + breadcrumbs: [ + { label: 'x', path: [] }, + { label: 'b', path: [2] }, + ], + }); + const table = tree.open([], 3); + assert.deepEqual(table, { + kind: 'object', + title: 'x$df', + params: { + view: 'v-2', + generation: 1, + path: [{ type: 'index', index: 3 }], + mode: 'paged', + }, + }); + assert.throws( + () => tree.open([], 9), + (e: unknown) => e instanceof JdvRequestError && e.code === 'unavailable', + ); + }); + + test('classifies atomic vectors by type', () => { + assert.equal(isVectorType('integer', ['factor']), true); + assert.equal(isVectorType('double', ['numeric']), true); + assert.equal(isVectorType('integer', ['matrix', 'array']), false); + assert.equal(isVectorType('list', ['list']), false); + assert.equal(isVectorType(undefined, []), false); + }); + + test('leaves have nothing to open unless they are tables', () => { + const tree = new TreeAdapter( + parseDescriptor({ + view: 'v-2', + generation: 1, + kind: 'tree', + access: 'random', + encoding: 'jdv-json', + capabilities: { ...capabilities, children: true }, + path: [], + class: ['list'], + type: 'list', + }), + 'x', + ); + const page = tree.children([], { + path: [], + offset: 0, + total_children: 2, + children: [ + { + index: 1, + label: 'n', + native_type: ['numeric'], + type: 'double', + summary: 'double (1): 1', + has_children: false, + viewable: true, + path: [{ type: 'index', index: 1 }], + }, + { + index: 2, + label: 'm', + class: ['matrix', 'array'], + type: 'integer', + summary: 'matrix', + has_children: false, + viewable: true, + path: [{ type: 'index', index: 2 }], + }, + ], + }); + assert.deepEqual( + page.children.map((child) => child.viewable), + [false, true], + ); + }); +}); diff --git a/src/test/node/jdvArrow.test.ts b/src/test/node/jdvArrow.test.ts new file mode 100644 index 000000000..33c1dde7b --- /dev/null +++ b/src/test/node/jdvArrow.test.ts @@ -0,0 +1,521 @@ +import * as assert from 'node:assert/strict'; +import * as net from 'node:net'; +import { tableFromArrays, tableToIPC } from 'apache-arrow'; +import { + JdvDescriptor, + JdvRequestError, + JsonObject, + gridColumnDefs, + parseDescriptor, +} from '../../jdvAdapter'; +import { SequentialStream, UNAVAILABLE_TEXT, formatScaled, formatTimestamp } from '../../jdvArrow'; +import { JdvHost, JdvPeer } from '../../jdvTransport'; + +const descriptor: JdvDescriptor = parseDescriptor({ + view: 'v-1', + generation: 1, + kind: 'grid', + access: 'sequential', + encoding: 'arrow-ipc', + capabilities: { + seek: false, + sort: false, + filter: false, + known_total: false, + next: true, + children: false, + }, + nrow: null, + ncol: 3, + max_page_rows: 2000, + columns: [ + { + index: 0, + position: 0, + name: 'a', + type: 'i', + ipc_index: 0, + unsupported: null, + sortable: false, + filterable: false, + }, + { + index: 1, + position: 1, + name: 's', + type: 'u', + ipc_index: 1, + unsupported: null, + sortable: false, + filterable: false, + }, + { + index: 2, + position: 2, + name: 'x', + type: '+w:2', + ipc_index: null, + unsupported: 'not supported', + sortable: false, + filterable: false, + }, + ], +}); + +/** A fake producer returning pages of `size` rows out of `total`. */ +function producer(total: number, size: number) { + let cursor = 0; + let calls = 0; + return { + calls: () => calls, + next: (maxRows: number) => { + calls++; + const rows = Math.min(size, maxRows, total - cursor); + const a = Int32Array.from({ length: rows }, (_, i) => cursor + i + 1); + const s = Array.from({ length: rows }, (_, i) => + (cursor + i) % 3 === 0 ? null : `s${cursor + i + 1}`, + ); + const body = rows > 0 ? tableToIPC(tableFromArrays({ a, s })) : undefined; + const result = { + view: 'v-1', + generation: 1, + first_row: cursor, + rows_fetched: rows, + rows_seen: cursor + rows, + done: cursor + rows >= total && rows === 0, + }; + cursor += rows; + return Promise.resolve({ result, body }); + }, + }; +} + +suite('jdv Arrow views', () => { + test('formats decimals and timestamps exactly', () => { + assert.equal(formatScaled(12345n, 2), '123.45'); + assert.equal(formatScaled(-5n, 3), '-0.005'); + assert.equal(formatScaled(12n, -2), '1200'); + assert.equal(formatTimestamp(1_500n, 1, 'UTC'), '1970-01-01T00:00:01.500Z'); + assert.equal(formatTimestamp(-1n, 3, null), '1969-12-31T23:59:59.999999999'); + assert.equal( + formatTimestamp(0n, 0, 'Asia/Tokyo'), + '1970-01-01T09:00:00+09:00 [Asia/Tokyo]', + ); + }); + + test('builds unsortable, unfilterable columns', () => { + const columns = gridColumnDefs(descriptor); + assert.deepEqual( + columns.map((column) => [column.field, column.type, column.sortable, column.filter]), + [ + ['0', 'textColumn', false, false], + ['1', 'numericColumn', false, false], + ['2', 'textColumn', false, false], + ['3', 'textColumn', false, false], + ], + ); + assert.equal(columns[1].cellDataType, false); + assert.equal(columns[2].cellDataType, undefined); + assert.match(columns[3].headerTooltip ?? '', /unsupported: not supported/); + }); + + test('reads the stream forward and serves blocks from received pages', async () => { + const source = producer(25, 10); + const stream = new SequentialStream(descriptor, source.next, 10); + await stream.ensure(5); + assert.equal(stream.rowsSeen, 10); + assert.equal(source.calls(), 1); + assert.deepEqual(stream.rows(0, 2), [ + { '0': '1', '1': '1', '2': null, '3': '' }, + { '0': '2', '1': '2', '2': 's2', '3': '' }, + ]); + // Concurrent requests share one stream/next at a time. + await Promise.all([stream.ensure(15), stream.ensure(20)]); + assert.equal(stream.rowsSeen, 20); + await stream.ensure(Infinity); + assert.equal(stream.done, true); + assert.equal(stream.rowsSeen, 25); + assert.deepEqual( + stream.rows(23, 100).map((row) => row['1']), + ['24', '25'], + ); + stream.dispose(); + }); + + test('reports evicted rows as unavailable', async () => { + const source = producer(40, 10); + // Room for about two pages. + const stream = new SequentialStream(descriptor, source.next, 10, 2 * (1024 + 800)); + await stream.ensure(40); + assert.ok(stream.unavailableBefore >= 10); + assert.equal(stream.rows(0, 1)[0]['1'], UNAVAILABLE_TEXT); + assert.equal(stream.rows(39, 40)[0]['1'], '40'); + stream.dispose(); + }); + + test('rejects inconsistent page metadata', async () => { + const stream = new SequentialStream( + descriptor, + () => + Promise.resolve({ + result: { first_row: 0, rows_fetched: 5, rows_seen: 4, done: false }, + }), + 10, + ); + await assert.rejects(stream.ensure(1), (e: unknown) => e instanceof JdvRequestError); + stream.dispose(); + }); + + test('returns after a round of empty non-EOF pages', async () => { + let calls = 0; + const stream = new SequentialStream( + descriptor, + () => { + calls++; + assert.ok(calls <= 64, 'empty pages must not start another fetch round'); + return Promise.resolve({ + result: { first_row: 0, rows_fetched: 0, rows_seen: 0, done: false }, + }); + }, + 10, + ); + await stream.ensure(10); + assert.equal(calls, 64); + assert.equal(stream.rowsSeen, 0); + assert.equal(stream.done, false); + stream.dispose(); + }); + + test('treats a gap page as progress', async () => { + let calls = 0; + const stream = new SequentialStream( + descriptor, + () => { + calls++; + const gap = calls === 64; + return Promise.resolve({ + result: { + first_row: gap ? 5 : 0, + rows_fetched: 0, + rows_seen: gap ? 5 : 0, + done: false, + }, + }); + }, + 10, + ); + await stream.ensure(1); + assert.equal(calls, 64); + assert.equal(stream.rowsSeen, 5); + assert.equal(stream.rows(0, 5).length, 5); + assert.equal(stream.rows(0, 5)[0]['1'], UNAVAILABLE_TEXT); + stream.dispose(); + }); +}); + +/** Connect to a host as R would, returning helpers to drive the protocol. */ +async function fakeR(socketPath: string) { + const socket = net.connect(socketPath); + await new Promise((resolve, reject) => { + socket.once('connect', () => resolve()); + socket.once('error', reject); + }); + let buffer = ''; + const lines: Record[] = []; + let waiter: (() => void) | undefined; + socket.on('data', (chunk) => { + buffer += chunk.toString('utf8'); + let newline; + while ((newline = buffer.indexOf('\n')) >= 0) { + lines.push(JSON.parse(buffer.slice(0, newline)) as Record); + buffer = buffer.slice(newline + 1); + waiter?.(); + } + }); + let frames = 0; + const next = async (predicate: (line: Record) => boolean) => { + for (;;) { + const index = lines.findIndex(predicate); + if (index >= 0) { + frames += index + 1; + const [line] = lines.splice(0, index + 1).slice(-1); + return line; + } + await new Promise((resolve) => { + waiter = resolve; + }); + } + }; + const send = (value: unknown, body?: Uint8Array) => { + socket.write(JSON.stringify(value) + '\n'); + if (body) { + socket.write(body); + } + }; + return { socket, next, send, ack: () => send({ method: 'ack', params: { frames } }) }; +} + +function helloParams(limits: Record = {}): JsonObject { + return { + protocol: 1, + session: 'producer-1', + wake: true, + producer: { name: 'jdv', version: '0.0.9000', runtime: { name: 'R', version: '4.5.2' } }, + limits: { + max_unacked_frames: 264, + max_unacked_bytes: 1048577, + queue_messages: 256, + queue_bytes: 16777216, + max_request_bytes: 1048576, + max_message_bytes: 10485760, + ...limits, + }, + }; +} + +suite('jdv transport', () => { + test('rejects requests that exceed byte limits before or after hello', async () => { + for (const limits of [ + { max_unacked_bytes: 160, max_request_bytes: 4096 }, + { max_unacked_bytes: 4096, max_request_bytes: 160 }, + ]) { + let peer: JdvPeer | undefined; + const host = await JdvHost.create({ + viewOpen: (connection) => { + peer = connection; + }, + viewClosed: () => undefined, + viewUpdateFailed: () => undefined, + viewTracking: () => undefined, + disconnected: () => undefined, + }); + try { + const r = await fakeR(host.socketPath); + while (!host.connection) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + peer = host.connection; + + const queuedTooLarge = peer!.request('grid/rows', { text: 'x'.repeat(200) }); + const queuedSmall = peer!.request('view/close', { view: 'v-1' }); + r.send({ method: 'hello', params: helloParams(limits) }); + await assert.rejects( + queuedTooLarge, + (e: unknown) => e instanceof JdvRequestError && e.code === 'limit_exceeded', + ); + const small = await r.next((line) => line.method === 'view/close'); + r.send({ id: small.id, result: {}, warnings: [] }); + await queuedSmall; + + await assert.rejects( + peer!.request('grid/rows', { text: 'x'.repeat(200) }), + (e: unknown) => e instanceof JdvRequestError && e.code === 'limit_exceeded', + ); + r.socket.destroy(); + } finally { + host.dispose(); + } + } + }); + + test('exchanges requests, wakes and Arrow bodies with R', async () => { + const opened: string[] = []; + const failures: JsonObject[] = []; + const tracking: JsonObject[] = []; + let peer: JdvPeer | undefined; + const host = await JdvHost.create({ + viewOpen: (connection, params) => { + peer = connection; + opened.push(String(params.view)); + }, + viewClosed: () => undefined, + viewUpdateFailed: (_connection, params) => { + failures.push(params); + }, + viewTracking: (_connection, params) => { + tracking.push(params); + }, + disconnected: () => undefined, + }); + try { + const r = await fakeR(host.socketPath); + r.send({ + method: 'hello', + params: helloParams({ max_unacked_frames: 4, max_unacked_bytes: 65536 }), + }); + r.send({ method: 'view/open', params: { view: 'v-1', generation: 1 } }); + while (!peer) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + assert.deepEqual(opened, ['v-1']); + + // A tracked update that failed is reported with the displayed generation. + r.send({ + method: 'view/update_failed', + params: { + view: 'v-1', + generation: 1, + code: 'limit_exceeded', + message: 'too large', + }, + }); + while (failures.length === 0) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + assert.equal(failures[0].message, 'too large'); + + // A tracking change arrives before the response of the request that caused it. + const tracked = peer.request('view/track', { + view: 'v-1', + generation: 1, + enabled: false, + }); + const trackRequest = await r.next((line) => line.method === 'view/track'); + assert.deepEqual(trackRequest.params, { view: 'v-1', generation: 1, enabled: false }); + const paused = { name: 'df', state: 'paused' }; + r.send({ + method: 'view/tracking', + params: { view: 'v-1', generation: 1, tracking: paused }, + }); + r.send({ + id: trackRequest.id, + result: { view: 'v-1', generation: 1, tracking: paused }, + warnings: [], + }); + assert.deepEqual(await tracked, { view: 'v-1', generation: 1, tracking: paused }); + assert.deepEqual(tracking, [{ view: 'v-1', generation: 1, tracking: paused }]); + + const response = peer.requestWithBody('stream/next', { + view: 'v-1', + generation: 1, + max_rows: 10, + }); + const request = await r.next((line) => line.method === 'stream/next'); + // Wakes repeat while the request is pending. + await r.next((line) => line.method === 'wake'); + r.ack(); + const body = Uint8Array.from([1, 2, 3, 10, 4]); + r.send( + { + id: request.id, + warnings: [], + result: { + view: 'v-1', + generation: 1, + first_row: 0, + rows_fetched: 1, + rows_seen: 1, + done: false, + transfer: { encoding: 'arrow-ipc', byte_length: body.length }, + }, + }, + body, + ); + r.send({ id: 'unknown', result: {}, warnings: [] }); + const { result, body: received } = await response; + assert.deepEqual(Array.from(received!), [1, 2, 3, 10, 4]); + assert.equal((result as { rows_seen: number }).rows_seen, 1); + + const failed = peer.request('grid/rows', { view: 'v-1', generation: 1 }); + const second = await r.next((line) => line.method === 'grid/rows'); + r.send({ id: second.id, error: { code: 'unsupported', message: 'no' }, warnings: [] }); + await assert.rejects( + failed, + (e: unknown) => e instanceof JdvRequestError && e.code === 'unsupported', + ); + + const pending = peer.request('view/close', { view: 'v-1', generation: 1 }); + r.socket.destroy(); + await assert.rejects( + pending, + (e: unknown) => e instanceof JdvRequestError && e.code === 'view_not_found', + ); + } finally { + host.dispose(); + } + }); + + test('rejects an invalid hello and treats unknown error codes as producer failures', async () => { + let reason = ''; + let peer: JdvPeer | undefined; + const host = await JdvHost.create({ + viewOpen: (connection) => { + peer = connection; + }, + viewClosed: () => undefined, + viewUpdateFailed: () => undefined, + viewTracking: () => undefined, + disconnected: (_peer, why) => { + reason = why; + }, + }); + try { + const bad = await fakeR(host.socketPath); + bad.send({ + method: 'hello', + params: { ...helloParams(), limits: { max_unacked_frames: 4 } }, + }); + await new Promise((resolve) => bad.socket.once('close', () => resolve())); + assert.match(reason, /Invalid jdv hello/); + + const good = await fakeR(host.socketPath); + good.send({ method: 'hello', params: helloParams() }); + good.send({ method: 'view/open', params: { view: 'v-1', generation: 1 } }); + while (!peer) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + const failed = peer.request('grid/rows', { view: 'v-1', generation: 1 }); + const request = await good.next((line) => line.method === 'grid/rows'); + good.send({ + id: request.id, + error: { code: 'some_future_code', message: 'boom' }, + warnings: [], + }); + await assert.rejects( + failed, + (e: unknown) => + e instanceof JdvRequestError && + e.code === 'producer_error' && + /boom \(some_future_code\)/.test(e.message), + ); + } finally { + host.dispose(); + } + }); + + test('sends wakes only when the producer asks for them', async () => { + let peer: JdvPeer | undefined; + const host = await JdvHost.create({ + viewOpen: (connection) => { + peer = connection; + }, + viewClosed: () => undefined, + viewUpdateFailed: () => undefined, + viewTracking: () => undefined, + disconnected: () => undefined, + }); + try { + const quiet = await fakeR(host.socketPath); + quiet.send({ method: 'hello', params: { ...helloParams(), wake: false } }); + quiet.send({ method: 'view/open', params: { view: 'v-1', generation: 1 } }); + while (!peer) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + void peer.request('view/close', { view: 'v-1', generation: 1 }).catch(() => undefined); + const request = await quiet.next((line) => line.method === 'view/close'); + // A later request arrives without a wake in between. + await new Promise((resolve) => setTimeout(resolve, 200)); + quiet.send({ id: request.id, result: {}, warnings: [] }); + quiet.ack(); + await assert.rejects( + Promise.race([ + quiet.next((line) => line.method === 'wake'), + new Promise((_, reject) => setTimeout(() => reject(new Error('no wake')), 150)), + ]), + /no wake/, + ); + } finally { + host.dispose(); + } + }); +}); diff --git a/src/test/node/jdvGrid.test.ts b/src/test/node/jdvGrid.test.ts new file mode 100644 index 000000000..17898c365 --- /dev/null +++ b/src/test/node/jdvGrid.test.ts @@ -0,0 +1,104 @@ +import * as assert from 'node:assert/strict'; +import { JdvDescriptor, JdvRequestError, JsonObject, parseDescriptor } from '../../jdvAdapter'; +import { fetchGridPage } from '../../jdvGrid'; + +const descriptor: JdvDescriptor = parseDescriptor({ + view: 'v', + generation: 1, + kind: 'grid', + access: 'random', + encoding: 'jdv-json', + capabilities: { seek: true, known_total: true }, + collation: 'C', + nrow: 5, + ncol: 2, + columns: [ + { position: 0, name: 'a', type: 'string' }, + { position: 1, name: 'b', type: 'string' }, + ], +}); +const message = { startRow: 0, endRow: 5, columns: ['1'] }; +const page = (offset: number, count: number) => ({ + total: 5, + total_unfiltered: 7, + offset, + limit: count, + data: { + source_row: Array.from({ length: count }, (_, index) => offset + index), + row_labels: null, + columns: [], + }, +}); + +suite('jdv grid block fetching', () => { + test('keeps selected columns across short pages', async () => { + const requests: JsonObject[] = []; + const result = await fetchGridPage(descriptor, message, (method, params) => { + assert.equal(method, 'grid/rows'); + requests.push(params); + return Promise.resolve(page(Number(params.offset), Math.min(2, Number(params.limit)))); + }); + assert.deepEqual( + requests.map((params) => [params.offset, params.limit, params.cols]), + [ + [0, 5, [0]], + [2, 3, [0]], + [4, 1, [0]], + ], + ); + assert.deepEqual( + result.rows.map((row) => row.__sourceRow), + [0, 1, 2, 3, 4], + ); + assert.equal(result.lastRow, 5); + assert.equal(result.totalUnfiltered, 7); + }); + + test('propagates limit_exceeded for selected, empty and all columns without retrying', async () => { + for (const columns of [['1'], [], undefined]) { + let calls = 0; + const error = new JdvRequestError('limit_exceeded', 'row too large'); + await assert.rejects( + fetchGridPage(descriptor, { ...message, columns }, (_method, params) => { + calls++; + assert.deepEqual( + params.cols, + columns?.map((field) => Number(field) - 1), + ); + return Promise.reject(error); + }), + (received: unknown) => received === error, + ); + assert.equal(calls, 1); + } + }); + + test('propagates other errors without retrying', async () => { + let calls = 0; + await assert.rejects( + fetchGridPage(descriptor, message, () => { + calls++; + return Promise.reject(new JdvRequestError('expired', 'expired')); + }), + /expired/, + ); + assert.equal(calls, 1); + }); + + test('stops on an empty page or the total row count', async () => { + let calls = 0; + const result = await fetchGridPage(descriptor, message, () => { + calls++; + return Promise.resolve(page(0, 0)); + }); + assert.equal(calls, 1); + assert.deepEqual(result.rows, []); + calls = 0; + const full = await fetchGridPage(descriptor, { ...message, endRow: 10 }, () => { + calls++; + return Promise.resolve(page(0, 5)); + }); + assert.equal(calls, 1); + assert.equal(full.rows.length, 5); + }); +}); diff --git a/src/test/node/jdvRowCount.test.ts b/src/test/node/jdvRowCount.test.ts new file mode 100644 index 000000000..9f771850e --- /dev/null +++ b/src/test/node/jdvRowCount.test.ts @@ -0,0 +1,79 @@ +import * as assert from 'node:assert/strict'; +import * as fs from 'node:fs'; +import * as vm from 'node:vm'; +import { getDataViewerScript } from '../../dataViewer'; + +interface Viewer { + applyJdvRowCount(session: string, value: typeof hint): void; + bindRowCount(state: ReturnType): void; + panels: Map>; +} + +// Load the actual viewer module with its VS Code dependencies stubbed out. +function viewer(): Viewer { + return vm.runInNewContext( + fs.readFileSync(require.resolve('../../jdvViewer'), 'utf8') + + '\n;({ applyJdvRowCount, bindRowCount, panels })', + { + exports: {}, + require: (name: string) => + name === './session' ? { getDocumentGeneration: () => 1 } : {}, + }, + ) as Viewer; +} +const hint = { address: 'socket-a', view: 'v-1', generation: 1, nrow: 1000 }; +function panel() { + const messages: { knownTotal: number }[] = []; + return { + hintScope: JSON.stringify(['session-a', 'socket-a']), + stream: {}, + descriptor: { view: 'v-1', generation: 1 }, + knownTotal: undefined, + panel: { + webview: { postMessage: (message: { knownTotal: number }) => messages.push(message) }, + }, + messages, + }; +} + +suite('jdv row-count hints', () => { + test('takes a hint received before view/open', () => { + const host = viewer(), + state = panel(); + host.applyJdvRowCount('session-a', hint); + host.bindRowCount(state); + assert.equal(state.knownTotal, 1000); + }); + + test('updates an open panel when its hint arrives later', () => { + const host = viewer(), + state = panel(); + host.panels.set(JSON.stringify(['session-a', 'v-1']), state); + host.applyJdvRowCount('session-a', hint); + assert.equal(state.knownTotal, 1000); + assert.equal(state.messages[0].knownTotal, 1000); + }); + + test('does not attach another scope or generation to the panel', () => { + const host = viewer(), + state = panel(); + host.panels.set(JSON.stringify(['session-a', 'v-1']), state); + host.applyJdvRowCount('session-b', hint); + host.applyJdvRowCount('session-a', { ...hint, address: 'socket-b' }); + host.applyJdvRowCount('session-a', { ...hint, generation: 2 }); + host.bindRowCount(state); + assert.equal(state.knownTotal, undefined); + assert.equal(state.messages.length, 0); + }); + + test('formats unknown, known and complete stream counts', () => { + const format = vm.runInNewContext(getDataViewerScript() + ';formatSequentialRowCount') as ( + loaded: number, + knownTotal: number | undefined, + done: boolean, + ) => string; + assert.equal(format(2000, undefined, false), '2,000 / ??? rows'); + assert.equal(format(2000, 1000000, false), '2,000 / 1,000,000 rows'); + assert.equal(format(1000000, 1000000, true), '1,000,000 rows'); + }); +}); diff --git a/src/test/node/jdvTable.test.ts b/src/test/node/jdvTable.test.ts new file mode 100644 index 000000000..d8367afbd --- /dev/null +++ b/src/test/node/jdvTable.test.ts @@ -0,0 +1,444 @@ +import * as assert from 'node:assert/strict'; +import { + createListAdapter, + isJdvOutput, + listRequest, + normalizeJdvDisplay, + queryJdvTablePage, +} from '../../interactive/jdvTable'; +import { + columnOperators, + filterNeedsValue, + filterOperatorLabel, + TableColumn, +} from '../../interactive/tableQuery'; + +const capabilities = { + seek: true, + sort: true, + filter: true, + known_total: true, + next: false, + children: false, + refresh: false, + track: false, +}; +const gridDescriptor = { + view: 'h-1', + generation: 1, + kind: 'grid', + access: 'random', + encoding: 'jdv-json', + capabilities, + collation: 'C', + nrow: 50, + ncol: 3, + columns: [ + { position: 0, name: 'n', type: 'integer', sortable: true, filterable: true }, + { position: 1, name: 'd', type: 'date', sortable: true, filterable: true }, + { position: 2, name: 'i', type: 'int64', sortable: true, filterable: true }, + ], +}; +const rowsPage = (offset: number, count: number, total = 50) => + JSON.stringify({ + view: 'h-1', + generation: 1, + total, + total_unfiltered: 50, + offset, + limit: count, + data: { + source_row: Array.from({ length: count }, (_, i) => offset + i), + row_labels: null, + columns: [ + { + schema: gridDescriptor.columns[0], + data: { + values: Array.from({ length: count }, (_, i) => offset + i + 1), + display: Array.from({ length: count }, (_, i) => String(offset + i + 1)), + na_rows: [], + }, + errors: {}, + }, + ], + }, + }); +const display = (extra: Record = {}) => ({ + kind: 'table', + handleId: 'h1', + descriptor: JSON.stringify(gridDescriptor), + page: rowsPage(0, 2), + sourceRows: 50, + ...extra, +}); + +suite('Interactive jdv tables', () => { + test('turns a handle preview into renderer data and keeps the descriptor', () => { + assert.equal(isJdvOutput(display()), true); + assert.equal(isJdvOutput({ kind: 'table', viewId: 'x' }), false); + const data = normalizeJdvDisplay(display()); + assert.equal(data.page, undefined); + assert.equal(data.totalRows, 50); + assert.equal(data.sourceRows, 50); + assert.deepEqual( + (data.columns as TableColumn[]).map((column) => column.field), + ['0', '1', '2', '3'], + ); + assert.deepEqual(data.rows, [ + { '0': '1', __sourceRow: 0, '1': '1' }, + { '0': '2', __sourceRow: 1, '1': '2' }, + ]); + assert.equal((data.descriptor as { view: string }).view, 'h-1'); + // Output that is not from a handle is left alone. + const legacy = { kind: 'table', viewId: 'v' }; + assert.equal(normalizeJdvDisplay(legacy), legacy); + assert.throws(() => normalizeJdvDisplay({ ...display(), kind: 'list' })); + }); + + test('combines raw short preview pages and keeps their display precision', () => { + const first = JSON.parse(rowsPage(0, 2)) as { + data: { columns: { data: { display: string[] } }[] }; + }; + first.data.columns[0].data.display[0] = '3.141592653589793'; + const onePage = normalizeJdvDisplay(display({ page: [JSON.stringify(first)] })); + assert.equal((onePage.rows as unknown[]).length, 2); + const data = normalizeJdvDisplay( + display({ page: [JSON.stringify(first), rowsPage(2, 2)] }), + ); + const rows = data.rows as Record[]; + assert.equal(rows.length, 4); + assert.deepEqual( + rows.map((row) => row.__sourceRow), + [0, 1, 2, 3], + ); + assert.equal(rows[0]['1'], '3.141592653589793'); + assert.equal(rows[3]['1'], '4'); + }); + + test('marks short and empty previews for completion from page zero', async () => { + for (const page of [[], [rowsPage(0, 2)]]) { + const data = normalizeJdvDisplay(display({ page })); + assert.equal(data.totalRows, 50); + assert.equal(data.previewPartial, true); + const offsets: number[] = []; + const result = await queryJdvTablePage(data, { start: 0, size: 20 }, (request) => { + const params = request.params.params as { offset: number; limit: number }; + offsets.push(params.offset); + return Promise.resolve({ json: rowsPage(params.offset, params.limit) }); + }); + assert.deepEqual(offsets, [0]); + assert.deepEqual( + (result.rows as Record[]).map((row) => row.__sourceRow), + Array.from({ length: 20 }, (_, i) => i), + ); + } + assert.equal(normalizeJdvDisplay(display({ page: rowsPage(0, 20) })).previewPartial, false); + assert.equal( + normalizeJdvDisplay(display({ page: rowsPage(0, 2, 2) })).previewPartial, + false, + ); + }); + + test('offers the operators of the column type', () => { + const columns = normalizeJdvDisplay(display()).columns as TableColumn[]; + assert.deepEqual(columnOperators(columns[1]).slice(-2), ['jdvIsNa', 'jdvNotNa']); + assert.equal(filterOperatorLabel('jdvIsNa'), 'is NA'); + assert.equal(filterOperatorLabel('jdvNotNa'), 'is not NA'); + assert.ok(columnOperators(columns[1]).includes('greaterThan')); + assert.ok(!columnOperators(columns[1]).includes('blank')); + assert.deepEqual(columnOperators(columns[3]), ['equals', 'jdvIsNa', 'jdvNotNa']); + assert.equal(filterOperatorLabel('blank'), 'is blank'); + assert.equal(filterOperatorLabel('notBlank'), 'is not blank'); + assert.equal(filterNeedsValue('jdvIsNa'), false); + // Text columns additionally offer empty-string filters, separate from NA. + const text = { + field: '1', + headerName: 's', + jdv: { filterOptions: ['contains', 'jdvIsEmpty', 'jdvNotEmpty'] }, + }; + assert.deepEqual(columnOperators(text), [ + 'contains', + 'jdvIsEmpty', + 'jdvNotEmpty', + 'jdvIsNa', + 'jdvNotNa', + ]); + assert.equal(filterOperatorLabel('jdvIsEmpty'), 'is empty'); + assert.equal(filterOperatorLabel('jdvNotEmpty'), 'is not empty'); + assert.equal(filterNeedsValue('jdvIsEmpty'), false); + assert.equal(filterNeedsValue('jdvNotEmpty'), false); + assert.equal(filterNeedsValue('contains'), true); + assert.ok(!columnOperators(columns[1]).includes('inRange')); + }); + + test('offers logical NA filters for jdv booleans and preserves legacy options', () => { + const descriptor = { + ...gridDescriptor, + ncol: 1, + columns: [ + { position: 0, name: 'flag', type: 'boolean', sortable: true, filterable: true }, + ], + }; + const data = normalizeJdvDisplay(display({ descriptor: JSON.stringify(descriptor) })); + const column = (data.columns as TableColumn[])[1]; + const operators = columnOperators(column); + assert.deepEqual(operators, ['true', 'false', 'jdvIsNa', 'jdvNotNa']); + assert.deepEqual(operators.map(filterOperatorLabel), [ + 'is TRUE', + 'is FALSE', + 'is NA', + 'is not NA', + ]); + assert.ok(operators.every((operator) => !filterNeedsValue(operator))); + assert.deepEqual( + columnOperators({ field: '1', headerName: 'flag', type: 'booleanColumn' }), + ['true', 'false', 'blank', 'notBlank'], + ); + const disabled = normalizeJdvDisplay( + display({ + descriptor: JSON.stringify({ + ...descriptor, + capabilities: { ...capabilities, filter: false }, + }), + }), + ); + assert.equal((disabled.columns as TableColumn[])[1].filter, false); + assert.equal((disabled.columns as TableColumn[])[1].jdv, undefined); + }); + + test('queries the handle for sorted, filtered and refilled pages', async () => { + const data = normalizeJdvDisplay(display()); + const requests: Record[] = []; + // The handle returns at most 3 rows per request, as when a page exceeds its byte budget. + const inspect = (request: { method: string; params: Record }) => { + assert.equal(request.method, 'jdv_handle_request'); + assert.equal(request.params.handle_id, 'h1'); + assert.equal(request.params.method, 'grid/rows'); + const params = request.params.params as Record; + requests.push(params); + return Promise.resolve({ json: rowsPage(params.offset, Math.min(3, params.limit)) }); + }; + const result = await queryJdvTablePage( + data, + { + start: 20, + size: 20, + sortModel: [{ colId: '1', sort: 'desc' }], + filterModel: { '2': { type: 'greaterThan', filter: '2026-01-01' } }, + }, + inspect, + ); + assert.equal((result.rows as unknown[]).length, 20); + assert.equal(result.startRow, 20); + assert.equal(result.totalRows, 50); + assert.equal(result.live, true); + assert.deepEqual( + requests.slice(0, 2).map((params) => [params.offset, params.limit]), + [ + [20, 20], + [23, 17], + ], + ); + assert.deepEqual(requests[0].sort, [{ col: 0, desc: true }]); + assert.deepEqual(requests[0].filter, { op: 'gt', col: 1, value: '2026-01-01' }); + assert.equal('view' in requests[0] || 'generation' in requests[0], false); + + requests.length = 0; + await queryJdvTablePage(data, { start: 0, size: 20 }, inspect); + assert.equal('sort' in requests[0] || 'filter' in requests[0], false); + }); + + test('fills a 100-row page after more than twenty short responses and stops on no progress', async () => { + const data = normalizeJdvDisplay(display()); + const offsets: number[] = []; + const inspect = (request: { params: Record }) => { + const params = request.params.params as { offset: number; limit: number }; + offsets.push(params.offset); + return Promise.resolve({ + json: rowsPage(params.offset, Math.min(1, params.limit), 150), + }); + }; + const result = await queryJdvTablePage(data, { start: 0, size: 100 }, inspect); + assert.equal((result.rows as unknown[]).length, 100); + assert.deepEqual( + (result.rows as Record[]).map((row) => row.__sourceRow), + Array.from({ length: 100 }, (_, i) => i), + ); + assert.equal(offsets.length, 100); + assert.equal(result.totalRows, 150); + assert.deepEqual(offsets.slice(0, 3), [0, 1, 2]); + const following = await queryJdvTablePage(data, { start: 100, size: 100 }, inspect); + assert.equal((following.rows as unknown[]).length, 50); + assert.deepEqual( + (following.rows as Record[]).map((row) => row.__sourceRow), + Array.from({ length: 50 }, (_, i) => i + 100), + ); + assert.deepEqual(offsets.slice(100, 103), [100, 101, 102]); + assert.equal(offsets.at(-1), 149); + + let calls = 0; + const noProgress = await queryJdvTablePage(data, { start: 0, size: 20 }, () => { + calls++; + return Promise.resolve({ json: rowsPage(0, 0) }); + }); + assert.equal(calls, 1); + assert.deepEqual(noProgress.rows, []); + }); + + test('surfaces a jdv limit_exceeded error instead of retrying it', async () => { + const data = normalizeJdvDisplay(display()); + let calls = 0; + await assert.rejects( + queryJdvTablePage(data, { start: 0, size: 20 }, () => { + calls++; + return Promise.reject(new Error('limit_exceeded: row is too large')); + }), + /limit_exceeded/, + ); + assert.equal(calls, 1); + }); + + test('rejects queries the column does not support', async () => { + const data = normalizeJdvDisplay(display()); + const inspect = () => Promise.reject(new Error('not reached')); + await assert.rejects( + queryJdvTablePage( + data, + { start: 0, filterModel: { '3': { type: 'lessThan', filter: '1' } } }, + inspect, + ), + /Invalid column filter/, + ); + await assert.rejects( + queryJdvTablePage( + data, + { start: 0, sortModel: [{ colId: '9', sort: 'asc' }] }, + inspect, + ), + /cannot be sorted/, + ); + await assert.rejects( + queryJdvTablePage(data, { start: 0, size: 7 }, inspect), + /Choose 20, 50 or 100/, + ); + }); + + test('serves list actions from the handle and opens tables in the Data Viewer', async () => { + const tree = { + view: 'h-2', + generation: 1, + kind: 'tree', + access: 'random', + encoding: 'jdv-json', + capabilities: { ...capabilities, sort: false, filter: false, children: true }, + path: [], + class: ['list'], + type: 'list', + }; + const children = (path: unknown[], items: unknown[]) => + JSON.stringify({ + view: 'h-2', + generation: 1, + path, + offset: 0, + children: items, + total_children: items.length, + }); + const root = children( + [], + [ + { + index: 1, + label: 'inner', + class: ['list'], + type: 'list', + summary: 'list (1)', + has_children: true, + viewable: true, + path: [{ type: 'index', index: 1 }], + }, + { + index: 2, + label: '', + class: ['data.frame'], + type: 'list', + summary: 'data.frame (1)', + has_children: true, + viewable: true, + path: [{ type: 'index', index: 2 }], + }, + ], + ); + const data = normalizeJdvDisplay({ + kind: 'list', + handleId: 'h2', + descriptor: JSON.stringify(tree), + page: root, + }); + assert.deepEqual( + (data.children as { label: string }[]).map((child) => child.label), + ['inner', '[[2]]'], + ); + const calls: Record[] = []; + const inspect = (request: { method: string; params: Record }) => { + calls.push(request.params); + return Promise.resolve({ + json: (request.params.params as { path: unknown[] }).path.length + ? children( + [{ type: 'index', index: 1 }], + [ + { + index: 1, + label: 'x', + class: ['numeric'], + type: 'double', + summary: 'double (1): 1', + has_children: false, + viewable: true, + path: [ + { type: 'index', index: 1 }, + { type: 'index', index: 1 }, + ], + }, + ], + ) + : root, + }); + }; + const adapter = await createListAdapter(data, inspect); + const navigate = await listRequest( + adapter, + data, + { action: 'listItem', path: [], index: 1 }, + inspect, + ); + assert.deepEqual( + (navigate.result as { navigation: { path: number[] } }).navigation.path, + [1], + ); + const page = await listRequest( + adapter, + data, + { action: 'listPage', path: [1], start: 1 }, + inspect, + ); + assert.equal((page.result as { children: { label: string }[] }).children[0].label, 'x'); + assert.deepEqual(calls.at(-1)?.params, { + path: [{ type: 'index', index: 1 }], + offset: 0, + limit: 200, + attributes: false, + }); + const table = await listRequest( + adapter, + data, + { action: 'listItem', path: [], index: 2 }, + inspect, + ); + assert.deepEqual(table.open, { path: [{ type: 'index', index: 2 }], title: 'List[[2]]' }); + assert.deepEqual( + (await listRequest(adapter, data, { action: 'list', path: [] }, inspect)).open, + { path: [], title: 'List' }, + ); + }); +}); diff --git a/src/test/node/jdvTransport.test.ts b/src/test/node/jdvTransport.test.ts new file mode 100644 index 000000000..0a772a45d --- /dev/null +++ b/src/test/node/jdvTransport.test.ts @@ -0,0 +1,155 @@ +import * as assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { Socket } from 'node:net'; +import { JdvPeer } from '../../jdvTransport'; +import { JsonObject } from '../../jdvAdapter'; + +class FakeSocket extends EventEmitter { + destroyed = false; + write(_bytes: Buffer): boolean { + return true; + } + destroy(): void { + this.destroyed = true; + } + setNoDelay(): this { + return this; + } +} + +function connection() { + const socket = new FakeSocket(); + const reasons: string[] = []; + const notifications: JsonObject[] = []; + const peer = new JdvPeer(socket as unknown as Socket, { + notification: (_peer, _method, params) => notifications.push(params), + closed: (_peer, reason) => reasons.push(reason), + }); + socket.emit( + 'data', + Buffer.from( + JSON.stringify({ + method: 'hello', + params: { + protocol: 1, + session: 'test', + producer: { name: 'test', version: '1' }, + limits: { + max_unacked_frames: 32, + max_unacked_bytes: 10485760, + queue_messages: 32, + queue_bytes: 10485760, + max_request_bytes: 10485760, + max_message_bytes: 10485760, + }, + }, + }) + '\n', + ), + ); + return { socket, peer, reasons, notifications }; +} + +suite('jdv transport framing', () => { + test('receives a large UTF-8 JSON line in tiny chunks', async () => { + const { socket, peer } = connection(); + try { + const response = peer.request('grid/rows', {}); + const value = 'λ🙂'.repeat(350000); + const frame = Buffer.from( + JSON.stringify({ id: 'vscode-r-1', result: { value } }) + '\n', + ); + for (let offset = 0; offset < frame.length; offset += 17) { + socket.emit('data', frame.subarray(offset, offset + 17)); + } + assert.deepEqual(await response, { value }); + assert.equal(peer.isClosed, false); + } finally { + peer.close('test complete'); + } + }); + + test('dispatches several messages and retains a partial next line', () => { + const { socket, peer, notifications } = connection(); + try { + const line = (value: number) => + JSON.stringify({ method: 'view/open', params: { value } }) + '\n'; + const third = line(3); + socket.emit('data', Buffer.from(line(1) + line(2) + third.slice(0, 10))); + assert.deepEqual(notifications, [{ value: 1 }, { value: 2 }]); + socket.emit('data', Buffer.from(third.slice(10))); + assert.deepEqual(notifications, [{ value: 1 }, { value: 2 }, { value: 3 }]); + } finally { + peer.close('test complete'); + } + }); + + test('receives Arrow headers and bodies across every split and in one-byte chunks', async () => { + const body = Buffer.from([0, 10, 255, 13, 123, 34, 0, 10]); + const result = { + view: 'v', + generation: 1, + transfer: { encoding: 'arrow-ipc', byte_length: body.length }, + }; + const header = Buffer.from(JSON.stringify({ id: 'vscode-r-1', result }) + '\n'); + const following = Buffer.from(JSON.stringify({ id: 'vscode-r-2', result: 'next' }) + '\n'); + const frame = Buffer.concat([header, body, following]); + const deliveries: Buffer[][] = []; + for (let split = 1; split < frame.length; split++) { + deliveries.push([frame.subarray(0, split), frame.subarray(split)]); + } + deliveries.push(Array.from(frame, (_value, index) => frame.subarray(index, index + 1))); + for (const chunks of deliveries) { + const { socket, peer } = connection(); + try { + const response = peer.requestWithBody('stream/next', { view: 'v', generation: 1 }); + const next = peer.request('grid/rows', {}); + for (const chunk of chunks) { + socket.emit('data', chunk); + } + assert.deepEqual(await response, { result, body: new Uint8Array(body) }); + assert.equal(await next, 'next'); + assert.equal(peer.isClosed, false); + } finally { + peer.close('test complete'); + } + } + }); + + test('closes on an oversized unterminated line or a complete oversized line', () => { + for (const newline of ['', '\n']) { + const { socket, peer, reasons } = connection(); + const line = Buffer.from( + JSON.stringify({ result: 'x'.repeat(10 * 1024 * 1024) }) + newline, + ); + for (let offset = 0; offset < line.length; offset += 16384) { + socket.emit('data', line.subarray(offset, offset + 16384)); + } + assert.equal(peer.isClosed, true); + assert.equal(socket.destroyed, true); + assert.match(reasons[0], /exceeds the size limit/); + } + }); + + test('closes on invalid binary frame lengths and missing response ids', () => { + for (const [id, length] of [ + ['x', -1], + ['x', 1.5], + ['x', 10 * 1024 * 1024], + [null, 1], + ]) { + const { socket, peer, reasons } = connection(); + socket.emit( + 'data', + Buffer.from( + JSON.stringify({ + id, + result: { transfer: { encoding: 'arrow-ipc', byte_length: length } }, + }) + '\n', + ), + ); + assert.equal(peer.isClosed, true); + assert.equal(socket.destroyed, true); + assert.match(reasons[0], /Invalid jdv binary frame/); + } + }); +}); diff --git a/src/test/suite/dataViewer.test.ts b/src/test/suite/dataViewer.test.ts index 5b2196959..9bd3b1b09 100644 --- a/src/test/suite/dataViewer.test.ts +++ b/src/test/suite/dataViewer.test.ts @@ -2,6 +2,7 @@ import * as assert from 'assert'; import * as vm from 'vm'; import type { ColDef, ColumnState, GridOptions, GridState } from 'ag-grid-community'; import { getDataViewerScript } from '../../dataViewer'; +import { gridColumnDefs, parseDescriptor } from '../../jdvAdapter'; interface SavedViewer { schema: string; @@ -238,6 +239,8 @@ suite('Data viewer', () => { assert.strictEqual(defs[3].cellDataType, 'dateString'); assert.strictEqual(defs[4].cellDataType, 'dateTimeString'); assert.strictEqual(defs[4].filter, 'agDateColumnFilter'); + assert.strictEqual((defs[3].filterParams as { includeTime: boolean }).includeTime, false); + assert.strictEqual((defs[4].filterParams as { includeTime: boolean }).includeTime, true); assert.strictEqual(defs[5].cellDataType, 'boolean'); assert.strictEqual(defs[6].cellDataType, 'bigint'); assert.strictEqual(defs[6].filter, 'agBigIntColumnFilter'); @@ -247,6 +250,52 @@ suite('Data viewer', () => { assert.strictEqual(defs[0].lockVisible, true); }); + test('keeps sequential Arrow numeric display strings precise and right aligned', () => { + const descriptor = parseDescriptor({ + view: 'arrow-1', + generation: 1, + kind: 'grid', + access: 'sequential', + encoding: 'arrow-ipc', + capabilities: { next: true }, + nrow: null, + ncol: 1, + max_page_rows: 100, + columns: [ + { + position: 0, + name: 'precise', + type: 'g', + unsupported: null, + sortable: false, + filterable: false, + }, + ], + }); + const arrowColumns = gridColumnDefs(descriptor) as unknown as ColDef[]; + const viewer = viewerScript().script; + viewer.prepareViewerColumns(arrowColumns); + assert.strictEqual(arrowColumns[1].type, 'numericColumn'); + assert.strictEqual(arrowColumns[1].cellDataType, false); + + const displayValues = [ + '3.141592653589793238462643383279', + '123456789012345678901234567890', + ]; + const rows = viewer.prepareViewerRows( + displayValues.map((value) => ({ '1': value })), + [], + ); + assert.deepStrictEqual( + rows.map((row) => row['1']), + displayValues, + ); + + const legacyNumeric: ColDef[] = [{ field: '2', type: 'numericColumn' }]; + viewer.prepareViewerColumns(legacyNumeric); + assert.strictEqual(legacyNumeric[0].cellDataType, 'number'); + }); + test('keeps integer64 precision and missing values when loading rows', () => { const script = viewerScript().script; const rows = script.prepareViewerRows( @@ -283,6 +332,119 @@ suite('Data viewer', () => { } }); + test('jdv columns offer only jdv operators and keep display text', () => { + const viewer = viewerScript(); + const jdvColumns = [ + { + field: '1', + headerName: 'n', + type: 'numericColumn', + filter: 'agNumberColumnFilter', + jdv: { filterOptions: ['equals', 'inRange'], naOptions: true, units: 'hours' }, + }, + { + field: '2', + headerName: 'i', + type: 'bigintColumn', + filter: 'agBigIntColumnFilter', + jdv: { filterOptions: ['equals'], naOptions: false }, + }, + { + field: '3', + headerName: 'd', + type: 'dateColumn', + filter: 'agDateColumnFilter', + jdv: { filterOptions: [], naOptions: true }, + }, + { + field: '4', + headerName: 'dt', + type: 'datetimeColumn', + filter: 'agDateColumnFilter', + jdv: { filterOptions: ['equals', 'inRange'], naOptions: true }, + }, + { + field: '5', + headerName: 's', + filter: 'agTextColumnFilter', + jdv: { filterOptions: ['contains', 'jdvIsEmpty', 'jdvNotEmpty'], naOptions: true }, + }, + ] as (ColDef & { jdv?: unknown })[]; + const bigintFields = viewer.script.prepareViewerColumns(jdvColumns); + assert.strictEqual(bigintFields.length, 0); + // Arrays created inside the vm context have a different prototype. + const options = (column: ColDef) => + Array.from( + (column.filterParams as { filterOptions: (string | { displayKey: string })[] }) + .filterOptions, + (option) => (typeof option === 'string' ? option : option.displayKey), + ); + assert.deepStrictEqual(options(jdvColumns[0]), [ + 'equals', + 'inRange', + 'jdvIsNa', + 'jdvNotNa', + ]); + assert.deepStrictEqual(options(jdvColumns[1]), ['equals']); + assert.deepStrictEqual(options(jdvColumns[2]), ['jdvIsNa', 'jdvNotNa']); + assert.deepStrictEqual(options(jdvColumns[4]), [ + 'contains', + 'jdvIsEmpty', + 'jdvNotEmpty', + 'jdvIsNa', + 'jdvNotNa', + ]); + assert.strictEqual( + (jdvColumns[0].filterParams as { inRangeInclusive: boolean }).inRangeInclusive, + true, + ); + assert.strictEqual( + (jdvColumns[0].filterParams as { filterPlaceholder: string }).filterPlaceholder, + 'Value (hours)', + ); + assert.strictEqual( + (jdvColumns[2].filterParams as { browserDatePicker: boolean }).browserDatePicker, + true, + ); + assert.strictEqual( + (jdvColumns[2].filterParams as { includeTime: boolean }).includeTime, + false, + ); + assert.strictEqual( + (jdvColumns[3].filterParams as { includeTime: boolean }).includeTime, + true, + ); + for (const column of jdvColumns) { + assert.strictEqual(column.cellDataType, undefined); + assert.strictEqual(column.jdv, undefined); + } + }); + + test('drillable jdv cells get an opening control only when they have a source row', () => { + const viewer = viewerScript(); + const columns = [ + { field: '1', headerName: 'x', jdvDrill: 3 }, + { field: '2', headerName: 'y' }, + ] as (ColDef & { jdvDrill?: number })[]; + viewer.script.prepareViewerColumns(columns); + assert.strictEqual(columns[0].jdvDrill, undefined); + assert.strictEqual((columns[0].context as { drillColumn: number }).drillColumn, 3); + const select = (column: ColDef, data: unknown, value: unknown) => + ( + column.cellRendererSelector as ( + params: unknown, + ) => { component: unknown } | undefined + )({ data, value }); + const drill = select(columns[0], { __sourceRow: 7 }, '1 item'); + assert.strictEqual(typeof drill?.component, 'function'); + assert.strictEqual( + select(columns[0], { __sourceRow: 7 }, null)?.component !== drill?.component, + true, + ); + assert.strictEqual(select(columns[0], {}, '1 item'), undefined); + assert.strictEqual(select(columns[1], { __sourceRow: 7 }, 'v'), undefined); + }); + test('high contrast light uses the light grid theme', () => { assert.strictEqual(viewerScript(undefined, {}, true).script.getAgTheme(), 'light'); assert.strictEqual(viewerScript().script.getAgTheme(), 'dark'); diff --git a/src/test/suite/session.test.ts b/src/test/suite/session.test.ts index 4b582ebd5..0f82f2566 100644 --- a/src/test/suite/session.test.ts +++ b/src/test/suite/session.test.ts @@ -988,6 +988,7 @@ suite('Session Communication', () => { assert.match(source, /auto_activate = TRUE/); assert.match(source, /tryCatch\(/); assert.match(source, /use_rstudioapi = (TRUE|FALSE)/); + assert.match(source, /track_variables = (TRUE|FALSE)/); const backgroundSource = await fsp.readFile(backgroundPreparation.scriptPath, 'utf8'); assert.ok( backgroundSource.includes( @@ -1421,7 +1422,7 @@ suite('Session Communication', () => { const scriptContent = await fsp.readFile(scriptPath, 'utf8'); assert.match( scriptContent, - /vscode_r_attach_sess\(endpoint, sess_src, managed_root, consent_dir, source_helper, installer_helper, "native", use_rstudioapi = (?:TRUE|FALSE)\)/, + /vscode_r_attach_sess\(endpoint, sess_src, managed_root, consent_dir, source_helper, installer_helper, "native", use_rstudioapi = (?:TRUE|FALSE), track_variables = (?:TRUE|FALSE)\)/, ); assert.doesNotMatch(scriptContent, /Sys\.(?:setenv|unsetenv)\(JGD_SOCKET/); }); diff --git a/src/test/suite/terminal.test.ts b/src/test/suite/terminal.test.ts index 975c7e9bc..970b81b87 100644 --- a/src/test/suite/terminal.test.ts +++ b/src/test/suite/terminal.test.ts @@ -304,6 +304,7 @@ suite('R Terminal', () => { assert.ok(discoveryFile); assert.strictEqual(options.env['SESS_ENDPOINT'], null); assert.strictEqual(options.env['SESS_RSTUDIOAPI'], 'TRUE'); + assert.strictEqual(options.env['SESS_TRACK_VARIABLES'], 'TRUE'); assert.strictEqual(options.env['SESS_PLOT_BACKEND'], 'httpgd'); assert.ok(options.env['R_PROFILE_USER']); assert.ok(options.env['R_PROFILE_USER'].endsWith(path.join('R', 'profile.R'))); diff --git a/src/viewerFollow.ts b/src/viewerFollow.ts new file mode 100644 index 000000000..50f9d39d9 --- /dev/null +++ b/src/viewerFollow.ts @@ -0,0 +1,58 @@ +'use strict'; + +// The "Follow" toggle shared by the Data Viewer and List Viewer toolbars. It is shown +// only for jdv views that can follow a variable (`tracking` in the descriptor). + +import type { JdvTracking } from './jdvAdapter'; + +/** Update the toggle for a tracking state. Runs in the webview, so it must be self-contained. */ +export function updateFollowButton(button: HTMLButtonElement, tracking: JdvTracking): void { + const lost = tracking.state === 'lost'; + button.disabled = lost; + button.setAttribute('aria-pressed', String(tracking.state === 'active')); + if (lost) { + button.title = + tracking.message || `The variable "${tracking.name}" is no longer available.`; + } else if (tracking.state === 'active') { + button.title = `Following "${tracking.name}": the view updates when the variable changes. Click to pause.`; + } else { + button.title = `Paused. Click to follow "${tracking.name}" again.`; + } +} + +/** + * Webview script that wires the toggle `buttonId`. `onStart` and `onError` are + * source snippets that show a status; `onError` can use `message.error`. + */ +export function getFollowScript( + tracking: JdvTracking, + buttonId: string, + onStart: string, + onError: string, +): string { + return ` + (() => { + const followButton = document.getElementById(${JSON.stringify(buttonId)}); + if (!followButton) { return; } + const updateFollowButton = ${updateFollowButton.toString()}; + let tracking = ${JSON.stringify(tracking).replace(/ { + followButton.disabled = true; + ${onStart} + vscode.postMessage({ message: 'viewer/track', documentGeneration, enabled: tracking.state !== 'active' }); + }); + // A rebuilt view replaces this document, so only state changes and failures arrive. + window.addEventListener('message', event => { + const message = event.data; + if (!message || message.documentGeneration !== documentGeneration) { return; } + if (message.message === 'viewer/tracking') { + tracking = message.tracking; + updateFollowButton(followButton, tracking); + } else if (message.message === 'viewer/trackFailed') { + updateFollowButton(followButton, tracking); + ${onError} + } + }); + })();`; +}