From c894474152a86fce374fcec0c3fe8c0c0def5db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 30 Oct 2026 14:56:35 +0100 Subject: [PATCH 1/8] Docs: merge getting-started pages and fix manual copy - Merge the Introduction, Installation and Getting started pages into a single "Getting started" page, with page-aliases so the old /installation/ and /getting-started/ URLs keep redirecting - Lead "Rendering a graph" with render_widget and demote render to the static/export path; drop the documented-but-nonexistent show_hover_tooltip parameter - Fix the integration examples: the Neo4j import, the from_neo4j row_limit name, from_dfs naming, a missing Snowflake session, and end each example with render_widget() - Clarify continuous color-space interpolation and fix the toggle_nodes_pinned example in "Customizing the visualization" - Fix the GDS tutorial's Credentials heading level (Sphinx warning) --- docs/antora/modules/ROOT/content-nav.adoc | 2 - .../modules/ROOT/pages/customizing.adoc | 10 +- .../modules/ROOT/pages/getting-started.adoc | 91 ------------- docs/antora/modules/ROOT/pages/index.adoc | 125 +++++++++++++++++- .../modules/ROOT/pages/installation.adoc | 26 ---- .../modules/ROOT/pages/integration/gds.adoc | 4 +- .../modules/ROOT/pages/integration/neo4j.adoc | 12 +- .../ROOT/pages/integration/pandas.adoc | 6 +- .../ROOT/pages/integration/snowflake.adoc | 8 +- docs/antora/modules/ROOT/pages/rendering.adoc | 38 ++++-- examples/gds-example.ipynb | 2 +- 11 files changed, 174 insertions(+), 150 deletions(-) delete mode 100644 docs/antora/modules/ROOT/pages/getting-started.adoc delete mode 100644 docs/antora/modules/ROOT/pages/installation.adoc diff --git a/docs/antora/modules/ROOT/content-nav.adoc b/docs/antora/modules/ROOT/content-nav.adoc index be2ceba7..bf63bcf6 100644 --- a/docs/antora/modules/ROOT/content-nav.adoc +++ b/docs/antora/modules/ROOT/content-nav.adoc @@ -1,6 +1,4 @@ * xref:index.adoc[] -* xref:installation.adoc[] -* xref:getting-started.adoc[] * xref:rendering.adoc[] * xref:customizing.adoc[] * xref:integration/index.adoc[] diff --git a/docs/antora/modules/ROOT/pages/customizing.adoc b/docs/antora/modules/ROOT/pages/customizing.adoc index 2b3631fb..076c6132 100644 --- a/docs/antora/modules/ROOT/pages/customizing.adoc +++ b/docs/antora/modules/ROOT/pages/customizing.adoc @@ -7,7 +7,7 @@ relationships of existing `VisualizationGraph` objects. If you have not yet created a `VisualizationGraph` object, refer to one of the following sections: -* xref:getting-started.adoc[Getting started] for creating a visualization graph from scratch using `neo4j-viz` +* xref:index.adoc[Getting started] for creating a visualization graph from scratch using `neo4j-viz` primitives like link:{api-docs-uri}/node/[Node] and link:{api-docs-uri}/relationship/[Relationship] and link:{api-docs-uri}/visualization-graph[`VisualizationGraph`] directly. * xref:integration/index.adoc[] for importing data from a Pandas DataFrame or Neo4j GDS projection. @@ -108,7 +108,7 @@ from palettable.wesanderson import Moonrise1_5 VG.color_nodes(field="caption", colors=Moonrise1_5.colors) # ColorSpace.DISCRETE is default ---- -In theses cases, all items with the same field value will get the same color. +In these cases, all items with the same field value will get the same color. If there are fewer colors than unique values for the `field` or `property` provided, the colors will be reused in a cycle. To avoid that, you could use a larger palette or extend one with additional colors. @@ -138,8 +138,8 @@ VG.color_relationships( ) ---- -In these cases, the nodes or relationships will be colored based on the value of the property, with the lowest values being colored with the first color and the highest values being colored with the last color. -Since we only provided five colors in the range, the granularity of the gradient will be limited to five steps. +The property values are min-max scaled, and each node or relationship is then colored by interpolating smoothly along the gradient: the lowest value gets the first color, the highest gets the last color, and values in between are blends of the neighboring color stops. +The number of stops controls the shape of the gradient (for example a multi-hue one), not the number of distinct colors. `palettable` and `matplotlib` are great libraries to use to create custom color gradients. @@ -253,7 +253,7 @@ In the following example, we pin the node with ID 1337 and unpin the node with I [source, python] ---- # VG is a VisualizationGraph object -VG.toggle_nodes_pinned(1337: True, 42: False)}) +VG.toggle_nodes_pinned({1337: True, 42: False}) ---- == Reading the selection diff --git a/docs/antora/modules/ROOT/pages/getting-started.adoc b/docs/antora/modules/ROOT/pages/getting-started.adoc deleted file mode 100644 index ac0fd2b8..00000000 --- a/docs/antora/modules/ROOT/pages/getting-started.adoc +++ /dev/null @@ -1,91 +0,0 @@ -= Getting started - -In this section, we will cover the very basics of creating a visualization graph using the `neo4j-viz` library. -We will use a small toy graph representing the purchase history of a few people and products. - -To follow along with the example below, please make sure you have xref:installation.adoc[installed the library] first. - -We start by instantiating the link:{api-docs-uri}/node/[Nodes] and link:{api-docs-uri}/relationship[Relationships] we want in our graph. -The only mandatory fields for a node are the “id”, and “source” and “target” for a relationship. -But the other fields can optionally be used to customize the appearance of the nodes and relationships in the visualization. - -Lastly we create a link:{api-docs-uri}/visualization-graph/[VisualizationGraph] object with the nodes and relationships we created, and call its `render` method to display the graph. - -[source, python, role=nocollapse] ----- -from neo4j_viz import Node, Relationship, VisualizationGraph - -nodes = [ - Node(id=0, size=10, caption="Person"), - Node(id=1, size=10, caption="Product"), - Node(id=2, size=20, caption="Product"), - Node(id=3, size=10, caption="Person"), - Node(id=4, size=10, caption="Product"), -] - -relationships = [ - Relationship( - source=0, - target=1, - caption="BUYS", - ), - Relationship( - source=0, - target=2, - caption="BUYS", - ), - Relationship( - source=3, - target=2, - caption="BUYS", - ), -] - -VG = VisualizationGraph(nodes=nodes, relationships=relationships) - -VG.render(initial_zoom=2) ----- - -[TIP] -==== -See the link:{tutorials-docs-uri}/getting-started/[Getting started tutorial] to interact with the graph. -==== - -image::getting-started-widget.png["The graph rendered as an interactive widget"] - -As we can see in the graph above, the radius of one of the nodes is larger than the others. -This is because we set the “size” field of the node to 20, while the others are set to 10. - -At this time the nodes are colored using the default colors. -If we want to distinguish between the different types of nodes, we can color them differently with the `color_nodes` method. -We can pass the field we want to use to color the nodes as an argument. -In this case, we will use the "caption" field. -Nodes with the same "caption" will have the same color. -We will use the default color scheme, which is the Neo4j color scheme. - -[source, python] ----- -VG.color_nodes(field="caption") - -VG.render(initial_zoom=2) ----- - -[TIP] -==== -See the link:{tutorials-docs-uri}/getting-started/[Getting started tutorial] to interact with the graph. -==== - -image::getting-started-widget-colored.png["The graph with nodes colored by caption"] - -We are now easily able to distinguish between the different types of nodes in the graph. - -== Next steps - -Now that we have covered the basics of creating and rendering a visualization graph, we can move on to more advanced topics. - -Here are some suggestions for what to do next: - -* Browse the link:{tutorials-docs-uri}[notebook examples] for more examples and use cases -* Learn about the xref:integration/index.adoc[integrations] with other libraries such as Neo4j GDS and Pandas -* Check out the xref:rendering.adoc[rendering options] and convenience methods of `VisualizationGraph` -* Read about our link:{api-docs-uri}/[API reference] diff --git a/docs/antora/modules/ROOT/pages/index.adoc b/docs/antora/modules/ROOT/pages/index.adoc index 491598c6..7383576e 100644 --- a/docs/antora/modules/ROOT/pages/index.adoc +++ b/docs/antora/modules/ROOT/pages/index.adoc @@ -1,4 +1,5 @@ -= Introduction += Getting started +:page-aliases: installation.adoc, getting-started.adoc The Python Graph Visualization library allows you to visualize graph data interactively in Python. @@ -10,4 +11,124 @@ The library wraps the JavaScript-based link:https://neo4j.com/docs/nvl/current/[ * xref:integration/snowflake.adoc[] The source code is available on link:https://github.com/neo4j/python-graph-visualization[GitHub]. -If you have a suggestion on how we can improve the library or want to report a problem, you can create a link:https://github.com/neo4j/python-graph-visualization/issues/new[new issue]. \ No newline at end of file +If you have a suggestion on how we can improve the library or want to report a problem, you can create a link:https://github.com/neo4j/python-graph-visualization/issues/new[new issue]. + +== Installation + +You can install the latest version of the library with `pip`: + +[source, console] +---- +pip install neo4j-viz +---- + +=== Optional dependencies + +There are a few optional dependencies that you can install to enable additional features of the library. + +==== Integration with external sources + +The library can import data directly from xref:integration/index.adoc[external sources]. +Each integration page includes installation instructions for the needed dependencies. + +==== Tutorials + +You need to install some additional dependencies to run the link:{tutorials-docs-uri}[tutorials]: + +[source, console] +---- +pip install neo4j-viz[notebook] +---- + +== Creating your first visualization + +In this section, we will cover the very basics of creating a visualization graph using the `neo4j-viz` library. +We will use a small toy graph representing the purchase history of a few people and products. + +We start by instantiating the link:{api-docs-uri}/node/[Nodes] and link:{api-docs-uri}/relationship[Relationships] we want in our graph. +The only mandatory fields for a node are the “id”, and “source” and “target” for a relationship. +But the other fields can optionally be used to customize the appearance of the nodes and relationships in the visualization. + +Lastly we create a link:{api-docs-uri}/visualization-graph/[VisualizationGraph] object with the nodes and relationships we created, and call its `render_widget` method to display the graph as an interactive widget. + +[source, python, role=nocollapse] +---- +from neo4j_viz import Node, Relationship, VisualizationGraph + +nodes = [ + Node(id=0, size=10, caption="Person"), + Node(id=1, size=10, caption="Product"), + Node(id=2, size=20, caption="Product"), + Node(id=3, size=10, caption="Person"), + Node(id=4, size=10, caption="Product"), +] + +relationships = [ + Relationship( + source=0, + target=1, + caption="BUYS", + ), + Relationship( + source=0, + target=2, + caption="BUYS", + ), + Relationship( + source=3, + target=2, + caption="BUYS", + ), +] + +VG = VisualizationGraph(nodes=nodes, relationships=relationships) + +VG.render_widget(initial_zoom=2) +---- + +[TIP] +==== +See the link:{tutorials-docs-uri}/getting-started/[Getting started tutorial] to interact with the graph. +==== + +image::getting-started-widget.png["The graph rendered as an interactive widget"] + +NOTE: `render_widget` is the recommended way to display a graph: it returns an interactive widget with two-way sync between Python and the browser, and features such as adding data to a graph that is already displayed, changing the layout or zoom dynamically, and reacting to selections and clicks are only available through it. +The `render` method instead returns a static HTML object and is not recommended for interactive use; see xref:rendering.adoc[Rendering a graph] for the full list of options. + +As we can see in the graph above, the radius of one of the nodes is larger than the others. +This is because we set the “size” field of the node to 20, while the others are set to 10. + +At this time the nodes are colored using the default colors. +If we want to distinguish between the different types of nodes, we can color them differently with the `color_nodes` method. +We can pass the field we want to use to color the nodes as an argument. +In this case, we will use the "caption" field. +Nodes with the same "caption" will have the same color. +We will use the default color scheme, which is the Neo4j color scheme. + +[source, python] +---- +VG.color_nodes(field="caption") + +VG.render_widget(initial_zoom=2) +---- + +[TIP] +==== +See the link:{tutorials-docs-uri}/getting-started/[Getting started tutorial] to interact with the graph. +==== + +image::getting-started-widget-colored.png["The graph with nodes colored by caption"] + +We are now easily able to distinguish between the different types of nodes in the graph. + +== Next steps + +Now that we have covered the basics of creating and rendering a visualization graph, we can move on to more advanced topics. + +Here are some suggestions for what to do next: + +* Browse the link:{tutorials-docs-uri}[tutorials] for more examples and use cases +* Learn about the xref:integration/index.adoc[integrations] with other libraries such as Neo4j GDS and Pandas +* Check out the xref:rendering.adoc[rendering options] and convenience methods of `VisualizationGraph` +* Read about our link:{api-docs-uri}/[API reference] diff --git a/docs/antora/modules/ROOT/pages/installation.adoc b/docs/antora/modules/ROOT/pages/installation.adoc deleted file mode 100644 index e1eb2c4a..00000000 --- a/docs/antora/modules/ROOT/pages/installation.adoc +++ /dev/null @@ -1,26 +0,0 @@ -= Installation - -You can install the latest version of the library with `pip`: - -[source, console] ----- -pip install neo4j-viz ----- - -== Optional dependencies - -There are a few optional dependencies that you can install to enable additional features of the library. - -=== Integration with external sources - -The library can import data directly from xref:integration/index.adoc[external sources]. -Each integration page includes installation instructions for the needed dependencies. - -=== Notebook tutorials - -You need to install some additional dependencies to run the link:{tutorials-docs-uri}[tutorials]: - -[source, console] ----- -pip install neo4j-viz[notebook] ----- \ No newline at end of file diff --git a/docs/antora/modules/ROOT/pages/integration/gds.adoc b/docs/antora/modules/ROOT/pages/integration/gds.adoc index e64148dc..19541eb0 100644 --- a/docs/antora/modules/ROOT/pages/integration/gds.adoc +++ b/docs/antora/modules/ROOT/pages/integration/gds.adoc @@ -1,7 +1,7 @@ = Neo4j Graph Data Science The `neo4j-viz` library provides a convenience method for importing data from the link:https://neo4j.com/docs/graph-data-science/current/[Neo4j Graph Data Science] (GDS) library. -It requires and additional dependency to be installed, which you can do by running: +It requires an additional dependency to be installed, which you can do by running: [source, console] ---- @@ -52,6 +52,8 @@ VG.resize_nodes(property="pagerank") # Color the nodes by the `componentId` property, so that the nodes are # colored by the connected component they belong to VG.color_nodes(property="componentId") + +VG.render_widget() ---- See the link:{tutorials-docs-uri}/gds-example[Visualizing Neo4j Graph Data Science (GDS) Graphs tutorial] for a more extensive example. diff --git a/docs/antora/modules/ROOT/pages/integration/neo4j.adoc b/docs/antora/modules/ROOT/pages/integration/neo4j.adoc index b171ebd0..ed7252a3 100644 --- a/docs/antora/modules/ROOT/pages/integration/neo4j.adoc +++ b/docs/antora/modules/ROOT/pages/integration/neo4j.adoc @@ -1,7 +1,7 @@ = Neo4j database The `neo4j-viz` library provides a convenience method for importing data from the Neo4j database via the link:https://neo4j.com/docs/python-manual/current/[Neo4j Python driver]. -It requires and additional dependency to be installed, which you can do by running: +It requires an additional dependency to be installed, which you can do by running: [source, console] ---- @@ -13,18 +13,18 @@ to import query results from Neo4j. The `from_neo4j` method takes one mandatory positional parameter: A `data` argument representing either a query result in the shape of a `neo4j.graph.Graph`, `neo4j.EagerResult` or `neo4j.Result`, or a `neo4j.Driver` in which case a simple default query will be executed internally to retrieve the graph data. -The optional `max_rows` parameter can be used to limit the number of relationships shown in the visualization. +The optional `row_limit` parameter can be used to limit the number of relationships shown in the visualization. By default, it is set to 10.000, meaning that if the database has more than 10.000 rows, a warning will be raised. Note, this only applies if the `data` parameter is a `neo4j.Driver`. == Example -In this small example, we import a graph from a Neo4j query result. +In this small example, we import a graph from a Neo4j query result and display it as an interactive widget. [source, python] ---- -from neo4j import GraphDatabase, RoutingControl, Result -from neo4j_viz.gds import from_gds +from neo4j import GraphDatabase, RoutingControl +from neo4j_viz.neo4j import from_neo4j # Modify this to match your Neo4j instance's URI and credentials URI = "neo4j://localhost:7687" @@ -39,6 +39,8 @@ with GraphDatabase.driver(URI, auth=auth) as driver: ) VG = from_neo4j(result) + +VG.render_widget() ---- See the link:{tutorials-docs-uri}/neo4j-example[Visualizing Neo4j Graphs tutorial] for a more extensive example. diff --git a/docs/antora/modules/ROOT/pages/integration/pandas.adoc b/docs/antora/modules/ROOT/pages/integration/pandas.adoc index 629cea22..6ce213c4 100644 --- a/docs/antora/modules/ROOT/pages/integration/pandas.adoc +++ b/docs/antora/modules/ROOT/pages/integration/pandas.adoc @@ -2,14 +2,14 @@ The `neo4j-viz` library provides a convenience method for importing data from link:https://pandas.pydata.org/[Pandas DataFrames]. These DataFrames can be created from many sources, such as CSV files. -It requires and additional dependency to be installed, which you can do by running: +It requires an additional dependency to be installed, which you can do by running: [source, console] ---- pip install neo4j-viz[pandas] ---- -Once you have installed the additional dependency, you can use the link:{api-docs-uri}/from_pandas[`from_pandas`] method +Once you have installed the additional dependency, you can use the link:{api-docs-uri}/from_pandas[`from_dfs`] method to import pandas DataFrames. The `from_dfs` method takes two mandatory positional parameters: @@ -49,4 +49,6 @@ relationships = DataFrame({ }) VG = from_dfs(nodes, relationships) + +VG.render_widget() ---- \ No newline at end of file diff --git a/docs/antora/modules/ROOT/pages/integration/snowflake.adoc b/docs/antora/modules/ROOT/pages/integration/snowflake.adoc index 1a65d061..c5d491ee 100644 --- a/docs/antora/modules/ROOT/pages/integration/snowflake.adoc +++ b/docs/antora/modules/ROOT/pages/integration/snowflake.adoc @@ -1,7 +1,7 @@ = Snowflake tables The `neo4j-viz` library provides a convenience method for importing data from link:https://docs.snowflake.com/[Snowflake tables]. -It requires and additional dependency to be installed, which you can do by running: +It requires an additional dependency to be installed, which you can do by running: [source, console] ---- @@ -17,7 +17,7 @@ The `from_snowflake` method takes two mandatory positional parameters: * A link:https://neo4j.com/docs/snowflake-graph-analytics/current/jobs/#jobs-project[project configuration] as a dictionary, that specifies how you want your tables to be projected as a graph. This configuration is the same as the project configuration of the link:https://neo4j.com/docs/snowflake-graph-analytics/current/[Neo4j Snowflake Graph Analytics application]. -You can further customize the visualization after the _VisualizationGraph_ has been created, by using the methods described in the link:{api-docs-uri}/customizing/[Customizing the visualization] section. +You can further customize the visualization after the _VisualizationGraph_ has been created, by using the methods described in the xref:customizing.adoc[Customizing the visualization] section. == Default behavior @@ -31,6 +31,8 @@ as an interactive widget using link:{api-docs-uri}/visualization-graph/#neo4j_vi [source, python] ---- +import os + from snowflake.snowpark import Session from neo4j_viz.snowflake import from_snowflake @@ -43,6 +45,8 @@ connection_parameters = { "warehouse": os.environ.get("SNOWFLAKE_WAREHOUSE"), } +session = Session.builder.configs(connection_parameters).create() + session.sql( "CREATE OR REPLACE TABLE EXAMPLE_DB.DATA_SCHEMA.PERSONS (NODEID VARCHAR);" ).collect() diff --git a/docs/antora/modules/ROOT/pages/rendering.adoc b/docs/antora/modules/ROOT/pages/rendering.adoc index 86e525b6..11afde47 100644 --- a/docs/antora/modules/ROOT/pages/rendering.adoc +++ b/docs/antora/modules/ROOT/pages/rendering.adoc @@ -5,22 +5,22 @@ to display the graph visualization. If you have not yet created a `VisualizationGraph` object, refer to one of the following sections: -* xref:getting-started.adoc[Getting started] for creating a visualization graph from scratch using `neo4j-viz` +* xref:index.adoc[Getting started] for creating a visualization graph from scratch using `neo4j-viz` primitives like link:{api-docs-uri}/node[Node] and link:{api-docs-uri}/relationship[Relationship] and link:{api-docs-uri}/visualization-graph[`VisualizationGraph`] directly. * xref:integration/index.adoc[] for importing data from a Pandas DataFrame or Neo4j GDS projection. -== The `render` method +== The `render_widget` method -Once you have a `VisualizationGraph` object, you can render it using the `render` method. -This will return a HTML object that will be displayed in an environment that supports HTML rendering, such as -Jupyter notebooks or streamlit application. +Once you have a `VisualizationGraph` object, you can display it using the `render_widget` method. +This returns a `GraphWidget`, an interactive widget with two-way sync between Python and the browser, that works in +JupyterLab, Notebook 7, VS Code, and Colab. -All parameter of the `render` method are optional, and the full list of parameters of them is listed in the API -reference: link:{api-docs-uri}/visualization-graph/#neo4j_viz.VisualizationGraph.render[neo4j_viz.VisualizationGraph.render()]. +All parameters of the `render_widget` method are optional, and the full list of them is listed in the API +reference: link:{api-docs-uri}/visualization-graph/#neo4j_viz.VisualizationGraph.render_widget[neo4j_viz.VisualizationGraph.render_widget()]. The most important parameters to be aware of are the `width` and `height` parameters, which control the size of -HTML object that will be rendered. +the widget. You can provide these either as a percentage of the available space (eg. `"80%"`), or as an absolute pixel value (eg. `"800px"`). @@ -40,17 +40,29 @@ In this case you might also want to pass `Renderer.WEB_GL` as the `renderer` to Relationships whose `source` or `target` is not part of the graph's nodes ("dangling" relationships) cannot be drawn, and a graph containing them may appear empty. -By default `render` emits a warning that lists the offending relationships so the problem is not silent. +By default `render_widget` emits a warning that lists the offending relationships so the problem is not silent. You can change this with the `on_dangling` parameter: pass `"error"` to raise a `ValueError` instead, or `"none"` to skip the check entirely. -By default a tooltip showing IDs and properties will be shown when mouse hovering over a node or relationship. -But you can disable this by passing `show_hover_tooltip=False`. +Unlike `render`, the widget also lets you update the graph after it has been displayed, for example to add nodes and +relationships, to change the layout or zoom, or to react to the user's selections and clicks. +Refer to xref:customizing.adoc[Customizing the visualization] for these methods. === Examples -Refer to the xref:getting-started.adoc[] and the link:{tutorials-docs-uri}[tutorials] for -examples of `render` method usage. +Refer to the xref:index.adoc[] and the link:{tutorials-docs-uri}[tutorials] for +examples of `render_widget` method usage. + +== The `render` method + +The `render` method returns an `IPython.display.HTML` object containing the graph as static HTML. +It is not recommended for interactive use; use `render_widget` instead. +Reach for `render` when you need to embed the graph in a document or web page, or to +xref:rendering.adoc#exporting-to-html[export it to an HTML file]. + +It accepts the same `width`, `height`, `layout`, `max_allowed_nodes`, and `on_dangling` parameters as `render_widget`. +The full list of parameters of the `render` method is listed in the API +reference: link:{api-docs-uri}/visualization-graph/#neo4j_viz.VisualizationGraph.render[neo4j_viz.VisualizationGraph.render()]. == Using with Streamlit diff --git a/examples/gds-example.ipynb b/examples/gds-example.ipynb index cefccf30..a12900e5 100644 --- a/examples/gds-example.ipynb +++ b/examples/gds-example.ipynb @@ -27,7 +27,7 @@ "id": "7f948daf", "metadata": {}, "source": [ - "### Credentials\n", + "## Credentials\n", "\n", "The connection details are read from an environment file named `creds.env`, next to this notebook.\n", "Copy `creds.env.example` to `creds.env` and fill in the URI, username, password, and database of your Neo4j instance, and optionally your Aura Graph Analytics credentials.\n", From d0d2dfe0059c11d57f76595fbff8c1aade3e7b2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 30 Oct 2026 15:05:28 +0100 Subject: [PATCH 2/8] Docs: move example images into examples/images Group the notebook and README images under examples/images/ so the example notebooks are easier to find. Update the notebook image references, the .nblink extra-media entries (now the images directory), MANIFEST.in and both READMEs, and ignore the subfolder nbsphinx_link copies into the docs tree. --- MANIFEST.in | 2 +- README.md | 2 +- docs/.gitignore | 1 + docs/source/tutorials/gds-example.nblink | 2 +- docs/source/tutorials/getting-started.nblink | 3 +-- docs/source/tutorials/neo4j-example.nblink | 2 +- docs/source/tutorials/snowflake-example.nblink | 3 +-- examples/gds-example.ipynb | 2 +- examples/getting-started.ipynb | 4 ++-- examples/{ => images}/example_graph.png | Bin examples/{ => images}/example_instruments_1.png | Bin examples/{ => images}/example_instruments_2.png | Bin examples/{ => images}/gds-example-graph.png | Bin .../getting-started-widget-colored.png | Bin examples/{ => images}/getting-started-widget.png | Bin examples/neo4j-example.ipynb | 2 +- examples/snowflake-example.ipynb | 4 ++-- python-wrapper/README.md | 2 +- scripts/regenerate_docs_images.py | 13 +++++++------ 19 files changed, 21 insertions(+), 21 deletions(-) rename examples/{ => images}/example_graph.png (100%) rename examples/{ => images}/example_instruments_1.png (100%) rename examples/{ => images}/example_instruments_2.png (100%) rename examples/{ => images}/gds-example-graph.png (100%) rename examples/{ => images}/getting-started-widget-colored.png (100%) rename examples/{ => images}/getting-started-widget.png (100%) diff --git a/MANIFEST.in b/MANIFEST.in index 38bd6261..6c78833a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include examples/example_graph.png +include examples/images/example_graph.png diff --git a/README.md b/README.md index 35e6011f..296b2fcb 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Alternatively, you can export the output to a file and view it in a web browser. The package wraps the [Neo4j Visualization JavaScript library (NVL)](https://neo4j.com/docs/nvl/current/). -![Example Graph](examples/example_graph.png) +![Example Graph](examples/images/example_graph.png) ## Some notable features diff --git a/docs/.gitignore b/docs/.gitignore index 0de8b858..8150e631 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -3,3 +3,4 @@ build # Antora docs antora/node_modules/ source/tutorials/*.png +source/tutorials/images/ diff --git a/docs/source/tutorials/gds-example.nblink b/docs/source/tutorials/gds-example.nblink index b4e7691c..da10582f 100644 --- a/docs/source/tutorials/gds-example.nblink +++ b/docs/source/tutorials/gds-example.nblink @@ -1,6 +1,6 @@ { "path": "../../../examples/gds-example.ipynb", "extra-media": [ - "../../../examples/gds-example-graph.png" + "../../../examples/images" ] } diff --git a/docs/source/tutorials/getting-started.nblink b/docs/source/tutorials/getting-started.nblink index af3261f3..166d065c 100644 --- a/docs/source/tutorials/getting-started.nblink +++ b/docs/source/tutorials/getting-started.nblink @@ -1,7 +1,6 @@ { "path": "../../../examples/getting-started.ipynb", "extra-media": [ - "../../../examples/getting-started-widget.png", - "../../../examples/getting-started-widget-colored.png" + "../../../examples/images" ] } diff --git a/docs/source/tutorials/neo4j-example.nblink b/docs/source/tutorials/neo4j-example.nblink index b38cc33f..ca7084b3 100644 --- a/docs/source/tutorials/neo4j-example.nblink +++ b/docs/source/tutorials/neo4j-example.nblink @@ -1,6 +1,6 @@ { "path": "../../../examples/neo4j-example.ipynb", "extra-media": [ - "../../../examples/example_graph.png" + "../../../examples/images" ] } diff --git a/docs/source/tutorials/snowflake-example.nblink b/docs/source/tutorials/snowflake-example.nblink index 2a7a9d05..cbd0f020 100644 --- a/docs/source/tutorials/snowflake-example.nblink +++ b/docs/source/tutorials/snowflake-example.nblink @@ -1,7 +1,6 @@ { "path": "../../../examples/snowflake-example.ipynb", "extra-media": [ - "../../../examples/example_instruments_1.png", - "../../../examples/example_instruments_2.png" + "../../../examples/images" ] } diff --git a/examples/gds-example.ipynb b/examples/gds-example.ipynb index a12900e5..7e78ee9e 100644 --- a/examples/gds-example.ipynb +++ b/examples/gds-example.ipynb @@ -339,7 +339,7 @@ "id": "a2fb5780", "metadata": {}, "source": [ - "![The Cora graph, sized by PageRank and colored by community](gds-example-graph.png)" + "![The Cora graph, sized by PageRank and colored by community](images/gds-example-graph.png)" ] }, { diff --git a/examples/getting-started.ipynb b/examples/getting-started.ipynb index d2429e10..cc461a30 100644 --- a/examples/getting-started.ipynb +++ b/examples/getting-started.ipynb @@ -70,7 +70,7 @@ "id": "69be3030", "metadata": {}, "source": [ - "![The graph rendered as an interactive widget](getting-started-widget.png)" + "![The graph rendered as an interactive widget](images/getting-started-widget.png)" ] }, { @@ -104,7 +104,7 @@ "id": "c61b6779", "metadata": {}, "source": [ - "![The same graph, with the nodes colored by caption](getting-started-widget-colored.png)" + "![The same graph, with the nodes colored by caption](images/getting-started-widget-colored.png)" ] }, { diff --git a/examples/example_graph.png b/examples/images/example_graph.png similarity index 100% rename from examples/example_graph.png rename to examples/images/example_graph.png diff --git a/examples/example_instruments_1.png b/examples/images/example_instruments_1.png similarity index 100% rename from examples/example_instruments_1.png rename to examples/images/example_instruments_1.png diff --git a/examples/example_instruments_2.png b/examples/images/example_instruments_2.png similarity index 100% rename from examples/example_instruments_2.png rename to examples/images/example_instruments_2.png diff --git a/examples/gds-example-graph.png b/examples/images/gds-example-graph.png similarity index 100% rename from examples/gds-example-graph.png rename to examples/images/gds-example-graph.png diff --git a/examples/getting-started-widget-colored.png b/examples/images/getting-started-widget-colored.png similarity index 100% rename from examples/getting-started-widget-colored.png rename to examples/images/getting-started-widget-colored.png diff --git a/examples/getting-started-widget.png b/examples/images/getting-started-widget.png similarity index 100% rename from examples/getting-started-widget.png rename to examples/images/getting-started-widget.png diff --git a/examples/neo4j-example.ipynb b/examples/neo4j-example.ipynb index 3b4789fd..df9aeb1f 100644 --- a/examples/neo4j-example.ipynb +++ b/examples/neo4j-example.ipynb @@ -193,7 +193,7 @@ "id": "3b912fb6", "metadata": {}, "source": [ - "![The example graph rendered as an interactive widget](example_graph.png)" + "![The example graph rendered as an interactive widget](images/example_graph.png)" ] }, { diff --git a/examples/snowflake-example.ipynb b/examples/snowflake-example.ipynb index 9346cccb..c8ac4548 100644 --- a/examples/snowflake-example.ipynb +++ b/examples/snowflake-example.ipynb @@ -260,7 +260,7 @@ "id": "aeb09ede04f946f2", "metadata": {}, "source": [ - "![Graph rendered with default styling](example_instruments_1.png)" + "![Graph rendered with default styling](images/example_instruments_1.png)" ] }, { @@ -306,7 +306,7 @@ "id": "e7591ac1fb3654a7", "metadata": {}, "source": [ - "![Graph rendered with one color per node](example_instruments_2.png)" + "![Graph rendered with one color per node](images/example_instruments_2.png)" ] }, { diff --git a/python-wrapper/README.md b/python-wrapper/README.md index f164c036..aa28fc77 100644 --- a/python-wrapper/README.md +++ b/python-wrapper/README.md @@ -16,7 +16,7 @@ Alternatively, you can export the output to a file and view it in a web browser. The package wraps the [Neo4j Visualization JavaScript library (NVL)](https://neo4j.com/docs/nvl/current/). -![Example Graph](https://raw.githubusercontent.com/neo4j/python-graph-visualization/main/examples/example_graph.png) +![Example Graph](https://raw.githubusercontent.com/neo4j/python-graph-visualization/main/examples/images/example_graph.png) ## Some notable features diff --git a/scripts/regenerate_docs_images.py b/scripts/regenerate_docs_images.py index c909a158..ae78e9d0 100755 --- a/scripts/regenerate_docs_images.py +++ b/scripts/regenerate_docs_images.py @@ -40,6 +40,7 @@ _ANTORA_IMAGES = GIT_ROOT / "docs" / "antora" / "modules" / "ROOT" / "images" _EXAMPLES = GIT_ROOT / "examples" +_EXAMPLE_IMAGES = _EXAMPLES / "images" # The widget wrapper element: the graph canvas plus its UI chrome (toolbar, # search, legend). Screenshotting it captures the widget as a user sees it. @@ -263,7 +264,7 @@ def _specs() -> list[ImageSpec]: return [ ImageSpec( name="example_graph", - target=_EXAMPLES / "example_graph.png", + target=_EXAMPLE_IMAGES / "example_graph.png", size=widget_size, cells=(_EXAMPLE_CREATE_CELLS, _render_cell(widget_size)), ), @@ -272,7 +273,7 @@ def _specs() -> list[ImageSpec]: target=_ANTORA_IMAGES / "getting-started-widget.png", size=widget_size, cells=(_TOY_GRAPH_CELLS, _render_cell(widget_size)), - also_copy=(_EXAMPLES / "getting-started-widget.png",), + also_copy=(_EXAMPLE_IMAGES / "getting-started-widget.png",), ), ImageSpec( name="getting-started-widget-colored", @@ -283,24 +284,24 @@ def _specs() -> list[ImageSpec]: "VG.color_nodes(field='caption')\n", _render_cell(widget_size), ), - also_copy=(_EXAMPLES / "getting-started-widget-colored.png",), + also_copy=(_EXAMPLE_IMAGES / "getting-started-widget-colored.png",), ), ImageSpec( name="gds-example-graph", - target=_EXAMPLES / "gds-example-graph.png", + target=_EXAMPLE_IMAGES / "gds-example-graph.png", size=widget_size, cells=(*_GDS_CELLS, _render_cell(widget_size)), requires_env="NEO4J_URI", ), ImageSpec( name="snowflake-example-graph", - target=_EXAMPLES / "example_instruments_1.png", + target=_EXAMPLE_IMAGES / "example_instruments_1.png", size=widget_size, cells=(_SNOWFLAKE_CELLS, _render_cell(widget_size)), ), ImageSpec( name="snowflake-example-graph-colored", - target=_EXAMPLES / "example_instruments_2.png", + target=_EXAMPLE_IMAGES / "example_instruments_2.png", size=widget_size, cells=( _SNOWFLAKE_CELLS, From b09d5dd6c4f9d10b31d6d57babde4c25574d0da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 30 Oct 2026 15:23:16 +0100 Subject: [PATCH 3/8] Docs: drop the broken notebook extra install step `pip install neo4j-viz[notebook]` documents an extra that does not exist (`notebook` is a uv dependency-group, not a project extra), so pip only warns and installs the base package. Every tutorial installs what it needs in its first notebook cells anyway. --- docs/antora/modules/ROOT/pages/index.adoc | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/docs/antora/modules/ROOT/pages/index.adoc b/docs/antora/modules/ROOT/pages/index.adoc index 7383576e..1642db31 100644 --- a/docs/antora/modules/ROOT/pages/index.adoc +++ b/docs/antora/modules/ROOT/pages/index.adoc @@ -25,21 +25,13 @@ pip install neo4j-viz === Optional dependencies There are a few optional dependencies that you can install to enable additional features of the library. +The link:{tutorials-docs-uri}[tutorials] install any additional dependencies they need in their first notebook cells, so there is nothing extra to install to follow along. ==== Integration with external sources The library can import data directly from xref:integration/index.adoc[external sources]. Each integration page includes installation instructions for the needed dependencies. -==== Tutorials - -You need to install some additional dependencies to run the link:{tutorials-docs-uri}[tutorials]: - -[source, console] ----- -pip install neo4j-viz[notebook] ----- - == Creating your first visualization In this section, we will cover the very basics of creating a visualization graph using the `neo4j-viz` library. From c3468587243364e3f3285790c84cfe2f4415cc12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 30 Oct 2026 15:32:20 +0100 Subject: [PATCH 4/8] Docs: allow patched Node 26 in the Antora docs toolchain The docs/antora engines cap below Node 26 was added because Antora 3.1.x silently no-ops on Node 26. Antora was bumped to 3.2.1 shortly after, and Node 26.13.0 now builds the site identically to Node 24 (only sitemap timestamps differ), so widen the range to match js-applet. The eMudhra-patched floor (>=26.12.1) is preserved, and the silent-no-op guard in the workflow is kept. --- .github/workflows/render-docs.yml | 4 ++-- docs/antora/package-lock.json | 2 +- docs/antora/package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/render-docs.yml b/.github/workflows/render-docs.yml index f2e90344..8d206716 100644 --- a/.github/workflows/render-docs.yml +++ b/.github/workflows/render-docs.yml @@ -82,6 +82,6 @@ jobs: run: | npm ci npm run verify - # Antora 3.1.x silently exits 0 without generating anything on - # non-LTS Node (e.g. 26); guard against that silent no-op. + # Antora can exit 0 without generating a site (observed with Antora + # 3.1.x on newer Node); guard against that silent no-op. test -f build/site/index.html || { echo '::error::Antora did not generate a site'; exit 1; } diff --git a/docs/antora/package-lock.json b/docs/antora/package-lock.json index fd6c1a58..bdd8a44f 100644 --- a/docs/antora/package-lock.json +++ b/docs/antora/package-lock.json @@ -25,7 +25,7 @@ "nodemon": "^3.1.0" }, "engines": { - "node": "^22.23.4 || ^24.22.1" + "node": "^22.23.4 || ^24.22.1 || >=26.12.1" } }, "node_modules/@antora/asciidoc-loader": { diff --git a/docs/antora/package.json b/docs/antora/package.json index c9bffb7a..30b3577f 100644 --- a/docs/antora/package.json +++ b/docs/antora/package.json @@ -24,7 +24,7 @@ "author": "Neo4j", "license": "ISC", "engines": { - "node": "^22.23.4 || ^24.22.1" + "node": "^22.23.4 || ^24.22.1 || >=26.12.1" }, "dependencies": { "@neo4j-antora/aliases-redirects": "^0.2.5", From 37e5752ef2fc9de762130200e1c33b81da9b500c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 30 Oct 2026 15:36:15 +0100 Subject: [PATCH 5/8] Docs: make the tutorial notebooks Colab-ready Add an install cell to the getting-started tutorial (it had none), and install neo4j-viz itself in the GDS tutorial (it was missing, so the notebook would ImportError on a fresh Colab runtime). Consolidate each notebook's install steps into a single `%pip install "neo4j-viz[extra]" ...` using the extras the integration pages document. --- examples/gds-example.ipynb | 4 +--- examples/getting-started.ipynb | 20 ++++++++++++++++++++ examples/neo4j-example.ipynb | 4 +--- examples/snowflake-example.ipynb | 3 +-- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/examples/gds-example.ipynb b/examples/gds-example.ipynb index 7e78ee9e..4dade1c1 100644 --- a/examples/gds-example.ipynb +++ b/examples/gds-example.ipynb @@ -17,9 +17,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install graphdatascience\n", - "%pip install matplotlib\n", - "%pip install python-dotenv" + "%pip install \"neo4j-viz[gds]\" matplotlib python-dotenv" ] }, { diff --git a/examples/getting-started.ipynb b/examples/getting-started.ipynb index cc461a30..134e9c76 100644 --- a/examples/getting-started.ipynb +++ b/examples/getting-started.ipynb @@ -25,6 +25,26 @@ "Lastly we create a `VisualizationGraph` object with the nodes and relationships we created, and call its `render_widget` method to display the graph as an interactive widget." ] }, + { + "cell_type": "markdown", + "id": "b9aab710", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Install the dependencies used in this tutorial:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "83d01d72", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install neo4j-viz" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/neo4j-example.ipynb b/examples/neo4j-example.ipynb index df9aeb1f..104e9431 100644 --- a/examples/neo4j-example.ipynb +++ b/examples/neo4j-example.ipynb @@ -29,9 +29,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install neo4j\n", - "%pip install neo4j-viz\n", - "%pip install python-dotenv" + "%pip install \"neo4j-viz[neo4j]\" python-dotenv" ] }, { diff --git a/examples/snowflake-example.ipynb b/examples/snowflake-example.ipynb index c8ac4548..34f85a4e 100644 --- a/examples/snowflake-example.ipynb +++ b/examples/snowflake-example.ipynb @@ -36,8 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install neo4j-viz[snowflake]\n", - "%pip install python-dotenv" + "%pip install \"neo4j-viz[snowflake]\" python-dotenv" ] }, { From ff5254412bd0177ab2f4cc1411f4efb2f416c3df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 30 Oct 2026 16:02:52 +0100 Subject: [PATCH 6/8] Test: retry lost cell runs in the browser harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The widget browser tests intermittently hung with a cell stuck showing [*]: while its kernel sat idle — the run command had gone to a session that was still being wired up (the previous test's kernel was still being torn down), so the kernel never executed the cell. While waiting for a cell the harness now also polls the kernel's execution state: if the kernel never picks the run up, the cell is re-issued via the toolbar Run button (which a focus-stealing dialog cannot swallow). Cells the kernel did run are left alone — the frontend prompt can lag well behind the kernel (e.g. while rendering an image for save()), and re-issuing those would just restart the work. Timeout errors now include the prompt and kernel state. Also shut down each notebook's session after a run so its kernel does not outlive it and interfere with the next notebook on the same server. --- python-wrapper/tests/browser_harness.py | 185 +++++++++++++++++++----- 1 file changed, 151 insertions(+), 34 deletions(-) diff --git a/python-wrapper/tests/browser_harness.py b/python-wrapper/tests/browser_harness.py index d393593b..51649dc2 100644 --- a/python-wrapper/tests/browser_harness.py +++ b/python-wrapper/tests/browser_harness.py @@ -25,6 +25,7 @@ from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path +from typing import Any import nbformat from playwright.sync_api import ( @@ -46,6 +47,13 @@ CELL_RUN_TIMEOUT = 20 CELL_RUN_ATTEMPTS = 4 CELL_COMPLETE_TIMEOUT = 180 +# A started cell whose kernel never picks it up within this long had its run +# lost (e.g. it went to a session that was still being wired up), so it gets +# re-issued. A cell the kernel did run is left alone: the frontend prompt can +# lag well behind it, e.g. while rendering an image for `save()`. +CELL_STALL_GRACE = 10 +# How often to poll the prompt and the kernel state while waiting for a cell. +CELL_POLL_INTERVAL = 0.5 WIDGET_MOUNT_TIMEOUT = 30 NOTEBOOK_RUN_TIMEOUT = 120 @@ -64,6 +72,9 @@ # dialog that grabbed keyboard focus. RUN_CELL_COMMAND = "notebook:run-cell-and-select-next" +# A finished cell shows its execution count in the prompt, e.g. "[3]:". +CELL_COMPLETE_RE = re.compile(r"^\[\d+\]:$") + class BrowserUnavailableError(RuntimeError): """Raised when neither system Chrome nor bundled Chromium can be launched.""" @@ -212,6 +223,46 @@ def _wait_until_ready(server: JupyterServer | MarimoServer, path: str) -> None: ) +def _session_for_notebook(server: JupyterServer, notebook: str) -> dict[str, Any] | None: + """Return the Lab session dict for ``notebook``, or None when it has no session.""" + try: + with urllib.request.urlopen(f"{server.url}/api/sessions", timeout=2) as response: + sessions = json.loads(response.read().decode("utf-8")) + except (urllib.error.URLError, OSError): + return None + if not isinstance(sessions, list): + return None + for session in sessions: + if isinstance(session, dict) and session.get("path") == notebook: + return dict(session) + return None + + +def _kernel_execution_state(server: JupyterServer, notebook: str) -> str | None: + """Return the notebook kernel's execution state, or None when it has no kernel. + + Uses the sessions REST API, which — unlike Lab's DOM — is version-independent. + """ + session = _session_for_notebook(server, notebook) + if session is None: + return None + state = (session.get("kernel") or {}).get("execution_state") + return state if isinstance(state, str) else None + + +def _delete_session(server: JupyterServer, notebook: str) -> None: + """Best-effort shutdown of the notebook's session so its kernel is not left running.""" + session_id = (_session_for_notebook(server, notebook) or {}).get("id") + if not isinstance(session_id, str): + return + try: + request = urllib.request.Request(f"{server.url}/api/sessions/{session_id}", method="DELETE") + with urllib.request.urlopen(request, timeout=5): + pass + except (urllib.error.URLError, OSError): + pass + + def _wait_for_kernel_idle(server: JupyterServer, notebook: str) -> None: """Wait until Lab has started the notebook's kernel. @@ -222,15 +273,8 @@ def _wait_for_kernel_idle(server: JupyterServer, notebook: str) -> None: """ deadline = time.monotonic() + KERNEL_START_TIMEOUT while time.monotonic() < deadline: - try: - with urllib.request.urlopen(f"{server.url}/api/sessions", timeout=2) as response: - sessions = json.loads(response.read().decode("utf-8")) - except (urllib.error.URLError, OSError): - sessions = [] - for session in sessions: - kernel = session.get("kernel") or {} - if session.get("path") == notebook and kernel.get("execution_state") == "idle": - return + if _kernel_execution_state(server, notebook) == "idle": + return time.sleep(0.25) raise RuntimeError( f"the kernel for {notebook} did not become idle within {KERNEL_START_TIMEOUT}s; " @@ -285,15 +329,15 @@ def _cell_start_error(prompt: Locator) -> AssertionError | None: return None -def _run_cell(page: Page, index: int) -> None: - cell = page.locator(".jp-Cell").nth(index) - prompt = cell.locator(".jp-InputArea-prompt") +def _issue_run(page: Page, cell: Locator, prompt: Locator) -> None: + """Press run for a cell until it leaves the unqueued state. - # The run command can be swallowed while JupyterLab is still settling — e.g. - # a kernel-picker dialog grabbing keyboard focus after the dialog sweep. If - # the prompt is still "[ ]:" nothing was queued, so re-trying is safe. Each - # attempt tries the keyboard first, then the toolbar's Run button, which a - # focus-stealing dialog cannot swallow. + The run command can be swallowed while JupyterLab is still settling — e.g. a + kernel-picker dialog grabbing keyboard focus after the dialog sweep. If the + prompt is still "[ ]:" nothing was queued, so re-trying is safe. Each attempt + tries the keyboard first, then the toolbar's Run button, which a + focus-stealing dialog cannot swallow. + """ start_error: AssertionError | None = None for _ in range(CELL_RUN_ATTEMPTS): _accept_visible_dialogs(page) @@ -301,26 +345,96 @@ def _run_cell(page: Page, index: int) -> None: page.keyboard.press("Shift+Enter") start_error = _cell_start_error(prompt) if start_error is None: - break + return run_button = page.locator(f'.jp-NotebookPanel-toolbar [data-command="{RUN_CELL_COMMAND}"]') if run_button.count(): run_button.first.click() start_error = _cell_start_error(prompt) if start_error is None: - break - if start_error is not None: - raise AssertionError( - f"cell {index} did not start executing (prompt is still {prompt.text_content()!r}; " - f"a dialog may be blocking input and the kernel may not have started)" - ) from start_error - # Also wait for the cell to finish (its prompt gets an execution count): a slow - # kernel — e.g. cold imports on a loaded CI runner — would otherwise eat into - # the widget-mount timeout below, which is only meant to cover the frontend - # rendering the already-displayed widget. - expect(prompt).to_have_text(re.compile(r"^\[\d+\]:$"), timeout=CELL_COMPLETE_TIMEOUT * 1000) - - -def _execute_notebook_in_browser(page: Page, cells: Sequence[str]) -> None: + return + raise AssertionError( + f"cell did not start executing (prompt is still {prompt.text_content()!r}; " + f"a dialog may be blocking input and the kernel may not have started)" + ) from start_error + + +def _reissue_run(page: Page, cell: Locator) -> None: + """Re-issue a run for a cell whose previous execution was lost. + + Uses the toolbar's Run button, which — unlike Shift+Enter — cannot be + swallowed by a dialog that grabbed keyboard focus. + """ + _accept_visible_dialogs(page) + cell.click() + run_button = page.locator(f'.jp-NotebookPanel-toolbar [data-command="{RUN_CELL_COMMAND}"]') + if run_button.count(): + run_button.first.click() + else: + page.keyboard.press("Shift+Enter") + + +def _cell_finished(prompt: Locator) -> bool: + return bool(CELL_COMPLETE_RE.match((prompt.text_content() or "").strip())) + + +def _wait_for_cell_completion( + page: Page, cell: Locator, prompt: Locator, server: JupyterServer, notebook: str, index: int +) -> None: + """Wait for a started cell to finish, re-issuing it if its run never reached the kernel. + + A slow kernel — e.g. cold imports on a loaded CI runner — legitimately keeps + the cell running, and the frontend prompt can lag well behind the kernel, so + we poll rather than wait blindly. If the kernel never picks the run up at + all, though, the run was lost before it could execute (typically because the + run command went to a session that was still being wired up), so it is + re-issued. Every cell this harness runs is idempotent — it rebuilds the + widget or rewrites the same files — so repeating one is safe. + """ + deadline = time.monotonic() + CELL_COMPLETE_TIMEOUT + attempts = 1 + ran = False + idle_since: float | None = None + while True: + if _cell_finished(prompt): + return + state = _kernel_execution_state(server, notebook) + now = time.monotonic() + if now >= deadline: + raise AssertionError( + f"cell {index} did not finish within {CELL_COMPLETE_TIMEOUT}s " + f"(prompt is {prompt.text_content()!r}, kernel is {state!r})" + ) + if state == "busy": + ran = True + idle_since = None + elif idle_since is None: + idle_since = now + elif not ran and now - idle_since > CELL_STALL_GRACE: + # The kernel never picked the run up, so it was lost before it could + # execute (e.g. the run command went to a session that was still + # being wired up): re-issue it. A run that did execute is left alone + # — the frontend can lag well behind the kernel, e.g. while it + # renders an image for ``save()``. + if attempts >= CELL_RUN_ATTEMPTS: + raise AssertionError( + f"cell {index} never reached the kernel after {attempts} attempts " + f"(prompt is {prompt.text_content()!r}, kernel is {state!r})" + ) + attempts += 1 + _reissue_run(page, cell) + idle_since = None + time.sleep(CELL_POLL_INTERVAL) + + +def _run_cell(page: Page, index: int, server: JupyterServer, notebook: str) -> None: + cell = page.locator(".jp-Cell").nth(index) + prompt = cell.locator(".jp-InputArea-prompt") + + _issue_run(page, cell, prompt) + _wait_for_cell_completion(page, cell, prompt, server, notebook, index) + + +def _execute_notebook_in_browser(page: Page, cells: Sequence[str], server: JupyterServer, notebook: str) -> None: expect(page.locator(".jp-Cell")).to_have_count(len(cells), timeout=LAB_LOAD_TIMEOUT * 1000) for index in range(len(cells)): if index == len(cells) - 1: @@ -330,7 +444,7 @@ def _execute_notebook_in_browser(page: Page, cells: Sequence[str]) -> None: # hit the "not rendered yet" reply. page.wait_for_selector(WIDGET_CANVAS_SELECTOR, timeout=WIDGET_MOUNT_TIMEOUT * 1000) _accept_visible_dialogs(page) - _run_cell(page, index) + _run_cell(page, index, server, notebook) def _await_notebook_markers(server: JupyterServer) -> None: @@ -458,7 +572,7 @@ def record_console(message: ConsoleMessage) -> None: page.goto(f"{server.url}/lab/workspaces/{workspace}/tree/{notebook_name}") _wait_for_kernel_idle(server, notebook_name) try: - _execute_notebook_in_browser(page, cells) + _execute_notebook_in_browser(page, cells, server, notebook_name) except Exception as exc: # Cell-execution failures are usually kernel-side, and the # pytest tmp artifacts never reach CI logs — inline the log. @@ -473,6 +587,9 @@ def record_console(message: ConsoleMessage) -> None: raise finally: browser.close() + # Best-effort: the kernel would otherwise outlive the run and can + # interfere with the next notebook driven by the same server. + _delete_session(server, notebook_name) if console_errors: raise RuntimeError(f"browser console errors: {console_errors}") From f1edd4b2cc62ea4e22eab337a01211a1efda2b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 30 Oct 2026 16:11:40 +0100 Subject: [PATCH 7/8] Test: revert the browser-harness cell retry, keep diagnostics Retrying a cell whose run never reached the kernel made things worse in CI: the kernel-state polling can miss a short busy window, so the retry fired spuriously and every Python matrix job failed. Restore the previous prompt-based wait (unchanged behaviour) and keep only: - best-effort session shutdown after a run, so a kernel does not outlive it, - richer failure output (the sessions JSON and the browser console tail), since the pytest tmp artifacts never reach CI logs and these failures are frontend/kernel communication problems. --- python-wrapper/tests/browser_harness.py | 123 +++++++----------------- 1 file changed, 33 insertions(+), 90 deletions(-) diff --git a/python-wrapper/tests/browser_harness.py b/python-wrapper/tests/browser_harness.py index 51649dc2..631970b1 100644 --- a/python-wrapper/tests/browser_harness.py +++ b/python-wrapper/tests/browser_harness.py @@ -47,13 +47,6 @@ CELL_RUN_TIMEOUT = 20 CELL_RUN_ATTEMPTS = 4 CELL_COMPLETE_TIMEOUT = 180 -# A started cell whose kernel never picks it up within this long had its run -# lost (e.g. it went to a session that was still being wired up), so it gets -# re-issued. A cell the kernel did run is left alone: the frontend prompt can -# lag well behind it, e.g. while rendering an image for `save()`. -CELL_STALL_GRACE = 10 -# How often to poll the prompt and the kernel state while waiting for a cell. -CELL_POLL_INTERVAL = 0.5 WIDGET_MOUNT_TIMEOUT = 30 NOTEBOOK_RUN_TIMEOUT = 120 @@ -223,18 +216,23 @@ def _wait_until_ready(server: JupyterServer | MarimoServer, path: str) -> None: ) -def _session_for_notebook(server: JupyterServer, notebook: str) -> dict[str, Any] | None: - """Return the Lab session dict for ``notebook``, or None when it has no session.""" +def _sessions(server: JupyterServer) -> list[dict[str, Any]]: + """Return the Lab sessions, or an empty list when the sessions API is unreachable.""" try: with urllib.request.urlopen(f"{server.url}/api/sessions", timeout=2) as response: sessions = json.loads(response.read().decode("utf-8")) except (urllib.error.URLError, OSError): - return None + return [] if not isinstance(sessions, list): - return None - for session in sessions: - if isinstance(session, dict) and session.get("path") == notebook: - return dict(session) + return [] + return [session for session in sessions if isinstance(session, dict)] + + +def _session_for_notebook(server: JupyterServer, notebook: str) -> dict[str, Any] | None: + """Return the Lab session dict for ``notebook``, or None when it has no session.""" + for session in _sessions(server): + if session.get("path") == notebook: + return session return None @@ -291,6 +289,11 @@ def _tail_text(path: Path, lines: int = 40) -> str: return "\n".join(content[-lines:]) +def _console_tail(messages: Sequence[str], lines: int = 40) -> str: + """Return the last ``lines`` browser console messages, for CI failure output.""" + return "\n".join(messages[-lines:]) or "" + + def _launch_chromium(playwright: Playwright) -> Browser: # Prefer the system Chrome (also preinstalled on CI runners) so that no # `playwright install` download is needed; fall back to Playwright's @@ -358,83 +361,19 @@ def _issue_run(page: Page, cell: Locator, prompt: Locator) -> None: ) from start_error -def _reissue_run(page: Page, cell: Locator) -> None: - """Re-issue a run for a cell whose previous execution was lost. - - Uses the toolbar's Run button, which — unlike Shift+Enter — cannot be - swallowed by a dialog that grabbed keyboard focus. - """ - _accept_visible_dialogs(page) - cell.click() - run_button = page.locator(f'.jp-NotebookPanel-toolbar [data-command="{RUN_CELL_COMMAND}"]') - if run_button.count(): - run_button.first.click() - else: - page.keyboard.press("Shift+Enter") - - -def _cell_finished(prompt: Locator) -> bool: - return bool(CELL_COMPLETE_RE.match((prompt.text_content() or "").strip())) - - -def _wait_for_cell_completion( - page: Page, cell: Locator, prompt: Locator, server: JupyterServer, notebook: str, index: int -) -> None: - """Wait for a started cell to finish, re-issuing it if its run never reached the kernel. - - A slow kernel — e.g. cold imports on a loaded CI runner — legitimately keeps - the cell running, and the frontend prompt can lag well behind the kernel, so - we poll rather than wait blindly. If the kernel never picks the run up at - all, though, the run was lost before it could execute (typically because the - run command went to a session that was still being wired up), so it is - re-issued. Every cell this harness runs is idempotent — it rebuilds the - widget or rewrites the same files — so repeating one is safe. - """ - deadline = time.monotonic() + CELL_COMPLETE_TIMEOUT - attempts = 1 - ran = False - idle_since: float | None = None - while True: - if _cell_finished(prompt): - return - state = _kernel_execution_state(server, notebook) - now = time.monotonic() - if now >= deadline: - raise AssertionError( - f"cell {index} did not finish within {CELL_COMPLETE_TIMEOUT}s " - f"(prompt is {prompt.text_content()!r}, kernel is {state!r})" - ) - if state == "busy": - ran = True - idle_since = None - elif idle_since is None: - idle_since = now - elif not ran and now - idle_since > CELL_STALL_GRACE: - # The kernel never picked the run up, so it was lost before it could - # execute (e.g. the run command went to a session that was still - # being wired up): re-issue it. A run that did execute is left alone - # — the frontend can lag well behind the kernel, e.g. while it - # renders an image for ``save()``. - if attempts >= CELL_RUN_ATTEMPTS: - raise AssertionError( - f"cell {index} never reached the kernel after {attempts} attempts " - f"(prompt is {prompt.text_content()!r}, kernel is {state!r})" - ) - attempts += 1 - _reissue_run(page, cell) - idle_since = None - time.sleep(CELL_POLL_INTERVAL) - - -def _run_cell(page: Page, index: int, server: JupyterServer, notebook: str) -> None: +def _run_cell(page: Page, index: int) -> None: cell = page.locator(".jp-Cell").nth(index) prompt = cell.locator(".jp-InputArea-prompt") _issue_run(page, cell, prompt) - _wait_for_cell_completion(page, cell, prompt, server, notebook, index) + # Also wait for the cell to finish (its prompt gets an execution count): a slow + # kernel — e.g. cold imports on a loaded CI runner — would otherwise eat into + # the widget-mount timeout below, which is only meant to cover the frontend + # rendering the already-displayed widget. + expect(prompt).to_have_text(CELL_COMPLETE_RE, timeout=CELL_COMPLETE_TIMEOUT * 1000) -def _execute_notebook_in_browser(page: Page, cells: Sequence[str], server: JupyterServer, notebook: str) -> None: +def _execute_notebook_in_browser(page: Page, cells: Sequence[str]) -> None: expect(page.locator(".jp-Cell")).to_have_count(len(cells), timeout=LAB_LOAD_TIMEOUT * 1000) for index in range(len(cells)): if index == len(cells) - 1: @@ -444,7 +383,7 @@ def _execute_notebook_in_browser(page: Page, cells: Sequence[str], server: Jupyt # hit the "not rendered yet" reply. page.wait_for_selector(WIDGET_CANVAS_SELECTOR, timeout=WIDGET_MOUNT_TIMEOUT * 1000) _accept_visible_dialogs(page) - _run_cell(page, index, server, notebook) + _run_cell(page, index) def _await_notebook_markers(server: JupyterServer) -> None: @@ -572,12 +511,16 @@ def record_console(message: ConsoleMessage) -> None: page.goto(f"{server.url}/lab/workspaces/{workspace}/tree/{notebook_name}") _wait_for_kernel_idle(server, notebook_name) try: - _execute_notebook_in_browser(page, cells, server, notebook_name) + _execute_notebook_in_browser(page, cells) except Exception as exc: - # Cell-execution failures are usually kernel-side, and the - # pytest tmp artifacts never reach CI logs — inline the log. + # Cell-execution failures are usually frontend/kernel + # communication problems, and the pytest tmp artifacts never + # reach CI logs — inline the state that explains them. raise RuntimeError( - f"{exc}\n--- jupyter-server.log tail ---\n{_tail_text(server.log_path)}" + f"{exc}\n" + f"--- sessions ---\n{json.dumps(_sessions(server), indent=2)}\n" + f"--- browser console tail ---\n{_console_tail(console_messages)}\n" + f"--- jupyter-server.log tail ---\n{_tail_text(server.log_path)}" ) from exc if check is not None: check(page) From a44bb12364b24af871e30e53ab45a7f480d47125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 30 Oct 2026 16:15:07 +0100 Subject: [PATCH 8/8] Test: retry a failed notebook run in the browser harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The widget browser tests intermittently hang with a cell stuck showing [*]: while its kernel sits idle — the JupyterLab frontend failed to reach the kernel, so the run never executed, and re-issuing the cell does not help because the request still cannot get through. Retry the whole run instead: a failed run is repeated from scratch — a fresh notebook, page, and kernel — up to NOTEBOOK_RUN_ATTEMPTS. BrowserUnavailable still propagates so callers can skip. Verified with a temporary test that fails on the first attempt and passes on the retry. --- python-wrapper/tests/browser_harness.py | 35 ++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/python-wrapper/tests/browser_harness.py b/python-wrapper/tests/browser_harness.py index 631970b1..65a287b2 100644 --- a/python-wrapper/tests/browser_harness.py +++ b/python-wrapper/tests/browser_harness.py @@ -49,6 +49,10 @@ CELL_COMPLETE_TIMEOUT = 180 WIDGET_MOUNT_TIMEOUT = 30 NOTEBOOK_RUN_TIMEOUT = 120 +# A notebook run is retried from scratch (fresh notebook, page, and kernel) +# because the JupyterLab frontend intermittently fails to reach the kernel, +# leaving a cell stuck showing "[*]:" forever. +NOTEBOOK_RUN_ATTEMPTS = 2 # Kernel-side marker files written by the last notebook cell: ``_done.txt`` on # success, ``_error.txt`` with the traceback on failure. @@ -473,9 +477,7 @@ def assert_no_ndl_leak(page: Page) -> None: ) -def run_notebook_in_browser( - server: JupyterServer, cells: Sequence[str], check: Callable[[Page], None] | None = None -) -> None: +def _run_notebook_once(server: JupyterServer, cells: Sequence[str], check: Callable[[Page], None] | None) -> None: """Create a notebook from ``cells`` on ``server`` and run it in headless Chrome. The last cell must write the ``_done.txt``/``_error.txt`` markers (see @@ -537,6 +539,33 @@ def record_console(message: ConsoleMessage) -> None: raise RuntimeError(f"browser console errors: {console_errors}") +def run_notebook_in_browser( + server: JupyterServer, cells: Sequence[str], check: Callable[[Page], None] | None = None +) -> None: + """Run a notebook on ``server`` in headless Chrome, retrying a failed run. + + A failed run is retried from scratch — a fresh notebook, page, and kernel — + because the JupyterLab frontend intermittently fails to reach the kernel, + leaving a cell stuck showing ``[*]:`` forever. See ``_run_notebook_once`` + for the contract of a single run. + """ + errors: list[str] = [] + for attempt in range(1, NOTEBOOK_RUN_ATTEMPTS + 1): + try: + _run_notebook_once(server, cells, check) + return + except BrowserUnavailableError: + raise + except Exception as exc: + errors.append(f"attempt {attempt}:\n{exc}") + if attempt < NOTEBOOK_RUN_ATTEMPTS: + print( + f"[harness] notebook run failed on attempt {attempt}; retrying with a fresh notebook, page, and kernel", + flush=True, + ) + raise RuntimeError(f"notebook did not run after {NOTEBOOK_RUN_ATTEMPTS} attempts:\n\n" + "\n\n".join(errors)) + + def run_marimo_app_in_browser(server: MarimoServer, check: Callable[[Page], None]) -> None: """Open the Marimo app served by ``server`` in headless Chrome and run ``check``.