From 3a5b212ef6125b3017c8e64f05d919fa3aaf2715 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 2 Oct 2026 14:41:21 +0300 Subject: [PATCH 01/16] Move notebook execution to a persistent JDI executor Launch the managing kernel directly and remove the launcher JVM. Keep user state in a separate executor across cells, imports and redefinitions. - Pass JJAVA_JVM_OPTS to the executor JVM. - Handle magics and extension lifecycle in the managing kernel. - Return ordinary expression results as text/plain. - Transfer portable display bundles over a loopback socket. - Report timeouts, interrupts and executor termination explicitly. - Wire executor stdout/stderr directly to JupyterIO. - Add regression coverage for state, recovery and display transport. --- CLAUDE.md | 2 +- DEV-GUIDE.md | 2 +- jjava-distro/assembly/zipped-kernel.xml | 4 - .../main/java/org/dflib/jjava/distro/Env.java | 6 +- .../java/org/dflib/jjava/distro/JJava.java | 1 + .../org/dflib/jjava/distro/CellOutput.java | 2 +- .../org/dflib/jjava/distro/KernelEnvIT.java | 34 ++- .../dflib/jjava/distro/KernelExecutionIT.java | 89 ++++++- .../org/dflib/jjava/distro/KernelMagicIT.java | 90 +++++-- .../dflib/jjava/distro/KernelStartupIT.java | 48 +++- .../src/test/resources/kernel-driver.py | 70 ++++-- .../jupyter/kernel/BaseNotebookStatics.java | 167 +++++++++++-- .../jupyter/kernel/display/DisplayData.java | 12 + .../jupyter/kernel/magic/MagicsResolver.java | 7 +- .../BaseNotebookStaticsDisplayTest.java | 163 ++++++++++++ .../org/dflib/jjava/kernel/JavaKernel.java | 202 ++++++++++++--- .../dflib/jjava/kernel/JavaKernelBuilder.java | 24 +- .../jjava/kernel/JavaNotebookStatics.java | 6 +- .../jjava/kernel/execution/CodeEvaluator.java | 235 ++++++++---------- .../execution/DisplayCallbackServer.java | 126 ++++++++++ .../ExecutorTerminationException.java | 7 + .../dflib/jjava/kernel/magics/TimeMagic.java | 58 +---- .../kernel/DisplayCallbackServerTest.java | 177 +++++++++++++ .../kernel/JavaKernelEvalBuilderTest.java | 79 ++++++ .../JavaKernelExtensionsLifecycleTest.java | 14 +- .../kernel/execution/CodeEvaluatorTest.java | 56 +++++ .../JdiExecutionControlRedefinitionTest.java | 34 +++ .../execution/JdiExecutionControlTest.java | 32 +++ .../kernel/test/ExtraClasspathExtension.java | 3 + .../jjava/kernel/test/EvalExtension.java | 3 + jjava-launcher/pom.xml | 72 ------ .../dflib/jjava/launcher/KernelLauncher.java | 116 --------- kernelspec/java/kernel.json | 3 +- pom.xml | 1 - 34 files changed, 1433 insertions(+), 512 deletions(-) create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStaticsDisplayTest.java create mode 100644 jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/DisplayCallbackServer.java create mode 100644 jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/ExecutorTerminationException.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/DisplayCallbackServerTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEvalBuilderTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/CodeEvaluatorTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlRedefinitionTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlTest.java delete mode 100644 jjava-launcher/pom.xml delete mode 100644 jjava-launcher/src/main/java/org/dflib/jjava/launcher/KernelLauncher.java diff --git a/CLAUDE.md b/CLAUDE.md index 4435e86..9d8b424 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Build, Test, Run - `mvn clean package` — full build; produces the shaded kernel jar and the kernelspec zip under `jjava-distro/target/`. -- `mvn clean verify` — what CI runs on Linux/macOS. Adds Failsafe integration tests (the `*IT.java` cases in `jjava-distro`, which spin up `eclipse-temurin` containers via Testcontainers). Testcontainers does not work on Windows; CI uses `mvn clean test` there. +- `mvn clean verify` — runs Failsafe integration tests (`*IT.java` in `jjava-distro`) via Testcontainers if Testcontainers can reach Docker (Windows GitHub-hosted runners cannot run WSL-backed containers). If Docker is unavailable, use `mvn clean test`. - `mvn test -pl jjava-kernel -am` — single-module unit tests (with required upstream modules built). - `mvn -pl jjava-kernel test -Dtest=JavaKernelExtensionsLifecycleTest` — run a single test class. - Surefire in `jjava-kernel` and `jjava-distro` is configured with `--add-opens jdk.jshell/jdk.jshell=ALL-UNNAMED` — required for JShell reflection. Anything that exercises `JavaKernel`/`CodeEvaluator` directly needs the same flag. diff --git a/DEV-GUIDE.md b/DEV-GUIDE.md index 892a5a0..2c64483 100644 --- a/DEV-GUIDE.md +++ b/DEV-GUIDE.md @@ -22,7 +22,7 @@ You will need a **JDK** >= 11 and proper credentials for the `sonatype-central` Two submodules of this project have different release strategies: -- the library modules (`jjava-jupyter`, `jjava-kernel`, `jjava-maven`, `jjava-launcher`) are released +- the library modules (`jjava-jupyter`, `jjava-kernel`, `jjava-maven`) are released on Maven Central via the Sonatype Central Portal - the `jjava-distro` kernel assembly is released through GitHub Releases, and from there picked up by the [Homebrew tap](https://github.com/dflib/homebrew-tap) and PyPI diff --git a/jjava-distro/assembly/zipped-kernel.xml b/jjava-distro/assembly/zipped-kernel.xml index 9bb0369..12e9e3f 100644 --- a/jjava-distro/assembly/zipped-kernel.xml +++ b/jjava-distro/assembly/zipped-kernel.xml @@ -8,10 +8,6 @@ ${project.build.directory}/jjava-distro-${project.version}.jar jjava.jar - - ../jjava-launcher/${project.build.directory}/jjava-launcher-${project.version}.jar - jjava-launcher.jar - ../kernelspec/java/kernel.json diff --git a/jjava-distro/src/main/java/org/dflib/jjava/distro/Env.java b/jjava-distro/src/main/java/org/dflib/jjava/distro/Env.java index 2d19ac8..2b8e75c 100644 --- a/jjava-distro/src/main/java/org/dflib/jjava/distro/Env.java +++ b/jjava-distro/src/main/java/org/dflib/jjava/distro/Env.java @@ -24,7 +24,6 @@ final class Env { public static final String JJAVA_STARTUP_SCRIPT = "JJAVA_STARTUP_SCRIPT"; public static final String JJAVA_LOAD_EXTENSIONS = "JJAVA_LOAD_EXTENSIONS"; - // not used by JJava, but rather by the kernel launcher script public static final String JJAVA_JVM_OPTS = "JJAVA_JVM_OPTS"; public static String timeout() { @@ -48,6 +47,11 @@ public static List compilerOpts() { return optsString != null ? Opts.splitOpts(optsString) : java.util.List.of(); } + public static List jvmOpts() { + String optsString = System.getenv(Env.JJAVA_JVM_OPTS); + return optsString != null ? Opts.splitOpts(optsString) : List.of(); + } + public static String extraClasspath() { return System.getenv(Env.JJAVA_CLASSPATH); } diff --git a/jjava-distro/src/main/java/org/dflib/jjava/distro/JJava.java b/jjava-distro/src/main/java/org/dflib/jjava/distro/JJava.java index 01059a8..0c92739 100644 --- a/jjava-distro/src/main/java/org/dflib/jjava/distro/JJava.java +++ b/jjava-distro/src/main/java/org/dflib/jjava/distro/JJava.java @@ -63,6 +63,7 @@ public static void main(String[] args) throws Exception { .extensionsEnabled(Env.extensionsEnabled()) .compilerOpts(Env.compilerOpts()) + .remoteVMOptions(Env.jvmOpts()) .timeout(timeout.time, timeout.timeUnit) .lineMagic("load", new LoadMagic()) diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/CellOutput.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/CellOutput.java index 91aa28f..7efa4cc 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/CellOutput.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/CellOutput.java @@ -87,7 +87,7 @@ public CellOutput assertError(String... errorFragments) { return this; } - private String field(String name) { + String field(String name) { return fields.getOrDefault(name, ""); } diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelEnvIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelEnvIT.java index 16bbcb7..1e0a7b0 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelEnvIT.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelEnvIT.java @@ -5,6 +5,7 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class KernelEnvIT extends ContainerizedKernelCase { @@ -25,9 +26,34 @@ public void compilerOpts() throws Exception { public void timeout() throws Exception { Map env = Map.of(Env.JJAVA_TIMEOUT, "3000"); String cell = "Thread.sleep(5000);"; - KernelRun run = executeInKernel(env, cell); + KernelRun run = executeInKernel(env, cell, "2 + 2"); run.cell(1).assertError("| " + cell, "Evaluation timed out after 3000 milliseconds."); + assertEquals("4", run.cell(2).result()); + } + + @Test + public void timeoutCoversTheWholeCell() throws Exception { + String cell = "Thread.sleep(2000); System.out.println(\"first complete\"); Thread.sleep(2000);"; + KernelRun run = executeInKernel(Map.of(Env.JJAVA_TIMEOUT, "3000"), "1 + 1", cell); + + assertEquals("2", run.cell(1).result()); + run.cell(2).assertError("Evaluation timed out after 3000 milliseconds."); + assertTrue(run.cell(2).stdout().contains("first complete"), run.cell(2).toString()); + } + + @Test + public void zeroTimeoutDisablesDeadline() throws Exception { + KernelRun run = executeInKernel(Map.of(Env.JJAVA_TIMEOUT, "0"), "Thread.sleep(100); 42").assertNoErrors(); + + assertEquals("42", run.cell(1).result()); + } + + @Test + public void negativeTimeoutDisablesDeadline() throws Exception { + KernelRun run = executeInKernel(Map.of(Env.JJAVA_TIMEOUT, "-1"), "Thread.sleep(100); 42").assertNoErrors(); + + assertEquals("42", run.cell(1).result()); } @Test @@ -38,7 +64,7 @@ public void classpath() throws Exception { "\"className = \" + Dummy.class.getName();" ).assertNoErrors(); - assertEquals("className = org.dflib.jjava.Dummy", run.cell(2).result()); + assertEquals("\"className = org.dflib.jjava.Dummy\"", run.cell(2).result()); } @Test @@ -46,7 +72,7 @@ public void startUpScriptsPath() throws Exception { Map env = Map.of(Env.JJAVA_STARTUP_SCRIPTS_PATH, CONTAINER_RESOURCES + "/test-ping.jshell"); KernelRun run = executeInKernel(env, "ping()").assertNoErrors(); - assertEquals("pong!", run.cell(1).result()); + assertEquals("\"pong!\"", run.cell(1).result()); } @Test @@ -54,7 +80,7 @@ public void startUpScript() throws Exception { Map env = Map.of(Env.JJAVA_STARTUP_SCRIPT, "public String ping() { return \"pong!\"; }"); KernelRun run = executeInKernel(env, "ping()").assertNoErrors(); - assertEquals("pong!", run.cell(1).result()); + assertEquals("\"pong!\"", run.cell(1).result()); } @Test diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java index 1c1d68a..790b5d5 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class KernelExecutionIT extends ContainerizedKernelCase { @@ -22,7 +23,7 @@ public void variableSurvivesLaterImports() throws Exception { ) ).assertNoErrors(); - assertEquals("com.fasterxml.jackson.databind.ObjectMapper", run.cell(2).result()); + assertEquals("\"com.fasterxml.jackson.databind.ObjectMapper\"", run.cell(2).result()); } @Test @@ -45,7 +46,89 @@ public void variableRedeclarationInSameCell() throws Exception { "var v = \"b\";\nv" ).assertNoErrors(); - assertEquals("a", run.cell(1).result()); - assertEquals("b", run.cell(2).result()); + assertEquals("\"a\"", run.cell(1).result()); + assertEquals("\"b\"", run.cell(2).result()); + } + + @Test + public void textPlainStringsAndCustomObjects() throws Exception { + KernelRun run = executeInKernel( + "\"quoted\"", + "\"line1\\nline2\"", + "class DisplayValue { public String toString() { return \"custom-value\"; } }", + "new DisplayValue()" + ).assertNoErrors(); + + assertEquals("\"quoted\"", run.cell(1).result()); + assertEquals("\"line1\\nline2\"", run.cell(2).result()); + assertEquals("custom-value", run.cell(4).result()); + } + + @Test + public void methodRedefinitionKeepsSavedValues() throws Exception { + KernelRun run = executeInKernel( + "int calculate() { return 1; }", + "int saved = calculate();", + "int calculate() { return 2; }", + "saved", + "calculate()" + ).assertNoErrors(); + + assertEquals("1", run.cell(4).result()); + assertEquals("2", run.cell(5).result()); + } + + @Test + public void classRedefinitionKeepsSavedValuesAndInstances() throws Exception { + KernelRun run = executeInKernel( + "class Reloadable { int value() { return 1; } }", + "Reloadable instance = new Reloadable();", + "int saved = instance.value();", + "class Reloadable { int value() { return 2; } }", + "saved", + "instance.value()", + "new Reloadable().value()" + ).assertNoErrors(); + + assertEquals("1", run.cell(5).result()); + assertEquals("2", run.cell(6).result()); + assertEquals("2", run.cell(7).result()); + } + + @Test + public void interruptStopsBusyLoopAndKeepsExecutor() throws Exception { + KernelRun run = executeInKernel( + "@@INTERRUPT_AFTER_MS=500@@while (true) {}", + "1 + 1" + ); + + run.cell(1).assertError("Evaluation interrupted."); + assertTrue(Long.parseLong(run.cell(1).field("interruptLatencyMs")) <= 2000, "interrupt was not handled within two seconds"); + assertEquals("2", run.cell(2).result()); + } + + @Test + public void executorStreamsReachNotebook() throws Exception { + KernelRun run = executeInKernel( + "System.out.print(\"stdout-1\"); System.err.print(\"stderr-1\");", + "System.out.print(\"stdout-2\"); System.err.print(\"stderr-2\");" + ).assertNoErrors(); + + assertEquals("stdout-1", run.cell(1).stdout()); + assertEquals("stderr-1", run.cell(1).stderr()); + assertEquals("stdout-2", run.cell(2).stdout()); + assertEquals("stderr-2", run.cell(2).stderr()); + } + + @Test + public void userExceptionDoesNotLoseExecutorState() throws Exception { + KernelRun run = executeInKernel( + "int saved = 7;", + "throw new IllegalStateException(\"outer\", new IllegalArgumentException(\"inner\"));", + "saved + 1" + ); + + run.cell(2).assertError("IllegalStateException", "outer", "IllegalArgumentException", "inner", "at .(#"); + assertEquals("8", run.cell(3).result()); } } diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelMagicIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelMagicIT.java index 6554657..6398678 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelMagicIT.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelMagicIT.java @@ -1,5 +1,7 @@ package org.dflib.jjava.distro; +import com.google.gson.Gson; +import com.google.gson.JsonObject; import org.junit.jupiter.api.Test; import org.testcontainers.containers.Container; @@ -8,24 +10,23 @@ public class KernelMagicIT extends ContainerizedKernelCase { + private static final Gson GSON = new Gson(); + @Deprecated @Test public void jars() throws Exception { - String jar = CONTAINER_RESOURCES + "/jakarta.annotation-api-3.0.0.jar"; - Container.ExecResult fetchResult = container.execInContainer( - "curl", "-L", "-s", "-S", "-f", - "https://repo1.maven.org/maven2/jakarta/annotation/jakarta.annotation-api/3.0.0/jakarta.annotation-api-3.0.0.jar", - "-o", jar - ); - assertEquals(0, fetchResult.getExitCode(), fetchResult.getStdout()); + String jar = CONTAINER_RESOURCES + "/dummy.jar"; + Container.ExecResult jarResult = container.execInContainer( + "jar", "--create", "--file", jar, "-C", TEST_CLASSPATH, "."); + assertEquals(0, jarResult.getExitCode(), jarResult.getStderr() + "\n" + jarResult.getStdout()); KernelRun run = executeInKernel( "%jars " + jar, - "import jakarta.annotation.Nullable;", - "Nullable.class.getName()" + "import org.dflib.jjava.Dummy;", + "Dummy.class.getName()" ).assertNoErrors(); - assertEquals("jakarta.annotation.Nullable", run.cell(3).result()); + assertEquals("\"org.dflib.jjava.Dummy\"", run.cell(3).result()); } @Test @@ -36,18 +37,30 @@ public void classpath() throws Exception { "\"className = \" + Dummy.class.getName();" ).assertNoErrors(); - assertEquals("className = org.dflib.jjava.Dummy", run.cell(3).result()); + assertEquals("\"className = org.dflib.jjava.Dummy\"", run.cell(3).result()); } @Test public void maven() throws Exception { KernelRun run = executeInKernel( - "%maven org.dflib:dflib-jupyter:1.0.0-RC1", - "System.getProperty(\"java.class.path\")" + "%maven jakarta.annotation:jakarta.annotation-api:3.0.0", + "import jakarta.annotation.Nullable;", + "Nullable.class.getName()" ).assertNoErrors(); - String classpath = run.cell(2).result(); - assertTrue(classpath.contains("dflib-jupyter-1.0.0-RC1.jar"), () -> "Unexpected classpath: " + classpath); + assertEquals("\"jakarta.annotation.Nullable\"", run.cell(3).result()); + } + + @Test + public void executorKernelHelperFailsExplicitlyWithoutLosingState() throws Exception { + KernelRun run = executeInKernel( + "int saved = 40;", + "kernel();", + "saved + 2" + ); + + run.cell(2).assertError("JavaKernel.kernel() cannot expose the managing JVM"); + assertEquals("42", run.cell(3).result()); } @Deprecated @@ -55,11 +68,48 @@ public void maven() throws Exception { public void mavenIvySyntax() throws Exception { KernelRun run = executeInKernel( "%maven jakarta.annotation#jakarta.annotation-api;3.0.0", - "System.getProperty(\"java.class.path\")" + "import jakarta.annotation.Nullable;", + "Nullable.class.getName()" ).assertNoErrors(); - String classpath = run.cell(2).result(); - assertTrue(classpath.contains("jakarta.annotation-api-3.0.0.jar"), () -> "Unexpected classpath: " + classpath); + assertEquals("\"jakarta.annotation.Nullable\"", run.cell(3).result()); + } + + @Test + public void timeMagicCompletes() throws Exception { + KernelRun run = executeInKernel("%time 1 + 1").assertNoErrors(); + + assertEquals("2", run.cell(1).result()); + assertTrue(run.cell(1).display().contains("Wall time:"), run.cell(1).display()); + } + + @Test + public void portableDisplayBundlesReachTheManager() throws Exception { + String cell = String.join("\n", + "import org.dflib.jjava.jupyter.kernel.display.DisplayData;", + "var bundle = new DisplayData(\"portable\");", + "bundle.putHTML(\"portable\");", + "bundle.putData(\"image/png\", \"AQID\");", + "String id = display(bundle);" + ); + KernelRun run = executeInKernel(cell, "updateDisplay(id, new DisplayData(\"updated\"));").assertNoErrors(); + + JsonObject mime = GSON.fromJson(run.cell(1).field("mime"), JsonObject.class); + assertEquals("portable", mime.get("text/plain").getAsString()); + assertEquals("portable", mime.get("text/html").getAsString()); + assertEquals("AQID", mime.get("image/png").getAsString()); + assertEquals("updated", run.cell(2).display()); + } + + @Test + public void nonPortableDisplayPayloadFailsVisibly() throws Exception { + KernelRun run = executeInKernel( + "var data = new org.dflib.jjava.jupyter.kernel.display.DisplayData(); data.putData(\"text/html\", new Object()); display(data);", + "40 + 2" + ); + + run.cell(1).assertError("Non-portable display value"); + assertEquals("42", run.cell(2).result()); } @Test @@ -69,7 +119,7 @@ public void load() throws Exception { "ping()" ).assertNoErrors(); - assertEquals("pong!", run.cell(2).result()); + assertEquals("\"pong!\"", run.cell(2).result()); } @Test @@ -80,6 +130,6 @@ public void loadFromPOM() throws Exception { "Nullable.class.getName()" ).assertNoErrors(); - assertEquals("jakarta.annotation.Nullable", run.cell(3).result()); + assertEquals("\"jakarta.annotation.Nullable\"", run.cell(3).result()); } } diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java index eceea46..ee7a7ee 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java @@ -10,10 +10,50 @@ public class KernelStartupIT extends ContainerizedKernelCase { @Test - public void startUp() throws Exception { - KernelRun run = executeInKernel("1000d + 1").assertNoErrors(); + public void directStartupUsesOneExecutor() throws Exception { + KernelRun run = executeInKernel( + "ProcessHandle.current().pid()", + "ProcessHandle.current().pid()", + "ProcessHandle.current().parent().get().info().commandLine().orElse(\"\")" + ).assertNoErrors(); - assertEquals("1001.0", run.cell(1).result()); + assertEquals(run.cell(1).result(), run.cell(2).result()); + assertTrue(run.cell(3).result().contains("jjava.jar"), run.cell(3).result()); + } + + @Test + public void executorTerminationMarksSessionLost() throws Exception { + KernelRun run = executeInKernel( + Map.of(Env.JJAVA_TIMEOUT, "3000"), + "Runtime.getRuntime().halt(17);", + "1 + 1" + ); + + run.cell(1).assertError("Executor process terminated", "Restart the kernel"); + run.cell(2).assertError("Executor process terminated", "Restart the kernel"); + } + + @Test + public void executorOutOfMemoryMarksSessionLost() throws Exception { + KernelRun run = executeInKernel( + Map.of(Env.JJAVA_JVM_OPTS, "-Xmx128m"), + "new byte[Integer.MAX_VALUE]", + "1 + 1" + ); + + run.cell(1).assertError("Executor process terminated", "Restart the kernel"); + run.cell(2).assertError("Executor process terminated", "Restart the kernel"); + } + + @Test + public void notebookInitializerImportsAndHelpers() throws Exception { + KernelRun run = executeInKernel( + "LocalDate.of(2025, 1, 2).getYear()", + "printf(\"helper-%s\", \"ready\");" + ).assertNoErrors(); + + assertEquals("2025", run.cell(1).result()); + assertEquals("helper-ready", run.cell(2).stdout()); } @Test @@ -25,6 +65,6 @@ public void startUp_scriptRequiresClasspath() throws Exception { KernelRun run = executeInKernel(env, "\"hash = \" + obj.hashCode()").assertNoErrors(); String result = run.cell(1).result(); - assertTrue(result.matches("hash = -?\\d+"), () -> "Unexpected result: " + result); + assertTrue(result.matches("\\\"hash = -?\\d+\\\""), () -> "Unexpected result: " + result); } } diff --git a/jjava-distro/src/test/resources/kernel-driver.py b/jjava-distro/src/test/resources/kernel-driver.py index e574ed3..b53932c 100644 --- a/jjava-distro/src/test/resources/kernel-driver.py +++ b/jjava-distro/src/test/resources/kernel-driver.py @@ -15,8 +15,11 @@ """ import base64 +import json import re import sys +import threading +import time from queue import Empty from jupyter_client.manager import start_new_kernel @@ -47,31 +50,50 @@ def receive(receiver, channel, cell_number): % (cell_number, channel, MESSAGE_TIMEOUT)) -def execute(client, cell_number, source): +def execute(client, manager, cell_number, source): + interrupt_match = re.match(r"^@@INTERRUPT_AFTER_MS=(\d+)@@", source) + interrupt_after_ms = int(interrupt_match.group(1)) if interrupt_match else None + if interrupt_match: + source = source[interrupt_match.end():].lstrip("\n") + + started = time.monotonic() msg_id = client.execute(source, allow_stdin=False) + timer = None + if interrupt_after_ms is not None: + timer = threading.Timer(interrupt_after_ms / 1000, manager.interrupt_kernel) + timer.daemon = True + timer.start() - outputs = {"result": "", "display": "", "stdout": "", "stderr": "", "error": ""} - while True: - message = receive(client.get_iopub_msg, "iopub", cell_number) - if message["parent_header"].get("msg_id") != msg_id: - continue - - message_type = message["msg_type"] - content = message["content"] - if message_type == "status": - if content["execution_state"] == "idle": - break - elif message_type == "stream": - key = "stdout" if content["name"] == "stdout" else "stderr" - outputs[key] += content["text"] - elif message_type == "execute_result": - outputs["result"] += content["data"].get("text/plain", "") - elif message_type in ("display_data", "update_display_data"): - outputs["display"] += content["data"].get("text/plain", "") - elif message_type == "error": - lines = ["%s: %s" % (content["ename"], content["evalue"])] - lines.extend(content["traceback"]) - outputs["error"] += ANSI.sub("", "\n".join(lines)) + outputs = {"result": "", "display": "", "mime": "", "stdout": "", "stderr": "", "error": ""} + try: + while True: + message = receive(client.get_iopub_msg, "iopub", cell_number) + if message["parent_header"].get("msg_id") != msg_id: + continue + + message_type = message["msg_type"] + content = message["content"] + if message_type == "status": + if content["execution_state"] == "idle": + break + elif message_type == "stream": + key = "stdout" if content["name"] == "stdout" else "stderr" + outputs[key] += content["text"] + elif message_type == "execute_result": + outputs["result"] += content["data"].get("text/plain", "") + elif message_type in ("display_data", "update_display_data"): + outputs["display"] += content["data"].get("text/plain", "") + outputs["mime"] += json.dumps(content["data"], separators=(",", ":")) + elif message_type == "error": + lines = ["%s: %s" % (content["ename"], content["evalue"])] + lines.extend(content["traceback"]) + outputs["error"] += ANSI.sub("", "\n".join(lines)) + finally: + if timer: + timer.cancel() + + if interrupt_after_ms is not None: + outputs["interruptLatencyMs"] = str(max(0, int((time.monotonic() - started) * 1000) - interrupt_after_ms)) # "wait_for_ready" re-sends "kernel_info_request" until the kernel answers, and consumes # only one of the replies, so the shell channel may still hold replies to those extra @@ -90,7 +112,7 @@ def main(cells): manager, client = start_new_kernel(kernel_name="java", startup_timeout=STARTUP_TIMEOUT) try: for i, source in enumerate(cells): - execute(client, i + 1, source) + execute(client, manager, i + 1, source) finally: client.stop_channels() manager.shutdown_kernel() diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java index 8acdf85..6850a81 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java @@ -1,9 +1,22 @@ package org.dflib.jjava.jupyter.kernel; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; import org.dflib.jjava.jupyter.kernel.display.DisplayData; import org.dflib.jjava.jupyter.kernel.magic.UndefinedMagicException; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Map; import java.util.UUID; /** @@ -12,16 +25,25 @@ */ public class BaseNotebookStatics { + private static final Gson GSON = new GsonBuilder().serializeNulls().create(); + public static void printf(String format, Object... args) { System.out.printf(format, args); } public static Object eval(String source) { - return BaseKernel.notebookKernel().evalBuilder(source).resolveMagics().eval(); + BaseKernel kernel = BaseKernel.notebookKernel; + if (kernel == null) { + throw new UnsupportedOperationException("eval() cannot be called from executor code in JDI mode"); + } + return kernel.evalBuilder(source).resolveMagics().eval(); } public static T lineMagic(String name, List args) { - BaseKernel kernel = BaseKernel.notebookKernel(); + BaseKernel kernel = BaseKernel.notebookKernel; + if (kernel == null) { + throw new UnsupportedOperationException("Magics cannot be called programmatically from executor code in JDI mode; use %magic syntax"); + } try { return kernel.getMagicsRegistry().evalLineMagic(kernel, name, args); @@ -33,7 +55,10 @@ public static T lineMagic(String name, List args) { } public static T cellMagic(String name, List args, String body) { - BaseKernel kernel = BaseKernel.notebookKernel(); + BaseKernel kernel = BaseKernel.notebookKernel; + if (kernel == null) { + throw new UnsupportedOperationException("Magics cannot be called programmatically from executor code in JDI mode; use %%magic syntax"); + } try { return kernel.getMagicsRegistry().evalCellMagic(kernel, name, args, body); @@ -45,47 +70,147 @@ public static T cellMagic(String name, List args, String body) { } public static DisplayData render(Object o) { - return BaseKernel.notebookKernel().getRenderer().render(o); + BaseKernel kernel = BaseKernel.notebookKernel; + return kernel != null ? kernel.getRenderer().render(o) : renderPortable(o); } public static DisplayData render(Object o, String... as) { - return BaseKernel.notebookKernel().getRenderer().renderAs(o, as); + BaseKernel kernel = BaseKernel.notebookKernel; + return kernel != null ? kernel.getRenderer().renderAs(o, as) : renderPortable(o); } public static String display(Object o) { + return displayData(render(o)); + } - DisplayData data = render(o); + public static String display(Object o, String... as) { + return displayData(render(o, as)); + } + + public static void updateDisplay(String id, Object o) { + updateDisplayData(id, render(o)); + } + public static void updateDisplay(String id, Object o, String... as) { + updateDisplayData(id, render(o, as)); + } + + private static String displayData(DisplayData data) { String id = data.getDisplayId(); if (id == null) { id = UUID.randomUUID().toString(); data.setDisplayId(id); } - BaseKernel.notebookKernel().display(data); + BaseKernel kernel = BaseKernel.notebookKernel; + if (kernel != null) { + kernel.display(data); + } else { + sendRemoteDisplay("display", id, data); + } return id; } - public static String display(Object o, String... as) { - DisplayData data = render(o, as); + private static void updateDisplayData(String id, DisplayData data) { + BaseKernel kernel = BaseKernel.notebookKernel; + if (kernel != null) { + kernel.getIO().display.updateDisplay(id, data); + } else { + sendRemoteDisplay("updateDisplay", id, data); + } + } - String id = data.getDisplayId(); - if (id == null) { - id = UUID.randomUUID().toString(); - data.setDisplayId(id); + private static void sendRemoteDisplay(String action, String id, DisplayData data) { + validateDisplayData(data); + int port = Integer.getInteger("jjava.display.port", -1); + if (port <= 0) { + throw new IllegalStateException("Remote display channel is not available"); } - BaseKernel.notebookKernel().display(data); - return id; + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(InetAddress.getLoopbackAddress(), port), 10_000); + socket.setSoTimeout(10_000); + try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8)); + BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) { + JsonObject json = new JsonObject(); + json.addProperty("action", action); + if ("updateDisplay".equals(action) && id != null) { + json.addProperty("id", id); + } + json.add("bundle", GSON.toJsonTree(data)); + + writer.write(json.toString()); + writer.newLine(); + writer.flush(); + + checkDisplayResponse(reader.readLine()); + } + } catch (IOException e) { + throw new RuntimeException("Failed to send display data: " + e.getMessage(), e); + } } - public static void updateDisplay(String id, Object o) { - DisplayData data = render(o); - BaseKernel.notebookKernel().getIO().display.updateDisplay(id, data); + private static void checkDisplayResponse(String response) throws IOException { + if (response == null) { + throw new IOException("Display callback closed without a response"); + } + if (!"ok".equals(response)) { + String message; + try { + message = GSON.fromJson(response, String.class); + } catch (RuntimeException e) { + message = response; + } + throw new IOException("Display callback failed: " + (message == null ? response : message)); + } } - public static void updateDisplay(String id, Object o, String... as) { - DisplayData data = render(o, as); - BaseKernel.notebookKernel().getIO().display.updateDisplay(id, data); + private static void validateDisplayData(DisplayData data) { + validatePortable(data.getData()); + validatePortable(data.getMetadata()); + validatePortable(data.getTransientData()); + } + + private static void validatePortable(Object value) { + if (value == null + || value instanceof CharSequence + || value instanceof Number + || value instanceof Boolean + || value instanceof Character) { + return; + } + if (value instanceof Map) { + for (Map.Entry entry : ((Map) value).entrySet()) { + if (!(entry.getKey() instanceof String)) { + Object key = entry.getKey(); + throw new IllegalArgumentException("Non-portable display value: " + + (key == null ? "null" : key.getClass().getName())); + } + validatePortable(entry.getValue()); + } + return; + } + if (value instanceof Iterable) { + ((Iterable) value).forEach(BaseNotebookStatics::validatePortable); + return; + } + if (value.getClass().isArray()) { + int len = java.lang.reflect.Array.getLength(value); + for (int i = 0; i < len; i++) { + validatePortable(java.lang.reflect.Array.get(value, i)); + } + return; + } + throw new IllegalArgumentException("Non-portable display value: " + value.getClass().getName()); + } + + private static DisplayData renderPortable(Object value) { + if (value instanceof DisplayData) { + return (DisplayData) value; + } + if (value instanceof CharSequence || value instanceof Number || value instanceof Boolean) { + return new DisplayData(String.valueOf(value)); + } + throw new IllegalArgumentException("Non-portable display value: " + value.getClass().getName()); } } diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/DisplayData.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/DisplayData.java index ed0ff3b..d31eb7f 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/DisplayData.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/DisplayData.java @@ -96,6 +96,18 @@ public boolean hasDataForType(MIMEType type) { return this.data.containsKey(type.toString()); } + public Map getData() { + return Collections.unmodifiableMap(data); + } + + public Map getMetadata() { + return Collections.unmodifiableMap(metadata); + } + + public Map getTransientData() { + return transientData == null ? Collections.emptyMap() : Collections.unmodifiableMap(transientData); + } + public void assign(DisplayData data) { this.data.putAll(data.data); diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicsResolver.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicsResolver.java index 9ef7745..a3bd48b 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicsResolver.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicsResolver.java @@ -46,7 +46,7 @@ String resolveLineMagics(String cellSource) { return out.toString(); } - ParsedCellMagic parseCellMagic(String cellSource) { + public ParsedCellMagic parseCellMagic(String cellSource) { Matcher m = cellMagicPattern.matcher(cellSource); if (!m.matches()) { @@ -62,6 +62,11 @@ ParsedCellMagic parseCellMagic(String cellSource) { bodyAfterMagic); } + public ParsedLineMagic parseLineMagic(String cellSource) { + Matcher matchedLine = lineMagicPattern.matcher(cellSource); + return matchedLine.matches() ? parseLineMagic(cellSource, matchedLine) : null; + } + private ParsedLineMagic parseLineMagic(String cellSource, Matcher matchedLine) { List split = split(matchedLine.group("args")); diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStaticsDisplayTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStaticsDisplayTest.java new file mode 100644 index 0000000..54d9591 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStaticsDisplayTest.java @@ -0,0 +1,163 @@ +package org.dflib.jjava.jupyter.kernel; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class BaseNotebookStaticsDisplayTest { + + private static final Gson GSON = new GsonBuilder().serializeNulls().create(); + + private BaseKernel previousKernel; + private String previousPort; + private ServerSocket serverSocket; + + @BeforeEach + public void setUp() throws IOException { + previousKernel = BaseKernel.notebookKernel; + previousPort = System.getProperty("jjava.display.port"); + BaseKernel.notebookKernel = null; + serverSocket = new ServerSocket(0, 1, InetAddress.getLoopbackAddress()); + serverSocket.setSoTimeout(10_000); + System.setProperty("jjava.display.port", Integer.toString(serverSocket.getLocalPort())); + } + + @AfterEach + public void tearDown() throws IOException { + try { + serverSocket.close(); + } finally { + BaseKernel.notebookKernel = previousKernel; + if (previousPort == null) { + System.clearProperty("jjava.display.port"); + } else { + System.setProperty("jjava.display.port", previousPort); + } + } + } + + @Test + public void sendsCompleteBundleAndRequiresSuccessAcknowledgement() throws Exception { + DisplayData data = new DisplayData(); + Map nested = new LinkedHashMap<>(); + nested.put("values", Arrays.asList("kept", true, null)); + nested.put("nullValue", null); + data.putData("application/json", nested); + data.putData("application/null", null); + data.putMetaData("render", nested); + data.putTransientData("custom", nested); + data.setDisplayId("display-id"); + + JsonObject request = exchange("ok", () -> assertEquals("display-id", BaseNotebookStatics.display(data))); + + assertEquals("display", request.get("action").getAsString()); + assertFalse(request.has("id")); + assertEquals(GSON.toJsonTree(data), request.get("bundle")); + } + + @Test + public void eofAndErrorAcknowledgementsFailVisibly() throws Exception { + JsonObject eof = exchange(null, () -> { + RuntimeException failure = assertThrows(RuntimeException.class, () -> BaseNotebookStatics.display("text")); + assertTrue(failure.getMessage().contains("closed without a response")); + }); + assertEquals("display", eof.get("action").getAsString()); + + JsonObject error = exchange("\"remote\\nerror\"", () -> { + RuntimeException failure = assertThrows(RuntimeException.class, () -> BaseNotebookStatics.display("text")); + assertTrue(failure.getMessage().contains("remote\nerror")); + }); + assertEquals("display", error.get("action").getAsString()); + } + + @Test + public void sendsUpdateIdAndCompleteBundle() throws Exception { + DisplayData data = new DisplayData("updated"); + data.putMetaData("render", Map.of("expanded", true)); + data.putTransientData("custom", "preserved"); + JsonObject request = exchange("ok", () -> BaseNotebookStatics.updateDisplay("requested-id", data)); + + assertEquals("updateDisplay", request.get("action").getAsString()); + assertEquals("requested-id", request.get("id").getAsString()); + assertEquals(GSON.toJsonTree(data), request.get("bundle")); + } + + @Test + public void rejectsNonPortableValuesBeforeConnectingAndAcceptsCharacterArrays() throws Exception { + DisplayData nestedValue = new DisplayData(); + nestedValue.putData("application/json", Arrays.asList(Map.of("bad", new Object()))); + assertNonPortable(nestedValue); + + DisplayData nonStringKeysData = new DisplayData(); + Map nonStringKeys = new LinkedHashMap<>(); + nonStringKeys.put(1, "bad"); + nonStringKeysData.putData("application/json", nonStringKeys); + assertNonPortable(nonStringKeysData); + + DisplayData transientValue = new DisplayData(); + transientValue.putTransientData("bad", new Object()); + assertNonPortable(transientValue); + + DisplayData characters = new DisplayData(); + characters.putData("application/characters", new char[]{'o', 'k'}); + JsonObject request = exchange("ok", () -> assertNotNull(BaseNotebookStatics.display(characters))); + JsonObject bundle = request.getAsJsonObject("bundle"); + assertEquals("[\"o\",\"k\"]", bundle.getAsJsonObject("data").get("application/characters").toString()); + assertTrue(bundle.getAsJsonObject("transient").has("display_id")); + } + + private void assertNonPortable(DisplayData data) { + IllegalArgumentException failure = assertThrows( + IllegalArgumentException.class, + () -> BaseNotebookStatics.display(data)); + assertTrue(failure.getMessage().contains("Non-portable display value")); + } + + private JsonObject exchange(String response, Runnable operation) throws Exception { + FutureTask request = new FutureTask<>(() -> { + try (Socket socket = serverSocket.accept()) { + socket.setSoTimeout(2_000); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8)); + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8))) { + JsonObject json = GSON.fromJson(reader.readLine(), JsonObject.class); + if (response != null) { + writer.write(response); + writer.newLine(); + writer.flush(); + } + return json; + } + } + }); + Thread server = new Thread(request, "test-display-callback"); + server.setDaemon(true); + server.start(); + operation.run(); + return request.get(2_000, TimeUnit.MILLISECONDS); + } +} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java index ac5e2b9..2108095 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java @@ -8,6 +8,7 @@ import jdk.jshell.SourceCodeAnalysis; import jdk.jshell.UnresolvedReferenceException; import org.dflib.jjava.jupyter.kernel.BaseKernel; +import org.dflib.jjava.jupyter.kernel.EvalBuilder; import org.dflib.jjava.jupyter.kernel.HelpLink; import org.dflib.jjava.jupyter.kernel.JupyterIO; import org.dflib.jjava.jupyter.kernel.LanguageInfo; @@ -17,6 +18,9 @@ import org.dflib.jjava.jupyter.kernel.display.Renderer; import org.dflib.jjava.jupyter.kernel.history.HistoryManager; import org.dflib.jjava.jupyter.kernel.magic.MagicTranspiler; +import org.dflib.jjava.jupyter.kernel.magic.ParsedCellMagic; +import org.dflib.jjava.jupyter.kernel.magic.ParsedLineMagic; +import org.dflib.jjava.jupyter.kernel.magic.UndefinedMagicException; import org.dflib.jjava.jupyter.kernel.magic.MagicsRegistry; import org.dflib.jjava.jupyter.kernel.magic.MagicsResolver; import org.dflib.jjava.jupyter.kernel.util.CharPredicate; @@ -24,11 +28,13 @@ import org.dflib.jjava.jupyter.kernel.util.StringStyler; import org.dflib.jjava.kernel.execution.CodeEvaluator; import org.dflib.jjava.kernel.execution.CompilationException; +import org.dflib.jjava.kernel.execution.DisplayCallbackServer; import org.dflib.jjava.kernel.execution.EvaluationInterruptedException; +import org.dflib.jjava.kernel.execution.ExecutorTerminationException; import org.dflib.jjava.kernel.execution.EvaluationTimeoutException; import org.dflib.jjava.kernel.execution.IncompleteSourceException; -import java.nio.charset.Charset; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -61,6 +67,8 @@ public static Builder builder() { private final JShell jShell; private final CodeEvaluator evaluator; + private volatile boolean shuttingDown; + private DisplayCallbackServer displayCallbackServer; protected JavaKernel( String name, @@ -94,6 +102,9 @@ protected JavaKernel( this.jShell = jShell; this.evaluator = evaluator; + jShell.onShutdown(ignored -> { + if (!shuttingDown) evaluator.markExecutorTerminated(); + }); } /** @@ -127,6 +138,8 @@ protected List formatError(Throwable e) { return formatEvaluationTimeoutException((EvaluationTimeoutException) e); } else if (e instanceof EvaluationInterruptedException) { return formatEvaluationInterruptedException((EvaluationInterruptedException) e); + } else if (e instanceof ExecutorTerminationException) { + return List.of(errorStyler.secondary(e.getMessage())); } else if (e instanceof RuntimeException && e.getCause() instanceof EvalException) { return formatEvalException((EvalException) e.getCause()); } else { @@ -241,11 +254,96 @@ private List formatEvaluationInterruptedException(EvaluationInterruptedE return fmt; } + @Override + public EvalBuilder evalBuilder(String source) { + return new JavaEvalBuilder<>(this, source, null, null, false); + } + @Override protected Object doEval(String source) { return evaluator.eval(jShell, source); } + private static final class JavaEvalBuilder implements EvalBuilder { + private final JavaKernel kernel; + private final String source; + private final ParsedLineMagic lineMagic; + private final ParsedCellMagic cellMagic; + private final boolean renderResults; + + private JavaEvalBuilder( + JavaKernel kernel, + String source, + ParsedLineMagic lineMagic, + ParsedCellMagic cellMagic, + boolean renderResults) { + this.kernel = kernel; + this.source = source; + this.lineMagic = lineMagic; + this.cellMagic = cellMagic; + this.renderResults = renderResults; + } + + @Override + public EvalBuilder resolveMagics() { + if (lineMagic != null || cellMagic != null) { + return this; + } + + ParsedCellMagic cell = kernel.getMagicsResolver().parseCellMagic(source); + if (cell != null) { + return new JavaEvalBuilder<>(kernel, source, null, cell, renderResults); + } + + ParsedLineMagic line = kernel.getMagicsResolver().parseLineMagic(source); + if (line != null && line.magicLinePrefix.isBlank()) { + return new JavaEvalBuilder<>(kernel, source, line, null, renderResults); + } + + return new JavaEvalBuilder<>(kernel, kernel.getMagicsResolver().resolve(source), null, null, renderResults); + } + + @SuppressWarnings("unchecked") + @Override + public EvalBuilder renderResults() { + return renderResults + ? (EvalBuilder) this + : new JavaEvalBuilder<>(kernel, source, lineMagic, cellMagic, true); + } + + @Override + public T eval() { + Object value = evalRaw(); + if (!renderResults) { + return (T) value; + } + if (value == null) { + return null; + } + return (T) (value instanceof DisplayData ? value : new DisplayData(String.valueOf(value))); + } + + private Object evalRaw() { + if (cellMagic == null && lineMagic == null) { + return kernel.doEval(source); + } + + try { + return cellMagic != null + ? kernel.getMagicsRegistry().evalCellMagic(kernel, cellMagic.name, cellMagic.args, cellMagic.cellBodyAfterMagic) + : kernel.getMagicsRegistry().evalLineMagic(kernel, lineMagic.name, lineMagic.args); + } catch (UndefinedMagicException e) { + throw e; + } catch (Exception e) { + String magicName = cellMagic != null ? cellMagic.name : lineMagic.name; + String kind = cellMagic != null ? "cell" : "line"; + throw new RuntimeException( + String.format("Exception running %s magic '%s': %s", kind, magicName, e.getMessage()), + e); + } + } + } + @Override public DisplayData inspect(String code, int at, boolean extraDetail) { // Move the code position to the end of the identifier to make the inspection work at any @@ -350,21 +448,25 @@ public String isComplete(String code) { @Override public void onShutdown(boolean isRestarting) { - super.onShutdown(isRestarting); - jShell.close(); + shuttingDown = true; + try { + super.onShutdown(isRestarting); + } finally { + try { + jShell.close(); + } finally { + try { + if (displayCallbackServer != null) displayCallbackServer.close(); + } finally { + evaluator.close(); + } + } + } } @Override public void interrupt() { - evaluator.interrupt(); - } - - /** - * Returns notebook ClassLoader, which in the case of JavaKernel is a JShell ClassLoader. - */ - @Override - protected ClassLoader getClassLoader() { - return evaluator.getClassLoader(); + evaluator.interrupt(jShell); } /** @@ -382,33 +484,61 @@ public static class Builder extends JavaKernelBuilder { private Builder() { } + @SuppressWarnings("resource") @Override public JavaKernel build() { + DisplayCallbackServer displayCallbackServer; + try { + displayCallbackServer = new DisplayCallbackServer(); + } catch (IOException e) { + throw new IllegalStateException("Unable to start display callback channel", e); + } - String name = buildName(); - Charset jupyterEncoding = buildJupyterIOEncoding(); - CodeEvaluator evaluator = buildCodeEvaluator(name); - - JShell jShell = buildJShell(evaluator); - LanguageInfo langInfo = buildLanguageInfo(); - MagicTranspiler magicTranspiler = buildMagicTranspiler(); - - return new JavaKernel( - name, - buildVersion(), - langInfo, - buildHelpLinks(), - buildHistoryManager(), - buildJupyterIO(jupyterEncoding), - buildCommManager(), - buildRenderer(), - buildMagicsResolver(magicTranspiler), - buildMagicsRegistry(), - buildExtensionsEnabled(), - buildErrorStyler(), - jShell, - evaluator - ); + JShell jShell = null; + CodeEvaluator evaluator = null; + try { + String name = buildName(); + JupyterIO io = buildJupyterIO(buildJupyterIOEncoding()); + evaluator = buildCodeEvaluator(); + jShell = buildJShell(io, List.of("-Djjava.display.port=" + displayCallbackServer.getPort())); + MagicTranspiler magicTranspiler = buildMagicTranspiler(); + + JavaKernel kernel = new JavaKernel( + name, + buildVersion(), + buildLanguageInfo(), + buildHelpLinks(), + buildHistoryManager(), + io, + buildCommManager(), + buildRenderer(), + buildMagicsResolver(magicTranspiler), + buildMagicsRegistry(), + buildExtensionsEnabled(), + buildErrorStyler(), + jShell, + evaluator + ); + kernel.displayCallbackServer = displayCallbackServer; + displayCallbackServer.setKernel(kernel); + return kernel; + } catch (RuntimeException | Error e) { + closeAfterFailure(e, jShell, displayCallbackServer, evaluator); + throw e; + } + } + + private static void closeAfterFailure(Throwable failure, AutoCloseable... resources) { + for (AutoCloseable resource : resources) { + if (resource == null) { + continue; + } + try { + resource.close(); + } catch (Throwable cleanup) { + if (cleanup != failure) failure.addSuppressed(cleanup); + } + } } protected List buildHelpLinks() { diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java index e52aeda..bab3786 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java @@ -1,7 +1,9 @@ package org.dflib.jjava.kernel; import jdk.jshell.JShell; +import jdk.jshell.execution.JdiExecutionControlProvider; import org.dflib.jjava.jupyter.kernel.BaseKernelBuilder; +import org.dflib.jjava.jupyter.kernel.JupyterIO; import org.dflib.jjava.jupyter.kernel.LanguageInfo; import org.dflib.jjava.jupyter.kernel.magic.MagicTranspiler; import org.dflib.jjava.jupyter.kernel.magic.MagicsResolver; @@ -23,9 +25,11 @@ public abstract class JavaKernelBuilder< protected long timeoutDuration; protected TimeUnit timeoutUnit; protected final List compilerOpts; + protected final List remoteVMOptions; protected JavaKernelBuilder() { this.compilerOpts = new ArrayList<>(); + this.remoteVMOptions = new ArrayList<>(List.of("-classpath", System.getProperty("java.class.path"))); } public B compilerOpts(Iterable opts) { @@ -33,6 +37,11 @@ public B compilerOpts(Iterable opts) { return (B) this; } + public B remoteVMOptions(Iterable opts) { + opts.forEach(this.remoteVMOptions::add); + return (B) this; + } + public B timeout(long timeoutDuration, TimeUnit timeoutUnit) { this.timeoutDuration = timeoutDuration; this.timeoutUnit = Objects.requireNonNull(timeoutUnit); @@ -42,20 +51,23 @@ public B timeout(long timeoutDuration, TimeUnit timeoutUnit) { @Override public abstract K build(); - protected JShell buildJShell(CodeEvaluator evaluator) { + protected JShell buildJShell(JupyterIO io, Iterable additionalRemoteVMOptions) { + List options = new ArrayList<>(remoteVMOptions); + additionalRemoteVMOptions.forEach(options::add); return JShell.builder() - .out(System.out) - .err(System.err) + .out(io.out) + .err(io.err) .in(System.in) - .executionEngine(evaluator.getExecControlProvider(), Map.of()) + .executionEngine(new JdiExecutionControlProvider(), Map.of()) + .remoteVMOptions(options.toArray(new String[0])) .compilerOptions(compilerOpts.toArray(new String[0])) .build(); } - protected CodeEvaluator buildCodeEvaluator(String name) { + protected CodeEvaluator buildCodeEvaluator() { long timeoutDuration = this.timeoutUnit != null ? this.timeoutDuration : -1; TimeUnit timeoutUnit = this.timeoutUnit != null ? this.timeoutUnit : TimeUnit.MILLISECONDS; - return new CodeEvaluator(name, timeoutDuration, timeoutUnit); + return new CodeEvaluator(timeoutDuration, timeoutUnit); } protected MagicsResolver buildMagicsResolver(MagicTranspiler transpiler) { diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaNotebookStatics.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaNotebookStatics.java index 9459014..efd8038 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaNotebookStatics.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaNotebookStatics.java @@ -19,6 +19,10 @@ public static JavaKernel getKernelInstance() { } public static JavaKernel kernel() { - return (JavaKernel) BaseKernel.notebookKernel(); + try { + return (JavaKernel) BaseKernel.notebookKernel(); + } catch (NullPointerException e) { + throw new UnsupportedOperationException("JavaKernel.kernel() cannot expose the managing JVM from executor code", e); + } } } diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java index e03b0ec..95c7fed 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java @@ -6,77 +6,55 @@ import jdk.jshell.Snippet; import jdk.jshell.SnippetEvent; import jdk.jshell.SourceCodeAnalysis; -import jdk.jshell.spi.ExecutionControl; -import jdk.jshell.spi.ExecutionControlProvider; -import jdk.jshell.spi.ExecutionEnv; import org.dflib.jjava.jupyter.kernel.BaseKernel; -import org.dflib.jjava.jupyter.telemetry.TelemetryCollector; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.List; -import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; -public class CodeEvaluator { +public class CodeEvaluator implements AutoCloseable { private static final Pattern WHITESPACE_PREFIX = Pattern.compile("(?:^|\r?\n)(?\\s*).*$"); private static final Pattern LAST_LINE = Pattern.compile("(?:^|\r?\n)(?.*)$"); - private static final String NO_MAGIC_RETURN = "\"__NO_MAGIC_RETURN\""; private static final String INDENTATION = " "; + private static final String EXECUTION_OUT_OF_MEMORY_NAME = "java.lang.OutOfMemoryError"; - private static final Method SNIPPET_CLASS_NAME_METHOD; - - static { - try { - SNIPPET_CLASS_NAME_METHOD = Snippet.class.getDeclaredMethod("classFullName"); - SNIPPET_CLASS_NAME_METHOD.setAccessible(true); - } catch (NoSuchMethodException e) { - throw new RuntimeException("Unable to access jdk.jshell.Snippet.classFullName() method.", e); - } - } - - private final String name; private final long timeoutDuration; private final TimeUnit timeoutUnit; - private final JJavaLoaderDelegate loaderDelegate; - private final JJavaExecutionControl execControl; + private final AtomicBoolean interrupted = new AtomicBoolean(); + private final ExecutorService evaluationExecutor; + private volatile ExecutorTerminationException executorFailure; - public CodeEvaluator(String name, long timeoutDuration, TimeUnit timeoutUnit) { - this.name = name; + public CodeEvaluator(long timeoutDuration, TimeUnit timeoutUnit) { this.timeoutDuration = timeoutDuration; this.timeoutUnit = timeoutUnit; - this.loaderDelegate = new JJavaLoaderDelegate(); - this.execControl = new JJavaExecutionControl(loaderDelegate, timeoutDuration, timeoutUnit); - } - - public void startThreadTelemetryCollection(TelemetryCollector collector) { - execControl.startThreadTelemetryCollection(collector); - } - - public void stopThreadTelemetryCollection() { - execControl.stopThreadTelemetryCollection(); - } - - /** - * Returns ExecutionControlProvider required by JShell. This links JShell execution engine to our code evaluator. - */ - public ExecutionControlProvider getExecControlProvider() { - return new SimpleExecControlProvider(name, execControl); + this.evaluationExecutor = Executors.newSingleThreadExecutor(r -> { + Thread thread = new Thread(r, "jjava-jshell-evaluation"); + thread.setDaemon(true); + return thread; + }); } public Object eval(JShell shell, String code) { - + checkExecutorFailure(); + interrupted.set(false); + long timeoutNanos = timeoutDuration > 0 ? timeoutUnit.toNanos(timeoutDuration) : -1; + long started = System.nanoTime(); SourceCodeAnalysis sca = shell.sourceCodeAnalysis(); Object lastResult = null; SourceCodeAnalysis.CompletionInfo info = sca.analyzeCompletion(code); while (info.completeness().isComplete()) { - - lastResult = evalSingle(shell, info.source()); + long remainingNanos = timeoutNanos < 0 ? -1 : timeoutNanos - (System.nanoTime() - started); + lastResult = evalSingle(shell, info.source(), remainingNanos); info = sca.analyzeCompletion(info.remaining()); } @@ -87,40 +65,43 @@ public Object eval(JShell shell, String code) { return lastResult; } - protected Object evalSingle(JShell shell, String code) { - - List events = shell.eval(code.strip()); - + protected Object evalSingle(JShell shell, String code, long timeoutNanos) { + List events = evaluate(shell, code, timeoutNanos); Object result = null; - // We iterate twice to make sure throwing an early exception doesn't leak the memory - // and we `takeResult` everything. for (SnippetEvent event : events) { - if (event.status() == Snippet.Status.OVERWRITTEN) { - // if a new snippet changed some other definition, drop the older one - dropSnippet(shell, event.snippet()); + if (event.causeSnippet() != null) { continue; } - String key = event.value(); - if (key == null) { - continue; + JShellException exception = event.exception(); + if (exception != null) { + if (exception instanceof EvalException) { + EvalException evalException = (EvalException) exception; + if (EXECUTION_OUT_OF_MEMORY_NAME.equals(evalException.getExceptionClassName())) { + throw executorTerminated(shell, exception); + } + throw new RuntimeException( + evalException.getExceptionClassName() + ", " + exception.getMessage(), + exception); + } + throw new RuntimeException(exception); } - Snippet.SubKind subKind = event.snippet().subKind(); + if (event.status() != Snippet.Status.RECOVERABLE_NOT_DEFINED && !event.status().isDefined()) { + throw new CompilationException(event); + } - // Only executable snippets make their way through the machinery we have setup in the - // JJavaExecutionControl. Declarations for example simply take their default value without - // being executed. - Object value = subKind.isExecutable() - ? execControl.takeResult(key) - : event.value(); + String value = event.value(); + if (value == null) { + continue; + } - switch (subKind) { + switch (event.snippet().subKind()) { case VAR_VALUE_SUBKIND: case OTHER_EXPRESSION_SUBKIND: case TEMP_VAR_EXPRESSION_SUBKIND: - result = NO_MAGIC_RETURN.equals(value) ? null : value; + result = value; break; default: result = null; @@ -128,60 +109,65 @@ protected Object evalSingle(JShell shell, String code) { } } - for (SnippetEvent event : events) { - // If fresh snippet - if (event.causeSnippet() == null) { - JShellException e = event.exception(); - if (e != null) { - - if (e instanceof EvalException) { - EvalException ee = (EvalException) e; - switch (ee.getExceptionClassName()) { - case JJavaExecutionControl.EXECUTION_TIMEOUT_NAME: - throw new EvaluationTimeoutException(timeoutDuration, timeoutUnit, code.trim()); - case JJavaExecutionControl.EXECUTION_INTERRUPTED_NAME: - throw new EvaluationInterruptedException(code.trim()); - default: - throw new RuntimeException(ee.getExceptionClassName() + ", " + e.getMessage(), e); - } - } + return result; + } - throw new RuntimeException(e); - } + private List evaluate(JShell shell, String code, long timeoutNanos) { + if (timeoutDuration > 0 && timeoutNanos <= 0) { + throw new EvaluationTimeoutException(timeoutDuration, timeoutUnit, code.trim()); + } - // Undefined snippets are generally bad, unless we can still recover from them. E.g., - // "Unresolved dependencies" errors are recoverable when those dependencies are defined in the later - // snippets. - if (event.status() != Snippet.Status.RECOVERABLE_NOT_DEFINED && !event.status().isDefined()) { - throw new CompilationException(event); - } + Future> evaluation = evaluationExecutor.submit(() -> shell.eval(code.strip())); + + List events; + try { + events = timeoutNanos > 0 ? evaluation.get(timeoutNanos, TimeUnit.NANOSECONDS) : evaluation.get(); + } catch (TimeoutException e) { + checkExecutorFailure(); + shell.stop(); + evaluation.cancel(true); + throw new EvaluationTimeoutException(timeoutDuration, timeoutUnit, code.trim()); + } catch (InterruptedException e) { + shell.stop(); + evaluation.cancel(true); + Thread.currentThread().interrupt(); + throw new EvaluationInterruptedException(code.trim()); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + checkExecutorFailure(); + if (interrupted.getAndSet(false)) { + throw new EvaluationInterruptedException(code.trim()); } + if (cause instanceof RuntimeException) throw (RuntimeException) cause; + if (cause instanceof Error) throw (Error) cause; + throw new RuntimeException(cause); } - return result; + if (interrupted.getAndSet(false)) { + throw new EvaluationInterruptedException(code.trim()); + } + checkExecutorFailure(); + return events; + } + + public void checkExecutorFailure() { + if (executorFailure != null) throw executorFailure; } - /** - * Try to clean up information linked to a code snippet and the snippet itself - */ - private void dropSnippet(JShell shell, Snippet snippet) { - shell.drop(snippet); - // snippet.classFullName() returns name of a wrapper class created for a snippet - String className = snippetClassName(snippet); - // check that this class is not used by other snippets - if (shell.snippets() - .map(this::snippetClassName) - .noneMatch(className::equals)) { - execControl.unloadClass(className); + public synchronized void markExecutorTerminated() { + if (executorFailure == null) { + executorFailure = new ExecutorTerminationException( + "Executor process terminated; session state is lost. Restart the kernel.", null); } } - private String snippetClassName(Snippet snippet) { - try { - return SNIPPET_CLASS_NAME_METHOD.invoke(snippet).toString(); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); + private synchronized ExecutorTerminationException executorTerminated(JShell shell, Throwable cause) { + if (executorFailure == null) { + executorFailure = new ExecutorTerminationException( + "Executor process terminated; session state is lost. Restart the kernel.", cause); + shell.close(); } + return executorFailure; } private String computeIndentation(String partialStatement) { @@ -252,32 +238,13 @@ public String isComplete(SourceCodeAnalysis sourceAnalyzer, String code) { } } - public void interrupt() { - execControl.interrupt(); - } - - public ClassLoader getClassLoader() { - return loaderDelegate.getClassLoader(); + public void interrupt(JShell shell) { + interrupted.set(true); + shell.stop(); } - static final class SimpleExecControlProvider implements ExecutionControlProvider { - - private final String name; - private final ExecutionControl control; - - public SimpleExecControlProvider(String name, ExecutionControl control) { - this.name = name; - this.control = control; - } - - @Override - public String name() { - return name; - } - - @Override - public ExecutionControl generate(ExecutionEnv env, Map parameters) { - return control; - } + @Override + public void close() { + evaluationExecutor.shutdownNow(); } } diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/DisplayCallbackServer.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/DisplayCallbackServer.java new file mode 100644 index 0000000..792d326 --- /dev/null +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/DisplayCallbackServer.java @@ -0,0 +1,126 @@ +package org.dflib.jjava.kernel.execution; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.kernel.JavaKernel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; + +public class DisplayCallbackServer implements Closeable { + + private static final Logger LOGGER = LoggerFactory.getLogger(DisplayCallbackServer.class); + private static final Gson GSON = new GsonBuilder().serializeNulls().create(); + private final ServerSocket serverSocket; + private volatile JavaKernel kernel; + + public DisplayCallbackServer() throws IOException { + this.serverSocket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()); + Thread listenerThread = new Thread(this::listen, "jjava-display-callback"); + listenerThread.setDaemon(true); + listenerThread.start(); + } + + public void setKernel(JavaKernel kernel) { + this.kernel = kernel; + } + + public int getPort() { + return serverSocket.getLocalPort(); + } + + private void listen() { + while (!serverSocket.isClosed()) { + try { + handleConnection(serverSocket.accept()); + } catch (IOException e) { + if (!serverSocket.isClosed()) { + LOGGER.warn("Display callback server accept failed", e); + } + } + } + } + + private void handleConnection(Socket socket) { + try (socket; + BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8)); + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8))) { + + socket.setSoTimeout(10_000); + String response; + try { + dispatch(reader.readLine()); + response = "ok"; + } catch (Exception e) { + String message = e.getMessage(); + response = GSON.toJson(message == null ? e.getClass().getSimpleName() : message); + } + writer.write(response); + writer.newLine(); + writer.flush(); + } catch (IOException e) { + if (!serverSocket.isClosed()) { + LOGGER.warn("Display callback connection failed", e); + } + } + } + + private void dispatch(String line) { + JsonObject json = GSON.fromJson(line, JsonObject.class); + if (json == null) { + throw new IllegalArgumentException("Display callback request must be a JSON object"); + } + + JsonElement actionElement = json.get("action"); + if (actionElement == null || !actionElement.isJsonPrimitive() + || !actionElement.getAsJsonPrimitive().isString()) { + throw new IllegalArgumentException("Display callback action must be display or updateDisplay"); + } + String action = actionElement.getAsString(); + if (!"display".equals(action) && !"updateDisplay".equals(action)) { + throw new IllegalArgumentException("Unsupported display callback action: " + action); + } + + String id = json.has("id") && !json.get("id").isJsonNull() ? json.get("id").getAsString() : null; + if ("updateDisplay".equals(action) && id == null) { + throw new IllegalArgumentException("updateDisplay requires a non-null id"); + } + + JsonElement bundleElement = json.get("bundle"); + if (bundleElement == null || bundleElement.isJsonNull()) { + throw new IllegalArgumentException("Display callback bundle must not be null"); + } + DisplayData data = GSON.fromJson(bundleElement, DisplayData.class); + + JavaKernel currentKernel = kernel; + if (currentKernel == null) { + throw new IllegalStateException("Display callback kernel is not available"); + } + if ("updateDisplay".equals(action)) { + currentKernel.getIO().display.updateDisplay(id, data); + } else { + currentKernel.display(data); + } + } + + @Override + public void close() { + try { + serverSocket.close(); + } catch (IOException ignored) { + } + } +} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/ExecutorTerminationException.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/ExecutorTerminationException.java new file mode 100644 index 0000000..5a44101 --- /dev/null +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/ExecutorTerminationException.java @@ -0,0 +1,7 @@ +package org.dflib.jjava.kernel.execution; + +public class ExecutorTerminationException extends RuntimeException { + public ExecutorTerminationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/magics/TimeMagic.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/magics/TimeMagic.java index c13bf4f..3f23238 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/magics/TimeMagic.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/magics/TimeMagic.java @@ -4,13 +4,8 @@ import org.dflib.jjava.jupyter.kernel.display.DisplayData; import org.dflib.jjava.jupyter.kernel.magic.CellMagic; import org.dflib.jjava.jupyter.kernel.magic.LineMagic; -import org.dflib.jjava.jupyter.telemetry.TelemetryCollector; -import org.dflib.jjava.jupyter.telemetry.WallAndCpuTimeCollector; -import org.dflib.jjava.jupyter.telemetry.WallTimeCollector; import org.dflib.jjava.kernel.JavaKernel; -import java.lang.management.ManagementFactory; -import java.lang.management.ThreadMXBean; import java.util.List; import java.util.UUID; @@ -32,68 +27,19 @@ public DisplayData eval(JavaKernel kernel, List args, String body) throw } private DisplayData timeAndRunCode(JavaKernel kernel, String code) { - ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); - - TelemetryCollector collector = WallAndCpuTimeCollector.canMeasureCpuTimes(threadMXBean) - ? new TMWallAndCpuTimeCollector(kernel, threadMXBean) - : new TMWallTimeCollector(kernel); - - kernel.getEvaluator().startThreadTelemetryCollection(collector); - + long started = System.nanoTime(); try { return kernel.evalBuilder(code) .resolveMagics() .renderResults() .eval(); } finally { - kernel.getEvaluator().stopThreadTelemetryCollection(); + displayWallTime(kernel, System.nanoTime() - started); } } - private static void displayCpuTime(BaseKernel kernel, long userTimeNanos, long sysTimeNanos, long totalTimeNanos) { - String cpuTimes = String.format("CPU times: user %.3f s, sys %.3f s, total %.3f s", - userTimeNanos / NANONS_IN_SEC, - sysTimeNanos / NANONS_IN_SEC, - totalTimeNanos / NANONS_IN_SEC); - - kernel.display(new DisplayData(cpuTimes).setDisplayId(UUID.randomUUID().toString())); - } - private static void displayWallTime(BaseKernel kernel, long wallTimeNanos) { String wallTime = String.format("Wall time: %.3f s", wallTimeNanos / NANONS_IN_SEC); kernel.display(new DisplayData(wallTime).setDisplayId(UUID.randomUUID().toString())); } - - static class TMWallTimeCollector extends WallTimeCollector { - private final BaseKernel kernel; - - public TMWallTimeCollector(BaseKernel kernel) { - this.kernel = kernel; - } - - @Override - protected void onStop(long wallTimeNanos) { - displayWallTime(kernel, wallTimeNanos); - } - } - - - static class TMWallAndCpuTimeCollector extends WallAndCpuTimeCollector { - private final BaseKernel kernel; - - public TMWallAndCpuTimeCollector(BaseKernel kernel, ThreadMXBean threadMXBean) { - super(threadMXBean); - this.kernel = kernel; - } - - @Override - protected void onStop(long wallTimeNanos, long userTimeNanos, long totalTimeNanos) { - // sanity check: CPU and user times can't be measured sometimes - if (userTimeNanos != -1 && totalTimeNanos != -1) { - displayCpuTime(kernel, userTimeNanos, totalTimeNanos - userTimeNanos, totalTimeNanos); - } - - displayWallTime(kernel, wallTimeNanos); - } - } } diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/DisplayCallbackServerTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/DisplayCallbackServerTest.java new file mode 100644 index 0000000..3a987b4 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/DisplayCallbackServerTest.java @@ -0,0 +1,177 @@ +package org.dflib.jjava.kernel; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import jdk.jshell.JShell; +import org.dflib.jjava.jupyter.channels.ShellReplyEnvironment; +import org.dflib.jjava.jupyter.kernel.JupyterIO; +import org.dflib.jjava.jupyter.kernel.LanguageInfo; +import org.dflib.jjava.jupyter.kernel.comm.CommManager; +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.Renderer; +import org.dflib.jjava.jupyter.kernel.magic.MagicTranspiler; +import org.dflib.jjava.jupyter.kernel.magic.MagicsRegistry; +import org.dflib.jjava.jupyter.kernel.magic.MagicsResolver; +import org.dflib.jjava.jupyter.kernel.util.StringStyler; +import org.dflib.jjava.jupyter.messages.Message; +import org.dflib.jjava.jupyter.messages.publish.PublishUpdateDisplayData; +import org.dflib.jjava.kernel.execution.CodeEvaluator; +import org.dflib.jjava.kernel.execution.DisplayCallbackServer; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class DisplayCallbackServerTest { + + private static final Gson GSON = new GsonBuilder().serializeNulls().create(); + + @Test + public void preservesStructuredBundleAndContinuesAfterMalformedRequest() throws Exception { + try (CallbackKernel kernel = new CallbackKernel(); + DisplayCallbackServer server = new DisplayCallbackServer()) { + server.setKernel(kernel); + + String error = GSON.fromJson(request(server, "not json"), String.class); + assertNotNull(error); + assertFalse(error.isBlank()); + assertNotEquals("ok", error); + + String request = displayRequest(); + JsonObject expectedBundle = GSON.fromJson(request, JsonObject.class).getAsJsonObject("bundle"); + assertEquals("ok", request(server, request)); + assertEquals(expectedBundle, GSON.toJsonTree(kernel.displayed)); + } + } + + @Test + public void returnsDispatchFailuresAndRoutesUpdatesThroughDisplayStream() throws Exception { + CapturingEnvironment env = new CapturingEnvironment(); + try (CallbackKernel kernel = new CallbackKernel(new CapturingIO(env)); + DisplayCallbackServer server = new DisplayCallbackServer()) { + server.setKernel(kernel); + kernel.displayFailure = new IllegalStateException("display failed\nnow"); + + assertEquals("\"display failed\\nnow\"", request(server, displayRequest())); + assertNull(kernel.displayed); + + String update = "{\"action\":\"updateDisplay\",\"id\":\"existing\"," + + "\"bundle\":{\"data\":{\"text/plain\":\"updated\"},\"metadata\":{},\"transient\":{}}}"; + assertEquals("ok", request(server, update)); + assertEquals(1, env.published.size()); + PublishUpdateDisplayData published = assertInstanceOf( + PublishUpdateDisplayData.class, + env.published.get(0).getContent()); + assertEquals("existing", published.getDisplayId()); + assertEquals("updated", published.getData().get("text/plain")); + } + } + + @Test + public void rejectsUpdatesWithoutIdsInsteadOfDisplayingThem() throws Exception { + try (CallbackKernel kernel = new CallbackKernel(); + DisplayCallbackServer server = new DisplayCallbackServer()) { + server.setKernel(kernel); + String response = request(server, + "{\"action\":\"updateDisplay\",\"bundle\":{\"data\":{},\"metadata\":{}}}"); + + assertTrue(GSON.fromJson(response, String.class).contains("requires a non-null id")); + assertNull(kernel.displayed); + } + } + + private String displayRequest() { + return "{\"action\":\"display\",\"bundle\":{" + + "\"data\":{\"application/json\":{\"values\":[1,true,null]},\"text/empty\":null}," + + "\"metadata\":{\"render\":{\"nested\":false}}," + + "\"transient\":{\"custom\":{\"values\":[\"kept\",null]}}}}"; + } + + private String request(DisplayCallbackServer server, String request) throws Exception { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(InetAddress.getLoopbackAddress(), server.getPort()), 10_000); + socket.setSoTimeout(10_000); + try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8)); + BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) { + writer.write(request); + writer.newLine(); + writer.flush(); + return reader.readLine(); + } + } + } + + private static class CapturingIO extends JupyterIO { + CapturingIO(CapturingEnvironment env) { + super(StandardCharsets.UTF_8); + setEnv(env); + } + } + + private static class CapturingEnvironment extends ShellReplyEnvironment { + final List> published = new ArrayList<>(); + + CapturingEnvironment() { + super(null, null, null, null); + } + + @Override + public void publish(Message message) { + published.add(message); + } + } + + private static class CallbackKernel extends JavaKernel implements AutoCloseable { + DisplayData displayed; + RuntimeException displayFailure; + + CallbackKernel() { + this(new JupyterIO(StandardCharsets.UTF_8)); + } + + CallbackKernel(JupyterIO io) { + super( + "test", "0", + new LanguageInfo.Builder("Java").build(), + Collections.emptyList(), null, + io, new CommManager(), new Renderer(), + new MagicsResolver("%", "%%", new MagicTranspiler()), + new MagicsRegistry(), false, new StringStyler.Builder().build(), + // These tests exercise callback I/O, not remote executor behavior. + JShell.builder().executionEngine("local").build(), + new CodeEvaluator(-1, TimeUnit.MILLISECONDS)); + } + + @Override + public void display(DisplayData data) { + if (displayFailure != null) { + throw displayFailure; + } + displayed = data; + } + + @Override + public void close() { + onShutdown(false); + } + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEvalBuilderTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEvalBuilderTest.java new file mode 100644 index 0000000..ed096e7 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEvalBuilderTest.java @@ -0,0 +1,79 @@ +package org.dflib.jjava.kernel; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.fail; + +public class JavaKernelEvalBuilderTest { + + private JavaKernel kernel; + + @BeforeEach + public void setUp() { + kernel = JavaKernel.builder().extensionsEnabled(false).build(); + } + + @AfterEach + public void tearDown() { + kernel.onShutdown(false); + } + + @Test + public void rendersManagerLineAndCellMagicsRegardlessOfBuilderOrder() { + AtomicInteger calls = new AtomicInteger(); + kernel.getMagicsRegistry().registerLineMagic("probe", (k, args) -> { + assertSame(kernel, k); + assertEquals(List.of("first", "second"), args); + return "magic-" + calls.incrementAndGet(); + }); + kernel.getMagicsRegistry().registerCellMagic("probeCell", (k, args, body) -> { + assertSame(kernel, k); + assertEquals(List.of("first", "second"), args); + assertEquals("cell body", body); + return "cell-result"; + }); + + DisplayData renderFirst = kernel.evalBuilder("%probe first second").renderResults().resolveMagics().eval(); + DisplayData resolveFirst = kernel.evalBuilder("%probe first second") + .resolveMagics() + .renderResults() + .renderResults() + .eval(); + DisplayData cellResult = kernel.evalBuilder("%%probeCell first second\ncell body") + .renderResults() + .resolveMagics() + .eval(); + + assertEquals("magic-1", renderFirst.getData().get("text/plain")); + assertEquals("magic-2", resolveFirst.getData().get("text/plain")); + assertEquals("cell-result", cellResult.getData().get("text/plain")); + } + + @Test + public void preservesNullAndDisplayDataMagicResults() { + kernel.getMagicsRegistry().registerLineMagic("nothing", (k, args) -> null); + assertNull(kernel.evalBuilder("%nothing").renderResults().resolveMagics().eval()); + + DisplayData returned = new DisplayData("already rendered"); + kernel.getMagicsRegistry().registerLineMagic("bundle", (k, args) -> returned); + assertSame(returned, kernel.evalBuilder("%bundle").resolveMagics().renderResults().eval()); + } + + @Test + public void doesNotUseManagerRendererForExecutorResults() { + kernel.getRenderer().createRegistration(String.class) + .register((value, context) -> fail("Executor result must not be rendered in the manager")); + + DisplayData executorResult = kernel.evalBuilder("40 + 2").renderResults().eval(); + assertEquals("42", executorResult.getData().get("text/plain")); + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelExtensionsLifecycleTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelExtensionsLifecycleTest.java index bed24db..4d1b067 100644 --- a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelExtensionsLifecycleTest.java +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelExtensionsLifecycleTest.java @@ -22,7 +22,9 @@ public void extraClasspathExtension() throws Exception { String extraClasspath = PathsHandler.joinPaths(List.of(extensionJar)); String extInstalledProp = "ext.installs:org.dflib.jjava.kernel.test.ExtraClasspathExtension"; + String extUninstalledProp = "ext.uninstalls:org.dflib.jjava.kernel.test.ExtraClasspathExtension"; System.clearProperty(extInstalledProp); + System.clearProperty(extUninstalledProp); JavaKernel kernel = JavaKernel .builder() @@ -42,6 +44,8 @@ public void extraClasspathExtension() throws Exception { } assertNull(System.getProperty(extInstalledProp)); + assertEquals("1", System.getProperty(extUninstalledProp)); + System.clearProperty(extUninstalledProp); } @Test @@ -63,10 +67,12 @@ public void evalExtension() throws Exception { kernel.addToClasspath(extraClasspath); Object installed = kernel.evalBuilder("evalExtensionInstalled").eval(); - assertEquals(true, installed, "EvalExtension should have been installed"); + assertEquals("true", installed, "EvalExtension should have been installed"); + + assertEquals("manager:argument", kernel.evalBuilder("%testManagerMagic argument").resolveMagics().eval()); Object result = kernel.evalBuilder("evalValue").eval(); - assertEquals("Test message", result.toString(), "eval() call was not successful"); + assertEquals("\"Test message\"", result.toString(), "eval() call was not successful"); } finally { kernel.onShutdown(false); } @@ -95,10 +101,10 @@ public void libraryExtension() throws Exception { kernel.addToClasspath(extraClasspath); Object installed = kernel.evalBuilder("externalLibraryExtensionInstalled").eval(); - assertEquals(true, installed, "ExternalLibraryExtension should have been installed"); + assertEquals("true", installed, "ExternalLibraryExtension should have been installed"); Object result = kernel.evalBuilder("externalLibraryValue").eval(); - assertEquals("Test message", result.toString(), "Library class method call was not successful"); + assertEquals("\"Test message\"", result.toString(), "Library class method call was not successful"); } finally { kernel.onShutdown(false); } diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/CodeEvaluatorTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/CodeEvaluatorTest.java new file mode 100644 index 0000000..d7f14cd --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/CodeEvaluatorTest.java @@ -0,0 +1,56 @@ +package org.dflib.jjava.kernel.execution; + +import jdk.jshell.EvalException; +import jdk.jshell.JShell; +import jdk.jshell.execution.JdiExecutionControlProvider; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class CodeEvaluatorTest { + + @Test + public void evaluatesLastExpressionAndLeavesDeclarationsWithoutResults() { + try (JShell shell = newShell(); CodeEvaluator evaluator = new CodeEvaluator(-1, TimeUnit.SECONDS)) { + assertEquals("42", evaluator.eval(shell, "int base = 40;\nbase + 2")); + assertNull(evaluator.eval(shell, "int declared = 1;")); + assertEquals("\"__NO_MAGIC_RETURN\"", evaluator.eval(shell, "\"__NO_MAGIC_RETURN\"")); + } + } + + @Test + public void userExceptionDoesNotPoisonSubsequentEvaluation() { + try (JShell shell = newShell(); CodeEvaluator evaluator = new CodeEvaluator(-1, TimeUnit.SECONDS)) { + RuntimeException failure = assertThrows( + RuntimeException.class, + () -> evaluator.eval(shell, "throw new IllegalStateException(\"expected\");")); + + assertInstanceOf(EvalException.class, failure.getCause()); + assertEquals("42", evaluator.eval(shell, "40 + 2")); + } + } + + @Test + public void timeoutDoesNotPoisonSubsequentEvaluation() { + try (JShell shell = newShell(); CodeEvaluator evaluator = new CodeEvaluator(5, TimeUnit.SECONDS)) { + assertEquals("2", evaluator.eval(shell, "1 + 1")); + assertThrows( + EvaluationTimeoutException.class, + () -> evaluator.eval(shell, "try { Thread.sleep(30000L); } catch (InterruptedException ignored) {}")); + + assertEquals("42", evaluator.eval(shell, "40 + 2")); + } + } + + private JShell newShell() { + return JShell.builder() + .executionEngine(new JdiExecutionControlProvider(), Map.of()) + .build(); + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlRedefinitionTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlRedefinitionTest.java new file mode 100644 index 0000000..c3ce56c --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlRedefinitionTest.java @@ -0,0 +1,34 @@ +package org.dflib.jjava.kernel.execution; + +import org.dflib.jjava.kernel.JavaKernel; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class JdiExecutionControlRedefinitionTest { + + @Test + public void classAndMethodRedefinitionKeepsSavedValues() { + JavaKernel kernel = JavaKernel.builder().extensionsEnabled(false).build(); + try { + kernel.evalBuilder("class Stateful { int value() { return 1; } }").eval(); + kernel.evalBuilder("Stateful instance = new Stateful();").eval(); + kernel.evalBuilder("int saved = instance.value();").eval(); + assertEquals("1", kernel.evalBuilder("saved").eval()); + + kernel.evalBuilder("class Stateful { int value() { return 2; } }").eval(); + + assertEquals("1", kernel.evalBuilder("saved").eval()); + assertEquals("2", kernel.evalBuilder("instance.value()").eval()); + assertEquals("2", kernel.evalBuilder("new Stateful().value()").eval()); + + kernel.evalBuilder("int currentValue() { return 10; }").eval(); + kernel.evalBuilder("int savedMethodValue = currentValue();").eval(); + kernel.evalBuilder("int currentValue() { return 20; }").eval(); + assertEquals("10", kernel.evalBuilder("savedMethodValue").eval()); + assertEquals("20", kernel.evalBuilder("currentValue()").eval()); + } finally { + kernel.onShutdown(false); + } + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlTest.java new file mode 100644 index 0000000..b85f9b5 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JdiExecutionControlTest.java @@ -0,0 +1,32 @@ +package org.dflib.jjava.kernel.execution; + +import org.dflib.jjava.kernel.JavaKernel; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +public class JdiExecutionControlTest { + + @Test + public void usesRemoteExecutorForPersistentStateImportsAndOptions() { + JavaKernel kernel = JavaKernel.builder() + .extensionsEnabled(false) + .remoteVMOptions(List.of("-Djjava.test.executor=remote")) + .build(); + try { + long executorPid = Long.parseLong(kernel.evalBuilder("ProcessHandle.current().pid()").eval()); + assertNotEquals(ProcessHandle.current().pid(), executorPid); + + kernel.evalBuilder("int saved = 40;").eval(); + kernel.evalBuilder("import java.time.LocalDate;").eval(); + assertEquals("42", kernel.evalBuilder("saved + 2").eval()); + assertEquals(Long.toString(executorPid), kernel.evalBuilder("ProcessHandle.current().pid()").eval()); + assertEquals("\"remote\"", kernel.evalBuilder("System.getProperty(\"jjava.test.executor\")").eval()); + } finally { + kernel.onShutdown(false); + } + } +} diff --git a/jjava-kernel/src/test/resources/extensions/classpath/org/dflib/jjava/kernel/test/ExtraClasspathExtension.java b/jjava-kernel/src/test/resources/extensions/classpath/org/dflib/jjava/kernel/test/ExtraClasspathExtension.java index 0fc927c..57e276d 100644 --- a/jjava-kernel/src/test/resources/extensions/classpath/org/dflib/jjava/kernel/test/ExtraClasspathExtension.java +++ b/jjava-kernel/src/test/resources/extensions/classpath/org/dflib/jjava/kernel/test/ExtraClasspathExtension.java @@ -21,5 +21,8 @@ public void install(BaseKernel kernel) { public void uninstall(BaseKernel kernel) { String key = "ext.installs:" + getClass().getName(); System.clearProperty(key); + String uninstallKey = "ext.uninstalls:" + getClass().getName(); + int count = Integer.parseInt(System.getProperty(uninstallKey, "0")); + System.setProperty(uninstallKey, String.valueOf(count + 1)); } } diff --git a/jjava-kernel/src/test/resources/extensions/eval/org/dflib/jjava/kernel/test/EvalExtension.java b/jjava-kernel/src/test/resources/extensions/eval/org/dflib/jjava/kernel/test/EvalExtension.java index 36f71a8..afcbcf8 100644 --- a/jjava-kernel/src/test/resources/extensions/eval/org/dflib/jjava/kernel/test/EvalExtension.java +++ b/jjava-kernel/src/test/resources/extensions/eval/org/dflib/jjava/kernel/test/EvalExtension.java @@ -2,6 +2,7 @@ import org.dflib.jjava.jupyter.Extension; import org.dflib.jjava.jupyter.kernel.BaseKernel; +import org.dflib.jjava.jupyter.kernel.magic.LineMagic; public class EvalExtension implements Extension { @@ -9,5 +10,7 @@ public class EvalExtension implements Extension { public void install(BaseKernel kernel) { kernel.evalBuilder("var evalValue = \"Test message\";").eval(); kernel.evalBuilder("var evalExtensionInstalled = true;").eval(); + LineMagic probe = (k, args) -> "manager:" + args.get(0); + kernel.getMagicsRegistry().registerLineMagic("testManagerMagic", probe); } } diff --git a/jjava-launcher/pom.xml b/jjava-launcher/pom.xml deleted file mode 100644 index 90c72c6..0000000 --- a/jjava-launcher/pom.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - 4.0.0 - - - org.dflib.jjava - jjava-parent - 1.0-SNAPSHOT - - - jjava-launcher - Java Jupyter Kernel launcher - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - org.dflib.jjava.launcher.KernelLauncher - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-artifacts - package - - copy - - - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - ../kernelspec/java - jjava-launcher.jar - - - - - - - - org.apache.maven.plugins - maven-clean-plugin - - - - ../kernelspec/java - - jjava-launcher.jar - - - - - - - - - diff --git a/jjava-launcher/src/main/java/org/dflib/jjava/launcher/KernelLauncher.java b/jjava-launcher/src/main/java/org/dflib/jjava/launcher/KernelLauncher.java deleted file mode 100644 index 348d91d..0000000 --- a/jjava-launcher/src/main/java/org/dflib/jjava/launcher/KernelLauncher.java +++ /dev/null @@ -1,116 +0,0 @@ -package org.dflib.jjava.launcher; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.logging.Logger; - -/** - * The launcher responsible for creating a new process to run the JJava kernel. - *

- * The command to run the kernel is constructed by combining the JVM executable, - * the JJAVA_JVM_OPTS environment variable (if set), and the kernel arguments. - */ -// TODO: stop using exceptions for flow control. We are dealing with a single class and a main method here. -// All the errors should be reported as exit code and messages printed to stderr -public class KernelLauncher { - - // Since "launcher" is kind of a part of the "distro", we are using JUL directly, - // unlike the rest of the framework that uses SLF4J, and is only linked to JUL in the "distro" - private static final Logger LOGGER = Logger.getLogger("KernelLauncher"); - - // More JUL stuff... This is copied verbatim from JJava class to mimic Jupyter log format - // TODO: does it make sense to pass this property to the child JJava process via -D, - // to avoid duplicating this logic? - private static final String JUL_JUPYTER_LOG_FORMAT = "[%4$.1s %1$tF %1$tT.%1$tL %3$s] %5$s%n"; - - private static final String JJAVA_JVM_OPTS = "JJAVA_JVM_OPTS"; - - private final List args; - - public KernelLauncher(List args) { - this.args = args; - } - - public static void main(String[] args) { - System.setProperty("java.util.logging.SimpleFormatter.format", JUL_JUPYTER_LOG_FORMAT); - KernelLauncher launcher = new KernelLauncher(Arrays.asList(args)); - int exitCode = launcher.launchKernel(); - System.exit(exitCode); - } - - /** - * Launches a new process to run the JJava kernel with the given arguments. - *

- * The kernel inherits the standard input, output and error streams from the parent process. - * The method blocks until the kernel terminates. - * It also provides shutdown hooks to terminate the kernel when the JVM terminates. - * - * @return the exit code of the kernel - */ - public int launchKernel() { - List command = buildCommand(args); - LOGGER.info(() -> "Starting kernel: " + command); - try { - ProcessBuilder pb = new ProcessBuilder(command); - pb.inheritIO(); - Process kernel = pb.start(); - addShutdownHook(kernel); - return kernel.waitFor(); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - private void addShutdownHook(Process kernel) { - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - LOGGER.fine("Terminating kernel..."); - kernel.destroy(); - try { - kernel.waitFor(); - LOGGER.info("Kernel terminated."); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - })); - } - - private List buildCommand(List args) { - if (args.size() < 2) { - throw new RuntimeException(buildErrorMessage(args)); - } - String kernelPath = args.get(0); - String connectionFile = args.get(1); - - List command = new ArrayList<>(); - command.add("java"); - - // Get JVM options from environment variable if provided - String jvmOptions = System.getenv(JJAVA_JVM_OPTS); - if (jvmOptions != null && !jvmOptions.isBlank()) { - String[] options = jvmOptions.split("\\s+"); - command.addAll(Arrays.asList(options)); - } - - // Add JVM option for JShell permissions - command.add("--add-opens"); - command.add("jdk.jshell/jdk.jshell=ALL-UNNAMED"); - command.add("-jar"); - command.add(kernelPath); - command.add(connectionFile); - return command; - } - - private String buildErrorMessage(List args) { - - switch (args.size()) { - case 0: - return "Missing arguments: %n"; - case 1: - return String.format("Missing arguments: %s %n", args.get(0)); - default: - return String.format("Arguments provided: %s%n", String.join(", ", args)); - } - } -} diff --git a/kernelspec/java/kernel.json b/kernelspec/java/kernel.json index 9ef084a..9700ce5 100644 --- a/kernelspec/java/kernel.json +++ b/kernelspec/java/kernel.json @@ -1,8 +1,9 @@ { "argv": [ "java", + "--add-opens", + "jdk.jshell/jdk.jshell=ALL-UNNAMED", "-jar", - "{resource_dir}/jjava-launcher.jar", "{resource_dir}/jjava.jar", "{connection_file}" ], diff --git a/pom.xml b/pom.xml index b4e10a4..13bec3b 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,6 @@ jjava-jupyter jjava-kernel jjava-maven - jjava-launcher jjava-distro From 6a9656deefbec0069ad032bfecabee2e0d0ddb46 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 9 Oct 2026 17:43:13 +0300 Subject: [PATCH 02/16] Update Java version requirement from 11 to 21 and refactor display handling --- .github/workflows/maven.yml | 7 +- .github/workflows/release-prepare.yml | 2 +- CLAUDE.md | 2 +- README.md | 2 +- .../jjava/distro/NotebookInitializer.java | 10 +- .../NotebookInitializerImportsTest.java | 23 ++ .../jupyter/kernel/BaseNotebookStatics.java | 152 +------------ .../kernel/ExecutorNotebookStatics.java | 102 +++++++++ .../display/protocol/DisplayDelivery.java | 10 + .../protocol/DisplayDeliveryBootstrap.java | 46 ++++ .../protocol/DisplayDeliveryException.java | 21 ++ .../display/protocol/DisplayProtocol.java | 204 +++++++++++++++++ .../display/protocol/DisplayRequest.java | 40 ++++ .../protocol/DisplayRequestHandler.java | 10 + .../uds/UnixDomainDisplayDelivery.java | 68 ++++++ .../uds/UnixDomainDisplayEndpoint.java | 143 ++++++++++++ .../jupyter/kernel/magic/MagicTranspiler.java | 6 +- .../BaseNotebookStaticsDisplayTest.java | 214 ++++++++---------- .../ExecutorNotebookStaticsFailureTest.java | 72 ++++++ ...bookStaticsRenderCharacterizationTest.java | 79 +++++++ .../ExecutorNotebookStaticsRoutingTest.java | 90 ++++++++ .../kernel/OutputBoundaryDependencyTest.java | 84 +++++++ .../DisplayDeliveryBootstrapTest.java | 44 ++++ .../protocol/DisplayDeliveryContractTest.java | 100 ++++++++ .../display/protocol/DisplayEncodingTest.java | 38 ++++ .../display/protocol/DisplayProtocolTest.java | 87 +++++++ .../InMemoryDisplayDeliveryContractTest.java | 23 ++ ...UnixDomainDisplayDeliveryContractTest.java | 33 +++ .../UnixDomainDisplayDeliveryTimeoutTest.java | 99 ++++++++ .../UnixDomainDisplayEndpointTest.java | 59 +++++ .../UnixDomainSocketPathAcceptanceTest.java | 131 +++++++++++ .../kernel/magic/MagicTranspilerTest.java | 26 +++ .../kernel/ExecutorJavaNotebookStatics.java | 22 ++ .../org/dflib/jjava/kernel/JavaKernel.java | 86 +++++-- .../dflib/jjava/kernel/JavaKernelBuilder.java | 24 ++ .../jjava/kernel/JavaNotebookStatics.java | 6 +- .../jjava/kernel/execution/CodeEvaluator.java | 10 +- .../execution/DisplayCallbackServer.java | 126 ----------- .../kernel/execution/EvaluationResult.java | 10 + .../kernel/CapturingShellEnvironment.java | 32 +++ .../kernel/DisplayCallbackServerTest.java | 177 --------------- .../kernel/JavaKernelDisplayFailureTest.java | 77 +++++++ .../JavaKernelEndpointLifecycleTest.java | 43 ++++ .../kernel/JavaKernelEvalBuilderTest.java | 17 +- .../kernel/JavaKernelStalledDisplayTest.java | 62 +++++ .../JavaNotebookStaticsRoutingTest.java | 28 +++ .../kernel/OutputBoundaryDependencyTest.java | 69 ++++++ .../kernel/RemoteDisplayRoundTripTest.java | 41 ++++ .../kernel/StandaloneMagicDispatchTest.java | 54 +++++ .../UnixDomainSocketJdiPropagationTest.java | 57 +++++ .../kernel/execution/CodeEvaluatorTest.java | 13 +- pom.xml | 2 +- 52 files changed, 2354 insertions(+), 629 deletions(-) create mode 100644 jjava-distro/src/test/java/org/dflib/jjava/distro/NotebookInitializerImportsTest.java create mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java create mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDelivery.java create mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrap.java create mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryException.java create mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocol.java create mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequest.java create mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequestHandler.java create mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayDelivery.java create mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayEndpoint.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsFailureTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRenderCharacterizationTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRoutingTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/OutputBoundaryDependencyTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrapTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryContractTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocolTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/InMemoryDisplayDeliveryContractTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryContractTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayEndpointTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java create mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspilerTest.java create mode 100644 jjava-kernel/src/main/java/org/dflib/jjava/kernel/ExecutorJavaNotebookStatics.java delete mode 100644 jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/DisplayCallbackServer.java create mode 100644 jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/EvaluationResult.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/jupyter/kernel/CapturingShellEnvironment.java delete mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/DisplayCallbackServerTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelDisplayFailureTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEndpointLifecycleTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelStalledDisplayTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaNotebookStaticsRoutingTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/OutputBoundaryDependencyTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/StandaloneMagicDispatchTest.java create mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/UnixDomainSocketJdiPropagationTest.java diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index c0bb677..ae63d05 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -12,11 +12,8 @@ jobs: strategy: matrix: - os: [ ubuntu-latest ] - java: [ 11, 17, 21, 25 ] - include: - - os: windows-latest - java: 25 + os: [ ubuntu-latest, windows-latest, macos-latest ] + java: [ 21, 25 ] steps: - name: Checkout diff --git a/.github/workflows/release-prepare.yml b/.github/workflows/release-prepare.yml index 2c68c5e..392cbe6 100644 --- a/.github/workflows/release-prepare.yml +++ b/.github/workflows/release-prepare.yml @@ -24,7 +24,7 @@ jobs: - name: Set up JDK uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 21 distribution: 'temurin' cache: 'maven' diff --git a/CLAUDE.md b/CLAUDE.md index 9d8b424..8a1d707 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - Surefire in `jjava-kernel` and `jjava-distro` is configured with `--add-opens jdk.jshell/jdk.jshell=ALL-UNNAMED` — required for JShell reflection. Anything that exercises `JavaKernel`/`CodeEvaluator` directly needs the same flag. - Local install of the freshly built kernel into Jupyter: see `DEV-GUIDE.md` (`unzip` the kernelspec zip, then `jupyter kernelspec install ... --name=java --user`). -Minimum toolchain: JDK 11 (`maven.compiler.release=11`). CI matrix builds against 11, 17, 21, 25. +Minimum toolchain: JDK 21 (`maven.compiler.release=21`). CI matrix builds against 21 and 25 on Linux, Windows, and macOS. ## Module Layout and Dependency Graph diff --git a/README.md b/README.md index 3189b6b..7380214 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ _JJava is an evolution of the earlier [IJava kernel](https://github.com/SpencerP ## Requirements -1. Java 11 or newer +1. Java 21 or newer 2. Python and a Jupyter-like environment to use the kernel in. ## Installation diff --git a/jjava-distro/src/main/java/org/dflib/jjava/distro/NotebookInitializer.java b/jjava-distro/src/main/java/org/dflib/jjava/distro/NotebookInitializer.java index 5019f19..e00463f 100644 --- a/jjava-distro/src/main/java/org/dflib/jjava/distro/NotebookInitializer.java +++ b/jjava-distro/src/main/java/org/dflib/jjava/distro/NotebookInitializer.java @@ -2,17 +2,17 @@ import org.dflib.jjava.jupyter.Extension; import org.dflib.jjava.jupyter.kernel.BaseKernel; -import org.dflib.jjava.jupyter.kernel.BaseNotebookStatics; -import org.dflib.jjava.kernel.JavaNotebookStatics; +import org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics; +import org.dflib.jjava.kernel.ExecutorJavaNotebookStatics; /** * An automatically-loaded extension that adds common imports to the notebook. This includes static method imports - * from {@link BaseNotebookStatics} and {@link JavaNotebookStatics}. + * from the executor helper classes {@link ExecutorNotebookStatics} and {@link ExecutorJavaNotebookStatics}. */ public class NotebookInitializer implements Extension { - private static final String BASE_STATICS_CLASS = BaseNotebookStatics.class.getName(); - private static final String JAVA_STATICS_CLASS = JavaNotebookStatics.class.getName(); + private static final String BASE_STATICS_CLASS = ExecutorNotebookStatics.class.getName(); + private static final String JAVA_STATICS_CLASS = ExecutorJavaNotebookStatics.class.getName(); private static final String STARTUP_SCRIPT = "import java.util.*;\n" + "import java.io.*;\n" + diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/NotebookInitializerImportsTest.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/NotebookInitializerImportsTest.java new file mode 100644 index 0000000..c5cebc6 --- /dev/null +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/NotebookInitializerImportsTest.java @@ -0,0 +1,23 @@ +package org.dflib.jjava.distro; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class NotebookInitializerImportsTest { + + @Test + void startupScriptImportsOnlyExecutorHelpers() throws Exception { + Field field = NotebookInitializer.class.getDeclaredField("STARTUP_SCRIPT"); + field.setAccessible(true); + String script = (String) field.get(null); + + assertTrue(script.contains("import static org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics.*;"), script); + assertTrue(script.contains("import static org.dflib.jjava.kernel.ExecutorJavaNotebookStatics.*;"), script); + assertFalse(script.contains("BaseNotebookStatics"), script); + assertFalse(script.contains("org.dflib.jjava.kernel.JavaNotebookStatics"), script); + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java index 6850a81..9d0c99d 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java @@ -1,50 +1,27 @@ package org.dflib.jjava.jupyter.kernel; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonObject; import org.dflib.jjava.jupyter.kernel.display.DisplayData; import org.dflib.jjava.jupyter.kernel.magic.UndefinedMagicException; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Map; import java.util.UUID; /** - * A collection of static methods for notebook code to interact with the kernel. The methods are automatically exposed - * in notebooks via a static import on bootstrap. + * Manager-side notebook helpers. They always use the running kernel directly; executor code uses + * {@link ExecutorNotebookStatics} instead. */ public class BaseNotebookStatics { - private static final Gson GSON = new GsonBuilder().serializeNulls().create(); - public static void printf(String format, Object... args) { System.out.printf(format, args); } public static Object eval(String source) { - BaseKernel kernel = BaseKernel.notebookKernel; - if (kernel == null) { - throw new UnsupportedOperationException("eval() cannot be called from executor code in JDI mode"); - } - return kernel.evalBuilder(source).resolveMagics().eval(); + return BaseKernel.notebookKernel().evalBuilder(source).resolveMagics().eval(); } public static T lineMagic(String name, List args) { - BaseKernel kernel = BaseKernel.notebookKernel; - if (kernel == null) { - throw new UnsupportedOperationException("Magics cannot be called programmatically from executor code in JDI mode; use %magic syntax"); - } - + BaseKernel kernel = BaseKernel.notebookKernel(); try { return kernel.getMagicsRegistry().evalLineMagic(kernel, name, args); } catch (UndefinedMagicException e) { @@ -55,11 +32,7 @@ public static T lineMagic(String name, List args) { } public static T cellMagic(String name, List args, String body) { - BaseKernel kernel = BaseKernel.notebookKernel; - if (kernel == null) { - throw new UnsupportedOperationException("Magics cannot be called programmatically from executor code in JDI mode; use %%magic syntax"); - } - + BaseKernel kernel = BaseKernel.notebookKernel(); try { return kernel.getMagicsRegistry().evalCellMagic(kernel, name, args, body); } catch (UndefinedMagicException e) { @@ -70,13 +43,11 @@ public static T cellMagic(String name, List args, String body) { } public static DisplayData render(Object o) { - BaseKernel kernel = BaseKernel.notebookKernel; - return kernel != null ? kernel.getRenderer().render(o) : renderPortable(o); + return BaseKernel.notebookKernel().getRenderer().render(o); } public static DisplayData render(Object o, String... as) { - BaseKernel kernel = BaseKernel.notebookKernel; - return kernel != null ? kernel.getRenderer().renderAs(o, as) : renderPortable(o); + return BaseKernel.notebookKernel().getRenderer().renderAs(o, as); } public static String display(Object o) { @@ -101,116 +72,11 @@ private static String displayData(DisplayData data) { id = UUID.randomUUID().toString(); data.setDisplayId(id); } - - BaseKernel kernel = BaseKernel.notebookKernel; - if (kernel != null) { - kernel.display(data); - } else { - sendRemoteDisplay("display", id, data); - } + BaseKernel.notebookKernel().display(data); return id; } private static void updateDisplayData(String id, DisplayData data) { - BaseKernel kernel = BaseKernel.notebookKernel; - if (kernel != null) { - kernel.getIO().display.updateDisplay(id, data); - } else { - sendRemoteDisplay("updateDisplay", id, data); - } - } - - private static void sendRemoteDisplay(String action, String id, DisplayData data) { - validateDisplayData(data); - int port = Integer.getInteger("jjava.display.port", -1); - if (port <= 0) { - throw new IllegalStateException("Remote display channel is not available"); - } - - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(InetAddress.getLoopbackAddress(), port), 10_000); - socket.setSoTimeout(10_000); - try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8)); - BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) { - JsonObject json = new JsonObject(); - json.addProperty("action", action); - if ("updateDisplay".equals(action) && id != null) { - json.addProperty("id", id); - } - json.add("bundle", GSON.toJsonTree(data)); - - writer.write(json.toString()); - writer.newLine(); - writer.flush(); - - checkDisplayResponse(reader.readLine()); - } - } catch (IOException e) { - throw new RuntimeException("Failed to send display data: " + e.getMessage(), e); - } - } - - private static void checkDisplayResponse(String response) throws IOException { - if (response == null) { - throw new IOException("Display callback closed without a response"); - } - if (!"ok".equals(response)) { - String message; - try { - message = GSON.fromJson(response, String.class); - } catch (RuntimeException e) { - message = response; - } - throw new IOException("Display callback failed: " + (message == null ? response : message)); - } - } - - private static void validateDisplayData(DisplayData data) { - validatePortable(data.getData()); - validatePortable(data.getMetadata()); - validatePortable(data.getTransientData()); - } - - private static void validatePortable(Object value) { - if (value == null - || value instanceof CharSequence - || value instanceof Number - || value instanceof Boolean - || value instanceof Character) { - return; - } - if (value instanceof Map) { - for (Map.Entry entry : ((Map) value).entrySet()) { - if (!(entry.getKey() instanceof String)) { - Object key = entry.getKey(); - throw new IllegalArgumentException("Non-portable display value: " - + (key == null ? "null" : key.getClass().getName())); - } - validatePortable(entry.getValue()); - } - return; - } - if (value instanceof Iterable) { - ((Iterable) value).forEach(BaseNotebookStatics::validatePortable); - return; - } - if (value.getClass().isArray()) { - int len = java.lang.reflect.Array.getLength(value); - for (int i = 0; i < len; i++) { - validatePortable(java.lang.reflect.Array.get(value, i)); - } - return; - } - throw new IllegalArgumentException("Non-portable display value: " + value.getClass().getName()); - } - - private static DisplayData renderPortable(Object value) { - if (value instanceof DisplayData) { - return (DisplayData) value; - } - if (value instanceof CharSequence || value instanceof Number || value instanceof Boolean) { - return new DisplayData(String.valueOf(value)); - } - throw new IllegalArgumentException("Non-portable display value: " + value.getClass().getName()); + BaseKernel.notebookKernel().getIO().display.updateDisplay(id, data); } } diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java new file mode 100644 index 0000000..2f4438f --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java @@ -0,0 +1,102 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryBootstrap; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; + +import java.util.List; +import java.util.UUID; + +/** + * Notebook helpers for executor (user cell) code. Fixed by import at wiring time: display requests go through + * {@link DisplayDeliveryBootstrap}, rendering is local, and the kernel is never read. + */ +public class ExecutorNotebookStatics { + + private ExecutorNotebookStatics() { + } + + public static void printf(String format, Object... args) { + // Deliberate: JShell forwards System.out to the cell output (FR-011). A logger would not reach the cell. + System.out.printf(format, args); // NOSONAR + } + + public static Object eval(String source) { + throw new UnsupportedOperationException("eval() cannot be called from executor code in JDI mode"); + } + + public static T lineMagic(String name, List args) { + throw new UnsupportedOperationException( + "Magics cannot be called programmatically from executor code in JDI mode; use %magic syntax"); + } + + public static T cellMagic(String name, List args, String body) { + throw new UnsupportedOperationException( + "Magics cannot be called programmatically from executor code in JDI mode; use %%magic syntax"); + } + + public static DisplayData render(Object o) { + return renderPortable(o); + } + + // Presentation hints are ignored by design (FR-012); the parameter keeps the manager's signature. + @SuppressWarnings("unused") + public static DisplayData render(Object o, String... as) { + return renderPortable(o); + } + + public static String display(Object o) { + return displayData(render(o)); + } + + public static String display(Object o, String... as) { + return displayData(render(o, as)); + } + + public static void updateDisplay(String id, Object o) { + updateDisplayData(id, render(o)); + } + + public static void updateDisplay(String id, Object o, String... as) { + updateDisplayData(id, render(o, as)); + } + + private static String displayData(DisplayData data) { + String id = data.getDisplayId(); + if (id == null) { + id = UUID.randomUUID().toString(); + data.setDisplayId(id); + } + deliver(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, id, data)); + return id; + } + + private static void updateDisplayData(String id, DisplayData data) { + deliver(DisplayRequest.of(DisplayRequest.Operation.UPDATE, id, data)); + } + + // one attempt, no retry; messages keep the existing prefixes + private static void deliver(DisplayRequest request) { + try { + DisplayDeliveryBootstrap.current().deliver(request); + } catch (DisplayDeliveryException e) { + switch (e.kind()) { + case UNAVAILABLE -> throw new IllegalStateException("Remote display channel is not available", e); + case REJECTED -> throw new RuntimeException(e.getMessage(), e); + case UNCERTAIN -> throw new RuntimeException("Failed to send display data: " + e.getMessage(), e); + } + throw e; + } + } + + private static DisplayData renderPortable(Object value) { + if (value instanceof DisplayData data) { + return data; + } + if (value instanceof CharSequence || value instanceof Number || value instanceof Boolean) { + return new DisplayData(String.valueOf(value)); + } + throw new IllegalArgumentException("Non-portable display value: " + value.getClass().getName()); + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDelivery.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDelivery.java new file mode 100644 index 0000000..0152354 --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDelivery.java @@ -0,0 +1,10 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +/** + * Executor-side boundary: hands one display request to the manager and returns only on confirmed delivery, otherwise + * throws a {@link DisplayDeliveryException}. + */ +public interface DisplayDelivery { + + void deliver(DisplayRequest request); +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrap.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrap.java new file mode 100644 index 0000000..9c28bb8 --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrap.java @@ -0,0 +1,46 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayDelivery; + +import java.nio.file.Path; +import java.util.Objects; + +/** + * Chooses the executor-side delivery from the {@code jjava.display.socket} system property. Without the property, + * every delivery fails as UNAVAILABLE. + */ +public final class DisplayDeliveryBootstrap { + + public static final String SOCKET_PROPERTY = "jjava.display.socket"; + + private static DisplayDelivery current; + + private DisplayDeliveryBootstrap() { + } + + public static synchronized DisplayDelivery current() { + if (current == null) { + current = fromSocketPath(System.getProperty(SOCKET_PROPERTY)); + } + return current; + } + + /** + * Replaces the delivery and returns the previous one. + */ + public static synchronized DisplayDelivery install(DisplayDelivery delivery) { + DisplayDelivery previous = current(); + current = Objects.requireNonNull(delivery); + return previous; + } + + static DisplayDelivery fromSocketPath(String socketPath) { + if (socketPath == null) { + return request -> { + throw new DisplayDeliveryException( + DisplayDeliveryException.Kind.UNAVAILABLE, "Remote display channel is not available", null); + }; + } + return new UnixDomainDisplayDelivery(Path.of(socketPath)); + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryException.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryException.java new file mode 100644 index 0000000..c2e4cb1 --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryException.java @@ -0,0 +1,21 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +public class DisplayDeliveryException extends IllegalStateException { + + public enum Kind { + UNAVAILABLE, + REJECTED, + UNCERTAIN + } + + private final Kind kind; + + public DisplayDeliveryException(Kind kind, String message, Throwable cause) { + super(message, cause); + this.kind = kind; + } + + public Kind kind() { + return kind; + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocol.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocol.java new file mode 100644 index 0000000..dbb26f3 --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocol.java @@ -0,0 +1,204 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import org.dflib.jjava.jupyter.kernel.display.DisplayData; + +import java.lang.reflect.Array; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; + +/** + * Transport-independent codec and portability rules of the cross-JVM display protocol. A request is one JSON object; + * a response is one token: {@code ok}, or a JSON string with the failure message. Nothing here depends on how the + * bytes travel. + */ +public final class DisplayProtocol { + + static final Gson GSON = new GsonBuilder().serializeNulls().create(); + + private static final String OK = "ok"; + private static final String ACTION = "action"; + private static final String TRANSIENT = "transient"; + + private DisplayProtocol() { + } + + static JsonObject snapshot(DisplayRequest.Operation operation, String displayId, DisplayData data) { + if (displayId == null) { + throw new IllegalArgumentException(operation == DisplayRequest.Operation.UPDATE + ? "updateDisplay requires a non-null id" + : "display requires a non-null id"); + } + + Set path = Collections.newSetFromMap(new IdentityHashMap()); + requirePortable(data.getData(), path); + requirePortable(data.getMetadata(), path); + requirePortable(data.getTransientData(), path); + + JsonObject bundle = new JsonObject(); + bundle.add("data", GSON.toJsonTree(data.getData())); + bundle.add("metadata", GSON.toJsonTree(data.getMetadata())); + bundle.add(TRANSIENT, GSON.toJsonTree(data.getTransientData())); + if (operation == DisplayRequest.Operation.DISPLAY) { + bundle.getAsJsonObject(TRANSIENT).addProperty("display_id", displayId); + } + return bundle; + } + + public static String encode(DisplayRequest request) { + JsonObject message = new JsonObject(); + if (request.operation() == DisplayRequest.Operation.UPDATE) { + message.addProperty(ACTION, "updateDisplay"); + message.addProperty("id", request.displayId()); + } else { + message.addProperty(ACTION, "display"); + } + message.add("bundle", request.bundle()); + return GSON.toJson(message); + } + + public static DisplayRequest decode(String line) { + JsonObject message; + try { + message = GSON.fromJson(line, JsonObject.class); + } catch (JsonParseException e) { + throw new IllegalArgumentException("Display request must be a JSON object", e); + } + if (message == null) { + throw new IllegalArgumentException("Display request must be a JSON object"); + } + + JsonElement bundle = message.get("bundle"); + if (bundle == null || !bundle.isJsonObject()) { + throw new IllegalArgumentException("Display bundle must not be null"); + } + + JsonElement action = message.get(ACTION); + String name = action != null && action.isJsonPrimitive() ? action.getAsString() : null; + if ("display".equals(name)) { + return new DisplayRequest( + DisplayRequest.Operation.DISPLAY, + transientDisplayId(bundle.getAsJsonObject()), + bundle.getAsJsonObject()); + } + if ("updateDisplay".equals(name)) { + JsonElement id = message.get("id"); + if (id == null || id.isJsonNull()) { + throw new IllegalArgumentException("updateDisplay requires a non-null id"); + } + return new DisplayRequest(DisplayRequest.Operation.UPDATE, id.getAsString(), bundle.getAsJsonObject()); + } + throw new IllegalArgumentException("Unsupported display action: " + name); + } + + /** + * Returns the response token for a handled request: {@code null} means success. + */ + public static String response(RuntimeException failure) { + if (failure == null) { + return OK; + } + String message = failure.getMessage() == null ? failure.getClass().getSimpleName() : failure.getMessage(); + return GSON.toJson(message); + } + + /** + * Interprets a response token; throws unless it is {@code ok}. + */ + public static void checkResponse(String response) { + if (response == null) { + throw new DisplayDeliveryException( + DisplayDeliveryException.Kind.UNCERTAIN, "Display callback closed without a response", null); + } + if (OK.equals(response)) { + return; + } + + String message; + try { + message = GSON.fromJson(response, String.class); + } catch (JsonParseException e) { + message = response; + } + throw new DisplayDeliveryException( + DisplayDeliveryException.Kind.REJECTED, + "Display callback failed: " + (message == null ? response : message), + null); + } + + private static String transientDisplayId(JsonObject bundle) { + JsonElement transientData = bundle.get(TRANSIENT); + JsonElement id = transientData != null && transientData.isJsonObject() + ? transientData.getAsJsonObject().get("display_id") + : null; + return id != null && id.isJsonPrimitive() ? id.getAsString() : null; + } + + private static void requirePortable(Object value, Set path) { + if (value == null || value instanceof CharSequence || value instanceof Boolean || value instanceof Character) { + return; + } + if (value instanceof Number number) { + requireFinite(number); + return; + } + + enter(value, path); + requireContainer(value, path); + path.remove(value); + } + + private static void requireContainer(Object value, Set path) { + if (value instanceof Map map) { + for (Map.Entry entry : map.entrySet()) { + requireStringKey(entry.getKey()); + requirePortable(entry.getValue(), path); + } + } else if (value instanceof Iterable items) { + for (Object item : items) { + requirePortable(item, path); + } + } else if (value.getClass().isArray()) { + for (int i = 0; i < Array.getLength(value); i++) { + requirePortable(Array.get(value, i), path); + } + } else { + throw new IllegalArgumentException("Non-portable display value: " + value.getClass().getName()); + } + } + + private static void requireStringKey(Object key) { + if (!(key instanceof String)) { + throw new IllegalArgumentException("Non-portable display value: " + + (key == null ? "null" : key.getClass().getName())); + } + } + + private static void requireFinite(Number number) { + if (isNonFinite(number)) { + throw new IllegalArgumentException("Non-portable display value: non-finite number"); + } + } + + private static void enter(Object container, Set path) { + if (!path.add(container)) { + throw new IllegalArgumentException("Non-portable display value: cyclic structure"); + } + } + + private static boolean isNonFinite(Number number) { + if (number instanceof Double d) { + return !Double.isFinite(d); + } + if (number instanceof Float f) { + return !Float.isFinite(f); + } + return false; + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequest.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequest.java new file mode 100644 index 0000000..7c12fed --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequest.java @@ -0,0 +1,40 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.dflib.jjava.jupyter.kernel.display.DisplayData; + +import java.util.Map; +import java.util.function.BiConsumer; + +public record DisplayRequest(Operation operation, String displayId, JsonObject bundle) { + + public enum Operation { + DISPLAY, + UPDATE + } + + public static DisplayRequest of(Operation operation, String displayId, DisplayData data) { + return new DisplayRequest(operation, displayId, DisplayProtocol.snapshot(operation, displayId, data)); + } + + public DisplayData toDisplayData() { + DisplayData out = new DisplayData(); + copy("data", out::putData); + copy("metadata", out::putMetaData); + copy("transient", out::putTransientData); + if (displayId != null) { + out.setDisplayId(displayId); + } + return out; + } + + private void copy(String section, BiConsumer sink) { + JsonElement values = bundle.get(section); + if (values != null && values.isJsonObject()) { + for (Map.Entry entry : values.getAsJsonObject().entrySet()) { + sink.accept(entry.getKey(), DisplayProtocol.GSON.fromJson(entry.getValue(), Object.class)); + } + } + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequestHandler.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequestHandler.java new file mode 100644 index 0000000..1f3a468 --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayRequestHandler.java @@ -0,0 +1,10 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +/** + * Manager-side boundary: applies one decoded display request to the notebook. Any exception it throws is reported to + * the executor as a rejection. + */ +public interface DisplayRequestHandler { + + void handle(DisplayRequest request); +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayDelivery.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayDelivery.java new file mode 100644 index 0000000..4a3a13a --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayDelivery.java @@ -0,0 +1,68 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol.uds; + +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDelivery; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayProtocol; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; + +import java.io.IOException; +import java.net.StandardProtocolFamily; +import java.net.UnixDomainSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.UNAVAILABLE; +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.UNCERTAIN; +import static org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint.DEADLINES; +import static org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint.DEADLINE_SECONDS; +import static org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint.closeQuietly; +import static org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint.readLine; + +/** + * Executor-side delivery over a Unix-domain socket. One deadline covers connect, write, and the response read. A + * failure before any payload byte is written is UNAVAILABLE; a failure after is UNCERTAIN. Nothing is retried. + */ +public final class UnixDomainDisplayDelivery implements DisplayDelivery { + + private static final String UNAVAILABLE_MESSAGE = "Remote display channel is not available"; + private static final String UNCERTAIN_MESSAGE = "Display delivery outcome is uncertain; nothing was retried"; + + private final Path socketPath; + + public UnixDomainDisplayDelivery(Path socketPath) { + this.socketPath = socketPath; + } + + @Override + public void deliver(DisplayRequest request) { + ByteBuffer out = ByteBuffer.wrap((DisplayProtocol.encode(request) + "\n").getBytes(StandardCharsets.UTF_8)); + + SocketChannel channel; + try { + channel = SocketChannel.open(StandardProtocolFamily.UNIX); + } catch (IOException e) { + throw new DisplayDeliveryException(UNAVAILABLE, UNAVAILABLE_MESSAGE, e); + } + + ScheduledFuture deadline = DEADLINES.schedule(() -> closeQuietly(channel), DEADLINE_SECONDS, TimeUnit.SECONDS); + try (channel) { + channel.connect(UnixDomainSocketAddress.of(socketPath)); + while (out.hasRemaining()) { + channel.write(out); + } + DisplayProtocol.checkResponse(readLine(channel)); + } catch (IOException e) { + boolean sent = out.position() > 0; + throw new DisplayDeliveryException( + sent ? UNCERTAIN : UNAVAILABLE, + sent ? UNCERTAIN_MESSAGE : UNAVAILABLE_MESSAGE, + e); + } finally { + deadline.cancel(false); + } + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayEndpoint.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayEndpoint.java new file mode 100644 index 0000000..2e6d98e --- /dev/null +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayEndpoint.java @@ -0,0 +1,143 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol.uds; + +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayProtocol; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequestHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.IOException; +import java.net.StandardProtocolFamily; +import java.net.UnixDomainSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * Manager-side Unix-domain endpoint. Accepts one request line per connection, serially, and answers with one response + * line. Each connection is bounded by a 10 s deadline. Closing the endpoint, or an accept failure, releases the + * channel and removes the socket file. + */ +public final class UnixDomainDisplayEndpoint implements Closeable { + + static final long DEADLINE_SECONDS = 10; + static final ScheduledExecutorService DEADLINES = Executors.newSingleThreadScheduledExecutor(r -> { + Thread thread = new Thread(r, "jjava-display-deadlines"); + thread.setDaemon(true); + return thread; + }); + + private static final Logger LOGGER = LoggerFactory.getLogger(UnixDomainDisplayEndpoint.class); + + private final ServerSocketChannel server; + private final Path path; + private final DisplayRequestHandler handler; + + public UnixDomainDisplayEndpoint(DisplayRequestHandler handler) throws IOException { + this(null, handler); + } + + public UnixDomainDisplayEndpoint(Path socketPath, DisplayRequestHandler handler) throws IOException { + this.handler = handler; + this.server = ServerSocketChannel.open(StandardProtocolFamily.UNIX); + this.server.bind(socketPath == null ? null : UnixDomainSocketAddress.of(socketPath)); + this.path = ((UnixDomainSocketAddress) server.getLocalAddress()).getPath(); + + Thread acceptor = new Thread(this::acceptLoop, "jjava-display-callback"); + acceptor.setDaemon(true); + acceptor.start(); + } + + public Path path() { + return path; + } + + private void acceptLoop() { + while (true) { + SocketChannel channel; + try { + channel = server.accept(); + } catch (IOException e) { + if (server.isOpen()) { + LOGGER.warn("Display endpoint accept failed; closing endpoint", e); + closeQuietly(this); + } + return; + } + try { + handle(channel); + } finally { + closeQuietly(channel); + } + } + } + + private void handle(SocketChannel channel) { + ScheduledFuture deadline = DEADLINES.schedule(() -> closeQuietly(channel), DEADLINE_SECONDS, TimeUnit.SECONDS); + try { + writeLine(channel, respond(channel)); + } catch (IOException e) { + LOGGER.warn("Display connection failed", e); + } finally { + deadline.cancel(false); + } + } + + private String respond(SocketChannel channel) throws IOException { + try { + handler.handle(DisplayProtocol.decode(readLine(channel))); + return DisplayProtocol.response(null); + } catch (RuntimeException e) { + return DisplayProtocol.response(e); + } + } + + @Override + public void close() throws IOException { + server.close(); + Files.deleteIfExists(path); + } + + static String readLine(ReadableByteChannel channel) throws IOException { + ByteArrayOutputStream line = new ByteArrayOutputStream(); + ByteBuffer buffer = ByteBuffer.allocate(8192); + while (true) { + buffer.clear(); + if (channel.read(buffer) < 0) { + return line.size() == 0 ? null : line.toString(StandardCharsets.UTF_8); + } + buffer.flip(); + while (buffer.hasRemaining()) { + byte b = buffer.get(); + if (b == '\n') { + return line.toString(StandardCharsets.UTF_8); + } + line.write(b); + } + } + } + + static void writeLine(SocketChannel channel, String text) throws IOException { + ByteBuffer out = ByteBuffer.wrap((text + "\n").getBytes(StandardCharsets.UTF_8)); + while (out.hasRemaining()) { + channel.write(out); + } + } + + static void closeQuietly(Closeable closeable) { + try { + closeable.close(); + } catch (IOException ignored) { + // best-effort close on a failure or deadline path; the caller has no recovery action + } + } +} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspiler.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspiler.java index 74d294c..1791ffe 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspiler.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspiler.java @@ -1,6 +1,6 @@ package org.dflib.jjava.jupyter.kernel.magic; -import org.dflib.jjava.jupyter.kernel.BaseNotebookStatics; +import org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics; import java.util.Base64; import java.util.regex.Matcher; @@ -15,8 +15,8 @@ public class MagicTranspiler { private static final Pattern UNESCAPED_QUOTE = Pattern.compile("(? delivered = new ArrayList<>(); private BaseKernel previousKernel; - private String previousPort; - private ServerSocket serverSocket; + private DisplayDelivery previousDelivery; + private CapturingEnvironment env; @BeforeEach - public void setUp() throws IOException { + void setUp() { previousKernel = BaseKernel.notebookKernel; - previousPort = System.getProperty("jjava.display.port"); - BaseKernel.notebookKernel = null; - serverSocket = new ServerSocket(0, 1, InetAddress.getLoopbackAddress()); - serverSocket.setSoTimeout(10_000); - System.setProperty("jjava.display.port", Integer.toString(serverSocket.getLocalPort())); + previousDelivery = DisplayDeliveryBootstrap.install(delivered::add); + + JupyterIO io = new JupyterIO(StandardCharsets.UTF_8); + env = new CapturingEnvironment(); + io.setEnv(env); + BaseKernel.notebookKernel = new StubKernel(io); } @AfterEach - public void tearDown() throws IOException { - try { - serverSocket.close(); - } finally { - BaseKernel.notebookKernel = previousKernel; - if (previousPort == null) { - System.clearProperty("jjava.display.port"); - } else { - System.setProperty("jjava.display.port", previousPort); - } - } + void tearDown() { + BaseKernel.notebookKernel = previousKernel; + DisplayDeliveryBootstrap.install(previousDelivery); } @Test - public void sendsCompleteBundleAndRequiresSuccessAcknowledgement() throws Exception { - DisplayData data = new DisplayData(); - Map nested = new LinkedHashMap<>(); - nested.put("values", Arrays.asList("kept", true, null)); - nested.put("nullValue", null); - data.putData("application/json", nested); - data.putData("application/null", null); - data.putMetaData("render", nested); - data.putTransientData("custom", nested); + void displayReturnsTheIdOfThePublishedBundle() { + DisplayData data = new DisplayData("x"); data.setDisplayId("display-id"); - JsonObject request = exchange("ok", () -> assertEquals("display-id", BaseNotebookStatics.display(data))); + String id = BaseNotebookStatics.display(data); - assertEquals("display", request.get("action").getAsString()); - assertFalse(request.has("id")); - assertEquals(GSON.toJsonTree(data), request.get("bundle")); + PublishDisplayData published = assertInstanceOf(PublishDisplayData.class, env.published.get(0).getContent()); + assertEquals(published.getDisplayId(), id); + assertTrue(delivered.isEmpty()); } @Test - public void eofAndErrorAcknowledgementsFailVisibly() throws Exception { - JsonObject eof = exchange(null, () -> { - RuntimeException failure = assertThrows(RuntimeException.class, () -> BaseNotebookStatics.display("text")); - assertTrue(failure.getMessage().contains("closed without a response")); - }); - assertEquals("display", eof.get("action").getAsString()); - - JsonObject error = exchange("\"remote\\nerror\"", () -> { - RuntimeException failure = assertThrows(RuntimeException.class, () -> BaseNotebookStatics.display("text")); - assertTrue(failure.getMessage().contains("remote\nerror")); - }); - assertEquals("display", error.get("action").getAsString()); + void updateDisplayUsesKernelWithTheTargetId() { + BaseNotebookStatics.updateDisplay("requested-id", new DisplayData("updated")); + + PublishUpdateDisplayData update = assertInstanceOf( + PublishUpdateDisplayData.class, + env.published.get(0).getContent()); + assertEquals("requested-id", update.getDisplayId()); + assertTrue(delivered.isEmpty()); } @Test - public void sendsUpdateIdAndCompleteBundle() throws Exception { - DisplayData data = new DisplayData("updated"); - data.putMetaData("render", Map.of("expanded", true)); - data.putTransientData("custom", "preserved"); - JsonObject request = exchange("ok", () -> BaseNotebookStatics.updateDisplay("requested-id", data)); - - assertEquals("updateDisplay", request.get("action").getAsString()); - assertEquals("requested-id", request.get("id").getAsString()); - assertEquals(GSON.toJsonTree(data), request.get("bundle")); - } + void managerWithoutKernelFailsWithNullPointerAndNeverDelivers() { + BaseKernel.notebookKernel = null; - @Test - public void rejectsNonPortableValuesBeforeConnectingAndAcceptsCharacterArrays() throws Exception { - DisplayData nestedValue = new DisplayData(); - nestedValue.putData("application/json", Arrays.asList(Map.of("bad", new Object()))); - assertNonPortable(nestedValue); - - DisplayData nonStringKeysData = new DisplayData(); - Map nonStringKeys = new LinkedHashMap<>(); - nonStringKeys.put(1, "bad"); - nonStringKeysData.putData("application/json", nonStringKeys); - assertNonPortable(nonStringKeysData); - - DisplayData transientValue = new DisplayData(); - transientValue.putTransientData("bad", new Object()); - assertNonPortable(transientValue); - - DisplayData characters = new DisplayData(); - characters.putData("application/characters", new char[]{'o', 'k'}); - JsonObject request = exchange("ok", () -> assertNotNull(BaseNotebookStatics.display(characters))); - JsonObject bundle = request.getAsJsonObject("bundle"); - assertEquals("[\"o\",\"k\"]", bundle.getAsJsonObject("data").get("application/characters").toString()); - assertTrue(bundle.getAsJsonObject("transient").has("display_id")); + assertThrows(NullPointerException.class, () -> BaseNotebookStatics.display("text")); + assertTrue(delivered.isEmpty()); } - private void assertNonPortable(DisplayData data) { - IllegalArgumentException failure = assertThrows( - IllegalArgumentException.class, - () -> BaseNotebookStatics.display(data)); - assertTrue(failure.getMessage().contains("Non-portable display value")); + private static class CapturingEnvironment extends ShellReplyEnvironment { + final List> published = new ArrayList<>(); + + CapturingEnvironment() { + super(null, null, null, null); + } + + @Override + public void publish(Message message) { + published.add(message); + } } - private JsonObject exchange(String response, Runnable operation) throws Exception { - FutureTask request = new FutureTask<>(() -> { - try (Socket socket = serverSocket.accept()) { - socket.setSoTimeout(2_000); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8)); - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8))) { - JsonObject json = GSON.fromJson(reader.readLine(), JsonObject.class); - if (response != null) { - writer.write(response); - writer.newLine(); - writer.flush(); - } - return json; - } - } - }); - Thread server = new Thread(request, "test-display-callback"); - server.setDaemon(true); - server.start(); - operation.run(); - return request.get(2_000, TimeUnit.MILLISECONDS); + private static class StubKernel extends BaseKernel { + StubKernel(JupyterIO io) { + super( + "test", "0", + new LanguageInfo.Builder("Java").build(), + List.of(), null, io, new CommManager(), new Renderer(), + new MagicsResolver("%", "%%", new MagicTranspiler()), + new MagicsRegistry(), false, new StringStyler.Builder().build()); + } + + @Override + protected Object doEval(String source) { + throw new UnsupportedOperationException(); + } + + @Override + public DisplayData inspect(String code, int at, boolean extraDetail) { + throw new UnsupportedOperationException(); + } + + @Override + public ReplacementOptions complete(String code, int at) { + throw new UnsupportedOperationException(); + } + + @Override + public String isComplete(String code) { + throw new UnsupportedOperationException(); + } } } diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsFailureTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsFailureTest.java new file mode 100644 index 0000000..61766ef --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsFailureTest.java @@ -0,0 +1,72 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDelivery; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryBootstrap; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ExecutorNotebookStaticsFailureTest { + + private DisplayDelivery previousDelivery; + private AtomicInteger attempts; + + @BeforeEach + void setUp() { + attempts = new AtomicInteger(); + previousDelivery = null; + } + + @AfterEach + void tearDown() { + if (previousDelivery != null) { + DisplayDeliveryBootstrap.install(previousDelivery); + } + } + + @Test + void unavailableMapsToRemoteChannelMessage() { + failingWith(DisplayDeliveryException.Kind.UNAVAILABLE, "Remote display channel is not available"); + + RuntimeException e = assertThrows(RuntimeException.class, () -> ExecutorNotebookStatics.display("text")); + + assertEquals("Remote display channel is not available", e.getMessage()); + assertEquals(1, attempts.get()); + } + + @Test + void rejectedKeepsDisplayCallbackPrefix() { + failingWith(DisplayDeliveryException.Kind.REJECTED, "Display callback failed: boom"); + + RuntimeException e = assertThrows(RuntimeException.class, () -> ExecutorNotebookStatics.display("text")); + + assertTrue(e.getMessage().startsWith("Display callback failed: "), e.getMessage()); + assertEquals(1, attempts.get()); + } + + @Test + void uncertainMapsToFailedToSendPrefixAndIsNotRetried() { + failingWith(DisplayDeliveryException.Kind.UNCERTAIN, "outcome unknown"); + + RuntimeException e = assertThrows(RuntimeException.class, () -> ExecutorNotebookStatics.updateDisplay("id", "text")); + + assertTrue(e.getMessage().startsWith("Failed to send display data: "), e.getMessage()); + assertEquals(1, attempts.get()); + } + + private void failingWith(DisplayDeliveryException.Kind kind, String message) { + previousDelivery = DisplayDeliveryBootstrap.install(request -> { + attempts.incrementAndGet(); + throw new DisplayDeliveryException(kind, message, null); + }); + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRenderCharacterizationTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRenderCharacterizationTest.java new file mode 100644 index 0000000..fbe2e45 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRenderCharacterizationTest.java @@ -0,0 +1,79 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDelivery; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryBootstrap; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Characterizes executor render behaviour (Gate 1 for the routing change). Rendering never publishes or delivers, so + * the recording delivery must stay empty throughout. + */ +class ExecutorNotebookStaticsRenderCharacterizationTest { + + private final List delivered = new ArrayList<>(); + private DisplayDelivery previousDelivery; + private BaseKernel previousKernel; + + @BeforeEach + void setUp() { + previousKernel = BaseKernel.notebookKernel; + BaseKernel.notebookKernel = null; + previousDelivery = DisplayDeliveryBootstrap.install(delivered::add); + } + + @AfterEach + void tearDown() { + DisplayDeliveryBootstrap.install(previousDelivery); + BaseKernel.notebookKernel = previousKernel; + } + + @Test + void renderNullThrowsNullPointerException() { + assertThrows(NullPointerException.class, () -> ExecutorNotebookStatics.render((Object) null)); + } + + @Test + void renderDisplayDataReturnsSameObject() { + DisplayData data = new DisplayData("x"); + assertSame(data, ExecutorNotebookStatics.render((Object) data)); + } + + @Test + void renderScalarsAsTextPlainOnly() { + assertEquals(Map.of("text/plain", "hi"), ExecutorNotebookStatics.render("hi").getData()); + assertEquals(Map.of("text/plain", "42"), ExecutorNotebookStatics.render(42).getData()); + assertEquals(Map.of("text/plain", "true"), ExecutorNotebookStatics.render(true).getData()); + } + + @Test + void renderCharacterThrowsIllegalArgumentException() { + assertThrows(IllegalArgumentException.class, () -> ExecutorNotebookStatics.render('c')); + } + + @Test + void renderIgnoresPresentationHints() { + assertEquals( + ExecutorNotebookStatics.render("x").getData(), + ExecutorNotebookStatics.render("x", "text/html").getData()); + } + + @Test + void renderNeverDelivers() { + ExecutorNotebookStatics.render("x"); + ExecutorNotebookStatics.render("x", "text/html"); + assertTrue(delivered.isEmpty()); + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRoutingTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRoutingTest.java new file mode 100644 index 0000000..79eb4fe --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRoutingTest.java @@ -0,0 +1,90 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDelivery; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryBootstrap; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ExecutorNotebookStaticsRoutingTest { + + private final List delivered = new ArrayList<>(); + private DisplayDelivery previousDelivery; + private BaseKernel previousKernel; + + @BeforeEach + void setUp() { + previousKernel = BaseKernel.notebookKernel; + BaseKernel.notebookKernel = null; + previousDelivery = DisplayDeliveryBootstrap.install(delivered::add); + } + + @AfterEach + void tearDown() { + DisplayDeliveryBootstrap.install(previousDelivery); + BaseKernel.notebookKernel = previousKernel; + } + + @Test + void displayDeliversThroughBootstrapWithoutReadingKernel() { + String id = ExecutorNotebookStatics.display("text"); + + assertEquals(1, delivered.size()); + assertEquals(DisplayRequest.Operation.DISPLAY, delivered.get(0).operation()); + assertEquals(id, delivered.get(0).displayId()); + } + + @Test + void updateDisplayDeliversUpdateWithTargetId() { + ExecutorNotebookStatics.updateDisplay("target", "text"); + + assertEquals(DisplayRequest.Operation.UPDATE, delivered.get(0).operation()); + assertEquals("target", delivered.get(0).displayId()); + } + + @Test + void updateDisplayWithNullIdFailsBeforeDelivering() { + assertThrows(IllegalArgumentException.class, () -> ExecutorNotebookStatics.updateDisplay(null, "text")); + assertTrue(delivered.isEmpty()); + } + + @Test + void unsupportedHelpersFailWithExistingMessages() { + assertEquals( + "eval() cannot be called from executor code in JDI mode", + assertThrows(UnsupportedOperationException.class, () -> ExecutorNotebookStatics.eval("1")).getMessage()); + assertEquals( + "Magics cannot be called programmatically from executor code in JDI mode; use %magic syntax", + assertThrows(UnsupportedOperationException.class, + () -> ExecutorNotebookStatics.lineMagic("m", List.of())).getMessage()); + assertEquals( + "Magics cannot be called programmatically from executor code in JDI mode; use %%magic syntax", + assertThrows(UnsupportedOperationException.class, + () -> ExecutorNotebookStatics.cellMagic("m", List.of(), "body")).getMessage()); + assertTrue(delivered.isEmpty()); + } + + @Test + void printfWritesStandardOutput() { + PrintStream original = System.out; + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + System.setOut(new PrintStream(captured, true, StandardCharsets.UTF_8)); + try { + ExecutorNotebookStatics.printf("x=%d", 7); + } finally { + System.setOut(original); + } + assertEquals("x=7", captured.toString(StandardCharsets.UTF_8)); + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/OutputBoundaryDependencyTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/OutputBoundaryDependencyTest.java new file mode 100644 index 0000000..b9b28bb --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/OutputBoundaryDependencyTest.java @@ -0,0 +1,84 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.spi.ToolProvider; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Dependency boundaries of the jupyter module (T056). Uses jdeps class-level output over target/classes. + */ +class OutputBoundaryDependencyTest { + + private static final String PROTOCOL = "org.dflib.jjava.jupyter.kernel.display.protocol."; + private static final String BASE_STATICS = "org.dflib.jjava.jupyter.kernel.BaseNotebookStatics"; + + @Test + void protocolClassesOutsideTransportPackagesDoNotUseSocketsOrJupyterChannels() { + List violations = new ArrayList<>(); + List edges = edges(Path.of("target", "classes")); + assertFalse(edges.isEmpty(), "jdeps produced no edges; the check would pass vacuously"); + for (String[] edge : edges) { + String from = edge[0]; + String to = edge[1]; + if (isDirectProtocolClass(from) && (to.startsWith("java.net.") + || to.startsWith("java.nio.channels.") + || to.startsWith("org.dflib.jjava.jupyter.channels.") + || to.startsWith("org.dflib.jjava.jupyter.messages.") + || to.equals("org.dflib.jjava.jupyter.kernel.DisplayStream") + || to.equals("org.dflib.jjava.jupyter.kernel.BaseKernel"))) { + violations.add(from + " -> " + to); + } + } + assertTrue(violations.isEmpty(), "forbidden edges: " + violations); + } + + @Test + void managerHelperNeverDeliversThroughTheExecutorBoundary() { + List violations = new ArrayList<>(); + for (String[] edge : edges(Path.of("target", "classes"))) { + String from = edge[0]; + String to = edge[1]; + if (from.startsWith(BASE_STATICS) && (to.startsWith(PROTOCOL + "uds.") + || to.startsWith(PROTOCOL + "tcp.") + || to.equals(PROTOCOL + "DisplayDelivery") + || to.equals(PROTOCOL + "DisplayDeliveryBootstrap") + || to.startsWith("java.net."))) { + violations.add(from + " -> " + to); + } + } + assertTrue(violations.isEmpty(), "forbidden edges: " + violations); + } + + // A class directly in the protocol package (not in uds/tcp subpackages). + private static boolean isDirectProtocolClass(String name) { + return name.startsWith(PROTOCOL) && name.indexOf('.', PROTOCOL.length()) < 0; + } + + static List edges(Path classes) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ToolProvider.findFirst("jdeps").orElseThrow().run( + new PrintStream(out, true, StandardCharsets.UTF_8), System.err, + "-verbose:class", classes.toString()); + + List edges = new ArrayList<>(); + for (String line : out.toString(StandardCharsets.UTF_8).split("\n")) { + int arrow = line.indexOf(" -> "); + if (arrow < 0) { + continue; + } + String from = line.substring(0, arrow).trim(); + String to = line.substring(arrow + 4).trim().split("\\s+")[0]; + edges.add(new String[]{from, to}); + } + return edges; + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrapTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrapTest.java new file mode 100644 index 0000000..cfc4cd0 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrapTest.java @@ -0,0 +1,44 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayDelivery; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.UNAVAILABLE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class DisplayDeliveryBootstrapTest { + + private static final DisplayRequest REQUEST = DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", new DisplayData("x")); + + @Test + void noSocketPropertyFailsEveryDeliveryUnavailable() { + DisplayDelivery unavailable = DisplayDeliveryBootstrap.fromSocketPath(null); + DisplayDeliveryException e = assertThrows(DisplayDeliveryException.class, () -> unavailable.deliver(REQUEST)); + assertEquals(UNAVAILABLE, e.kind()); + } + + @Test + void socketPropertySelectsUdsDelivery() { + assertInstanceOf(UnixDomainDisplayDelivery.class, DisplayDeliveryBootstrap.fromSocketPath("/tmp/x.sock")); + } + + @Test + void installReturnsPreviousAndIsUsedForSubsequentCalls() { + DisplayDelivery previous = DisplayDeliveryBootstrap.current(); + List seen = new ArrayList<>(); + try { + assertSame(previous, DisplayDeliveryBootstrap.install(seen::add)); + DisplayDeliveryBootstrap.current().deliver(REQUEST); + assertEquals(1, seen.size()); + } finally { + DisplayDeliveryBootstrap.install(previous); + } + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryContractTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryContractTest.java new file mode 100644 index 0000000..e36cbcb --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryContractTest.java @@ -0,0 +1,100 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.REJECTED; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Contract every delivery must satisfy, shared by the transports and the in-memory substitute. Transport-specific + * behaviour (unavailable endpoint, stalled peer) is tested in the transport's own test classes. + */ +abstract class DisplayDeliveryContractTest { + + private static final Gson GSON = new GsonBuilder().serializeNulls().create(); + + protected abstract DisplayDelivery delivery(DisplayRequestHandler handler); + + @Test + void bundleSurvivesDelivery() { + List received = new ArrayList<>(); + DisplayData sent = bundle(); + + delivery(received::add).deliver(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id-1", sent)); + + DisplayRequest request = received.get(0); + assertEquals("id-1", request.displayId()); + DisplayData got = request.toDisplayData(); + assertEquals(GSON.toJsonTree(sent.getData()), GSON.toJsonTree(got.getData())); + assertEquals(GSON.toJsonTree(sent.getMetadata()), GSON.toJsonTree(got.getMetadata())); + assertEquals("id-1", got.getDisplayId()); + } + + @Test + void updateCarriesTargetId() { + List received = new ArrayList<>(); + DisplayData data = new DisplayData("updated"); + data.putTransientData("display_id", "other"); + + delivery(received::add).deliver(DisplayRequest.of(DisplayRequest.Operation.UPDATE, "target", data)); + + DisplayRequest request = received.get(0); + assertEquals(DisplayRequest.Operation.UPDATE, request.operation()); + assertEquals("target", request.displayId()); + assertEquals("target", request.toDisplayData().getDisplayId()); + } + + @Test + void sequentialOperationsKeepOrder() { + List ids = new ArrayList<>(); + DisplayDelivery delivery = delivery(r -> ids.add(r.displayId())); + + List expected = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + String id = String.valueOf(i); + expected.add(id); + delivery.deliver(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, id, new DisplayData(id))); + } + assertEquals(expected, ids); + } + + @Test + void handlerFailureIsRejectedWithItsMessage() { + DisplayDelivery delivery = delivery(r -> { + throw new IllegalStateException("boom"); + }); + DisplayRequest request = DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "x", new DisplayData("t")); + + DisplayDeliveryException e = assertThrows( + DisplayDeliveryException.class, + () -> delivery.deliver(request)); + assertEquals(REJECTED, e.kind()); + assertTrue(e.getMessage().contains("boom")); + } + + private static DisplayData bundle() { + DisplayData data = new DisplayData("portable"); + data.putHTML("portable"); + data.putData("image/png", "AQID"); + + Map json = new LinkedHashMap<>(); + json.put("values", Arrays.asList("kept", null)); + json.put("nothing", null); + data.putData("application/json", json); + + data.putMetaData("render", Map.of("expanded", "yes")); + data.putTransientData("custom", "preserved"); + return data; + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java new file mode 100644 index 0000000..b6e186e --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java @@ -0,0 +1,38 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.junit.jupiter.api.Test; + +import java.util.Base64; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +class DisplayEncodingTest { + + @Test + void multilineNonAsciiAndPngSurviveRoundTripUnchanged() { + String multiline = "line one\nline two\r\nend"; + String nonAscii = "héllo — 日本語 🎉"; + String png = Base64.getEncoder().encodeToString(new byte[]{(byte) 0x89, 'P', 'N', 'G', 0x0A, 0x1A, 0x0A}); + + DisplayData data = new DisplayData(multiline); + data.putData("text/x-non-ascii", nonAscii); + data.putData("image/png", png); + + String line = DisplayProtocol.encode(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + DisplayData decoded = DisplayProtocol.decode(line).toDisplayData(); + + assertEquals(multiline, decoded.getData().get("text/plain")); + assertEquals(nonAscii, decoded.getData().get("text/x-non-ascii")); + assertEquals(png, decoded.getData().get("image/png")); + } + + @Test + void encodedRequestNeverContainsRawLineFeed() { + DisplayData data = new DisplayData("a\nb\nc"); + String line = DisplayProtocol.encode(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + + assertFalse(line.contains("\n"), line); + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocolTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocolTest.java new file mode 100644 index 0000000..a84efc1 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocolTest.java @@ -0,0 +1,87 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DisplayProtocolTest { + + @Test + void roundTripsDisplayAndUpdate() { + DisplayData data = new DisplayData("hello"); + data.putHTML("hi"); + + DisplayRequest display = DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id-1", data); + DisplayRequest decodedDisplay = DisplayProtocol.decode(DisplayProtocol.encode(display)); + assertEquals(DisplayRequest.Operation.DISPLAY, decodedDisplay.operation()); + assertEquals("id-1", decodedDisplay.displayId()); + assertEquals("hi", decodedDisplay.toDisplayData().getData().get("text/html")); + + DisplayRequest update = DisplayRequest.of(DisplayRequest.Operation.UPDATE, "id-1", data); + DisplayRequest decodedUpdate = DisplayProtocol.decode(DisplayProtocol.encode(update)); + assertEquals(DisplayRequest.Operation.UPDATE, decodedUpdate.operation()); + assertEquals("id-1", decodedUpdate.displayId()); + } + + @Test + void rejectsCyclicStructure() { + List cycle = new ArrayList<>(); + cycle.add(cycle); + DisplayData data = new DisplayData(); + data.putData("application/json", cycle); + + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + assertTrue(e.getMessage().contains("cyclic structure")); + } + + @Test + void rejectsNonFiniteNumbers() { + for (double value : new double[]{Double.NaN, Double.POSITIVE_INFINITY}) { + DisplayData data = new DisplayData(); + data.putData("application/json", value); + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + assertTrue(e.getMessage().contains("non-finite number")); + } + } + + @Test + void rejectsNonStringMapKeys() { + Map map = new LinkedHashMap<>(); + map.put(1, "bad"); + DisplayData data = new DisplayData(); + data.putData("application/json", map); + + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + assertTrue(e.getMessage().contains("Non-portable display value")); + } + + @Test + void rejectsUpdateWithoutId() { + DisplayData data = new DisplayData("x"); + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> DisplayRequest.of(DisplayRequest.Operation.UPDATE, null, data)); + assertEquals("updateDisplay requires a non-null id", e.getMessage()); + } + + @Test + void rejectsUnknownActionAndMalformedJson() { + assertThrows(IllegalArgumentException.class, + () -> DisplayProtocol.decode("{\"action\":\"explode\",\"bundle\":{}}")); + assertThrows(IllegalArgumentException.class, () -> DisplayProtocol.decode("not json")); + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/InMemoryDisplayDeliveryContractTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/InMemoryDisplayDeliveryContractTest.java new file mode 100644 index 0000000..e053a7e --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/InMemoryDisplayDeliveryContractTest.java @@ -0,0 +1,23 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +/** + * Runs the delivery contract against an in-memory substitute that round-trips every request and response through the + * protocol text, exactly as a real transport would. + */ +class InMemoryDisplayDeliveryContractTest extends DisplayDeliveryContractTest { + + @Override + protected DisplayDelivery delivery(DisplayRequestHandler handler) { + return request -> { + DisplayRequest received = DisplayProtocol.decode(DisplayProtocol.encode(request)); + String response; + try { + handler.handle(received); + response = DisplayProtocol.response(null); + } catch (RuntimeException e) { + response = DisplayProtocol.response(e); + } + DisplayProtocol.checkResponse(response); + }; + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryContractTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryContractTest.java new file mode 100644 index 0000000..ae3b019 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryContractTest.java @@ -0,0 +1,33 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayDelivery; +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint; +import org.junit.jupiter.api.AfterEach; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; + +class UnixDomainDisplayDeliveryContractTest extends DisplayDeliveryContractTest { + + private final List endpoints = new ArrayList<>(); + + @Override + protected DisplayDelivery delivery(DisplayRequestHandler handler) { + try { + UnixDomainDisplayEndpoint endpoint = new UnixDomainDisplayEndpoint(handler); + endpoints.add(endpoint); + return new UnixDomainDisplayDelivery(endpoint.path()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @AfterEach + void closeEndpoints() throws IOException { + for (UnixDomainDisplayEndpoint endpoint : endpoints) { + endpoint.close(); + } + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java new file mode 100644 index 0000000..80f3440 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java @@ -0,0 +1,99 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayDelivery; +import org.junit.jupiter.api.Test; + +import java.net.StandardProtocolFamily; +import java.net.UnixDomainSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.UNAVAILABLE; +import static org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException.Kind.UNCERTAIN; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Production bounds: a single 10 s RESPONSE deadline, judged with a 5 s scheduling tolerance (15 s). + */ +class UnixDomainDisplayDeliveryTimeoutTest { + + private static final long BOUND_MILLIS = 15_000; + + @Test + void missingSocketFailsUnavailableWithinConnectBound() { + Path missing = Path.of(System.getProperty("java.io.tmpdir"), "jjava-missing-" + System.nanoTime() + ".sock"); + UnixDomainDisplayDelivery delivery = new UnixDomainDisplayDelivery(missing); + DisplayRequest request = request(new DisplayData("x")); + long start = System.nanoTime(); + + DisplayDeliveryException e = assertThrows(DisplayDeliveryException.class, () -> delivery.deliver(request)); + + assertEquals(UNAVAILABLE, e.kind()); + assertTrue(elapsedMillis(start) < BOUND_MILLIS); + } + + @Test + void peerThatNeverReadsLargeRequestFailsUncertainWithinBound() throws Exception { + Path socket = Files.createTempDirectory("uds-timeout").resolve("s.sock"); + try (ServerSocketChannel server = listen(socket)) { + // Never accepts: connect completes through the listen backlog, then the large write stalls. + UnixDomainDisplayDelivery delivery = new UnixDomainDisplayDelivery(socket); + DisplayRequest request = request(new DisplayData("x".repeat(8 << 20))); + long start = System.nanoTime(); + + DisplayDeliveryException e = assertThrows(DisplayDeliveryException.class, () -> delivery.deliver(request)); + + assertEquals(UNCERTAIN, e.kind()); + assertTrue(elapsedMillis(start) < BOUND_MILLIS); + } + } + + @Test + void peerThatReadsButNeverRepliesFailsUncertainWithinBound() throws Exception { + Path socket = Files.createTempDirectory("uds-timeout").resolve("s.sock"); + try (ServerSocketChannel server = listen(socket)) { + Thread peer = new Thread(() -> holdAfterRead(server)); + peer.setDaemon(true); + peer.start(); + + UnixDomainDisplayDelivery delivery = new UnixDomainDisplayDelivery(socket); + DisplayRequest request = request(new DisplayData("small")); + long start = System.nanoTime(); + + DisplayDeliveryException e = assertThrows(DisplayDeliveryException.class, () -> delivery.deliver(request)); + + assertEquals(UNCERTAIN, e.kind()); + assertTrue(elapsedMillis(start) < BOUND_MILLIS); + } + } + + // Reads the request, then blocks until the client closes its end at the deadline (read returns -1). + private static void holdAfterRead(ServerSocketChannel server) { + try (SocketChannel accepted = server.accept()) { + accepted.read(ByteBuffer.allocate(1 << 20)); + accepted.read(ByteBuffer.allocate(1)); + } catch (Exception ignored) { + // the peer thread only exists to stall the client; its own failure has no observer + } + } + + private static ServerSocketChannel listen(Path socket) throws Exception { + ServerSocketChannel server = ServerSocketChannel.open(StandardProtocolFamily.UNIX); + server.bind(UnixDomainSocketAddress.of(socket)); + return server; + } + + private static DisplayRequest request(DisplayData data) { + return DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data); + } + + private static long elapsedMillis(long startNanos) { + return (System.nanoTime() - startNanos) / 1_000_000; + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayEndpointTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayEndpointTest.java new file mode 100644 index 0000000..7a8ba1d --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayEndpointTest.java @@ -0,0 +1,59 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.dflib.jjava.jupyter.kernel.display.DisplayData; +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.StandardProtocolFamily; +import java.net.UnixDomainSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class UnixDomainDisplayEndpointTest { + + @Test + void servesOneRequestLinePerConnectionInOrderAndRemovesSocketOnShutdown() throws Exception { + List ids = Collections.synchronizedList(new ArrayList<>()); + UnixDomainDisplayEndpoint endpoint = new UnixDomainDisplayEndpoint(r -> ids.add(r.displayId())); + Path socket = endpoint.path(); + try { + assertEquals("ok", send(socket, line("first"))); + assertEquals("ok", send(socket, line("second"))); + assertEquals(List.of("first", "second"), ids); + assertTrue(send(socket, "not json").startsWith("\"")); + } finally { + endpoint.close(); + } + assertFalse(Files.exists(socket)); + } + + private static String line(String id) { + return DisplayProtocol.encode(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, id, new DisplayData("x"))); + } + + private static String send(Path socket, String line) throws IOException { + try (SocketChannel channel = SocketChannel.open(StandardProtocolFamily.UNIX)) { + channel.connect(UnixDomainSocketAddress.of(socket)); + channel.write(ByteBuffer.wrap((line + "\n").getBytes(StandardCharsets.UTF_8))); + + ByteArrayOutputStream response = new ByteArrayOutputStream(); + ByteBuffer one = ByteBuffer.allocate(1); + while (channel.read(one.clear()) > 0 && one.get(0) != '\n') { + response.write(one.get(0)); + } + return response.toString(StandardCharsets.UTF_8); + } + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java new file mode 100644 index 0000000..4dbae02 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java @@ -0,0 +1,131 @@ +package org.dflib.jjava.jupyter.kernel.display.protocol; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.StandardProtocolFamily; +import java.net.UnixDomainSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Gate test (tasks T008): decides whether Unix-domain sockets work on this OS/JDK. Writes one line per check to + * target/uds-gate/{os}-jdk{N}.txt. The gate passes only if every check passes. + */ +public class UnixDomainSocketPathAcceptanceTest { + + private static void readFully(SocketChannel ch, ByteBuffer buf) throws IOException { + while (buf.hasRemaining()) { + if (ch.read(buf) < 0) { + break; + } + } + } + + @Test + void unixDomainSocketGate() throws Exception { + String os = System.getProperty("os.name").toLowerCase().replace(' ', '-'); + int jdk = Runtime.version().feature(); + List report = new ArrayList<>(); + report.add("os=" + System.getProperty("os.name") + " jdk=" + jdk + + " java.io.tmpdir=" + System.getProperty("java.io.tmpdir") + + " jdk.net.unixdomain.tmpdir=" + System.getProperty("jdk.net.unixdomain.tmpdir")); + boolean ok = true; + try { + Path standard = Files.createTempDirectory("uds-gate"); + Path spaced = Files.createTempDirectory("uds gate spaces "); + Path lengthened = nestedDir(Files.createTempDirectory("uds-gate-len")); + ok &= check(report, "standard-temp", standard.resolve("s.sock")); + ok &= check(report, "spaces-in-path", spaced.resolve("s.sock")); + ok &= check(report, "lengthened-path", lengthened.resolve("s.sock")); + ok &= permissionCheck(report, standard); + ok &= acceptFailureCleanup(report, standard.resolve("f.sock")); + report.add("short-private-dir=" + (System.getProperty("jdk.net.unixdomain.tmpdir") == null + ? "not-configured" : "configured")); + } catch (Throwable t) { + ok = false; + report.add("error=" + t); + throw t; + } finally { + report.add("gate=" + (ok ? "PASS" : "FAIL")); + Path out = Path.of("target", "uds-gate", os + "-jdk" + jdk + ".txt"); + Files.createDirectories(out.getParent()); + Files.write(out, report); + } + assertTrue(ok, String.join("\n", report)); + } + + // one extra segment: lengthens the standard temp path but stays under the ~108-byte AF_UNIX limit + private static Path nestedDir(Path base) throws Exception { + return Files.createDirectories(base.resolve("uds-gate-length-check")); + } + + private static boolean check(List report, String label, Path socket) throws Exception { + int bytes = socket.toString().getBytes(StandardCharsets.UTF_8).length; + try { + try (ServerSocketChannel server = ServerSocketChannel.open(StandardProtocolFamily.UNIX)) { + server.bind(UnixDomainSocketAddress.of(socket)); + try (SocketChannel client = SocketChannel.open(StandardProtocolFamily.UNIX)) { + client.connect(UnixDomainSocketAddress.of(socket)); + try (SocketChannel accepted = server.accept()) { + client.write(StandardCharsets.UTF_8.encode("ping\n")); + ByteBuffer buf = ByteBuffer.allocate(5); + readFully(accepted, buf); + assertEquals("ping\n", StandardCharsets.UTF_8.decode((ByteBuffer) buf.flip()).toString()); + } + } + } + boolean removed = !Files.exists(socket) || Files.deleteIfExists(socket); + report.add(label + "=" + (removed ? "PASS" : "FAIL") + " pathBytes=" + bytes + " socket=" + socket); + return removed; + } catch (Exception e) { + report.add(label + "=FAIL pathBytes=" + bytes + " error=" + e); + Files.deleteIfExists(socket); + return false; + } + } + + private static boolean permissionCheck(List report, Path dir) throws Exception { + boolean posix = Files.getFileStore(dir).supportsFileAttributeView("posix"); + report.add("permissions=" + (posix ? Files.getPosixFilePermissions(dir) : "not-posix")); + return true; + } + + private static boolean acceptFailureCleanup(List report, Path socket) throws Exception { + ServerSocketChannel server = ServerSocketChannel.open(StandardProtocolFamily.UNIX); + server.bind(UnixDomainSocketAddress.of(socket)); + ExecutorService exec = Executors.newSingleThreadExecutor(); + try { + Future accept = exec.submit(server::accept); + server.close(); + try { + accept.get(5, TimeUnit.SECONDS); + fail("accept returned after close"); + } catch (ExecutionException expected) { + // accept failed after close, as required + } + boolean removed = !Files.exists(socket) || Files.deleteIfExists(socket); + assertFalse(Files.exists(socket)); + report.add("accept-failure-cleanup=" + (removed ? "PASS" : "FAIL")); + return removed; + } finally { + exec.shutdownNow(); + } + } +} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspilerTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspilerTest.java new file mode 100644 index 0000000..dba2bc4 --- /dev/null +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspilerTest.java @@ -0,0 +1,26 @@ +package org.dflib.jjava.jupyter.kernel.magic; + +import org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class MagicTranspilerTest { + + private static final String EXECUTOR = ExecutorNotebookStatics.class.getName(); + + @Test + void generatedCellAndLineCallsTargetExecutorHelpers() { + MagicTranspiler transpiler = new MagicTranspiler(); + + String cell = transpiler.transpileCell(new ParsedCellMagic("m", List.of(), "body")); + String line = transpiler.transpileLine(new ParsedLineMagic("m", List.of(), "x = ", "%m")); + + assertTrue(cell.startsWith(EXECUTOR + ".cellMagic("), cell); + assertTrue(line.startsWith(EXECUTOR + ".lineMagic("), line); + assertFalse(cell.contains("BaseNotebookStatics") || line.contains("BaseNotebookStatics")); + } +} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/ExecutorJavaNotebookStatics.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/ExecutorJavaNotebookStatics.java new file mode 100644 index 0000000..9b25b35 --- /dev/null +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/ExecutorJavaNotebookStatics.java @@ -0,0 +1,22 @@ +package org.dflib.jjava.kernel; + +/** + * Kernel access for executor (user cell) code. Always fails; the managing JVM's kernel is never exposed to user code. + */ +public class ExecutorJavaNotebookStatics { + + private ExecutorJavaNotebookStatics() { + } + + /** + * @deprecated in favor of {@link #kernel()} + */ + @Deprecated(since = "1.0", forRemoval = true) + public static JavaKernel getKernelInstance() { + return kernel(); + } + + public static JavaKernel kernel() { + throw new UnsupportedOperationException("JavaKernel.kernel() cannot expose the managing JVM from executor code"); + } +} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java index 2108095..80f6ec0 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java @@ -28,13 +28,17 @@ import org.dflib.jjava.jupyter.kernel.util.StringStyler; import org.dflib.jjava.kernel.execution.CodeEvaluator; import org.dflib.jjava.kernel.execution.CompilationException; -import org.dflib.jjava.kernel.execution.DisplayCallbackServer; import org.dflib.jjava.kernel.execution.EvaluationInterruptedException; import org.dflib.jjava.kernel.execution.ExecutorTerminationException; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequestHandler; +import org.dflib.jjava.jupyter.kernel.display.protocol.uds.UnixDomainDisplayEndpoint; +import org.dflib.jjava.kernel.execution.EvaluationResult; import org.dflib.jjava.kernel.execution.EvaluationTimeoutException; import org.dflib.jjava.kernel.execution.IncompleteSourceException; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -68,7 +72,7 @@ public static Builder builder() { private final JShell jShell; private final CodeEvaluator evaluator; private volatile boolean shuttingDown; - private DisplayCallbackServer displayCallbackServer; + private UnixDomainDisplayEndpoint displayEndpoint; protected JavaKernel( String name, @@ -264,6 +268,21 @@ protected Object doEval(String source) { return evaluator.eval(jShell, source); } + /** + * Maps an evaluation or magic result to its Jupyter presentation: absent evaluation text produces no output, + * evaluation text becomes text/plain only, and any other magic value is rendered as text. + */ + static DisplayData toExecuteResult(Object value) { + if (value instanceof EvaluationResult result) { + String text = result.text(); + return text == null ? null : new DisplayData(text); + } + if (value == null || value instanceof DisplayData) { + return (DisplayData) value; + } + return new DisplayData(String.valueOf(value)); + } + private static final class JavaEvalBuilder implements EvalBuilder { private final JavaKernel kernel; private final String source; @@ -312,15 +331,13 @@ public EvalBuilder renderResults() { } @Override + @SuppressWarnings("unchecked") public T eval() { Object value = evalRaw(); if (!renderResults) { - return (T) value; + return (T) (value instanceof EvaluationResult result ? result.text() : value); } - if (value == null) { - return null; - } - return (T) (value instanceof DisplayData ? value : new DisplayData(String.valueOf(value))); + return (T) toExecuteResult(value); } private Object evalRaw() { @@ -456,7 +473,7 @@ public void onShutdown(boolean isRestarting) { jShell.close(); } finally { try { - if (displayCallbackServer != null) displayCallbackServer.close(); + closeDisplayEndpoint(); } finally { evaluator.close(); } @@ -464,6 +481,17 @@ public void onShutdown(boolean isRestarting) { } } + private void closeDisplayEndpoint() { + if (displayEndpoint == null) { + return; + } + try { + displayEndpoint.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + @Override public void interrupt() { evaluator.interrupt(jShell); @@ -484,23 +512,20 @@ public static class Builder extends JavaKernelBuilder { private Builder() { } - @SuppressWarnings("resource") @Override public JavaKernel build() { - DisplayCallbackServer displayCallbackServer; - try { - displayCallbackServer = new DisplayCallbackServer(); - } catch (IOException e) { - throw new IllegalStateException("Unable to start display callback channel", e); - } - + JavaKernel[] kernelRef = new JavaKernel[1]; + UnixDomainDisplayEndpoint endpoint = null; JShell jShell = null; CodeEvaluator evaluator = null; try { + endpoint = openDisplayEndpoint(kernelRef); + afterDisplayEndpointCreated.run(); + String name = buildName(); JupyterIO io = buildJupyterIO(buildJupyterIOEncoding()); evaluator = buildCodeEvaluator(); - jShell = buildJShell(io, List.of("-Djjava.display.port=" + displayCallbackServer.getPort())); + jShell = buildJShell(io, List.of("-Djjava.display.socket=" + endpoint.path())); MagicTranspiler magicTranspiler = buildMagicTranspiler(); JavaKernel kernel = new JavaKernel( @@ -519,15 +544,34 @@ public JavaKernel build() { jShell, evaluator ); - kernel.displayCallbackServer = displayCallbackServer; - displayCallbackServer.setKernel(kernel); + kernel.displayEndpoint = endpoint; + kernelRef[0] = kernel; return kernel; } catch (RuntimeException | Error e) { - closeAfterFailure(e, jShell, displayCallbackServer, evaluator); + closeAfterFailure(e, jShell, endpoint, evaluator); throw e; } } + private UnixDomainDisplayEndpoint openDisplayEndpoint(JavaKernel[] kernelRef) { + try { + DisplayRequestHandler handler = displayHandler != null + ? displayHandler + : request -> applyDisplay(kernelRef[0], request); + return new UnixDomainDisplayEndpoint(displayEndpointPath, handler); + } catch (IOException e) { + throw new IllegalStateException("Unable to start display channel", e); + } + } + + private static void applyDisplay(JavaKernel kernel, DisplayRequest request) { + if (request.operation() == DisplayRequest.Operation.UPDATE) { + kernel.getIO().display.updateDisplay(request.displayId(), request.toDisplayData()); + } else { + kernel.display(request.toDisplayData()); + } + } + private static void closeAfterFailure(Throwable failure, AutoCloseable... resources) { for (AutoCloseable resource : resources) { if (resource == null) { @@ -535,7 +579,7 @@ private static void closeAfterFailure(Throwable failure, AutoCloseable... resour } try { resource.close(); - } catch (Throwable cleanup) { + } catch (Exception cleanup) { if (cleanup != failure) failure.addSuppressed(cleanup); } } diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java index bab3786..819f08c 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java @@ -3,12 +3,14 @@ import jdk.jshell.JShell; import jdk.jshell.execution.JdiExecutionControlProvider; import org.dflib.jjava.jupyter.kernel.BaseKernelBuilder; +import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequestHandler; import org.dflib.jjava.jupyter.kernel.JupyterIO; import org.dflib.jjava.jupyter.kernel.LanguageInfo; import org.dflib.jjava.jupyter.kernel.magic.MagicTranspiler; import org.dflib.jjava.jupyter.kernel.magic.MagicsResolver; import org.dflib.jjava.kernel.execution.CodeEvaluator; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -18,6 +20,7 @@ /** * A common builder superclass for JJavaKernel and subclasses. */ +@SuppressWarnings("unchecked") public abstract class JavaKernelBuilder< B extends JavaKernelBuilder, K extends JavaKernel> extends BaseKernelBuilder { @@ -27,6 +30,12 @@ public abstract class JavaKernelBuilder< protected final List compilerOpts; protected final List remoteVMOptions; + // test seams: override the display endpoint address, and observe or fail the build after the endpoint exists + Path displayEndpointPath; + Runnable afterDisplayEndpointCreated = () -> { + }; + DisplayRequestHandler displayHandler; + protected JavaKernelBuilder() { this.compilerOpts = new ArrayList<>(); this.remoteVMOptions = new ArrayList<>(List.of("-classpath", System.getProperty("java.class.path"))); @@ -42,6 +51,21 @@ public B remoteVMOptions(Iterable opts) { return (B) this; } + B displayEndpointPath(Path path) { + this.displayEndpointPath = path; + return (B) this; + } + + B afterDisplayEndpointCreated(Runnable hook) { + this.afterDisplayEndpointCreated = hook; + return (B) this; + } + + B displayHandler(DisplayRequestHandler handler) { + this.displayHandler = handler; + return (B) this; + } + public B timeout(long timeoutDuration, TimeUnit timeoutUnit) { this.timeoutDuration = timeoutDuration; this.timeoutUnit = Objects.requireNonNull(timeoutUnit); diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaNotebookStatics.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaNotebookStatics.java index efd8038..9459014 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaNotebookStatics.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaNotebookStatics.java @@ -19,10 +19,6 @@ public static JavaKernel getKernelInstance() { } public static JavaKernel kernel() { - try { - return (JavaKernel) BaseKernel.notebookKernel(); - } catch (NullPointerException e) { - throw new UnsupportedOperationException("JavaKernel.kernel() cannot expose the managing JVM from executor code", e); - } + return (JavaKernel) BaseKernel.notebookKernel(); } } diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java index 95c7fed..34db118 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java @@ -42,14 +42,14 @@ public CodeEvaluator(long timeoutDuration, TimeUnit timeoutUnit) { }); } - public Object eval(JShell shell, String code) { + public EvaluationResult eval(JShell shell, String code) { checkExecutorFailure(); interrupted.set(false); long timeoutNanos = timeoutDuration > 0 ? timeoutUnit.toNanos(timeoutDuration) : -1; long started = System.nanoTime(); SourceCodeAnalysis sca = shell.sourceCodeAnalysis(); - Object lastResult = null; + EvaluationResult lastResult = EvaluationResult.NONE; SourceCodeAnalysis.CompletionInfo info = sca.analyzeCompletion(code); while (info.completeness().isComplete()) { @@ -65,9 +65,9 @@ public Object eval(JShell shell, String code) { return lastResult; } - protected Object evalSingle(JShell shell, String code, long timeoutNanos) { + protected EvaluationResult evalSingle(JShell shell, String code, long timeoutNanos) { List events = evaluate(shell, code, timeoutNanos); - Object result = null; + String result = null; for (SnippetEvent event : events) { if (event.causeSnippet() != null) { @@ -109,7 +109,7 @@ protected Object evalSingle(JShell shell, String code, long timeoutNanos) { } } - return result; + return new EvaluationResult(result); } private List evaluate(JShell shell, String code, long timeoutNanos) { diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/DisplayCallbackServer.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/DisplayCallbackServer.java deleted file mode 100644 index 792d326..0000000 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/DisplayCallbackServer.java +++ /dev/null @@ -1,126 +0,0 @@ -package org.dflib.jjava.kernel.execution; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import org.dflib.jjava.jupyter.kernel.display.DisplayData; -import org.dflib.jjava.kernel.JavaKernel; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.InetAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.charset.StandardCharsets; - -public class DisplayCallbackServer implements Closeable { - - private static final Logger LOGGER = LoggerFactory.getLogger(DisplayCallbackServer.class); - private static final Gson GSON = new GsonBuilder().serializeNulls().create(); - private final ServerSocket serverSocket; - private volatile JavaKernel kernel; - - public DisplayCallbackServer() throws IOException { - this.serverSocket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()); - Thread listenerThread = new Thread(this::listen, "jjava-display-callback"); - listenerThread.setDaemon(true); - listenerThread.start(); - } - - public void setKernel(JavaKernel kernel) { - this.kernel = kernel; - } - - public int getPort() { - return serverSocket.getLocalPort(); - } - - private void listen() { - while (!serverSocket.isClosed()) { - try { - handleConnection(serverSocket.accept()); - } catch (IOException e) { - if (!serverSocket.isClosed()) { - LOGGER.warn("Display callback server accept failed", e); - } - } - } - } - - private void handleConnection(Socket socket) { - try (socket; - BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8)); - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8))) { - - socket.setSoTimeout(10_000); - String response; - try { - dispatch(reader.readLine()); - response = "ok"; - } catch (Exception e) { - String message = e.getMessage(); - response = GSON.toJson(message == null ? e.getClass().getSimpleName() : message); - } - writer.write(response); - writer.newLine(); - writer.flush(); - } catch (IOException e) { - if (!serverSocket.isClosed()) { - LOGGER.warn("Display callback connection failed", e); - } - } - } - - private void dispatch(String line) { - JsonObject json = GSON.fromJson(line, JsonObject.class); - if (json == null) { - throw new IllegalArgumentException("Display callback request must be a JSON object"); - } - - JsonElement actionElement = json.get("action"); - if (actionElement == null || !actionElement.isJsonPrimitive() - || !actionElement.getAsJsonPrimitive().isString()) { - throw new IllegalArgumentException("Display callback action must be display or updateDisplay"); - } - String action = actionElement.getAsString(); - if (!"display".equals(action) && !"updateDisplay".equals(action)) { - throw new IllegalArgumentException("Unsupported display callback action: " + action); - } - - String id = json.has("id") && !json.get("id").isJsonNull() ? json.get("id").getAsString() : null; - if ("updateDisplay".equals(action) && id == null) { - throw new IllegalArgumentException("updateDisplay requires a non-null id"); - } - - JsonElement bundleElement = json.get("bundle"); - if (bundleElement == null || bundleElement.isJsonNull()) { - throw new IllegalArgumentException("Display callback bundle must not be null"); - } - DisplayData data = GSON.fromJson(bundleElement, DisplayData.class); - - JavaKernel currentKernel = kernel; - if (currentKernel == null) { - throw new IllegalStateException("Display callback kernel is not available"); - } - if ("updateDisplay".equals(action)) { - currentKernel.getIO().display.updateDisplay(id, data); - } else { - currentKernel.display(data); - } - } - - @Override - public void close() { - try { - serverSocket.close(); - } catch (IOException ignored) { - } - } -} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/EvaluationResult.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/EvaluationResult.java new file mode 100644 index 0000000..e41c181 --- /dev/null +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/EvaluationResult.java @@ -0,0 +1,10 @@ +package org.dflib.jjava.kernel.execution; + +/** + * The ordinary evaluation outcome of a cell: the JShell text of its last value or expression. A null text means the + * result is absent. It carries no Jupyter display, message, channel, or protocol types. + */ +public record EvaluationResult(String text) { + + public static final EvaluationResult NONE = new EvaluationResult(null); +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/jupyter/kernel/CapturingShellEnvironment.java b/jjava-kernel/src/test/java/org/dflib/jjava/jupyter/kernel/CapturingShellEnvironment.java new file mode 100644 index 0000000..9921e92 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/jupyter/kernel/CapturingShellEnvironment.java @@ -0,0 +1,32 @@ +package org.dflib.jjava.jupyter.kernel; + +import org.dflib.jjava.jupyter.channels.ShellReplyEnvironment; +import org.dflib.jjava.jupyter.messages.Message; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Test helper: records every message a kernel publishes. Lives in the kernel package so it can reach the protected + * {@code setEnv} of {@link JupyterIO}. + */ +public class CapturingShellEnvironment extends ShellReplyEnvironment { + + public final List> published = Collections.synchronizedList(new ArrayList<>()); + + private CapturingShellEnvironment() { + super(null, null, null, null); + } + + public static CapturingShellEnvironment attach(JupyterIO io) { + CapturingShellEnvironment env = new CapturingShellEnvironment(); + io.setEnv(env); + return env; + } + + @Override + public void publish(Message message) { + published.add(message); + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/DisplayCallbackServerTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/DisplayCallbackServerTest.java deleted file mode 100644 index 3a987b4..0000000 --- a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/DisplayCallbackServerTest.java +++ /dev/null @@ -1,177 +0,0 @@ -package org.dflib.jjava.kernel; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonObject; -import jdk.jshell.JShell; -import org.dflib.jjava.jupyter.channels.ShellReplyEnvironment; -import org.dflib.jjava.jupyter.kernel.JupyterIO; -import org.dflib.jjava.jupyter.kernel.LanguageInfo; -import org.dflib.jjava.jupyter.kernel.comm.CommManager; -import org.dflib.jjava.jupyter.kernel.display.DisplayData; -import org.dflib.jjava.jupyter.kernel.display.Renderer; -import org.dflib.jjava.jupyter.kernel.magic.MagicTranspiler; -import org.dflib.jjava.jupyter.kernel.magic.MagicsRegistry; -import org.dflib.jjava.jupyter.kernel.magic.MagicsResolver; -import org.dflib.jjava.jupyter.kernel.util.StringStyler; -import org.dflib.jjava.jupyter.messages.Message; -import org.dflib.jjava.jupyter.messages.publish.PublishUpdateDisplayData; -import org.dflib.jjava.kernel.execution.CodeEvaluator; -import org.dflib.jjava.kernel.execution.DisplayCallbackServer; -import org.junit.jupiter.api.Test; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class DisplayCallbackServerTest { - - private static final Gson GSON = new GsonBuilder().serializeNulls().create(); - - @Test - public void preservesStructuredBundleAndContinuesAfterMalformedRequest() throws Exception { - try (CallbackKernel kernel = new CallbackKernel(); - DisplayCallbackServer server = new DisplayCallbackServer()) { - server.setKernel(kernel); - - String error = GSON.fromJson(request(server, "not json"), String.class); - assertNotNull(error); - assertFalse(error.isBlank()); - assertNotEquals("ok", error); - - String request = displayRequest(); - JsonObject expectedBundle = GSON.fromJson(request, JsonObject.class).getAsJsonObject("bundle"); - assertEquals("ok", request(server, request)); - assertEquals(expectedBundle, GSON.toJsonTree(kernel.displayed)); - } - } - - @Test - public void returnsDispatchFailuresAndRoutesUpdatesThroughDisplayStream() throws Exception { - CapturingEnvironment env = new CapturingEnvironment(); - try (CallbackKernel kernel = new CallbackKernel(new CapturingIO(env)); - DisplayCallbackServer server = new DisplayCallbackServer()) { - server.setKernel(kernel); - kernel.displayFailure = new IllegalStateException("display failed\nnow"); - - assertEquals("\"display failed\\nnow\"", request(server, displayRequest())); - assertNull(kernel.displayed); - - String update = "{\"action\":\"updateDisplay\",\"id\":\"existing\"," - + "\"bundle\":{\"data\":{\"text/plain\":\"updated\"},\"metadata\":{},\"transient\":{}}}"; - assertEquals("ok", request(server, update)); - assertEquals(1, env.published.size()); - PublishUpdateDisplayData published = assertInstanceOf( - PublishUpdateDisplayData.class, - env.published.get(0).getContent()); - assertEquals("existing", published.getDisplayId()); - assertEquals("updated", published.getData().get("text/plain")); - } - } - - @Test - public void rejectsUpdatesWithoutIdsInsteadOfDisplayingThem() throws Exception { - try (CallbackKernel kernel = new CallbackKernel(); - DisplayCallbackServer server = new DisplayCallbackServer()) { - server.setKernel(kernel); - String response = request(server, - "{\"action\":\"updateDisplay\",\"bundle\":{\"data\":{},\"metadata\":{}}}"); - - assertTrue(GSON.fromJson(response, String.class).contains("requires a non-null id")); - assertNull(kernel.displayed); - } - } - - private String displayRequest() { - return "{\"action\":\"display\",\"bundle\":{" - + "\"data\":{\"application/json\":{\"values\":[1,true,null]},\"text/empty\":null}," - + "\"metadata\":{\"render\":{\"nested\":false}}," - + "\"transient\":{\"custom\":{\"values\":[\"kept\",null]}}}}"; - } - - private String request(DisplayCallbackServer server, String request) throws Exception { - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(InetAddress.getLoopbackAddress(), server.getPort()), 10_000); - socket.setSoTimeout(10_000); - try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8)); - BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) { - writer.write(request); - writer.newLine(); - writer.flush(); - return reader.readLine(); - } - } - } - - private static class CapturingIO extends JupyterIO { - CapturingIO(CapturingEnvironment env) { - super(StandardCharsets.UTF_8); - setEnv(env); - } - } - - private static class CapturingEnvironment extends ShellReplyEnvironment { - final List> published = new ArrayList<>(); - - CapturingEnvironment() { - super(null, null, null, null); - } - - @Override - public void publish(Message message) { - published.add(message); - } - } - - private static class CallbackKernel extends JavaKernel implements AutoCloseable { - DisplayData displayed; - RuntimeException displayFailure; - - CallbackKernel() { - this(new JupyterIO(StandardCharsets.UTF_8)); - } - - CallbackKernel(JupyterIO io) { - super( - "test", "0", - new LanguageInfo.Builder("Java").build(), - Collections.emptyList(), null, - io, new CommManager(), new Renderer(), - new MagicsResolver("%", "%%", new MagicTranspiler()), - new MagicsRegistry(), false, new StringStyler.Builder().build(), - // These tests exercise callback I/O, not remote executor behavior. - JShell.builder().executionEngine("local").build(), - new CodeEvaluator(-1, TimeUnit.MILLISECONDS)); - } - - @Override - public void display(DisplayData data) { - if (displayFailure != null) { - throw displayFailure; - } - displayed = data; - } - - @Override - public void close() { - onShutdown(false); - } - } -} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelDisplayFailureTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelDisplayFailureTest.java new file mode 100644 index 0000000..3d61adb --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelDisplayFailureTest.java @@ -0,0 +1,77 @@ +package org.dflib.jjava.kernel; + +import org.dflib.jjava.jupyter.kernel.CapturingShellEnvironment; +import org.dflib.jjava.jupyter.messages.publish.PublishDisplayData; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Non-portable payloads are rejected in the executor before any delivery: nothing is published, and the next cell + * still evaluates. The stalled-peer case is not covered yet (needs a blocking endpoint handler seam). + */ +class JavaKernelDisplayFailureTest { + + private static final String IMPORT = "import static org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics.*;\n"; + + @Test + void nonPortablePayloadsAreRejectedWithoutPublishingAndKernelContinues() { + JavaKernel kernel = JavaKernel.builder().extensionsEnabled(false).build(); + try { + CapturingShellEnvironment env = CapturingShellEnvironment.attach(kernel.getIO()); + + // a cyclic list and a non-finite double, both built in the child + // Rejected by renderPortable before the protocol check: a raw collection is not a supported display value. + assertFailsWithMessage(kernel, + "java.util.List cyclic = new java.util.ArrayList<>(); cyclic.add(cyclic); display(cyclic);", + "Non-portable display value"); + assertFailsWithMessage(kernel, + "display(java.util.List.of(Double.NaN));", + "Non-portable display value"); + + assertTrue(env.published.stream().noneMatch(m -> m.getContent() instanceof PublishDisplayData), + "a rejected payload was published"); + + assertEquals("42", kernel.evalBuilder("40 + 2").resolveMagics().eval()); + } finally { + kernel.onShutdown(false); + } + } + + // Pinned current behavior: a bare scalar is rendered to text before the protocol's non-finite check, which applies to containers. + @Test + void bareNonFiniteScalarIsPublishedAsTextNaN() { + JavaKernel kernel = JavaKernel.builder().extensionsEnabled(false).build(); + try { + CapturingShellEnvironment env = CapturingShellEnvironment.attach(kernel.getIO()); + + kernel.evalBuilder(IMPORT + "display(Double.NaN);").resolveMagics().eval(); + + PublishDisplayData published = assertInstanceOf(PublishDisplayData.class, env.published.get(0).getContent()); + assertEquals("NaN", published.getData().get("text/plain")); + } finally { + kernel.onShutdown(false); + } + } + + private static void assertFailsWithMessage(JavaKernel kernel, String code, String fragment) { + var evaluation = kernel.evalBuilder(IMPORT + code).resolveMagics(); + RuntimeException failure = org.junit.jupiter.api.Assertions.assertThrows( + RuntimeException.class, + evaluation::eval); + assertTrue(String.valueOf(failure.getMessage()).contains(fragment) + || causeChainContains(failure, fragment), + "expected message containing '" + fragment + "' but was: " + failure); + } + + private static boolean causeChainContains(Throwable t, String fragment) { + for (Throwable c = t; c != null; c = c.getCause()) { + if (String.valueOf(c.getMessage()).contains(fragment)) { + return true; + } + } + return false; + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEndpointLifecycleTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEndpointLifecycleTest.java new file mode 100644 index 0000000..1bb90e2 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEndpointLifecycleTest.java @@ -0,0 +1,43 @@ +package org.dflib.jjava.kernel; + +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class JavaKernelEndpointLifecycleTest { + + @Test + void endpointSocketIsRemovedOnShutdown() throws Exception { + Path socket = Files.createTempDirectory("uds-lifecycle").resolve("display.sock"); + + JavaKernel kernel = JavaKernel.builder().extensionsEnabled(false).displayEndpointPath(socket).build(); + assertTrue(Files.exists(socket)); + + kernel.onShutdown(false); + assertFalse(Files.exists(socket)); + } + + @Test + void buildFailureAfterEndpointCreationRemovesEndpoint() throws Exception { + Path socket = Files.createTempDirectory("uds-lifecycle").resolve("display.sock"); + AtomicBoolean hookRan = new AtomicBoolean(); + + assertThrows(IllegalStateException.class, () -> JavaKernel.builder() + .extensionsEnabled(false) + .displayEndpointPath(socket) + .afterDisplayEndpointCreated(() -> { + hookRan.set(true); + throw new IllegalStateException("boom"); + }) + .build()); + + assertTrue(hookRan.get()); + assertFalse(Files.exists(socket)); + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEvalBuilderTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEvalBuilderTest.java index ed096e7..5446776 100644 --- a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEvalBuilderTest.java +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelEvalBuilderTest.java @@ -18,17 +18,17 @@ public class JavaKernelEvalBuilderTest { private JavaKernel kernel; @BeforeEach - public void setUp() { + void setUp() { kernel = JavaKernel.builder().extensionsEnabled(false).build(); } @AfterEach - public void tearDown() { + void tearDown() { kernel.onShutdown(false); } @Test - public void rendersManagerLineAndCellMagicsRegardlessOfBuilderOrder() { + void rendersManagerLineAndCellMagicsRegardlessOfBuilderOrder() { AtomicInteger calls = new AtomicInteger(); kernel.getMagicsRegistry().registerLineMagic("probe", (k, args) -> { assertSame(kernel, k); @@ -59,7 +59,7 @@ public void rendersManagerLineAndCellMagicsRegardlessOfBuilderOrder() { } @Test - public void preservesNullAndDisplayDataMagicResults() { + void preservesNullAndDisplayDataMagicResults() { kernel.getMagicsRegistry().registerLineMagic("nothing", (k, args) -> null); assertNull(kernel.evalBuilder("%nothing").renderResults().resolveMagics().eval()); @@ -69,7 +69,14 @@ public void preservesNullAndDisplayDataMagicResults() { } @Test - public void doesNotUseManagerRendererForExecutorResults() { + void textualNullResultIsPresentWhileAbsentResultIsNot() { + DisplayData textualNull = kernel.evalBuilder("String nothing = null;\nnothing").renderResults().eval(); + assertEquals("null", textualNull.getData().get("text/plain")); + assertNull(kernel.evalBuilder("int declared = 1;").renderResults().eval()); + } + + @Test + void doesNotUseManagerRendererForExecutorResults() { kernel.getRenderer().createRegistration(String.class) .register((value, context) -> fail("Executor result must not be rendered in the manager")); diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelStalledDisplayTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelStalledDisplayTest.java new file mode 100644 index 0000000..04ddf7e --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaKernelStalledDisplayTest.java @@ -0,0 +1,62 @@ +package org.dflib.jjava.kernel; + +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * A manager that accepts a display request but never answers. The executor must fail within the response bound and + * report the outcome as uncertain (never retried), and the kernel must still evaluate afterwards. + */ +class JavaKernelStalledDisplayTest { + + private static final String IMPORT = "import static org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics.*;\n"; + // executor RESPONSE bound is 10 s; allow a scheduling tolerance of 5 s + private static final long BOUND_MILLIS = 15_000; + + @Test + void stalledManagerFailsUncertainWithinBoundAndKernelStaysUsable() { + CountDownLatch release = new CountDownLatch(1); + JavaKernel kernel = JavaKernel.builder() + .extensionsEnabled(false) + .displayHandler(request -> awaitQuietly(release)) + .build(); + try { + var stalledDisplay = kernel.evalBuilder(IMPORT + "display(\"stalled\")").resolveMagics(); + long start = System.nanoTime(); + RuntimeException failure = assertThrows(RuntimeException.class, stalledDisplay::eval); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + + assertTrue(elapsedMillis < BOUND_MILLIS, "took " + elapsedMillis + " ms"); + assertTrue(chainContains(failure, "Failed to send display data"), String.valueOf(failure)); + + release.countDown(); + assertEquals("42", kernel.evalBuilder("40 + 2").resolveMagics().eval()); + } finally { + release.countDown(); + kernel.onShutdown(false); + } + } + + private static void awaitQuietly(CountDownLatch latch) { + try { + latch.await(60, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private static boolean chainContains(Throwable t, String fragment) { + for (Throwable c = t; c != null; c = c.getCause()) { + if (String.valueOf(c.getMessage()).contains(fragment)) { + return true; + } + } + return false; + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaNotebookStaticsRoutingTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaNotebookStaticsRoutingTest.java new file mode 100644 index 0000000..a2ede71 --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/JavaNotebookStaticsRoutingTest.java @@ -0,0 +1,28 @@ +package org.dflib.jjava.kernel; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class JavaNotebookStaticsRoutingTest { + + @Test + void executorKernelAlwaysFailsWithoutReadingKernel() { + assertEquals( + "JavaKernel.kernel() cannot expose the managing JVM from executor code", + assertThrows(UnsupportedOperationException.class, ExecutorJavaNotebookStatics::kernel).getMessage()); + } + + @SuppressWarnings("removal") + @Test + void executorGetKernelInstanceAlwaysFails() { + assertThrows(UnsupportedOperationException.class, ExecutorJavaNotebookStatics::getKernelInstance); + } + + @Test + void managerKernelWithoutRunningKernelFailsWithNullPointerFromKernelAccess() { + // Accepted outcome (spec Story 4 #8): no lifecycle message; the kernel access itself fails. + assertThrows(NullPointerException.class, JavaNotebookStatics::kernel); + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/OutputBoundaryDependencyTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/OutputBoundaryDependencyTest.java new file mode 100644 index 0000000..afe114f --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/OutputBoundaryDependencyTest.java @@ -0,0 +1,69 @@ +package org.dflib.jjava.kernel; + +import jdk.jshell.JShell; +import org.dflib.jjava.kernel.execution.CodeEvaluator; +import org.dflib.jjava.kernel.execution.EvaluationResult; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.spi.ToolProvider; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Evaluation-result boundary (T057): the evaluator and its result model must not depend on Jupyter display, + * message, or channel types, and eval returns EvaluationResult. + */ +class OutputBoundaryDependencyTest { + + @Test + void evaluatorAndResultModelDoNotDependOnJupyterPresentation() { + List violations = new ArrayList<>(); + List edges = edges(Path.of("target", "classes")); + assertFalse(edges.isEmpty(), "jdeps produced no edges; the check would pass vacuously"); + for (String[] edge : edges) { + String from = edge[0]; + String to = edge[1]; + boolean evaluator = from.startsWith(CodeEvaluator.class.getName()); + boolean result = from.startsWith(EvaluationResult.class.getName()); + if ((evaluator || result) && (to.startsWith("org.dflib.jjava.jupyter.kernel.display.") + || to.startsWith("org.dflib.jjava.jupyter.messages.") + || to.startsWith("org.dflib.jjava.jupyter.channels."))) { + violations.add(from + " -> " + to); + } + } + assertTrue(violations.isEmpty(), "forbidden edges: " + violations); + } + + @Test + void evalReturnsEvaluationResult() throws Exception { + assertEquals(EvaluationResult.class, + CodeEvaluator.class.getMethod("eval", JShell.class, String.class).getReturnType()); + } + + private static List edges(Path classes) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ToolProvider.findFirst("jdeps").orElseThrow().run( + new PrintStream(out, true, StandardCharsets.UTF_8), System.err, + "-verbose:class", classes.toString()); + + List edges = new ArrayList<>(); + for (String line : out.toString(StandardCharsets.UTF_8).split("\n")) { + int arrow = line.indexOf(" -> "); + if (arrow < 0) { + continue; + } + edges.add(new String[]{ + line.substring(0, arrow).trim(), + line.substring(arrow + 4).trim().split("\\s+")[0]}); + } + return edges; + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java new file mode 100644 index 0000000..e7032da --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java @@ -0,0 +1,41 @@ +package org.dflib.jjava.kernel; + +import org.dflib.jjava.jupyter.kernel.CapturingShellEnvironment; +import org.dflib.jjava.jupyter.messages.publish.PublishDisplayData; +import org.dflib.jjava.jupyter.messages.publish.PublishUpdateDisplayData; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * A display and an update made by user code in the executor JVM are published by the manager. + */ +class RemoteDisplayRoundTripTest { + + private static final String IMPORT = "import static org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics.*;\n"; + + @Test + void executorDisplayAndUpdatePublishOnManager() { + JavaKernel kernel = JavaKernel.builder().extensionsEnabled(false).build(); + try { + CapturingShellEnvironment env = CapturingShellEnvironment.attach(kernel.getIO()); + + // eval() returns the JShell text of the String result, so the id arrives with its quotes + String id = ((String) kernel.evalBuilder(IMPORT + "display(\"hello\")").resolveMagics().eval()).replace("\"", ""); + + PublishDisplayData display = assertInstanceOf(PublishDisplayData.class, env.published.get(0).getContent()); + assertEquals(id, display.getDisplayId()); + assertEquals("hello", display.getData().get("text/plain")); + + kernel.evalBuilder(IMPORT + "updateDisplay(\"" + id + "\", \"updated\")").resolveMagics().eval(); + + PublishUpdateDisplayData update = assertInstanceOf( + PublishUpdateDisplayData.class, + env.published.get(1).getContent()); + assertEquals(id, update.getDisplayId()); + } finally { + kernel.onShutdown(false); + } + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/StandaloneMagicDispatchTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/StandaloneMagicDispatchTest.java new file mode 100644 index 0000000..2e850fe --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/StandaloneMagicDispatchTest.java @@ -0,0 +1,54 @@ +package org.dflib.jjava.kernel; + +import org.dflib.jjava.jupyter.kernel.BaseKernel; +import org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics; +import org.dflib.jjava.jupyter.kernel.magic.CellMagic; +import org.dflib.jjava.jupyter.kernel.magic.LineMagic; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Standalone magics are dispatched in the manager. If the generated call had been sent to the executor, the executor + * helper would throw UnsupportedOperationException, so a normal result also shows the executor never received it. + * Inline magics are the opposite: they resolve to an executor helper call. + */ +class StandaloneMagicDispatchTest { + + @Test + void standaloneLineAndCellMagicsDispatchInTheManager() { + List seen = new ArrayList<>(); + LineMagic line = (kernel, args) -> { + seen.add(kernel); + return "line"; + }; + CellMagic cell = (kernel, args, body) -> { + seen.add(kernel); + return body; + }; + + JavaKernel kernel = JavaKernel.builder() + .extensionsEnabled(false) + .lineMagic("echoLine", line) + .cellMagic("echoCell", cell) + .build(); + try { + assertEquals("line", kernel.evalBuilder("%echoLine hi").resolveMagics().eval()); + assertEquals("body", kernel.evalBuilder("%%echoCell x\nbody").resolveMagics().eval()); + + assertEquals(2, seen.size()); + assertSame(kernel, seen.get(0)); + assertSame(kernel, seen.get(1)); + + String inline = kernel.getMagicsResolver().resolve("x = %echoLine hi"); + assertTrue(inline.contains(ExecutorNotebookStatics.class.getName() + ".lineMagic("), inline); + } finally { + kernel.onShutdown(false); + } + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/UnixDomainSocketJdiPropagationTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/UnixDomainSocketJdiPropagationTest.java new file mode 100644 index 0000000..d8e9f2d --- /dev/null +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/UnixDomainSocketJdiPropagationTest.java @@ -0,0 +1,57 @@ +package org.dflib.jjava.kernel; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Gate test (tasks T009): a real JDI executor receives the kernel-owned Unix-domain socket path, which contains spaces, + * and the child reads the exact value and connects to it. The kernel owns the endpoint path and passes it to the + * executor as {@code -Djjava.display.socket}. Writes its result to target/uds-gate/{os}-jdk{N}-propagation.txt. + */ +class UnixDomainSocketJdiPropagationTest { + + @Test + void childReceivesExactPathAndConnects() throws Exception { + String os = System.getProperty("os.name").toLowerCase().replace(' ', '-'); + int jdk = Runtime.version().feature(); + Path dir = Files.createTempDirectory("uds gate prop "); + Path socket = dir.resolve("display.sock"); + String value = socket.toString(); + boolean ok = false; + try { + JavaKernel kernel = JavaKernel.builder() + .extensionsEnabled(false) + .displayEndpointPath(socket) + .build(); + try { + // base64 avoids escaping the Windows path inside the JShell source + String b64 = Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8)); + String code = "var ch = java.nio.channels.SocketChannel.open(java.net.StandardProtocolFamily.UNIX);\n" + + "ch.connect(java.net.UnixDomainSocketAddress.of(System.getProperty(\"jjava.display.socket\")));\n" + + "ch.close();\n" + + "System.getProperty(\"jjava.display.socket\").equals(" + + "new String(java.util.Base64.getDecoder().decode(\"" + b64 + "\"), java.nio.charset.StandardCharsets.UTF_8))"; + Object result = kernel.evalBuilder(code).resolveMagics().eval(); + assertEquals("true", String.valueOf(result)); + ok = true; + } finally { + kernel.onShutdown(false); + } + } finally { + Path out = Path.of("target", "uds-gate", os + "-jdk" + jdk + "-propagation.txt"); + Files.createDirectories(out.getParent()); + Files.write(out, List.of("os=" + System.getProperty("os.name") + " jdk=" + jdk + " path=" + value, + "propagation=" + (ok ? "PASS" : "FAIL"))); + Files.deleteIfExists(socket); + } + assertTrue(ok); + } +} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/CodeEvaluatorTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/CodeEvaluatorTest.java index d7f14cd..a007380 100644 --- a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/CodeEvaluatorTest.java +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/CodeEvaluatorTest.java @@ -10,7 +10,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; public class CodeEvaluatorTest { @@ -18,9 +17,9 @@ public class CodeEvaluatorTest { @Test public void evaluatesLastExpressionAndLeavesDeclarationsWithoutResults() { try (JShell shell = newShell(); CodeEvaluator evaluator = new CodeEvaluator(-1, TimeUnit.SECONDS)) { - assertEquals("42", evaluator.eval(shell, "int base = 40;\nbase + 2")); - assertNull(evaluator.eval(shell, "int declared = 1;")); - assertEquals("\"__NO_MAGIC_RETURN\"", evaluator.eval(shell, "\"__NO_MAGIC_RETURN\"")); + assertEquals("42", evaluator.eval(shell, "int base = 40;\nbase + 2").text()); + assertEquals(EvaluationResult.NONE, evaluator.eval(shell, "int declared = 1;")); + assertEquals("\"__NO_MAGIC_RETURN\"", evaluator.eval(shell, "\"__NO_MAGIC_RETURN\"").text()); } } @@ -32,19 +31,19 @@ public void userExceptionDoesNotPoisonSubsequentEvaluation() { () -> evaluator.eval(shell, "throw new IllegalStateException(\"expected\");")); assertInstanceOf(EvalException.class, failure.getCause()); - assertEquals("42", evaluator.eval(shell, "40 + 2")); + assertEquals("42", evaluator.eval(shell, "40 + 2").text()); } } @Test public void timeoutDoesNotPoisonSubsequentEvaluation() { try (JShell shell = newShell(); CodeEvaluator evaluator = new CodeEvaluator(5, TimeUnit.SECONDS)) { - assertEquals("2", evaluator.eval(shell, "1 + 1")); + assertEquals("2", evaluator.eval(shell, "1 + 1").text()); assertThrows( EvaluationTimeoutException.class, () -> evaluator.eval(shell, "try { Thread.sleep(30000L); } catch (InterruptedException ignored) {}")); - assertEquals("42", evaluator.eval(shell, "40 + 2")); + assertEquals("42", evaluator.eval(shell, "40 + 2").text()); } } diff --git a/pom.xml b/pom.xml index 13bec3b..78cae47 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,7 @@ UTF-8 UTF-8 - 11 + 21 3.9.9 2.4.0 From b8d2469af8a89ed110dee84ee574ad736c305030 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 9 Oct 2026 17:51:03 +0300 Subject: [PATCH 03/16] Add manual workflow trigger to Maven build action --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index ae63d05..49c2e0a 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,6 +1,7 @@ name: 'build test' on: + workflow_dispatch: push: branches: [ main ] pull_request: From 7725378845ef02608dfd86831f77d870bd220c02 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 9 Oct 2026 18:00:44 +0300 Subject: [PATCH 04/16] Test: pad Unix domain socket path to exactly 100 bytes to avoid platform-specific AF_UNIX limits --- .../protocol/UnixDomainSocketPathAcceptanceTest.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java index 4dbae02..56721e7 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java @@ -71,9 +71,16 @@ void unixDomainSocketGate() throws Exception { assertTrue(ok, String.join("\n", report)); } - // one extra segment: lengthens the standard temp path but stays under the ~108-byte AF_UNIX limit + // Pads the directory so the socket path is exactly LENGTHENED_TARGET_BYTES, which is under the smallest AF_UNIX + // limit (macOS sun_path, about 104 bytes). Sizing by the platform default is not safe: a fixed segment overshot it. + private static final int LENGTHENED_TARGET_BYTES = 100; + private static Path nestedDir(Path base) throws Exception { - return Files.createDirectories(base.resolve("uds-gate-length-check")); + int room = LENGTHENED_TARGET_BYTES - base.toString().getBytes(StandardCharsets.UTF_8).length - "/s.sock".length(); + if (room < 2) { + throw new IllegalStateException("temp path leaves no room to lengthen: " + base); + } + return Files.createDirectories(base.resolve("l".repeat(room - 1))); } private static boolean check(List report, String label, Path socket) throws Exception { From b116277c96f3dde93ae92afb455af0900f071335 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 9 Oct 2026 18:01:40 +0300 Subject: [PATCH 05/16] Add fail-fast: false to Maven build matrix --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 49c2e0a..5fcb2b1 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -15,6 +15,7 @@ jobs: matrix: os: [ ubuntu-latest, windows-latest, macos-latest ] java: [ 21, 25 ] + fail-fast: false steps: - name: Checkout From a6ac958e435be06ab02d1cb6808393133138fd77 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 9 Oct 2026 18:11:29 +0300 Subject: [PATCH 06/16] Add cause reporting to UnixDomainDisplayDeliveryTimeoutTest --- .../display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java index 80f3440..471c5fb 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java @@ -49,7 +49,8 @@ void peerThatNeverReadsLargeRequestFailsUncertainWithinBound() throws Exception DisplayDeliveryException e = assertThrows(DisplayDeliveryException.class, () -> delivery.deliver(request)); - assertEquals(UNCERTAIN, e.kind()); + // the cause is reported so a platform-specific failure mode is visible in CI output + assertEquals(UNCERTAIN, e.kind(), () -> "cause: " + e.getCause()); assertTrue(elapsedMillis(start) < BOUND_MILLIS); } } From 40c08d0d5e7f6e6e2053d67c2e7201e9aa3cd2e4 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 9 Oct 2026 18:18:34 +0300 Subject: [PATCH 07/16] Track connection state explicitly --- .../display/protocol/uds/UnixDomainDisplayDelivery.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayDelivery.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayDelivery.java index 4a3a13a..817a173 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayDelivery.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayDelivery.java @@ -49,17 +49,20 @@ public void deliver(DisplayRequest request) { } ScheduledFuture deadline = DEADLINES.schedule(() -> closeQuietly(channel), DEADLINE_SECONDS, TimeUnit.SECONDS); + // Once connected, a write may have partly reached the peer even if the call never returned (a deadline close + // interrupts it, and the byte count only advances on return). So UNAVAILABLE is reserved for failures before connect. + boolean connected = false; try (channel) { channel.connect(UnixDomainSocketAddress.of(socketPath)); + connected = true; while (out.hasRemaining()) { channel.write(out); } DisplayProtocol.checkResponse(readLine(channel)); } catch (IOException e) { - boolean sent = out.position() > 0; throw new DisplayDeliveryException( - sent ? UNCERTAIN : UNAVAILABLE, - sent ? UNCERTAIN_MESSAGE : UNAVAILABLE_MESSAGE, + connected ? UNCERTAIN : UNAVAILABLE, + connected ? UNCERTAIN_MESSAGE : UNAVAILABLE_MESSAGE, e); } finally { deadline.cancel(false); From 70ad807b365d6d868e375d7b51b46b846e1f77aa Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 9 Oct 2026 18:30:57 +0300 Subject: [PATCH 08/16] Remove macOS from Maven build matrix --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 5fcb2b1..3a4a6b1 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - os: [ ubuntu-latest, windows-latest, macos-latest ] + os: [ ubuntu-latest, windows-latest ] java: [ 21, 25 ] fail-fast: false From d6fa51e41c7020423aace4d9e39f88f6ca0713a0 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Thu, 15 Oct 2026 09:52:08 +0300 Subject: [PATCH 09/16] Add test for display after executor termination --- CLAUDE.md | 2 +- .../java/org/dflib/jjava/distro/KernelStartupIT.java | 12 ++++++++++++ .../jjava/distro/NotebookInitializerImportsTest.java | 2 +- .../main/java/org/dflib/jjava/kernel/JavaKernel.java | 2 ++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8a1d707..9f1a7ff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - Surefire in `jjava-kernel` and `jjava-distro` is configured with `--add-opens jdk.jshell/jdk.jshell=ALL-UNNAMED` — required for JShell reflection. Anything that exercises `JavaKernel`/`CodeEvaluator` directly needs the same flag. - Local install of the freshly built kernel into Jupyter: see `DEV-GUIDE.md` (`unzip` the kernelspec zip, then `jupyter kernelspec install ... --name=java --user`). -Minimum toolchain: JDK 21 (`maven.compiler.release=21`). CI matrix builds against 21 and 25 on Linux, Windows, and macOS. +Minimum toolchain: JDK 21 (`maven.compiler.release=21`). CI matrix builds against 21 and 25. ## Module Layout and Dependency Graph diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java index ee7a7ee..649bbe6 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java @@ -33,6 +33,18 @@ public void executorTerminationMarksSessionLost() throws Exception { run.cell(2).assertError("Executor process terminated", "Restart the kernel"); } + @Test + public void displayAfterExecutorTerminationReportsExecutorUnavailable() throws Exception { + KernelRun run = executeInKernel( + Map.of(Env.JJAVA_TIMEOUT, "3000"), + "Runtime.getRuntime().halt(17);", + "display(\"after termination\")" + ); + + run.cell(1).assertError("Executor process terminated", "Restart the kernel"); + run.cell(2).assertError("Executor process terminated", "Restart the kernel"); + } + @Test public void executorOutOfMemoryMarksSessionLost() throws Exception { KernelRun run = executeInKernel( diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/NotebookInitializerImportsTest.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/NotebookInitializerImportsTest.java index c5cebc6..8d5afcf 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/NotebookInitializerImportsTest.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/NotebookInitializerImportsTest.java @@ -10,7 +10,7 @@ class NotebookInitializerImportsTest { @Test - void startupScriptImportsOnlyExecutorHelpers() throws Exception { + public void startupScriptImportsOnlyExecutorHelpers() throws Exception { Field field = NotebookInitializer.class.getDeclaredField("STARTUP_SCRIPT"); field.setAccessible(true); String script = (String) field.get(null); diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java index 80f6ec0..8e1cc39 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernel.java @@ -548,6 +548,8 @@ public JavaKernel build() { kernelRef[0] = kernel; return kernel; } catch (RuntimeException | Error e) { + // Deliberately catches Error too: an OutOfMemoryError or similar during build must still release the + // JShell, display endpoint, and evaluator it has already created, and then be rethrown unchanged. closeAfterFailure(e, jShell, endpoint, evaluator); throw e; } From 1f0738b3f62b5216687936da5a49e0b1c289cea4 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Thu, 15 Oct 2026 10:11:47 +0300 Subject: [PATCH 10/16] Clean up --- .../BaseNotebookStaticsDisplayTest.java | 8 +- .../ExecutorNotebookStaticsFailureTest.java | 10 +- ...bookStaticsRenderCharacterizationTest.java | 14 +- .../ExecutorNotebookStaticsRoutingTest.java | 12 +- .../kernel/OutputBoundaryDependencyTest.java | 9 +- .../DisplayDeliveryBootstrapTest.java | 8 +- .../protocol/DisplayDeliveryContractTest.java | 10 +- .../display/protocol/DisplayEncodingTest.java | 6 +- .../display/protocol/DisplayProtocolTest.java | 14 +- .../InMemoryDisplayDeliveryContractTest.java | 2 +- ...UnixDomainDisplayDeliveryContractTest.java | 2 +- .../UnixDomainDisplayDeliveryTimeoutTest.java | 8 +- .../UnixDomainDisplayEndpointTest.java | 4 +- .../UnixDomainSocketPathAcceptanceTest.java | 138 ------------------ .../kernel/magic/MagicTranspilerTest.java | 4 +- .../kernel/OutputBoundaryDependencyTest.java | 10 +- .../UnixDomainSocketJdiPropagationTest.java | 57 -------- 17 files changed, 56 insertions(+), 260 deletions(-) delete mode 100644 jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java delete mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/UnixDomainSocketJdiPropagationTest.java diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStaticsDisplayTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStaticsDisplayTest.java index 98faab0..2e60651 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStaticsDisplayTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStaticsDisplayTest.java @@ -30,7 +30,7 @@ /** * Manager-side helpers: display and update go to the running kernel directly and never through DisplayDelivery. */ -class BaseNotebookStaticsDisplayTest { +public class BaseNotebookStaticsDisplayTest { private final List delivered = new ArrayList<>(); private BaseKernel previousKernel; @@ -38,7 +38,7 @@ class BaseNotebookStaticsDisplayTest { private CapturingEnvironment env; @BeforeEach - void setUp() { + public void setUp() { previousKernel = BaseKernel.notebookKernel; previousDelivery = DisplayDeliveryBootstrap.install(delivered::add); @@ -67,7 +67,7 @@ void displayReturnsTheIdOfThePublishedBundle() { } @Test - void updateDisplayUsesKernelWithTheTargetId() { + public void updateDisplayUsesKernelWithTheTargetId() { BaseNotebookStatics.updateDisplay("requested-id", new DisplayData("updated")); PublishUpdateDisplayData update = assertInstanceOf( @@ -78,7 +78,7 @@ void updateDisplayUsesKernelWithTheTargetId() { } @Test - void managerWithoutKernelFailsWithNullPointerAndNeverDelivers() { + public void managerWithoutKernelFailsWithNullPointerAndNeverDelivers() { BaseKernel.notebookKernel = null; assertThrows(NullPointerException.class, () -> BaseNotebookStatics.display("text")); diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsFailureTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsFailureTest.java index 61766ef..6fbae95 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsFailureTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsFailureTest.java @@ -1,10 +1,8 @@ package org.dflib.jjava.jupyter.kernel; -import org.dflib.jjava.jupyter.kernel.display.DisplayData; import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDelivery; import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryBootstrap; import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayDeliveryException; -import org.dflib.jjava.jupyter.kernel.display.protocol.DisplayRequest; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -15,7 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -class ExecutorNotebookStaticsFailureTest { +public class ExecutorNotebookStaticsFailureTest { private DisplayDelivery previousDelivery; private AtomicInteger attempts; @@ -34,7 +32,7 @@ void tearDown() { } @Test - void unavailableMapsToRemoteChannelMessage() { + public void unavailableMapsToRemoteChannelMessage() { failingWith(DisplayDeliveryException.Kind.UNAVAILABLE, "Remote display channel is not available"); RuntimeException e = assertThrows(RuntimeException.class, () -> ExecutorNotebookStatics.display("text")); @@ -44,7 +42,7 @@ void unavailableMapsToRemoteChannelMessage() { } @Test - void rejectedKeepsDisplayCallbackPrefix() { + public void rejectedKeepsDisplayCallbackPrefix() { failingWith(DisplayDeliveryException.Kind.REJECTED, "Display callback failed: boom"); RuntimeException e = assertThrows(RuntimeException.class, () -> ExecutorNotebookStatics.display("text")); @@ -54,7 +52,7 @@ void rejectedKeepsDisplayCallbackPrefix() { } @Test - void uncertainMapsToFailedToSendPrefixAndIsNotRetried() { + public void uncertainMapsToFailedToSendPrefixAndIsNotRetried() { failingWith(DisplayDeliveryException.Kind.UNCERTAIN, "outcome unknown"); RuntimeException e = assertThrows(RuntimeException.class, () -> ExecutorNotebookStatics.updateDisplay("id", "text")); diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRenderCharacterizationTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRenderCharacterizationTest.java index fbe2e45..2005140 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRenderCharacterizationTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRenderCharacterizationTest.java @@ -21,7 +21,7 @@ * Characterizes executor render behaviour (Gate 1 for the routing change). Rendering never publishes or delivers, so * the recording delivery must stay empty throughout. */ -class ExecutorNotebookStaticsRenderCharacterizationTest { +public class ExecutorNotebookStaticsRenderCharacterizationTest { private final List delivered = new ArrayList<>(); private DisplayDelivery previousDelivery; @@ -41,37 +41,37 @@ void tearDown() { } @Test - void renderNullThrowsNullPointerException() { + public void renderNullThrowsNullPointerException() { assertThrows(NullPointerException.class, () -> ExecutorNotebookStatics.render((Object) null)); } @Test - void renderDisplayDataReturnsSameObject() { + public void renderDisplayDataReturnsSameObject() { DisplayData data = new DisplayData("x"); assertSame(data, ExecutorNotebookStatics.render((Object) data)); } @Test - void renderScalarsAsTextPlainOnly() { + public void renderScalarsAsTextPlainOnly() { assertEquals(Map.of("text/plain", "hi"), ExecutorNotebookStatics.render("hi").getData()); assertEquals(Map.of("text/plain", "42"), ExecutorNotebookStatics.render(42).getData()); assertEquals(Map.of("text/plain", "true"), ExecutorNotebookStatics.render(true).getData()); } @Test - void renderCharacterThrowsIllegalArgumentException() { + public void renderCharacterThrowsIllegalArgumentException() { assertThrows(IllegalArgumentException.class, () -> ExecutorNotebookStatics.render('c')); } @Test - void renderIgnoresPresentationHints() { + public void renderIgnoresPresentationHints() { assertEquals( ExecutorNotebookStatics.render("x").getData(), ExecutorNotebookStatics.render("x", "text/html").getData()); } @Test - void renderNeverDelivers() { + public void renderNeverDelivers() { ExecutorNotebookStatics.render("x"); ExecutorNotebookStatics.render("x", "text/html"); assertTrue(delivered.isEmpty()); diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRoutingTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRoutingTest.java index 79eb4fe..fbbfbd4 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRoutingTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStaticsRoutingTest.java @@ -17,7 +17,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -class ExecutorNotebookStaticsRoutingTest { +public class ExecutorNotebookStaticsRoutingTest { private final List delivered = new ArrayList<>(); private DisplayDelivery previousDelivery; @@ -37,7 +37,7 @@ void tearDown() { } @Test - void displayDeliversThroughBootstrapWithoutReadingKernel() { + public void displayDeliversThroughBootstrapWithoutReadingKernel() { String id = ExecutorNotebookStatics.display("text"); assertEquals(1, delivered.size()); @@ -46,7 +46,7 @@ void displayDeliversThroughBootstrapWithoutReadingKernel() { } @Test - void updateDisplayDeliversUpdateWithTargetId() { + public void updateDisplayDeliversUpdateWithTargetId() { ExecutorNotebookStatics.updateDisplay("target", "text"); assertEquals(DisplayRequest.Operation.UPDATE, delivered.get(0).operation()); @@ -54,13 +54,13 @@ void updateDisplayDeliversUpdateWithTargetId() { } @Test - void updateDisplayWithNullIdFailsBeforeDelivering() { + public void updateDisplayWithNullIdFailsBeforeDelivering() { assertThrows(IllegalArgumentException.class, () -> ExecutorNotebookStatics.updateDisplay(null, "text")); assertTrue(delivered.isEmpty()); } @Test - void unsupportedHelpersFailWithExistingMessages() { + public void unsupportedHelpersFailWithExistingMessages() { assertEquals( "eval() cannot be called from executor code in JDI mode", assertThrows(UnsupportedOperationException.class, () -> ExecutorNotebookStatics.eval("1")).getMessage()); @@ -76,7 +76,7 @@ void unsupportedHelpersFailWithExistingMessages() { } @Test - void printfWritesStandardOutput() { + public void printfWritesStandardOutput() { PrintStream original = System.out; ByteArrayOutputStream captured = new ByteArrayOutputStream(); System.setOut(new PrintStream(captured, true, StandardCharsets.UTF_8)); diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/OutputBoundaryDependencyTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/OutputBoundaryDependencyTest.java index b9b28bb..aae893b 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/OutputBoundaryDependencyTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/OutputBoundaryDependencyTest.java @@ -13,16 +13,13 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -/** - * Dependency boundaries of the jupyter module (T056). Uses jdeps class-level output over target/classes. - */ -class OutputBoundaryDependencyTest { +public class OutputBoundaryDependencyTest { private static final String PROTOCOL = "org.dflib.jjava.jupyter.kernel.display.protocol."; private static final String BASE_STATICS = "org.dflib.jjava.jupyter.kernel.BaseNotebookStatics"; @Test - void protocolClassesOutsideTransportPackagesDoNotUseSocketsOrJupyterChannels() { + public void protocolClassesOutsideTransportPackagesDoNotUseSocketsOrJupyterChannels() { List violations = new ArrayList<>(); List edges = edges(Path.of("target", "classes")); assertFalse(edges.isEmpty(), "jdeps produced no edges; the check would pass vacuously"); @@ -42,7 +39,7 @@ void protocolClassesOutsideTransportPackagesDoNotUseSocketsOrJupyterChannels() { } @Test - void managerHelperNeverDeliversThroughTheExecutorBoundary() { + public void managerHelperNeverDeliversThroughTheExecutorBoundary() { List violations = new ArrayList<>(); for (String[] edge : edges(Path.of("target", "classes"))) { String from = edge[0]; diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrapTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrapTest.java index cfc4cd0..067ea0c 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrapTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryBootstrapTest.java @@ -13,24 +13,24 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; -class DisplayDeliveryBootstrapTest { +public class DisplayDeliveryBootstrapTest { private static final DisplayRequest REQUEST = DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", new DisplayData("x")); @Test - void noSocketPropertyFailsEveryDeliveryUnavailable() { + public void noSocketPropertyFailsEveryDeliveryUnavailable() { DisplayDelivery unavailable = DisplayDeliveryBootstrap.fromSocketPath(null); DisplayDeliveryException e = assertThrows(DisplayDeliveryException.class, () -> unavailable.deliver(REQUEST)); assertEquals(UNAVAILABLE, e.kind()); } @Test - void socketPropertySelectsUdsDelivery() { + public void socketPropertySelectsUdsDelivery() { assertInstanceOf(UnixDomainDisplayDelivery.class, DisplayDeliveryBootstrap.fromSocketPath("/tmp/x.sock")); } @Test - void installReturnsPreviousAndIsUsedForSubsequentCalls() { + public void installReturnsPreviousAndIsUsedForSubsequentCalls() { DisplayDelivery previous = DisplayDeliveryBootstrap.current(); List seen = new ArrayList<>(); try { diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryContractTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryContractTest.java index e36cbcb..473c59f 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryContractTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayDeliveryContractTest.java @@ -20,14 +20,14 @@ * Contract every delivery must satisfy, shared by the transports and the in-memory substitute. Transport-specific * behaviour (unavailable endpoint, stalled peer) is tested in the transport's own test classes. */ -abstract class DisplayDeliveryContractTest { +public abstract class DisplayDeliveryContractTest { private static final Gson GSON = new GsonBuilder().serializeNulls().create(); protected abstract DisplayDelivery delivery(DisplayRequestHandler handler); @Test - void bundleSurvivesDelivery() { + public void bundleSurvivesDelivery() { List received = new ArrayList<>(); DisplayData sent = bundle(); @@ -42,7 +42,7 @@ void bundleSurvivesDelivery() { } @Test - void updateCarriesTargetId() { + public void updateCarriesTargetId() { List received = new ArrayList<>(); DisplayData data = new DisplayData("updated"); data.putTransientData("display_id", "other"); @@ -56,7 +56,7 @@ void updateCarriesTargetId() { } @Test - void sequentialOperationsKeepOrder() { + public void sequentialOperationsKeepOrder() { List ids = new ArrayList<>(); DisplayDelivery delivery = delivery(r -> ids.add(r.displayId())); @@ -70,7 +70,7 @@ void sequentialOperationsKeepOrder() { } @Test - void handlerFailureIsRejectedWithItsMessage() { + public void handlerFailureIsRejectedWithItsMessage() { DisplayDelivery delivery = delivery(r -> { throw new IllegalStateException("boom"); }); diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java index b6e186e..56b6851 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java @@ -8,10 +8,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -class DisplayEncodingTest { +public class DisplayEncodingTest { @Test - void multilineNonAsciiAndPngSurviveRoundTripUnchanged() { + public void multilineNonAsciiAndPngSurviveRoundTripUnchanged() { String multiline = "line one\nline two\r\nend"; String nonAscii = "héllo — 日本語 🎉"; String png = Base64.getEncoder().encodeToString(new byte[]{(byte) 0x89, 'P', 'N', 'G', 0x0A, 0x1A, 0x0A}); @@ -29,7 +29,7 @@ void multilineNonAsciiAndPngSurviveRoundTripUnchanged() { } @Test - void encodedRequestNeverContainsRawLineFeed() { + public void encodedRequestNeverContainsRawLineFeed() { DisplayData data = new DisplayData("a\nb\nc"); String line = DisplayProtocol.encode(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocolTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocolTest.java index a84efc1..294d460 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocolTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocolTest.java @@ -12,10 +12,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -class DisplayProtocolTest { +public class DisplayProtocolTest { @Test - void roundTripsDisplayAndUpdate() { + public void roundTripsDisplayAndUpdate() { DisplayData data = new DisplayData("hello"); data.putHTML("hi"); @@ -32,7 +32,7 @@ void roundTripsDisplayAndUpdate() { } @Test - void rejectsCyclicStructure() { + public void rejectsCyclicStructure() { List cycle = new ArrayList<>(); cycle.add(cycle); DisplayData data = new DisplayData(); @@ -45,7 +45,7 @@ void rejectsCyclicStructure() { } @Test - void rejectsNonFiniteNumbers() { + public void rejectsNonFiniteNumbers() { for (double value : new double[]{Double.NaN, Double.POSITIVE_INFINITY}) { DisplayData data = new DisplayData(); data.putData("application/json", value); @@ -57,7 +57,7 @@ void rejectsNonFiniteNumbers() { } @Test - void rejectsNonStringMapKeys() { + public void rejectsNonStringMapKeys() { Map map = new LinkedHashMap<>(); map.put(1, "bad"); DisplayData data = new DisplayData(); @@ -70,7 +70,7 @@ void rejectsNonStringMapKeys() { } @Test - void rejectsUpdateWithoutId() { + public void rejectsUpdateWithoutId() { DisplayData data = new DisplayData("x"); IllegalArgumentException e = assertThrows( IllegalArgumentException.class, @@ -79,7 +79,7 @@ void rejectsUpdateWithoutId() { } @Test - void rejectsUnknownActionAndMalformedJson() { + public void rejectsUnknownActionAndMalformedJson() { assertThrows(IllegalArgumentException.class, () -> DisplayProtocol.decode("{\"action\":\"explode\",\"bundle\":{}}")); assertThrows(IllegalArgumentException.class, () -> DisplayProtocol.decode("not json")); diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/InMemoryDisplayDeliveryContractTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/InMemoryDisplayDeliveryContractTest.java index e053a7e..8351e92 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/InMemoryDisplayDeliveryContractTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/InMemoryDisplayDeliveryContractTest.java @@ -4,7 +4,7 @@ * Runs the delivery contract against an in-memory substitute that round-trips every request and response through the * protocol text, exactly as a real transport would. */ -class InMemoryDisplayDeliveryContractTest extends DisplayDeliveryContractTest { +public class InMemoryDisplayDeliveryContractTest extends DisplayDeliveryContractTest { @Override protected DisplayDelivery delivery(DisplayRequestHandler handler) { diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryContractTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryContractTest.java index ae3b019..09b9465 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryContractTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryContractTest.java @@ -9,7 +9,7 @@ import java.util.ArrayList; import java.util.List; -class UnixDomainDisplayDeliveryContractTest extends DisplayDeliveryContractTest { +public class UnixDomainDisplayDeliveryContractTest extends DisplayDeliveryContractTest { private final List endpoints = new ArrayList<>(); diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java index 471c5fb..fa54db1 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayDeliveryTimeoutTest.java @@ -21,12 +21,12 @@ /** * Production bounds: a single 10 s RESPONSE deadline, judged with a 5 s scheduling tolerance (15 s). */ -class UnixDomainDisplayDeliveryTimeoutTest { +public class UnixDomainDisplayDeliveryTimeoutTest { private static final long BOUND_MILLIS = 15_000; @Test - void missingSocketFailsUnavailableWithinConnectBound() { + public void missingSocketFailsUnavailableWithinConnectBound() { Path missing = Path.of(System.getProperty("java.io.tmpdir"), "jjava-missing-" + System.nanoTime() + ".sock"); UnixDomainDisplayDelivery delivery = new UnixDomainDisplayDelivery(missing); DisplayRequest request = request(new DisplayData("x")); @@ -39,7 +39,7 @@ void missingSocketFailsUnavailableWithinConnectBound() { } @Test - void peerThatNeverReadsLargeRequestFailsUncertainWithinBound() throws Exception { + public void peerThatNeverReadsLargeRequestFailsUncertainWithinBound() throws Exception { Path socket = Files.createTempDirectory("uds-timeout").resolve("s.sock"); try (ServerSocketChannel server = listen(socket)) { // Never accepts: connect completes through the listen backlog, then the large write stalls. @@ -56,7 +56,7 @@ void peerThatNeverReadsLargeRequestFailsUncertainWithinBound() throws Exception } @Test - void peerThatReadsButNeverRepliesFailsUncertainWithinBound() throws Exception { + public void peerThatReadsButNeverRepliesFailsUncertainWithinBound() throws Exception { Path socket = Files.createTempDirectory("uds-timeout").resolve("s.sock"); try (ServerSocketChannel server = listen(socket)) { Thread peer = new Thread(() -> holdAfterRead(server)); diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayEndpointTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayEndpointTest.java index 7a8ba1d..04f560e 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayEndpointTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainDisplayEndpointTest.java @@ -21,10 +21,10 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -class UnixDomainDisplayEndpointTest { +public class UnixDomainDisplayEndpointTest { @Test - void servesOneRequestLinePerConnectionInOrderAndRemovesSocketOnShutdown() throws Exception { + public void servesOneRequestLinePerConnectionInOrderAndRemovesSocketOnShutdown() throws Exception { List ids = Collections.synchronizedList(new ArrayList<>()); UnixDomainDisplayEndpoint endpoint = new UnixDomainDisplayEndpoint(r -> ids.add(r.displayId())); Path socket = endpoint.path(); diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java deleted file mode 100644 index 56721e7..0000000 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/UnixDomainSocketPathAcceptanceTest.java +++ /dev/null @@ -1,138 +0,0 @@ -package org.dflib.jjava.jupyter.kernel.display.protocol; - -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.StandardProtocolFamily; -import java.net.UnixDomainSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.ServerSocketChannel; -import java.nio.channels.SocketChannel; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * Gate test (tasks T008): decides whether Unix-domain sockets work on this OS/JDK. Writes one line per check to - * target/uds-gate/{os}-jdk{N}.txt. The gate passes only if every check passes. - */ -public class UnixDomainSocketPathAcceptanceTest { - - private static void readFully(SocketChannel ch, ByteBuffer buf) throws IOException { - while (buf.hasRemaining()) { - if (ch.read(buf) < 0) { - break; - } - } - } - - @Test - void unixDomainSocketGate() throws Exception { - String os = System.getProperty("os.name").toLowerCase().replace(' ', '-'); - int jdk = Runtime.version().feature(); - List report = new ArrayList<>(); - report.add("os=" + System.getProperty("os.name") + " jdk=" + jdk - + " java.io.tmpdir=" + System.getProperty("java.io.tmpdir") - + " jdk.net.unixdomain.tmpdir=" + System.getProperty("jdk.net.unixdomain.tmpdir")); - boolean ok = true; - try { - Path standard = Files.createTempDirectory("uds-gate"); - Path spaced = Files.createTempDirectory("uds gate spaces "); - Path lengthened = nestedDir(Files.createTempDirectory("uds-gate-len")); - ok &= check(report, "standard-temp", standard.resolve("s.sock")); - ok &= check(report, "spaces-in-path", spaced.resolve("s.sock")); - ok &= check(report, "lengthened-path", lengthened.resolve("s.sock")); - ok &= permissionCheck(report, standard); - ok &= acceptFailureCleanup(report, standard.resolve("f.sock")); - report.add("short-private-dir=" + (System.getProperty("jdk.net.unixdomain.tmpdir") == null - ? "not-configured" : "configured")); - } catch (Throwable t) { - ok = false; - report.add("error=" + t); - throw t; - } finally { - report.add("gate=" + (ok ? "PASS" : "FAIL")); - Path out = Path.of("target", "uds-gate", os + "-jdk" + jdk + ".txt"); - Files.createDirectories(out.getParent()); - Files.write(out, report); - } - assertTrue(ok, String.join("\n", report)); - } - - // Pads the directory so the socket path is exactly LENGTHENED_TARGET_BYTES, which is under the smallest AF_UNIX - // limit (macOS sun_path, about 104 bytes). Sizing by the platform default is not safe: a fixed segment overshot it. - private static final int LENGTHENED_TARGET_BYTES = 100; - - private static Path nestedDir(Path base) throws Exception { - int room = LENGTHENED_TARGET_BYTES - base.toString().getBytes(StandardCharsets.UTF_8).length - "/s.sock".length(); - if (room < 2) { - throw new IllegalStateException("temp path leaves no room to lengthen: " + base); - } - return Files.createDirectories(base.resolve("l".repeat(room - 1))); - } - - private static boolean check(List report, String label, Path socket) throws Exception { - int bytes = socket.toString().getBytes(StandardCharsets.UTF_8).length; - try { - try (ServerSocketChannel server = ServerSocketChannel.open(StandardProtocolFamily.UNIX)) { - server.bind(UnixDomainSocketAddress.of(socket)); - try (SocketChannel client = SocketChannel.open(StandardProtocolFamily.UNIX)) { - client.connect(UnixDomainSocketAddress.of(socket)); - try (SocketChannel accepted = server.accept()) { - client.write(StandardCharsets.UTF_8.encode("ping\n")); - ByteBuffer buf = ByteBuffer.allocate(5); - readFully(accepted, buf); - assertEquals("ping\n", StandardCharsets.UTF_8.decode((ByteBuffer) buf.flip()).toString()); - } - } - } - boolean removed = !Files.exists(socket) || Files.deleteIfExists(socket); - report.add(label + "=" + (removed ? "PASS" : "FAIL") + " pathBytes=" + bytes + " socket=" + socket); - return removed; - } catch (Exception e) { - report.add(label + "=FAIL pathBytes=" + bytes + " error=" + e); - Files.deleteIfExists(socket); - return false; - } - } - - private static boolean permissionCheck(List report, Path dir) throws Exception { - boolean posix = Files.getFileStore(dir).supportsFileAttributeView("posix"); - report.add("permissions=" + (posix ? Files.getPosixFilePermissions(dir) : "not-posix")); - return true; - } - - private static boolean acceptFailureCleanup(List report, Path socket) throws Exception { - ServerSocketChannel server = ServerSocketChannel.open(StandardProtocolFamily.UNIX); - server.bind(UnixDomainSocketAddress.of(socket)); - ExecutorService exec = Executors.newSingleThreadExecutor(); - try { - Future accept = exec.submit(server::accept); - server.close(); - try { - accept.get(5, TimeUnit.SECONDS); - fail("accept returned after close"); - } catch (ExecutionException expected) { - // accept failed after close, as required - } - boolean removed = !Files.exists(socket) || Files.deleteIfExists(socket); - assertFalse(Files.exists(socket)); - report.add("accept-failure-cleanup=" + (removed ? "PASS" : "FAIL")); - return removed; - } finally { - exec.shutdownNow(); - } - } -} diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspilerTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspilerTest.java index dba2bc4..3a1396b 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspilerTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/magic/MagicTranspilerTest.java @@ -8,12 +8,12 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -class MagicTranspilerTest { +public class MagicTranspilerTest { private static final String EXECUTOR = ExecutorNotebookStatics.class.getName(); @Test - void generatedCellAndLineCallsTargetExecutorHelpers() { + public void generatedCellAndLineCallsTargetExecutorHelpers() { MagicTranspiler transpiler = new MagicTranspiler(); String cell = transpiler.transpileCell(new ParsedCellMagic("m", List.of(), "body")); diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/OutputBoundaryDependencyTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/OutputBoundaryDependencyTest.java index afe114f..968ef74 100644 --- a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/OutputBoundaryDependencyTest.java +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/OutputBoundaryDependencyTest.java @@ -17,14 +17,10 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -/** - * Evaluation-result boundary (T057): the evaluator and its result model must not depend on Jupyter display, - * message, or channel types, and eval returns EvaluationResult. - */ -class OutputBoundaryDependencyTest { +public class OutputBoundaryDependencyTest { @Test - void evaluatorAndResultModelDoNotDependOnJupyterPresentation() { + public void evaluatorAndResultModelDoNotDependOnJupyterPresentation() { List violations = new ArrayList<>(); List edges = edges(Path.of("target", "classes")); assertFalse(edges.isEmpty(), "jdeps produced no edges; the check would pass vacuously"); @@ -43,7 +39,7 @@ void evaluatorAndResultModelDoNotDependOnJupyterPresentation() { } @Test - void evalReturnsEvaluationResult() throws Exception { + public void evalReturnsEvaluationResult() throws Exception { assertEquals(EvaluationResult.class, CodeEvaluator.class.getMethod("eval", JShell.class, String.class).getReturnType()); } diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/UnixDomainSocketJdiPropagationTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/UnixDomainSocketJdiPropagationTest.java deleted file mode 100644 index d8e9f2d..0000000 --- a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/UnixDomainSocketJdiPropagationTest.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.dflib.jjava.kernel; - -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Base64; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Gate test (tasks T009): a real JDI executor receives the kernel-owned Unix-domain socket path, which contains spaces, - * and the child reads the exact value and connects to it. The kernel owns the endpoint path and passes it to the - * executor as {@code -Djjava.display.socket}. Writes its result to target/uds-gate/{os}-jdk{N}-propagation.txt. - */ -class UnixDomainSocketJdiPropagationTest { - - @Test - void childReceivesExactPathAndConnects() throws Exception { - String os = System.getProperty("os.name").toLowerCase().replace(' ', '-'); - int jdk = Runtime.version().feature(); - Path dir = Files.createTempDirectory("uds gate prop "); - Path socket = dir.resolve("display.sock"); - String value = socket.toString(); - boolean ok = false; - try { - JavaKernel kernel = JavaKernel.builder() - .extensionsEnabled(false) - .displayEndpointPath(socket) - .build(); - try { - // base64 avoids escaping the Windows path inside the JShell source - String b64 = Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8)); - String code = "var ch = java.nio.channels.SocketChannel.open(java.net.StandardProtocolFamily.UNIX);\n" - + "ch.connect(java.net.UnixDomainSocketAddress.of(System.getProperty(\"jjava.display.socket\")));\n" - + "ch.close();\n" - + "System.getProperty(\"jjava.display.socket\").equals(" - + "new String(java.util.Base64.getDecoder().decode(\"" + b64 + "\"), java.nio.charset.StandardCharsets.UTF_8))"; - Object result = kernel.evalBuilder(code).resolveMagics().eval(); - assertEquals("true", String.valueOf(result)); - ok = true; - } finally { - kernel.onShutdown(false); - } - } finally { - Path out = Path.of("target", "uds-gate", os + "-jdk" + jdk + "-propagation.txt"); - Files.createDirectories(out.getParent()); - Files.write(out, List.of("os=" + System.getProperty("os.name") + " jdk=" + jdk + " path=" + value, - "propagation=" + (ok ? "PASS" : "FAIL"))); - Files.deleteIfExists(socket); - } - assertTrue(ok); - } -} From 0ff9285208dc3889c3328d5423809c1fe97b87b7 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Thu, 15 Oct 2026 12:39:13 +0300 Subject: [PATCH 11/16] Remove old telemetry infrastructure and execution control --- jjava-distro/pom.xml | 9 - .../jupyter/telemetry/TelemetryCollector.java | 31 --- .../telemetry/WallAndCpuTimeCollector.java | 75 ------- .../jupyter/telemetry/WallTimeCollector.java | 30 --- jjava-kernel/pom.xml | 1 - .../execution/JJavaExecutionControl.java | 204 ------------------ .../kernel/execution/JJavaLoaderDelegate.java | 131 ----------- .../execution/JJavaLoaderDelegateTest.java | 28 --- kernelspec/java/kernel.json | 2 - 9 files changed, 511 deletions(-) delete mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/TelemetryCollector.java delete mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallAndCpuTimeCollector.java delete mode 100644 jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallTimeCollector.java delete mode 100644 jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaExecutionControl.java delete mode 100644 jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java delete mode 100644 jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegateTest.java diff --git a/jjava-distro/pom.xml b/jjava-distro/pom.xml index 72080a1..2c7e925 100644 --- a/jjava-distro/pom.xml +++ b/jjava-distro/pom.xml @@ -133,15 +133,6 @@ - - org.apache.maven.plugins - maven-surefire-plugin - - - --add-opens jdk.jshell/jdk.jshell=ALL-UNNAMED - - - org.apache.maven.plugins maven-assembly-plugin diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/TelemetryCollector.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/TelemetryCollector.java deleted file mode 100644 index 93f1439..0000000 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/TelemetryCollector.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.dflib.jjava.jupyter.telemetry; - -/** - * Aggregates some telemetry number for a sequence of "measurements". Instead of a single method wrapping a measure code - * invocation lambda, each measurement is done via two separate methods: {@link #measurementStart()} and - * {@link #measurementEnd(Object)}. This is to make the collector minimally invasive and not adding to the call stack - * trace. - */ -public interface TelemetryCollector { - - I measurementStart(); - - void measurementEnd(I startState); - - void stop(); - - TelemetryCollector DO_NOTHING = new TelemetryCollector<>() { - @Override - public void measurementEnd(Object startState) { - } - - @Override - public Object measurementStart() { - return null; - } - - @Override - public void stop() { - } - }; -} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallAndCpuTimeCollector.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallAndCpuTimeCollector.java deleted file mode 100644 index 346b8a1..0000000 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallAndCpuTimeCollector.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.dflib.jjava.jupyter.telemetry; - -import java.lang.management.ThreadMXBean; -import java.util.concurrent.atomic.AtomicLong; - -public abstract class WallAndCpuTimeCollector implements TelemetryCollector { - - private final ThreadMXBean threadMXBean; - private final AtomicLong totalWT; - private final AtomicLong totalUT; - private final AtomicLong totalCT; - - public static boolean canMeasureCpuTimes(ThreadMXBean threadMXBean) { - // according to JavaDocs, isThreadCpuTimeEnabled() can throw in some cases - try { - return threadMXBean.isCurrentThreadCpuTimeSupported() && threadMXBean.isThreadCpuTimeEnabled(); - } catch (Exception e) { - return false; - } - } - - public WallAndCpuTimeCollector(ThreadMXBean threadMXBean) { - this.threadMXBean = threadMXBean; - this.totalWT = new AtomicLong(); - this.totalUT = new AtomicLong(); - this.totalCT = new AtomicLong(); - } - - @Override - public Measure measurementStart() { - return new Measure( - System.nanoTime(), - threadMXBean.getCurrentThreadUserTime(), - threadMXBean.getCurrentThreadCpuTime() - ); - } - - @Override - public void measurementEnd(Measure m) { - - long wt = System.nanoTime(); - long ut = threadMXBean.getCurrentThreadUserTime(); - long ct = threadMXBean.getCurrentThreadCpuTime(); - - totalWT.addAndGet(wt - m.wt); - - // sanity check - if (m.ut != -1 && ut != -1 && m.ct != -1 && ct != -1) { - totalUT.addAndGet(ut - m.ut); - totalCT.addAndGet(ct - m.ct); - } else { - totalUT.set(-1); - totalCT.set(-1); - } - } - - @Override - public void stop() { - onStop(totalWT.get(), totalUT.get(), totalCT.get()); - } - - protected abstract void onStop(long wallTimeNanos, long userTimeNanos, long totalTimeNanos); - - public static class Measure { - final long wt; - final long ut; - final long ct; - - public Measure(long wt, long ut, long ct) { - this.ct = ct; - this.wt = wt; - this.ut = ut; - } - } -} diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallTimeCollector.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallTimeCollector.java deleted file mode 100644 index 403bac9..0000000 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/telemetry/WallTimeCollector.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.dflib.jjava.jupyter.telemetry; - -import java.util.concurrent.atomic.AtomicLong; - -public abstract class WallTimeCollector implements TelemetryCollector { - - private final AtomicLong totalWallTime; - - public WallTimeCollector() { - this.totalWallTime = new AtomicLong(); - } - - @Override - public Long measurementStart() { - return System.nanoTime(); - } - - @Override - public void measurementEnd(Long startState) { - long delta = System.nanoTime() - startState; - totalWallTime.addAndGet(delta); - } - - @Override - public void stop() { - onStop(totalWallTime.get()); - } - - protected abstract void onStop(long wallTimeNanos); -} diff --git a/jjava-kernel/pom.xml b/jjava-kernel/pom.xml index c54c2ab..cee0a59 100644 --- a/jjava-kernel/pom.xml +++ b/jjava-kernel/pom.xml @@ -37,7 +37,6 @@ org.apache.maven.plugins maven-surefire-plugin - --add-opens jdk.jshell/jdk.jshell=ALL-UNNAMED false diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaExecutionControl.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaExecutionControl.java deleted file mode 100644 index f85b735..0000000 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaExecutionControl.java +++ /dev/null @@ -1,204 +0,0 @@ -package org.dflib.jjava.kernel.execution; - -import jdk.jshell.EvalException; -import jdk.jshell.execution.DirectExecutionControl; -import jdk.jshell.spi.SPIResolutionException; -import org.dflib.jjava.jupyter.telemetry.TelemetryCollector; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import java.util.concurrent.CancellationException; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * An ExecutionControl very similar to {@link jdk.jshell.execution.LocalExecutionControl} but which - * also logs the actual result of an invocation before being serialized. - */ -class JJavaExecutionControl extends DirectExecutionControl { - - // generate a semi-unique thread name prefix for each JVM run for easier detection of JJavaExecutionControl-produced threads - private static final String THREAD_NAME_PREFIX = "jjava-exec-" - + ThreadLocalRandom.current().ints(6, 'a', 'z' + 1).collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) - + "-"; - - /** - * A special "class name" for a {@link jdk.jshell.spi.ExecutionControl.UserException} such that it may be - * identified after serialization into an {@link jdk.jshell.EvalException} via {@link - * EvalException#getExceptionClassName()}. - */ - // Has spaces to not collide with a class name - public static final String EXECUTION_TIMEOUT_NAME = "Execution Timeout"; - - /** - * A special "class name" for a {@link jdk.jshell.spi.ExecutionControl.UserException} such that it may be - * identified after serialization into an {@link jdk.jshell.EvalException} via {@link - * EvalException#getExceptionClassName()} - */ - public static final String EXECUTION_INTERRUPTED_NAME = "Execution Interrupted"; - - private static final AtomicInteger EXECUTOR_THREAD_ID = new AtomicInteger(0); - - private final ExecutorService executor; - private final long timeoutDuration; - private final TimeUnit timeoutUnit; - private final Map> running; - private final Map results; - private final JJavaLoaderDelegate loaderDelegate; - private final ThreadLocal> telemetryCollector; - - public JJavaExecutionControl(JJavaLoaderDelegate loaderDelegate, long timeoutDuration, TimeUnit timeoutUnit) { - super(loaderDelegate); - - this.loaderDelegate = loaderDelegate; - this.running = new ConcurrentHashMap<>(); - this.results = new ConcurrentHashMap<>(); - - this.timeoutDuration = timeoutDuration; - this.timeoutUnit = timeoutDuration > 0 ? Objects.requireNonNull(timeoutUnit) : TimeUnit.MILLISECONDS; - this.executor = Executors.newCachedThreadPool(r -> new Thread(r, THREAD_NAME_PREFIX + EXECUTOR_THREAD_ID.getAndIncrement())); - this.telemetryCollector = new ThreadLocal<>(); - } - - @Override - public void stop() { - executor.shutdownNow(); - } - - public Object takeResult(String id) { - Object result = this.results.remove(id); - if (result == null) { - throw new IllegalStateException("No result with key: " + id); - } - - return result; - } - - public void unloadClass(String className) { - loaderDelegate.unloadClass(className); - } - - public void interrupt() { - running.forEach((id, f) -> f.cancel(true)); - } - - public void startThreadTelemetryCollection(TelemetryCollector collector) { - telemetryCollector.set(Objects.requireNonNull(collector)); - } - - public void stopThreadTelemetryCollection() { - TelemetryCollector collector = telemetryCollector.get(); - if (collector != null) { - collector.stop(); - telemetryCollector.set(null); - } - } - - /** - * This method was hijacked and actually only returns a key that can be later retrieved via - * {@link #takeResult(String)}. This should be called for every invocation as the objects are saved and not taking - * them will leak the memory. - * - * @returns the key to use for {@link #takeResult(String) looking up the result}. - */ - @Override - protected String invoke(Method doitMethod) throws Exception { - String id = UUID.randomUUID().toString(); - Object value; - - TelemetryCollector tc = threadTelemetryCollector(); - Object m = tc.measurementStart(); - try { - value = doInvoke(id, doitMethod); - } finally { - // use the same collector used for "measurementStart" call, so if the thread local is reset inside - // "doInvoke", we are still calling the matching collector - tc.measurementEnd(m); - } - - results.put(id, value); - return id; - } - - private RunException wrapInRunException(Throwable cause) { - if (cause instanceof SPIResolutionException) { - return new ResolutionException(((SPIResolutionException) cause).id(), cause.getStackTrace()); - } - - UserException ue = new UserException(String.valueOf(cause.getMessage()), cause.getClass().getName(), cause.getStackTrace()); - if (cause.getCause() != null) { - ue.initCause(wrapInRunException(cause.getCause())); - } - return ue; - } - - private Object doInvoke(String id, Method doitMethod) throws Exception { - - Future task = isNestedCall() - // run on the same thread if the invocation is done within another invocation - ? CompletableFuture.completedFuture(doitMethod.invoke(null)) - : executor.submit(() -> doitMethod.invoke(null)); - - running.put(id, task); - - try { - return timeoutDuration > 0 ? task.get(timeoutDuration, timeoutUnit) : task.get(); - } catch (CancellationException e) { - // If canceled this means that stop() or interrupt() was invoked. - if (executor.isShutdown()) { - // If the executor is shutdown, the situation is the former in which - // case the protocol is to throw an ExecutionControl.StoppedException. - throw new StoppedException(); - } else { - // The execution was purposely interrupted. - throw new UserException("Execution interrupted.", EXECUTION_INTERRUPTED_NAME, e.getStackTrace()); - } - } catch (ExecutionException e) { - - Throwable cause = e.getCause(); - if (cause instanceof InvocationTargetException) { - cause = cause.getCause(); - } - if (cause == null) { - throw new UserException("null", "Unknown Invocation Exception", e.getStackTrace()); - } else { - throw wrapInRunException(cause); - } - } catch (TimeoutException e) { - String message = String.format("Execution timed out after configured timeout of %d %s.", - timeoutDuration, - timeoutUnit.toString().toLowerCase()); - throw new UserException(message, EXECUTION_TIMEOUT_NAME, e.getStackTrace()); - } finally { - running.remove(id, task); - } - } - - private boolean isNestedCall() { - return Thread.currentThread().getName().startsWith(THREAD_NAME_PREFIX); - } - - private TelemetryCollector threadTelemetryCollector() { - TelemetryCollector tc = telemetryCollector.get(); - return tc != null ? tc : TelemetryCollector.DO_NOTHING; - } - - @Override - public String toString() { - return "JJavaExecutionControl{" + - "timeoutTime=" + timeoutDuration + - ", timeoutUnit=" + timeoutUnit + - '}'; - } -} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java deleted file mode 100644 index 8f7e39c..0000000 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java +++ /dev/null @@ -1,131 +0,0 @@ -package org.dflib.jjava.kernel.execution; - -import jdk.jshell.execution.LoaderDelegate; -import jdk.jshell.spi.ExecutionControl; -import org.dflib.jjava.jupyter.kernel.util.PathsHandler; - -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.Path; -import java.security.CodeSource; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -class JJavaLoaderDelegate implements LoaderDelegate { - - private static final String CLASSPATH_PROPERTY = "java.class.path"; - private static final String PATH_SEPARATOR = System.getProperty("path.separator"); - - private final Map declaredClasses; - private final Map> loadedClasses; - private final JJavaClassLoader classLoader; - - public JJavaLoaderDelegate() { - this.declaredClasses = new ConcurrentHashMap<>(); - this.loadedClasses = new ConcurrentHashMap<>(); - this.classLoader = new JJavaClassLoader(); - Thread.currentThread().setContextClassLoader(classLoader); - } - - @Override - public void load(ExecutionControl.ClassBytecodes[] cbcs) throws ExecutionControl.ClassInstallException { - boolean[] installed = new boolean[cbcs.length]; - - // Must record all defined classes before attempting to load them. Otherwise, classes depending on other, - // not yet loaded classes, may fail (see https://github.com/dflib/jjava/issues/65) - for (ExecutionControl.ClassBytecodes cbc : cbcs) { - declaredClasses.put(cbc.name(), cbc.bytecodes()); - } - - int i = 0; - for (ExecutionControl.ClassBytecodes cbc : cbcs) { - try { - Class loaderClass = classLoader.findClass(cbc.name()); - loadedClasses.put(cbc.name(), loaderClass); - } catch (ClassNotFoundException e) { - throw new ExecutionControl.ClassInstallException("Unable to load class " + cbc.name() - + ": " + e.getMessage(), installed); - } - installed[i++] = true; - } - } - - @Override - public void classesRedefined(ExecutionControl.ClassBytecodes[] cbcs) { - for (ExecutionControl.ClassBytecodes cbc : cbcs) { - declaredClasses.put(cbc.name(), cbc.bytecodes()); - } - } - - @Override - public void addToClasspath(String path) throws ExecutionControl.InternalException { - for (String next : PathsHandler.split(path)) { - try { - classLoader.addURL(Path.of(next).toUri().toURL()); - - System.setProperty( - CLASSPATH_PROPERTY, - System.getProperty(CLASSPATH_PROPERTY) + PATH_SEPARATOR + next); - - } catch (MalformedURLException e) { - throw new ExecutionControl.InternalException("Unable to resolve classpath " + next - + ": " + e.getMessage()); - } - } - } - - @Override - public Class findClass(String name) throws ClassNotFoundException { - Class klass = loadedClasses.get(name); - if (klass == null && declaredClasses.containsKey(name)) { - // check if it was removed - klass = loadClass(name); - } - if (klass == null) { - throw new ClassNotFoundException(name + " not found"); - } - return klass; - } - - private Class loadClass(String name) throws ClassNotFoundException { - return classLoader.loadClass(name); - } - - public void unloadClass(String name) { - loadedClasses.remove(name); - declaredClasses.remove(name); - } - - public ClassLoader getClassLoader() { - return classLoader; - } - - class JJavaClassLoader extends URLClassLoader { - - public JJavaClassLoader() { - super(new URL[0]); - } - - // redefine here for access from the parent class. Otherwise, the "protected" method would be inaccessible - @Override - protected void addURL(URL url) { - super.addURL(url); - } - - @Override - protected Class findClass(String name) throws ClassNotFoundException { - byte[] data = declaredClasses.get(name); - if (data == null) { - return super.findClass(name); - } - try { - return super.defineClass(name, data, 0, data.length, (CodeSource) null); - } catch (LinkageError er) { - // rethrow as ClassNotFoundException to let the caller properly handle this case - // this error could be thrown in some cases (like static method signature change) - throw new ClassNotFoundException(name, er); - } - } - } -} diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegateTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegateTest.java deleted file mode 100644 index 0e0f86e..0000000 --- a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegateTest.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.dflib.jjava.kernel.execution; - -import jdk.jshell.spi.ExecutionControl; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.net.URISyntaxException; - -import static org.junit.jupiter.api.Assertions.*; - -public class JJavaLoaderDelegateTest { - - @Test - public void addToClasspath() throws URISyntaxException, ExecutionControl.InternalException { - File extraCp = new File(getClass().getResource("cp").toURI()); - assertTrue(extraCp.isDirectory()); - - JJavaLoaderDelegate ld = new JJavaLoaderDelegate(); - - String cp1 = System.getProperty("java.class.path"); - - ld.addToClasspath(extraCp.getAbsolutePath()); - String cp2 = System.getProperty("java.class.path"); - assertEquals(cp1 + System.getProperty("path.separator") + extraCp.getAbsolutePath(), cp2); - - // TODO: test that classes from this location were actually loaded - } -} diff --git a/kernelspec/java/kernel.json b/kernelspec/java/kernel.json index 9700ce5..f5374ed 100644 --- a/kernelspec/java/kernel.json +++ b/kernelspec/java/kernel.json @@ -1,8 +1,6 @@ { "argv": [ "java", - "--add-opens", - "jdk.jshell/jdk.jshell=ALL-UNNAMED", "-jar", "{resource_dir}/jjava.jar", "{connection_file}" From 5fd5783a4aee3c57a46a4a6894b915a1214711e3 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Thu, 15 Oct 2026 12:49:30 +0300 Subject: [PATCH 12/16] Fix number precision loss --- .../display/protocol/DisplayProtocol.java | 6 ++++- .../display/protocol/DisplayEncodingTest.java | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocol.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocol.java index dbb26f3..18f5dd0 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocol.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayProtocol.java @@ -5,6 +5,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; +import com.google.gson.ToNumberPolicy; import org.dflib.jjava.jupyter.kernel.display.DisplayData; import java.lang.reflect.Array; @@ -20,7 +21,10 @@ */ public final class DisplayProtocol { - static final Gson GSON = new GsonBuilder().serializeNulls().create(); + static final Gson GSON = new GsonBuilder() + .serializeNulls() + .setObjectToNumberStrategy(ToNumberPolicy.LAZILY_PARSED_NUMBER) + .create(); private static final String OK = "ok"; private static final String ACTION = "action"; diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java index 56b6851..1b812bf 100644 --- a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java +++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/protocol/DisplayEncodingTest.java @@ -3,13 +3,36 @@ import org.dflib.jjava.jupyter.kernel.display.DisplayData; import org.junit.jupiter.api.Test; +import java.math.BigDecimal; import java.util.Base64; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; public class DisplayEncodingTest { + @Test + public void nestedJsonNumbersSurviveRoundTripWithoutPrecisionLoss() { + long integer = 9007199254740993L; + BigDecimal decimal = new BigDecimal("12345678901234567890.123456789012345678901234567890"); + DisplayData data = new DisplayData(); + data.putData("application/json", Map.of("integer", integer, "decimal", decimal)); + + String line = DisplayProtocol.encode(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", data)); + DisplayData decoded = DisplayProtocol.decode(line).toDisplayData(); + Map numbers = (Map) decoded.getData().get("application/json"); + + assertEquals(Long.toString(integer), ((Number) numbers.get("integer")).toString()); + assertEquals(decimal.toString(), ((Number) numbers.get("decimal")).toString()); + + String reencoded = DisplayProtocol.encode(DisplayRequest.of(DisplayRequest.Operation.DISPLAY, "id", decoded)); + var json = DisplayProtocol.decode(reencoded).bundle().getAsJsonObject("data") + .getAsJsonObject("application/json"); + assertEquals(Long.toString(integer), json.get("integer").getAsNumber().toString()); + assertEquals(decimal.toString(), json.get("decimal").getAsNumber().toString()); + } + @Test public void multilineNonAsciiAndPngSurviveRoundTripUnchanged() { String multiline = "line one\nline two\r\nend"; From 5c8934d3a0f1e226e964b48bc3b6afca8a27c691 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Thu, 15 Oct 2026 13:00:29 +0300 Subject: [PATCH 13/16] Clean up socket file on endpoint initialization failure --- .../uds/UnixDomainDisplayEndpoint.java | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayEndpoint.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayEndpoint.java index 2e6d98e..57a260a 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayEndpoint.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/protocol/uds/UnixDomainDisplayEndpoint.java @@ -49,12 +49,27 @@ public UnixDomainDisplayEndpoint(DisplayRequestHandler handler) throws IOExcepti public UnixDomainDisplayEndpoint(Path socketPath, DisplayRequestHandler handler) throws IOException { this.handler = handler; this.server = ServerSocketChannel.open(StandardProtocolFamily.UNIX); - this.server.bind(socketPath == null ? null : UnixDomainSocketAddress.of(socketPath)); - this.path = ((UnixDomainSocketAddress) server.getLocalAddress()).getPath(); - - Thread acceptor = new Thread(this::acceptLoop, "jjava-display-callback"); - acceptor.setDaemon(true); - acceptor.start(); + Path boundPath = null; + try { + this.server.bind(socketPath == null ? null : UnixDomainSocketAddress.of(socketPath)); + boundPath = socketPath; + boundPath = ((UnixDomainSocketAddress) server.getLocalAddress()).getPath(); + this.path = boundPath; + + Thread acceptor = new Thread(this::acceptLoop, "jjava-display-callback"); + acceptor.setDaemon(true); + acceptor.start(); + } catch (IOException | RuntimeException | Error e) { + closeQuietly(server); + if (boundPath != null) { + try { + Files.deleteIfExists(boundPath); + } catch (IOException cleanup) { + e.addSuppressed(cleanup); + } + } + throw e; + } } public Path path() { From d7808a2bd4da098f553a5b978b2f87e5b3f0e2fb Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Thu, 15 Oct 2026 13:13:56 +0300 Subject: [PATCH 14/16] Use Jupyter input stream for JShell input stream --- .../src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java index 819f08c..5414407 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/JavaKernelBuilder.java @@ -81,7 +81,7 @@ protected JShell buildJShell(JupyterIO io, Iterable additionalRemoteVMOp return JShell.builder() .out(io.out) .err(io.err) - .in(System.in) + .in(io.in) .executionEngine(new JdiExecutionControlProvider(), Map.of()) .remoteVMOptions(options.toArray(new String[0])) .compilerOptions(compilerOpts.toArray(new String[0])) From fd42549aa553b549fc1f8553fc6649f8a091340b Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Thu, 15 Oct 2026 13:14:06 +0300 Subject: [PATCH 15/16] Clean up --- .../dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java | 4 +--- .../org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java index 2f4438f..94dd89d 100644 --- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java +++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/ExecutorNotebookStatics.java @@ -18,8 +18,7 @@ private ExecutorNotebookStatics() { } public static void printf(String format, Object... args) { - // Deliberate: JShell forwards System.out to the cell output (FR-011). A logger would not reach the cell. - System.out.printf(format, args); // NOSONAR + System.out.printf(format, args); } public static Object eval(String source) { @@ -40,7 +39,6 @@ public static DisplayData render(Object o) { return renderPortable(o); } - // Presentation hints are ignored by design (FR-012); the parameter keeps the manager's signature. @SuppressWarnings("unused") public static DisplayData render(Object o, String... as) { return renderPortable(o); diff --git a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java index e7032da..497f9d6 100644 --- a/jjava-kernel/src/test/java/org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java +++ b/jjava-kernel/src/test/java/org/dflib/jjava/kernel/RemoteDisplayRoundTripTest.java @@ -11,12 +11,12 @@ /** * A display and an update made by user code in the executor JVM are published by the manager. */ -class RemoteDisplayRoundTripTest { +public class RemoteDisplayRoundTripTest { private static final String IMPORT = "import static org.dflib.jjava.jupyter.kernel.ExecutorNotebookStatics.*;\n"; @Test - void executorDisplayAndUpdatePublishOnManager() { + public void executorDisplayAndUpdatePublishOnManager() { JavaKernel kernel = JavaKernel.builder().extensionsEnabled(false).build(); try { CapturingShellEnvironment env = CapturingShellEnvironment.attach(kernel.getIO()); From 8bb350bac01ecd2b0e9869bd63a6d59670cc8c96 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Thu, 15 Oct 2026 17:42:31 +0300 Subject: [PATCH 16/16] Remove workflow_dispatch trigger from Maven build --- .github/workflows/maven.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 3a4a6b1..165f5d7 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,7 +1,6 @@ name: 'build test' on: - workflow_dispatch: push: branches: [ main ] pull_request: