From a43afd1d0e154a1592decf5614d40f4399345775 Mon Sep 17 00:00:00 2001 From: Kary Zheng <150742834+kz930@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:36:20 +0000 Subject: [PATCH] fix(visualization): take the union of Network Graph's two node columns (#7327) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What changes were proposed in this PR? Network Graph built its node set with `set(sources + destinations)`. On two pandas Series `+` is element-wise, so the set held each source glued to its destination rather than the union of the two columns; those glued values were added to the graph as nodes, and the genuine nodes only arrived afterwards with the edges. This takes the union instead, in first-appearance order — a `set` iterates strings in an order that varies between processes, which would leave the node sequence unstable from run to run. One line of code, plus a comment recording why neither `+` nor `set` is right here. ### Any related issues, documentation, discussions? Closes #7325. ### How was this PR tested? `NetworkGraphOpDescSpec` gains a case asserting the node set is built as a union: reverting the one-line change leaves it the only failing test. It also pins the ordered de-duplication, since a `set` would satisfy "union" while reordering the nodes between processes. ``` sbt "WorkflowOperator/testOnly org.apache.texera.amber.operator.visualization.networkGraph.NetworkGraphOpDescSpec" ``` Six cases, all passing. Beyond that, the operator's generated module was dumped and executed over ten rows carrying the edges `n3-to-n4`, `n1-to-n2` and `n2-to-n3`. Before the change it produced seven nodes — the four real ones plus `n3n4`, `n2n3` and `n1n2`, each reporting zero connections, with `n2n3` sitting in the same picture as the genuine edge from `n2` to `n3`. After it, four nodes with the correct connection counts and no isolated dots. Before the change: Screenshot 2026-08-07 at 1 50 52 PM Selecting an integer column as the source and a string column as the destination aborted the run with `TypeError: unsupported operand type(s) for +: 'int' and 'str'` before the change and renders normally after it. Before the change: Screenshot 2026-08-07 at 1 51 17 PM ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (Claude Opus 5) --------- (backported from commit 357296db573a0b51b3d662d17a4113fce9ed50f7) Co-authored-by: Claude Opus 5 (1M context) --- .../networkGraph/NetworkGraphOpDesc.scala | 4 +- .../networkGraph/NetworkGraphOpDescSpec.scala | 103 ++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDescSpec.scala diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala index 4a5ea6725e6..06939671656 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDesc.scala @@ -94,7 +94,9 @@ class NetworkGraphOpDesc extends PythonOperatorDescriptor { | if not table.empty: | sources = table[$source] | destinations = table[$destination] - | nodes = set(sources + destinations) + | # Union of the two columns, in first-appearance order. Adding the + | # Series pairs them off element-wise; a set reorders per run. + | nodes = list(dict.fromkeys(pd.concat([sources, destinations]).tolist())) | G = nx.Graph() | for node in nodes: | G.add_node(node) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDescSpec.scala new file mode 100644 index 00000000000..fc666c61abd --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/networkGraph/NetworkGraphOpDescSpec.scala @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.visualization.networkGraph + +import org.scalatest.BeforeAndAfter +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.util.Base64 + +class NetworkGraphOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { + + var opDesc: NetworkGraphOpDesc = _ + + before { + opDesc = new NetworkGraphOpDesc() + } + + private def b64(s: String): String = + Base64.getEncoder.encodeToString(s.getBytes(StandardCharsets.UTF_8)) + + private def carries(output: String, name: String): Boolean = + output.contains(name) || output.contains(b64(name)) + + private def fieldPart(msg: String): String = + msg.toLowerCase.replace("cannot be empty", "") + + it should "throw AssertionError naming the Source Column when both fields are empty" in { + val ex = intercept[AssertionError](opDesc.manipulateTable()) + ex.getMessage should not be null + ex.getMessage should include("cannot be empty") + fieldPart(ex.getMessage) should include("source") + } + + it should "throw AssertionError naming the Destination Column when only source is set" in { + opDesc.source = "from_node" + val ex = intercept[AssertionError](opDesc.manipulateTable()) + ex.getMessage should not be null + ex.getMessage should include("cannot be empty") + fieldPart(ex.getMessage) should include("destination") + } + + it should "throw AssertionError naming the Source Column when only destination is set" in { + opDesc.destination = "to_node" + val ex = intercept[AssertionError](opDesc.manipulateTable()) + ex.getMessage should not be null + ex.getMessage should include("cannot be empty") + fieldPart(ex.getMessage) should include("source") + } + + it should "render both configured columns when source and destination are set" in { + opDesc.source = "from_node" + opDesc.destination = "to_node" + val plain = opDesc.manipulateTable().plain + assert(carries(plain, "from_node")) + assert(carries(plain, "to_node")) + plain should include("dropna") + } + + it should "generate python code carrying source, destination, and title" in { + opDesc.source = "from_node" + opDesc.destination = "to_node" + opDesc.title = "My Graph" + val code = opDesc.generatePythonCode() + assert(carries(code, "from_node")) + assert(carries(code, "to_node")) + assert(carries(code, "My Graph")) + code should include("class ProcessTableOperator(UDFTableOperator)") + } + + it should "build the node set as a union rather than by adding the two columns" in { + opDesc.source = "from_node" + opDesc.destination = "to_node" + val code = opDesc.generatePythonCode() + + // `sources + destinations` is element-wise on two Series, so it glued each + // source to its destination and those strings entered the graph as nodes. + code should not include "set(sources + destinations)" + code should include("pd.concat([sources, destinations])") + + // Ordered de-duplication, not a set: a set iterates strings in an order that + // varies between processes, which would move the nodes from run to run. + code should include("dict.fromkeys") + } +}