Skip to content

Commit 6a101ad

Browse files
kz930claude
andauthored
feat(visualization): export the charts that draw over a table's columns (apache#8365)
### What changes were proposed in this PR? Sixteen plots whose shape comes from the columns they are pointed at: the bar, line and pie, the histograms, the scatter family, the box and violin, the strip and the ECDF, the heat map, the candlestick, the funnel and the dot plot. A line's series configuration and an error band's are one type extending the other, so they travel together. ### Any related issues, documentation, discussions? Part of apache#8325, 14 of 27; that issue lists the set in order. It needs apache#8327 for the trait, so it does not compile until that lands, and the rows these operators add to the verification runner follow with the harness rather than as whole new files here. Closes apache#8417, the task this change is the whole of. ### How was this PR tested? Each operator asserts the block it emits in its own spec. Once the harness lands, each is also run through the engine and through its generated script, on every configuration its schema offers, and the two answers compared. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 5) --------- Co-authored-by: Claude Opus 5 (1M context) <[email protected]>
1 parent 2351cb8 commit 6a101ad

19 files changed

Lines changed: 800 additions & 34 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/DotPlot/DotPlotOpDesc.scala‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,17 +22,21 @@ package org.apache.texera.amber.operator.visualization.DotPlot
2222
import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription}
2323
import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle
2424
import org.apache.texera.amber.core.tuple.{AttributeType, Schema}
25-
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext
25+
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{
26+
PythonTemplateBuilderStringContext,
27+
pyStringLiteral
28+
}
2629
import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString
2730
import org.apache.texera.amber.core.workflow.PortIdentity
2831
import org.apache.texera.amber.operator.PythonOperatorDescriptor
32+
import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode
2933
import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName
3034
import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo}
3135
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder
3236

3337
import javax.validation.constraints.NotNull
3438

35-
class DotPlotOpDesc extends PythonOperatorDescriptor {
39+
class DotPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode {
3640

3741
@JsonProperty(value = "Count Attribute", required = true)
3842
@JsonSchemaTitle("Count Attribute")
@@ -100,4 +104,41 @@ class DotPlotOpDesc extends PythonOperatorDescriptor {
100104
finalCode.encode
101105
}
102106

107+
// Output is an HTML chart, not a tabular DataFrame.
108+
// The translator skips it in the leaf-DataFrame print block.
109+
override def producesDataFrame(): Boolean = false
110+
111+
override def generateStandaloneCode(): String = {
112+
val countLit = pyStringLiteral(countAttribute)
113+
// The error page is written to the same file a plotted chart lands in, so a
114+
// reason for "no chart" is where the reader looks for the chart — printing it
115+
// to the terminal alone left that file absent. render_error's continuation
116+
// line keeps the runtime path's indentation, since the HTML is triple-quoted
117+
// and those spaces reach the browser.
118+
s"""def render_error(error_msg):
119+
| return '''<h1>DotPlot is not available.</h1>
120+
| <p>Reasons are: {} </p>
121+
| '''.format(error_msg)
122+
|
123+
|def fail(error_msg):
124+
| with open(outputHtml, "w", encoding="utf-8") as output:
125+
| output.write(render_error(error_msg))
126+
| print(f"Dot plot error: {error_msg}")
127+
|
128+
|if in1df.empty:
129+
| fail("Input table is empty.")
130+
|else:
131+
| df = in1df.groupby([$countLit])[$countLit].count().reset_index(name='counts')
132+
| if df.empty:
133+
| fail("No valid rows left (every row has at least 1 missing value).")
134+
| else:
135+
| fig = px.strip(df, x='counts', y=$countLit, orientation='h', color=$countLit,
136+
| color_discrete_sequence=px.colors.qualitative.Dark2)
137+
| fig.update_traces(marker=dict(size=12, line=dict(width=2, color='DarkSlateGrey')))
138+
| fig.update_layout(margin=dict(t=0, b=0, l=0, r=0))
139+
| fig.write_json(outputJson)
140+
| fig.write_html(outputHtml)
141+
| print("Dot plot saved to " + outputHtml)""".stripMargin
142+
}
143+
103144
}

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ScatterMatrixChart/ScatterMatrixChartOpDesc.scala‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.ScatterMatrixChart
2222
import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription}
2323
import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle}
2424
import org.apache.texera.amber.core.tuple.{AttributeType, Schema}
25-
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext
25+
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{
26+
PythonTemplateBuilderStringContext,
27+
pyStringLiteral
28+
}
2629
import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString
2730
import org.apache.texera.amber.core.workflow.PortIdentity
2831
import org.apache.texera.amber.operator.PythonOperatorDescriptor
32+
import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode
2933
import org.apache.texera.amber.operator.metadata.annotations.{
3034
AutofillAttributeName,
3135
AutofillAttributeNameList
@@ -46,7 +50,7 @@ import javax.validation.constraints.{NotEmpty, NotNull}
4650
}
4751
}
4852
""")
49-
class ScatterMatrixChartOpDesc extends PythonOperatorDescriptor {
53+
class ScatterMatrixChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode {
5054

5155
@JsonProperty(value = "Selected Attributes", required = true)
5256
@JsonSchemaTitle("Selected Attributes")
@@ -114,4 +118,20 @@ class ScatterMatrixChartOpDesc extends PythonOperatorDescriptor {
114118
finalcode.encode
115119
}
116120

121+
// Output is an HTML visualization, not a tabular DataFrame.
122+
// The translator skips it in the leaf-DataFrame print block.
123+
override def producesDataFrame(): Boolean = false
124+
125+
override def generateStandaloneCode(): String = {
126+
val dimensions = selectedAttributes.map(pyStringLiteral).mkString(", ")
127+
// No empty-input guard, matching generatePythonCode: it goes straight to
128+
// the figure, so an empty table raises out of px.scatter_matrix on both
129+
// paths. A guard here would only be a divergence — and the one removed
130+
// here printed to stdout without writing any output at all.
131+
s"""fig = px.scatter_matrix(in1df, dimensions=[$dimensions], color=${pyStringLiteral(color)})
132+
|fig.update_layout(margin=dict(t=0, b=0, l=0, r=0))
133+
|fig.write_json(outputJson)
134+
|fig.write_html(outputHtml)
135+
|print("Scatter matrix saved to " + outputHtml)""".stripMargin
136+
}
117137
}

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.barChart
2222
import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription}
2323
import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle}
2424
import org.apache.texera.amber.core.tuple.{AttributeType, Schema}
25-
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext
25+
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{
26+
PythonTemplateBuilderStringContext,
27+
pyStringLiteral
28+
}
2629
import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString
2730
import org.apache.texera.amber.core.workflow.PortIdentity
2831
import org.apache.texera.amber.operator.PythonOperatorDescriptor
32+
import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode
2933
import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName
3034
import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo}
3135
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder
@@ -42,7 +46,7 @@ import javax.validation.constraints.NotNull
4246
}
4347
}
4448
""")
45-
class BarChartOpDesc extends PythonOperatorDescriptor {
49+
class BarChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode {
4650

4751
@JsonProperty(value = "value", required = true)
4852
@JsonSchemaTitle("Value Column")
@@ -155,4 +159,56 @@ class BarChartOpDesc extends PythonOperatorDescriptor {
155159
finalCode.encode
156160
}
157161

162+
// Output is an HTML chart, not a tabular DataFrame.
163+
// The translator skips it in the leaf-DataFrame print block.
164+
override def producesDataFrame(): Boolean = false
165+
166+
override def generateStandaloneCode(): String = {
167+
val hasCategory = categoryColumn.nonEmpty && categoryColumn != "No Selection"
168+
val colorArg = if (hasCategory) pyStringLiteral(categoryColumn) else "None"
169+
val patternArg = if (pattern.nonEmpty) pyStringLiteral(pattern) else "None"
170+
val fieldsLit = pyStringLiteral(fields)
171+
val valueLit = pyStringLiteral(value)
172+
173+
val barArgs =
174+
if (horizontalOrientation)
175+
s"""y=$fieldsLit, x=$valueLit, color=$colorArg, pattern_shape=$patternArg, orientation='h'"""
176+
else
177+
s"""y=$valueLit, x=$fieldsLit, color=$colorArg, pattern_shape=$patternArg"""
178+
179+
// The error page is written to the same file a plotted chart lands in, so a
180+
// reason for "no chart" is where the reader looks for the chart — printing it
181+
// to the terminal alone left that file absent. render_error's continuation
182+
// line keeps the runtime path's indentation, since the HTML is triple-quoted
183+
// and those spaces reach the browser.
184+
s"""def render_error(error_msg):
185+
| return '''<h1>Bar chart is not available.</h1>
186+
| <p>Reason is: {} </p>
187+
| '''.format(error_msg)
188+
|
189+
|def fail(error_msg):
190+
| with open(outputHtml, "w", encoding="utf-8") as output:
191+
| output.write(render_error(error_msg))
192+
| print(f"Bar chart error: {error_msg}")
193+
|
194+
|if $fieldsLit == $valueLit:
195+
| fail("Fields should not have the same value.")
196+
|elif in1df.empty:
197+
| # Checked before the dropna, unlike the runtime path: an empty table read
198+
| # back from JSONL carries no columns at all, so naming them in `subset`
199+
| # would raise a KeyError instead of reporting the empty table.
200+
| fail("Table should not have any empty/null values or fields.")
201+
|else:
202+
| # Bound to a name of its own: the same frame can feed another branch
203+
| # of the plan, which must still see every row.
204+
| chart_df = in1df.dropna(subset=[$valueLit, $fieldsLit])
205+
| if chart_df.empty:
206+
| fail("Table should not have any empty/null values or fields.")
207+
| else:
208+
| fig = go.Figure(px.bar(chart_df, $barArgs))
209+
| fig.update_layout(margin=dict(l=0, r=0, t=0, b=0))
210+
| fig.write_json(outputJson)
211+
| fig.write_html(outputHtml)
212+
| print("Bar chart saved to " + outputJson + " and " + outputHtml)""".stripMargin
213+
}
158214
}

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.boxViolinPlot
2222
import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription, JsonPropertyOrder}
2323
import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle}
2424
import org.apache.texera.amber.core.tuple.{AttributeType, Schema}
25-
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext
25+
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{
26+
PythonTemplateBuilderStringContext,
27+
pyStringLiteral
28+
}
2629
import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString
2730
import org.apache.texera.amber.core.workflow.PortIdentity
2831
import org.apache.texera.amber.operator.PythonOperatorDescriptor
32+
import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode
2933
import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName
3034
import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo}
3135
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder
@@ -42,7 +46,7 @@ import javax.validation.constraints.NotNull
4246
}
4347
}
4448
""")
45-
class BoxViolinPlotOpDesc extends PythonOperatorDescriptor {
49+
class BoxViolinPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode {
4650

4751
@JsonProperty(value = "value", required = true)
4852
@JsonSchemaTitle("Value Column")
@@ -152,4 +156,56 @@ class BoxViolinPlotOpDesc extends PythonOperatorDescriptor {
152156
finalCode.encode
153157
}
154158

159+
// Output is a Plotly visualization, not a tabular DataFrame.
160+
// The translator skips it in the leaf-DataFrame print block.
161+
override def producesDataFrame(): Boolean = false
162+
163+
override def generateStandaloneCode(): String = {
164+
val horizontal = if (horizontalOrientation) "True" else "False"
165+
val violin = if (violinPlot) "True" else "False"
166+
val quartileMethod =
167+
if (quartileType == null) "linear" else quartileType.getQuartiletype
168+
val valueLit = pyStringLiteral(value)
169+
170+
// The error page is written to the same file a plotted chart lands in, so a
171+
// reason for "no chart" is where the reader looks for the chart — printing it
172+
// to the terminal alone left that file absent. render_error's continuation
173+
// line keeps the runtime path's indentation, since the HTML is triple-quoted
174+
// and those spaces reach the browser.
175+
s"""def render_error(error_msg):
176+
| return '''<h1>Box/Violin Plot is not available.</h1>
177+
| <p>Reason is: {} </p>
178+
| '''.format(error_msg)
179+
|
180+
|def fail(error_msg):
181+
| with open(outputHtml, "w", encoding="utf-8") as output:
182+
| output.write(render_error(error_msg))
183+
| print(f"Box/Violin Plot error: {error_msg}")
184+
|
185+
|if in1df.empty:
186+
| fail("input table is empty.")
187+
|else:
188+
| # Bound to a name of its own: the same frame can feed another branch
189+
| # of the plan, which must still see every row.
190+
| chart_df = in1df.dropna(subset=[$valueLit])
191+
| if chart_df.empty:
192+
| fail("value column contains only non-positive numbers or nulls.")
193+
| else:
194+
| if $violin:
195+
| if $horizontal:
196+
| fig = px.violin(chart_df, x=$valueLit, box=True, points='all')
197+
| else:
198+
| fig = px.violin(chart_df, y=$valueLit, box=True, points='all')
199+
| else:
200+
| if $horizontal:
201+
| fig = px.box(chart_df, x=$valueLit, boxmode="overlay", points='all')
202+
| else:
203+
| fig = px.box(chart_df, y=$valueLit, boxmode="overlay", points='all')
204+
| fig.update_traces(quartilemethod=${pyStringLiteral(quartileMethod)}, col=1)
205+
| fig.update_layout(margin=dict(t=0, b=0, l=0, r=0))
206+
| fig.write_json(outputJson)
207+
| fig.write_html(outputHtml)
208+
| print("Box/Violin Plot saved to " + outputJson + " and " + outputHtml)""".stripMargin
209+
}
210+
155211
}

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.bubbleChart
2222
import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription}
2323
import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle}
2424
import org.apache.texera.amber.core.tuple.{AttributeType, Schema}
25-
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext
25+
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{
26+
PythonTemplateBuilderStringContext,
27+
pyStringLiteral
28+
}
2629
import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString
2730
import org.apache.texera.amber.core.workflow.PortIdentity
2831
import org.apache.texera.amber.operator.PythonOperatorDescriptor
32+
import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode
2933
import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName
3034
import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo}
3135
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder
@@ -49,7 +53,7 @@ import javax.validation.constraints.NotNull
4953
}
5054
}
5155
""")
52-
class BubbleChartOpDesc extends PythonOperatorDescriptor {
56+
class BubbleChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode {
5357

5458
@JsonProperty(value = "xValue", required = true)
5559
@JsonSchemaTitle("X-Column")
@@ -157,4 +161,56 @@ class BubbleChartOpDesc extends PythonOperatorDescriptor {
157161
|"""
158162
finalCode.encode
159163
}
164+
165+
// Output is an HTML chart, not a tabular DataFrame.
166+
// The translator skips it in the leaf-DataFrame print block.
167+
override def producesDataFrame(): Boolean = false
168+
169+
override def generateStandaloneCode(): String = {
170+
// Same rule as the platform path: an unset column is "no color" even with the
171+
// toggle on, else px.scatter(color='') fails.
172+
val colorArg =
173+
if (enableColor && colorCategory.nonEmpty) s""", color=${pyStringLiteral(colorCategory)}"""
174+
else ""
175+
val xLit = pyStringLiteral(xValue)
176+
val yLit = pyStringLiteral(yValue)
177+
val zLit = pyStringLiteral(zValue)
178+
179+
// The error page is written to the same file a plotted chart lands in, so a
180+
// reason for "no chart" is where the reader looks for the chart — printing it
181+
// to the terminal alone left that file absent. The heading is the
182+
// runtime path's, TreeMap and all, so both paths say the same thing.
183+
// render_error's continuation line keeps the runtime path's indentation, since
184+
// the HTML is triple-quoted and those spaces reach the browser.
185+
s"""def render_error(error_msg):
186+
| return '''<h1>TreeMap is not available.</h1>
187+
| <p>Reasons are: {} </p>
188+
| '''.format(error_msg)
189+
|
190+
|def fail(error_msg):
191+
| with open(outputHtml, "w", encoding="utf-8") as output:
192+
| output.write(render_error(error_msg))
193+
| print(f"Bubble chart error: {error_msg}")
194+
|
195+
|if in1df.empty:
196+
| fail("Input table is empty.")
197+
|else:
198+
| # Bound to a name of its own: the same frame can feed another branch
199+
| # of the plan, which must still see every row.
200+
| chart_df = in1df.dropna(subset=[$xLit, $yLit, $zLit])
201+
| if chart_df.empty:
202+
| fail("No valid rows left (every row has at least 1 missing value).")
203+
| else:
204+
| fig = go.Figure(px.scatter(
205+
| chart_df,
206+
| x=$xLit,
207+
| y=$yLit,
208+
| size=$zLit,
209+
| size_max=100$colorArg
210+
| ))
211+
| fig.update_layout(margin=dict(l=0, r=0, b=0, t=0))
212+
| fig.write_json(outputJson)
213+
| fig.write_html(outputHtml)
214+
| print("Bubble chart saved to " + outputHtml)""".stripMargin
215+
}
160216
}

0 commit comments

Comments
 (0)