Skip to content

Commit 506397a

Browse files
kz930claude
andauthored
feat(machine-learning): export the Machine Learning Scorer (#8367)
### What changes were proposed in this PR? The scorer reads one label through two columns, the actual and the predicted, and reports the metrics a task admits. The export says which task it is scoring the same way the operator does, so the two paths select the same metrics from the same pair. Two behaviour fixes travel with it. It reports the settings it cannot score rather than failing deep inside the generated Python, and it scores the rows an empty cell leaves usable rather than ending the run on one. ### Any related issues, documentation, discussions? Part of #8325, 19 of 27; that issue lists the set in order. It needs #8327 for the trait, so it does not compile until that lands, and the rows these operators add to the verification runner follow with the harness rather than as whole new files here. Closes #8302, closes #8058. Closes #8421, the task this change is the whole of. ### How was this PR tested? Each operator asserts the block it emits in its own spec. Once the harness lands, each is also run through the engine and through its generated script, on every configuration its schema offers, and the two answers compared. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 5) --------- Co-authored-by: Claude Opus 5 (1M context) <[email protected]>
1 parent 6d0931c commit 506397a

2 files changed

Lines changed: 394 additions & 3 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala‎

Lines changed: 156 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,46 @@ import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema}
2929
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext
3030
import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString
3131
import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity}
32-
import org.apache.texera.amber.operator.PythonOperatorDescriptor
33-
import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, HideAnnotation}
32+
import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator}
33+
import org.apache.texera.amber.operator.metadata.annotations.{
34+
AutofillAttributeName,
35+
HideAnnotation,
36+
SampleColumn
37+
}
3438
import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo}
39+
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral
3540

36-
class MachineLearningScorerOpDesc extends PythonOperatorDescriptor {
41+
// Whichever metric list the task shows must hold at least one metric: an empty
42+
// one renders as `metric_list = ['']` and the metric lookup fails on the empty
43+
// name. The rule is conditional because only one of the two lists is visible at
44+
// a time, and a flat `required` on both can never be satisfied.
45+
@JsonSchemaInject(json = """
46+
{
47+
"allOf": [
48+
{
49+
"if": {
50+
"required": ["isRegression"],
51+
"properties": { "isRegression": { "const": false } }
52+
},
53+
"then": {
54+
"required": ["classificationFlag"],
55+
"properties": { "classificationFlag": { "minItems": 1 } }
56+
}
57+
},
58+
{
59+
"if": {
60+
"required": ["isRegression"],
61+
"properties": { "isRegression": { "const": true } }
62+
},
63+
"then": {
64+
"required": ["regressionFlag"],
65+
"properties": { "regressionFlag": { "minItems": 1 } }
66+
}
67+
}
68+
]
69+
}
70+
""")
71+
class MachineLearningScorerOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator {
3772
@JsonProperty(required = true, defaultValue = "false")
3873
@JsonSchemaTitle("Regression")
3974
@JsonPropertyDescription(
@@ -45,12 +80,16 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor {
4580
@JsonSchemaTitle("Actual Value")
4681
@JsonPropertyDescription("Specify the label attribute")
4782
@AutofillAttributeName
83+
// Test-only steering: the pair has to be one label read twice, and the
84+
// first-unused pick names two unrelated columns.
85+
@SampleColumn("a\"b\\c_species")
4886
var actualValueColumn: EncodableString = ""
4987

5088
@JsonProperty(required = true)
5189
@JsonSchemaTitle("Predicted Value")
5290
@JsonPropertyDescription("Specify the attribute generated by the model")
5391
@AutofillAttributeName
92+
@SampleColumn("a\"b\\c_species_pred")
5493
var predictValueColumn: EncodableString = ""
5594

5695
@JsonProperty(required = false, value = "classificationFlag")
@@ -85,9 +124,61 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor {
85124
inputPorts = List(InputPort()),
86125
outputPorts = List(OutputPort())
87126
)
127+
128+
/** The two scored columns hold one quantity measured twice, so they have to be
129+
* comparable. Left to sklearn, a mismatched pair either raises `Mix of label
130+
* input types` from deep inside a metric or, for Accuracy, quietly scores 0 —
131+
* neither of which points back at the two columns the user picked.
132+
*
133+
* Types only, not values: a DOUBLE label column of 0.0 / 1.0 is a legitimate
134+
* classification target even though a continuous one is not, and the schema
135+
* cannot tell the two apart.
136+
*/
137+
private def validateScoredColumns(inputSchemas: Map[PortIdentity, Schema]): Unit = {
138+
val numeric = Set(AttributeType.INTEGER, AttributeType.LONG, AttributeType.DOUBLE)
139+
// Blank names are what the form reports before the user has picked; the
140+
// schema's own `required` already says so, and saying it twice would put a
141+
// second message on a half-filled operator.
142+
if (actualValueColumn.isEmpty || predictValueColumn.isEmpty) return
143+
inputSchemas.get(operatorInfo.inputPorts.head.id).foreach { schema =>
144+
Seq(("Actual Value", actualValueColumn), ("Predicted Value", predictValueColumn))
145+
.foreach {
146+
case (label, column) =>
147+
if (!schema.containsAttribute(column))
148+
throw new RuntimeException(s"$label column '$column' is not in the input table")
149+
}
150+
val actualType = schema.getAttribute(actualValueColumn).getType
151+
val predictType = schema.getAttribute(predictValueColumn).getType
152+
if (isRegression) {
153+
Seq(
154+
("Actual Value", actualValueColumn, actualType),
155+
("Predicted Value", predictValueColumn, predictType)
156+
)
157+
.foreach {
158+
case (label, column, attributeType) =>
159+
if (!numeric.contains(attributeType))
160+
throw new RuntimeException(
161+
s"A regression metric needs a numeric $label column, but '$column' is " +
162+
s"${attributeType.getName}"
163+
)
164+
}
165+
} else {
166+
val comparable = (numeric.contains(actualType) && numeric.contains(predictType)) ||
167+
actualType == predictType
168+
if (!comparable)
169+
throw new RuntimeException(
170+
s"Actual Value '$actualValueColumn' (${actualType.getName}) and Predicted Value " +
171+
s"'$predictValueColumn' (${predictType.getName}) hold different kinds of label, " +
172+
"so a classification metric cannot compare them"
173+
)
174+
}
175+
}
176+
}
177+
88178
override def getOutputSchemas(
89179
inputSchemas: Map[PortIdentity, Schema]
90180
): Map[PortIdentity, Schema] = {
181+
validateScoredColumns(inputSchemas)
91182
val metrics = if (isRegression) {
92183
regressionMetrics.map(_.getName())
93184
} else {
@@ -176,6 +267,14 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor {
176267
|
177268
| @overrides
178269
| def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
270+
| # A row missing either value has nothing to score, and the metrics
271+
| # refuse the empty cell rather than passing over it.
272+
| table = table.dropna(subset=[$actualValueColumn, $predictValueColumn])
273+
| # Nothing survived the drop. The metrics answer that badly and each in
274+
| # its own way: the regression ones raise from inside scikit-learn, and
275+
| # the classification ones return a NaN score, which reads as a result.
276+
| if table.empty:
277+
| raise ValueError("No rows left to score: every row is missing the actual value, the predicted value, or both.")
179278
| y_true = table[$actualValueColumn]
180279
| y_pred = table[$predictValueColumn]
181280
|
@@ -193,4 +292,58 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor {
193292
finalcode.encode
194293
}
195294

295+
override def generateStandaloneCode(): String = {
296+
val isRegressionStr = if (isRegression) "True" else "False"
297+
s"""import pandas as pd
298+
|from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, mean_squared_error, root_mean_squared_error, mean_absolute_error, r2_score
299+
|
300+
|def classification_metrics(y_true, y_pred, metric_list, labels):
301+
| if 'Accuracy' in metric_list:
302+
| labels.insert(0, 'Overall')
303+
| result = {metric: [None] * len(labels) for metric in metric_list}
304+
| metrics_func = {'Precision Score': precision_score, 'Recall Score': recall_score, 'F1 Score': f1_score}
305+
|
306+
| for metric in metric_list:
307+
| if metric == 'Accuracy':
308+
| result['Accuracy'][0] = round(accuracy_score(y_true, y_pred), 4)
309+
| else:
310+
| for i, label in enumerate(labels):
311+
| if label != 'Overall':
312+
| prediction = metrics_func[metric](y_true, y_pred, average=None, labels=[label])
313+
| result[metric][i] = round(prediction[0], 4)
314+
|
315+
| labels = ['class_' + str(label) if type(label) != str else label for label in labels]
316+
| result['Class'] = labels
317+
| # Class first and the metrics in the order they were picked, which is
318+
| # the column order this operator declares and so the order of the rows
319+
| # the workflow shows.
320+
| return pd.DataFrame(result)[['Class'] + metric_list]
321+
|
322+
|def regression_metrics(y_true, y_pred, metric_list):
323+
| result = dict()
324+
| metrics_func = {'MSE': mean_squared_error, 'RMSE': root_mean_squared_error, 'MAE': mean_absolute_error, 'R2': r2_score}
325+
| for metric in metric_list:
326+
| result[metric] = metrics_func[metric](y_true, y_pred)
327+
| return pd.DataFrame(result, index=[0])
328+
|
329+
|# Bound to a name of its own: the same frame can feed another branch of the
330+
|# plan, which must still see every row.
331+
|scored_df = in1df.dropna(subset=[${pyStringLiteral(actualValueColumn)}, ${pyStringLiteral(
332+
predictValueColumn
333+
)}])
334+
|# Nothing survived the drop. The metrics answer that badly and each in its own
335+
|# way: the regression ones raise from inside scikit-learn, and the
336+
|# classification ones return a NaN score, which reads as a result.
337+
|if scored_df.empty:
338+
| raise ValueError("No rows left to score: every row is missing the actual value, the predicted value, or both.")
339+
|y_true = scored_df[${pyStringLiteral(actualValueColumn)}]
340+
|y_pred = scored_df[${pyStringLiteral(predictValueColumn)}]
341+
|metric_list = [${getSelectedMetrics()}]
342+
|if $isRegressionStr:
343+
| out1df = regression_metrics(y_true, y_pred, metric_list)
344+
|else:
345+
| labels = list(set(y_true))
346+
| out1df = classification_metrics(y_true, y_pred, metric_list, labels)""".stripMargin
347+
}
348+
196349
}

0 commit comments

Comments
 (0)