@@ -29,11 +29,46 @@ import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema}
2929import org .apache .texera .amber .pybuilder .PythonTemplateBuilder .PythonTemplateBuilderStringContext
3030import org .apache .texera .amber .pybuilder .PyStringTypes .EncodableString
3131import org .apache .texera .amber .core .workflow .{InputPort , OutputPort , PortIdentity }
32- import org .apache .texera .amber .operator .PythonOperatorDescriptor
33- import org .apache .texera .amber .operator .metadata .annotations .{AutofillAttributeName , HideAnnotation }
32+ import org .apache .texera .amber .operator .{PythonOperatorDescriptor , StandaloneCodeGenerator }
33+ import org .apache .texera .amber .operator .metadata .annotations .{
34+ AutofillAttributeName ,
35+ HideAnnotation ,
36+ SampleColumn
37+ }
3438import org .apache .texera .amber .operator .metadata .{OperatorGroupConstants , OperatorInfo }
39+ import org .apache .texera .amber .pybuilder .PythonTemplateBuilder .pyStringLiteral
3540
36- class MachineLearningScorerOpDesc extends PythonOperatorDescriptor {
41+ // Whichever metric list the task shows must hold at least one metric: an empty
42+ // one renders as `metric_list = ['']` and the metric lookup fails on the empty
43+ // name. The rule is conditional because only one of the two lists is visible at
44+ // a time, and a flat `required` on both can never be satisfied.
45+ @ JsonSchemaInject (json = """
46+ {
47+ "allOf": [
48+ {
49+ "if": {
50+ "required": ["isRegression"],
51+ "properties": { "isRegression": { "const": false } }
52+ },
53+ "then": {
54+ "required": ["classificationFlag"],
55+ "properties": { "classificationFlag": { "minItems": 1 } }
56+ }
57+ },
58+ {
59+ "if": {
60+ "required": ["isRegression"],
61+ "properties": { "isRegression": { "const": true } }
62+ },
63+ "then": {
64+ "required": ["regressionFlag"],
65+ "properties": { "regressionFlag": { "minItems": 1 } }
66+ }
67+ }
68+ ]
69+ }
70+ """ )
71+ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator {
3772 @ JsonProperty (required = true , defaultValue = " false" )
3873 @ JsonSchemaTitle (" Regression" )
3974 @ JsonPropertyDescription (
@@ -45,12 +80,16 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor {
4580 @ JsonSchemaTitle (" Actual Value" )
4681 @ JsonPropertyDescription (" Specify the label attribute" )
4782 @ AutofillAttributeName
83+ // Test-only steering: the pair has to be one label read twice, and the
84+ // first-unused pick names two unrelated columns.
85+ @ SampleColumn (" a\" b\\ c_species" )
4886 var actualValueColumn : EncodableString = " "
4987
5088 @ JsonProperty (required = true )
5189 @ JsonSchemaTitle (" Predicted Value" )
5290 @ JsonPropertyDescription (" Specify the attribute generated by the model" )
5391 @ AutofillAttributeName
92+ @ SampleColumn (" a\" b\\ c_species_pred" )
5493 var predictValueColumn : EncodableString = " "
5594
5695 @ JsonProperty (required = false , value = " classificationFlag" )
@@ -85,9 +124,61 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor {
85124 inputPorts = List (InputPort ()),
86125 outputPorts = List (OutputPort ())
87126 )
127+
128+ /** The two scored columns hold one quantity measured twice, so they have to be
129+ * comparable. Left to sklearn, a mismatched pair either raises `Mix of label
130+ * input types` from deep inside a metric or, for Accuracy, quietly scores 0 —
131+ * neither of which points back at the two columns the user picked.
132+ *
133+ * Types only, not values: a DOUBLE label column of 0.0 / 1.0 is a legitimate
134+ * classification target even though a continuous one is not, and the schema
135+ * cannot tell the two apart.
136+ */
137+ private def validateScoredColumns (inputSchemas : Map [PortIdentity , Schema ]): Unit = {
138+ val numeric = Set (AttributeType .INTEGER , AttributeType .LONG , AttributeType .DOUBLE )
139+ // Blank names are what the form reports before the user has picked; the
140+ // schema's own `required` already says so, and saying it twice would put a
141+ // second message on a half-filled operator.
142+ if (actualValueColumn.isEmpty || predictValueColumn.isEmpty) return
143+ inputSchemas.get(operatorInfo.inputPorts.head.id).foreach { schema =>
144+ Seq ((" Actual Value" , actualValueColumn), (" Predicted Value" , predictValueColumn))
145+ .foreach {
146+ case (label, column) =>
147+ if (! schema.containsAttribute(column))
148+ throw new RuntimeException (s " $label column ' $column' is not in the input table " )
149+ }
150+ val actualType = schema.getAttribute(actualValueColumn).getType
151+ val predictType = schema.getAttribute(predictValueColumn).getType
152+ if (isRegression) {
153+ Seq (
154+ (" Actual Value" , actualValueColumn, actualType),
155+ (" Predicted Value" , predictValueColumn, predictType)
156+ )
157+ .foreach {
158+ case (label, column, attributeType) =>
159+ if (! numeric.contains(attributeType))
160+ throw new RuntimeException (
161+ s " A regression metric needs a numeric $label column, but ' $column' is " +
162+ s " ${attributeType.getName}"
163+ )
164+ }
165+ } else {
166+ val comparable = (numeric.contains(actualType) && numeric.contains(predictType)) ||
167+ actualType == predictType
168+ if (! comparable)
169+ throw new RuntimeException (
170+ s " Actual Value ' $actualValueColumn' ( ${actualType.getName}) and Predicted Value " +
171+ s " ' $predictValueColumn' ( ${predictType.getName}) hold different kinds of label, " +
172+ " so a classification metric cannot compare them"
173+ )
174+ }
175+ }
176+ }
177+
88178 override def getOutputSchemas (
89179 inputSchemas : Map [PortIdentity , Schema ]
90180 ): Map [PortIdentity , Schema ] = {
181+ validateScoredColumns(inputSchemas)
91182 val metrics = if (isRegression) {
92183 regressionMetrics.map(_.getName())
93184 } else {
@@ -176,6 +267,14 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor {
176267 |
177268 | @overrides
178269 | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
270+ | # A row missing either value has nothing to score, and the metrics
271+ | # refuse the empty cell rather than passing over it.
272+ | table = table.dropna(subset=[ $actualValueColumn, $predictValueColumn])
273+ | # Nothing survived the drop. The metrics answer that badly and each in
274+ | # its own way: the regression ones raise from inside scikit-learn, and
275+ | # the classification ones return a NaN score, which reads as a result.
276+ | if table.empty:
277+ | raise ValueError("No rows left to score: every row is missing the actual value, the predicted value, or both.")
179278 | y_true = table[ $actualValueColumn]
180279 | y_pred = table[ $predictValueColumn]
181280 |
@@ -193,4 +292,58 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor {
193292 finalcode.encode
194293 }
195294
295+ override def generateStandaloneCode (): String = {
296+ val isRegressionStr = if (isRegression) " True" else " False"
297+ s """ import pandas as pd
298+ |from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, mean_squared_error, root_mean_squared_error, mean_absolute_error, r2_score
299+ |
300+ |def classification_metrics(y_true, y_pred, metric_list, labels):
301+ | if 'Accuracy' in metric_list:
302+ | labels.insert(0, 'Overall')
303+ | result = {metric: [None] * len(labels) for metric in metric_list}
304+ | metrics_func = {'Precision Score': precision_score, 'Recall Score': recall_score, 'F1 Score': f1_score}
305+ |
306+ | for metric in metric_list:
307+ | if metric == 'Accuracy':
308+ | result['Accuracy'][0] = round(accuracy_score(y_true, y_pred), 4)
309+ | else:
310+ | for i, label in enumerate(labels):
311+ | if label != 'Overall':
312+ | prediction = metrics_func[metric](y_true, y_pred, average=None, labels=[label])
313+ | result[metric][i] = round(prediction[0], 4)
314+ |
315+ | labels = ['class_' + str(label) if type(label) != str else label for label in labels]
316+ | result['Class'] = labels
317+ | # Class first and the metrics in the order they were picked, which is
318+ | # the column order this operator declares and so the order of the rows
319+ | # the workflow shows.
320+ | return pd.DataFrame(result)[['Class'] + metric_list]
321+ |
322+ |def regression_metrics(y_true, y_pred, metric_list):
323+ | result = dict()
324+ | metrics_func = {'MSE': mean_squared_error, 'RMSE': root_mean_squared_error, 'MAE': mean_absolute_error, 'R2': r2_score}
325+ | for metric in metric_list:
326+ | result[metric] = metrics_func[metric](y_true, y_pred)
327+ | return pd.DataFrame(result, index=[0])
328+ |
329+ |# Bound to a name of its own: the same frame can feed another branch of the
330+ |# plan, which must still see every row.
331+ |scored_df = in1df.dropna(subset=[ ${pyStringLiteral(actualValueColumn)}, ${pyStringLiteral(
332+ predictValueColumn
333+ )}])
334+ |# Nothing survived the drop. The metrics answer that badly and each in its own
335+ |# way: the regression ones raise from inside scikit-learn, and the
336+ |# classification ones return a NaN score, which reads as a result.
337+ |if scored_df.empty:
338+ | raise ValueError("No rows left to score: every row is missing the actual value, the predicted value, or both.")
339+ |y_true = scored_df[ ${pyStringLiteral(actualValueColumn)}]
340+ |y_pred = scored_df[ ${pyStringLiteral(predictValueColumn)}]
341+ |metric_list = [ ${getSelectedMetrics()}]
342+ |if $isRegressionStr:
343+ | out1df = regression_metrics(y_true, y_pred, metric_list)
344+ |else:
345+ | labels = list(set(y_true))
346+ | out1df = classification_metrics(y_true, y_pred, metric_list, labels) """ .stripMargin
347+ }
348+
196349}
0 commit comments