Skip to content

Commit f9835ba

Browse files
kz930claude
andcommitted
fix(operator): read the model port row by row in the exported script
Sklearn Testing scored with the first model the port carried and wrote that one number down every row, where the executor scores each model row on its own. Sklearn Prediction read the first row as well, where the executor keeps the last model the port hands it. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent ad54f24 commit f9835ba

4 files changed

Lines changed: 82 additions & 11 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnPredictionOpDesc.scala‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,21 +136,31 @@ class SklearnPredictionOpDesc extends PythonOperatorDescriptor with StandaloneCo
136136
| out1df.loc[_complete, $resultLit] = _predicted""".stripMargin
137137
}
138138

139+
/** Python that takes the model the executor predicts with.
140+
*
141+
* The executor keeps the model of every row the model port hands it, each one
142+
* overwriting the last, so the model it predicts with is the one on the final
143+
* row. Reading the first row instead would answer with a different model
144+
* whenever that port carries more than one.
145+
*/
146+
private def takeTheModelTheExecutorKeeps(modelLit: String): String =
147+
s"model = in1df[$modelLit].iloc[-1]"
148+
139149
override def generateStandaloneCode(): String = {
140150
val modelLit = pyStringLiteral(model)
141151
val resultLit = pyStringLiteral(resultAttribute)
142152
if (groundTruthAttribute.nonEmpty) {
143153
s"""from sklearn.pipeline import Pipeline
144154
|
145-
|model = in1df[$modelLit].iloc[0]
155+
|${takeTheModelTheExecutorKeeps(modelLit)}
146156
|out1df = in2df.copy()
147157
|X = in2df.drop(${pyStringLiteral(groundTruthAttribute)}, axis=1)
148158
|$narrowToFittedFeatures
149159
|${predictLeavingIncompleteRowsEmpty(resultLit, asText = false)}""".stripMargin
150160
} else {
151161
s"""from sklearn.pipeline import Pipeline
152162
|
153-
|model = in1df[$modelLit].iloc[0]
163+
|${takeTheModelTheExecutorKeeps(modelLit)}
154164
|out1df = in2df.copy()
155165
|X = in2df
156166
|$narrowToFittedFeatures

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDesc.scala‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,6 @@ class SklearnTestingOpDesc
134134
val targetLit = pyStringLiteral(target)
135135
s"""from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, root_mean_squared_error, mean_absolute_error, r2_score
136136
|
137-
|model = in1df[$modelLit].iloc[0]
138137
|out1df = in1df.copy()
139138
|# The same drop the executor makes before it scores: the model arrives
140139
|# already fitted, so this operator cannot ask which estimator it holds and
@@ -150,16 +149,21 @@ class SklearnTestingOpDesc
150149
|# predict wants (n_samples, n_features). Squeezing the frame first collapsed
151150
|# a single feature, or a single surviving row, to one dimension, which
152151
|# scikit-learn rejects.
153-
|predictions = model.predict(X)
152+
|#
153+
|# The model port carries one model per row and the executor scores each of
154+
|# them on its own, so a row has to be given the score of the model it holds.
155+
|# One score written down the column would report the first model's answer
156+
|# for all of them.
157+
|predictions = [_model.predict(X) for _model in in1df[$modelLit]]
154158
|if $isRegressionStr:
155-
| out1df["R2"] = r2_score(Y, predictions)
156-
| out1df["RMSE"] = root_mean_squared_error(Y, predictions)
157-
| out1df["MAE"] = mean_absolute_error(Y, predictions)
159+
| out1df["R2"] = [r2_score(Y, _p) for _p in predictions]
160+
| out1df["RMSE"] = [root_mean_squared_error(Y, _p) for _p in predictions]
161+
| out1df["MAE"] = [mean_absolute_error(Y, _p) for _p in predictions]
158162
|else:
159-
| out1df["accuracy"] = round(accuracy_score(Y, predictions), 4)
160-
| out1df["f1"] = f1_score(Y, predictions, average="weighted")
161-
| out1df["precision"] = precision_score(Y, predictions, average="weighted")
162-
| out1df["recall"] = recall_score(Y, predictions, average="weighted")""".stripMargin
163+
| out1df["accuracy"] = [round(accuracy_score(Y, _p), 4) for _p in predictions]
164+
| out1df["f1"] = [f1_score(Y, _p, average="weighted") for _p in predictions]
165+
| out1df["precision"] = [precision_score(Y, _p, average="weighted") for _p in predictions]
166+
| out1df["recall"] = [recall_score(Y, _p, average="weighted") for _p in predictions]""".stripMargin
163167
}
164168

165169
}

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPredictionOpDescSpec.scala‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,22 @@ class SklearnPredictionOpDescSpec extends AnyFlatSpec with Matchers {
188188
}
189189
}
190190

191+
// The executor keeps the model of every row the model port hands it, each one
192+
// overwriting the last, so it predicts with the model on the final row. A model
193+
// port carrying more than one row is where reading the first row instead would
194+
// answer with a different model than the run did.
195+
it should "predict with the last model the model port carries, as the executor does" in {
196+
val d = new SklearnPredictionOpDesc
197+
d.model = "model"
198+
d.resultAttribute = "prediction"
199+
// The executor's own line, which holds one model rather than collecting them.
200+
d.generatePythonCode() should include("self.model = tuple_[")
201+
Seq("y", "").foreach { groundTruth =>
202+
d.groundTruthAttribute = groundTruth
203+
d.generateStandaloneCode() should include("model = in1df[\"model\"].iloc[-1]")
204+
}
205+
}
206+
191207
"SklearnPredictionOpDesc" should
192208
"round-trip its config fields through the polymorphic base" in {
193209
val d = new SklearnPredictionOpDesc

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDescSpec.scala‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,47 @@ class SklearnTestingOpDescSpec extends AnyFlatSpec with Matchers {
266266
}
267267
}
268268

269+
// The model port carries one model per row, and the executor takes each of
270+
// those rows on its own. Two models that answer differently are what tells a
271+
// per-row score from one score written down the column.
272+
it should "score each model row with the model that row holds" in {
273+
val python = resolvePython().getOrElse(cancel("No runnable python executable"))
274+
if (!canImport(python, "pandas, sklearn")) cancel(s"'$python' cannot import pandas and sklearn")
275+
276+
val op = new SklearnTestingOpDesc
277+
op.model = "model"
278+
op.target = "target"
279+
280+
val driver =
281+
s"""import pandas as pd
282+
|from sklearn.tree import DecisionTreeClassifier
283+
|
284+
|# Two models that disagree on every row: one was trained to answer 0,
285+
|# the other to answer 1. Against labels that are all 0 the first scores
286+
|# 1.0 and the second 0.0.
287+
|features = pd.DataFrame({"f1": [0, 1]})
288+
|answers_0 = DecisionTreeClassifier(random_state=0).fit(features, [0, 0])
289+
|answers_1 = DecisionTreeClassifier(random_state=0).fit(features, [1, 1])
290+
|
291+
|in1df = pd.DataFrame({"model": [answers_0, answers_1]})
292+
|in2df = pd.DataFrame({"f1": [0, 1], "target": [0, 0]})
293+
|${op.generateStandaloneCode()}
294+
|
295+
|print([round(float(a), 10) for a in out1df["accuracy"]])
296+
|""".stripMargin
297+
298+
val script = Files.createTempFile("sklearn-testing-two-models-", ".py")
299+
script.toFile.deleteOnExit()
300+
Files.write(script, driver.getBytes(StandardCharsets.UTF_8))
301+
val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start()
302+
val out = Source.fromInputStream(process.getInputStream).mkString
303+
process.waitFor(180, TimeUnit.SECONDS)
304+
withClue(s"python said:\n$out\nscript:\n$driver") {
305+
process.exitValue() shouldBe 0
306+
out.linesIterator.toSeq should contain("[1.0, 0.0]")
307+
}
308+
}
309+
269310
private def resolvePython(): Option[String] = {
270311
def fromConfig: Option[String] =
271312
Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption

0 commit comments

Comments
 (0)