Skip to content

Commit 8c41b1d

Browse files
kz930claude
andcommitted
Merge feat/verify-derive-a-configuration into feat/verify-sweep-configuration-variants
Co-Authored-By: Claude Opus 5.5 <[email protected]>
2 parents c50bb9c + ab851d6 commit 8c41b1d

2 files changed

Lines changed: 192 additions & 33 deletions

File tree

‎workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGenerator.scala‎

Lines changed: 118 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -103,12 +103,18 @@ object ConfigGenerator {
103103
}
104104
}
105105

106-
/**
107-
* Like [[generate]], but also sweeps every enum field: returns the base
108-
* config plus one variant per non-default enum value (one enum flipped at a
109-
* time — linear, NOT the combinatorial product). Lets the runner exercise
110-
* each enum branch (e.g. LineChart's line mode = line / dots / line+dots)
111-
* instead of only the default. The label identifies the flipped value.
106+
/** Every column at `port` a list knob may hold: the ones its `attributeTypeRules`
107+
* admits, or all of them when there is no rule, minus the ones a single-column
108+
* knob beside it already took. A rule that matches nothing is refused.
109+
* The same fill for a required and an optional field, so the two cannot drift.
110+
*
111+
* Subtracting `used` is what [[resolveColumn]] already does for a single-column
112+
* knob, and the two knobs answer to the same rule: a column means something
113+
* different to each field that names it, so handing one column to two of them
114+
* writes a config nobody would. Radar Chart's name column arrived inside its own
115+
* value columns that way, and sklearn's label inside the features it is fitted
116+
* against. Not marked used in turn, since a list knob wants every column its rule
117+
* admits and marking them would leave a later single-column knob nothing to take.
112118
*/
113119
def generateVariants(
114120
opClass: Class[_ <: LogicalOp],
@@ -810,24 +816,55 @@ object ConfigGenerator {
810816
used: collection.Set[(Int, String)],
811817
siblings: JsonNode
812818
): Either[String, JsonNode] =
813-
columnNames(schemas, port).map { names =>
814-
val filtered = allowedTypes(f, siblings) match {
819+
columnNames(schemas, port).flatMap { names =>
820+
val admitted: Either[String, Seq[String]] = columnTypes(f, schemas, siblings) match {
815821
case Some(types) =>
816822
val matching = schemas
817823
.get(port)
818824
.map(_.getAttributes.filter(a => types.contains(a.getType)).map(_.getName))
819825
.getOrElse(Seq.empty)
820-
if (matching.nonEmpty) matching else names
821-
case None => names
826+
if (matching.nonEmpty) Right(matching)
827+
else Left(noAllowedColumn(f, schemas, port, types))
828+
case None => Right(names)
829+
}
830+
admitted.map { columns =>
831+
val free = columns.filterNot(name => used.contains((port, name)))
832+
val arr = objectMapper.createArrayNode()
833+
(if (free.nonEmpty) free else columns).foreach(arr.add)
834+
arr
822835
}
823-
val free = filtered.filterNot(name => used.contains((port, name)))
824-
val arr = objectMapper.createArrayNode()
825-
(if (free.nonEmpty) free else filtered).foreach(arr.add)
826-
arr
827836
}
828837

829838
private def pointerOf(f: Field, path: String): String = s"$path/${jsonNameOf(f)}"
830839

840+
/** Why no column at `port` can fill `f`.
841+
*
842+
* Taking a column the rule does not admit wrote a configuration nobody would:
843+
* the operator states which types the field takes, and a generator that reads
844+
* that and then ignores it is testing something the operator does not
845+
* describe. Nothing reaches this today, every type a rule names having a
846+
* column in every fixture, so its whole job is to say so the day one of them
847+
* narrows, rather than to quietly choose another column and report the
848+
* operator as covered.
849+
*/
850+
private def noAllowedColumn(
851+
f: Field,
852+
schemas: Map[Int, Schema],
853+
port: Int,
854+
types: Set[AttributeType]
855+
): String = {
856+
val present = schemas
857+
.get(port)
858+
.map(_.getAttributes.map(a => s"${a.getName}:${a.getType}").mkString(", "))
859+
.getOrElse("no schema")
860+
if (types.isEmpty)
861+
s"${jsonNameOf(f)}'s type rules leave no type a column at port $port could have " +
862+
s"(port holds: $present)"
863+
else
864+
s"${jsonNameOf(f)} accepts ${types.mkString("/")}, and no column at port $port " +
865+
s"has one of those types (port holds: $present)"
866+
}
867+
831868
/** The key a field carries in the config JSON. */
832869
private def jsonNameOf(f: Field): String =
833870
Option(f.getAnnotation(classOf[JsonProperty]))
@@ -844,8 +881,8 @@ object ConfigGenerator {
844881
Option(f.getAnnotation(classOf[JsonProperty])).exists(_.required) ||
845882
requiredUnder(scope, siblings).contains(jsonNameOf(f))
846883

847-
/** The nested-row type a field holds — its `List[Row]` / `Option[Row]` element
848-
* type, or its own type when the field IS the row. `None` for a scalar field.
884+
/** The first value a field offers under `examples` — a legal sample the operator
885+
* states itself, so nothing here has to invent one.
849886
*/
850887
private def rowType(f: Field): Option[Class[_]] = {
851888
val t = f.getType
@@ -950,9 +987,8 @@ object ConfigGenerator {
950987
.map(_.path(key))
951988
.filterNot(_.isMissingNode)
952989

953-
/** A type whose value the user types in freely — the fills of
954-
* [[optionalScalarFills]]. Boolean is excluded: the enum sweep already covers
955-
* both of its values.
990+
/** Maps each registered operator class to its `operatorType` discriminator,
991+
* read from [[LogicalOp]]'s `@JsonSubTypes` (the same registry Jackson uses).
956992
*/
957993
private def isFreeScalar(t: Class[_]): Boolean =
958994
t == classOf[String] || t == classOf[Int] || t == classOf[java.lang.Integer] ||
@@ -1857,14 +1893,14 @@ object ConfigGenerator {
18571893
)
18581894
}
18591895
case None =>
1860-
allowedTypes(f, siblings) match {
1896+
columnTypes(f, schemas, siblings) match {
18611897
case Some(types) =>
18621898
schemas
18631899
.get(port)
18641900
.map(_.getAttributes.filter(a => types.contains(a.getType)).map(_.getName)) match {
18651901
case Some(cols) if cols.nonEmpty =>
18661902
Right(take(cols.find(c => !used.contains((port, c))).getOrElse(cols.head)))
1867-
case _ => firstUnused(schemas, port, used) // no type-matching column; fall back
1903+
case _ => Left(noAllowedColumn(f, schemas, port, types))
18681904
}
18691905
case None => firstUnused(schemas, port, used)
18701906
}
@@ -1883,11 +1919,8 @@ object ConfigGenerator {
18831919
* means the ones that do all bind, so their sets intersect.
18841920
*/
18851921
private def allowedTypes(f: Field, siblings: JsonNode): Option[Set[AttributeType]] =
1886-
Option(f.getDeclaringClass.getAnnotation(classOf[JsonSchemaInject]))
1887-
.map(_.json)
1888-
.filter(_.nonEmpty)
1889-
.flatMap(js => Try(objectMapper.readTree(js)).toOption)
1890-
.map(_.path("attributeTypeRules").path(jsonNameOf(f)))
1922+
Some(typeRules(f.getDeclaringClass).path(jsonNameOf(f)))
1923+
.filterNot(_.isMissingNode)
18911924
.flatMap { rule =>
18921925
val branches =
18931926
if (rule.path("allOf").isArray) rule.path("allOf").elements().asScala.toSeq
@@ -1898,6 +1931,61 @@ object ConfigGenerator {
18981931
bound.reduceOption(_ intersect _).filter(_.nonEmpty)
18991932
}
19001933

1934+
/** A class's `attributeTypeRules`, keyed by field JSON name; a missing node when
1935+
* it states none.
1936+
*/
1937+
private def typeRules(clazz: Class[_]): JsonNode =
1938+
Option(clazz.getAnnotation(classOf[JsonSchemaInject]))
1939+
.map(_.json)
1940+
.filter(_.nonEmpty)
1941+
.flatMap(js => Try(objectMapper.readTree(js)).toOption)
1942+
.map(_.path("attributeTypeRules"))
1943+
.getOrElse(objectMapper.missingNode())
1944+
1945+
/** Every type rule on `f`, the `enum` it names and the `$data` link, intersected.
1946+
* Empty when the rules leave no type, which refuses the field rather than letting
1947+
* it ignore them; `None` when it has none.
1948+
*/
1949+
private def columnTypes(
1950+
f: Field,
1951+
schemas: Map[Int, Schema],
1952+
siblings: JsonNode
1953+
): Option[Set[AttributeType]] =
1954+
(allowedTypes(f, siblings).toSeq ++ linkedTypes(f, schemas, siblings))
1955+
.reduceOption(_ intersect _)
1956+
1957+
/** What `"const": {"$data": "other"}` lets `f` hold: the type of the column the
1958+
* other knob names, which is how a join says its two keys must be alike.
1959+
*
1960+
* The link binds both fields, whichever one states it, since the generator fills
1961+
* them in declaration order and Hash Join states it on the knob filled first. Once
1962+
* the other knob holds a column, only that column's type; before then, only a type
1963+
* its port also has, so the pair can still be completed.
1964+
*/
1965+
private def linkedTypes(
1966+
f: Field,
1967+
schemas: Map[Int, Schema],
1968+
siblings: JsonNode
1969+
): Seq[Set[AttributeType]] = {
1970+
val clazz = f.getDeclaringClass
1971+
val name = jsonNameOf(f)
1972+
val partners = typeRules(clazz).fields().asScala.toSeq.flatMap { rule =>
1973+
val ref = rule.getValue.path("const").path("$data").asText("")
1974+
if (rule.getKey == name && ref.nonEmpty) Some(ref)
1975+
else if (ref == name) Some(rule.getKey)
1976+
else None
1977+
}
1978+
for {
1979+
other <- partners
1980+
field <- configFields(clazz).find(jsonNameOf(_) == other).toSeq
1981+
spec <- autofillSpec(field).toSeq
1982+
schema <- schemas.get(spec.port).toSeq
1983+
} yield Option(siblings.get(other)).map(_.asText).filter(schema.containsAttribute) match {
1984+
case Some(col) => Set(schema.getAttribute(col).getType)
1985+
case None => schema.getAttributes.map(_.getType).toSet
1986+
}
1987+
}
1988+
19011989
/** The [[AttributeType]]s an `enum` array names, or `None` if it names none. */
19021990
private def typeSet(enumNode: JsonNode): Option[Set[AttributeType]] =
19031991
if (!enumNode.isArray) None
@@ -1968,13 +2056,10 @@ object ConfigGenerator {
19682056
).toOption.map(s => SchemaScope(s, s)).getOrElse(empty)
19692057
}
19702058

1971-
/** What filling a field needs beyond the field itself: the input schemas a column
1972-
* picker resolves against, the columns already spoken for, and the row count a
1973-
* range-less number is sized from. Carried into the enum walk so that a value which
1974-
* makes a field apply can fill it the way the base pass would have.
1975-
*
1976-
* Empty for a caller sweeping an already-configured operator: such a config states
1977-
* both sides of a conditional itself, so nothing there is left to fill.
2059+
/** The fields an object's conditional `allOf` requires of `row` as it stands. The
2060+
* condition is JSON Schema's own `properties`/`const`, not the `valEnum` form a
2061+
* Texera rule uses, because this one is read by the validator rather than by the
2062+
* form.
19782063
*/
19792064
private final case class FillContext(
19802065
schemas: Map[Int, Schema] = Map.empty,

‎workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/ConfigGeneratorSpec.scala‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema}
2323
import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc
2424
import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc
2525
import org.apache.texera.amber.operator.intersect.IntersectOpDesc
26+
import org.apache.texera.amber.operator.intervalJoin.IntervalJoinOpDesc
2627
import org.apache.texera.amber.operator.regex.RegexOpDesc
2728
import org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpDesc
2829
import org.apache.texera.amber.operator.source.scan.csvOld.CSVOldScanSourceOpDesc
@@ -64,6 +65,49 @@ class ConfigGeneratorSpec extends AnyFlatSpec with Matchers {
6465
op.joinType should not be null
6566
}
6667

68+
// Hash Join's left key takes the right key's type through `$data`. The two
69+
// ports listing their columns in opposite orders joined an INTEGER to a STRING.
70+
it should "join two keys of one type when the ports list them in different orders" in {
71+
val left = new Schema(
72+
new Attribute("id", AttributeType.INTEGER),
73+
new Attribute("name", AttributeType.STRING)
74+
)
75+
val right = new Schema(
76+
new Attribute("label", AttributeType.STRING),
77+
new Attribute("key", AttributeType.INTEGER)
78+
)
79+
val result = ConfigGenerator.generate(classOf[HashJoinOpDesc[Any]], Map(0 -> left, 1 -> right))
80+
withClue(result) { result.isRight shouldBe true }
81+
val op = result.toOption.get.asInstanceOf[HashJoinOpDesc[Any]]
82+
left.getAttribute(op.buildAttributeName).getType shouldBe
83+
right.getAttribute(op.probeAttributeName).getType
84+
}
85+
86+
it should "refuse a join whose two ports share no type" in {
87+
val left = new Schema(new Attribute("id", AttributeType.INTEGER))
88+
val right = new Schema(new Attribute("label", AttributeType.STRING))
89+
val result = ConfigGenerator.generate(classOf[HashJoinOpDesc[Any]], Map(0 -> left, 1 -> right))
90+
result.isLeft shouldBe true
91+
}
92+
93+
// Interval Join states the same link the other way round: the right key takes
94+
// the left key's type.
95+
it should "give Interval Join's right key the left key's type" in {
96+
val left = new Schema(
97+
new Attribute("name", AttributeType.STRING),
98+
new Attribute("t", AttributeType.INTEGER)
99+
)
100+
val right = new Schema(
101+
new Attribute("label", AttributeType.STRING),
102+
new Attribute("u", AttributeType.INTEGER)
103+
)
104+
val result = ConfigGenerator.generate(classOf[IntervalJoinOpDesc], Map(0 -> left, 1 -> right))
105+
withClue(result) { result.isRight shouldBe true }
106+
val op = result.toOption.get.asInstanceOf[IntervalJoinOpDesc]
107+
op.leftAttributeName shouldBe "t"
108+
op.rightAttributeName shouldBe "u"
109+
}
110+
67111
it should "build a non-empty nested predicate list with a valid column and enum" in {
68112
val result = ConfigGenerator.generate(classOf[SpecializedFilterOpDesc], Map(0 -> schema))
69113
result.isRight shouldBe true
@@ -125,6 +169,36 @@ class ConfigGeneratorSpec extends AnyFlatSpec with Matchers {
125169
Set("id", "score", "open", "high", "low", "close") should contain(op.color.toString)
126170
}
127171

172+
// A rule states the types the operator takes. Filling the field from a column
173+
// outside them writes a configuration nobody would, and the run would then
174+
// report the operator as covered on it. Nothing reaches this against today's
175+
// fixtures, every type a rule names having a column in each of them, so what
176+
// these two pin is that a fixture narrowed later says so.
177+
private val textOnly = Map(
178+
0 -> new Schema(
179+
new Attribute("iso_country", AttributeType.STRING),
180+
new Attribute("name", AttributeType.STRING)
181+
)
182+
)
183+
184+
it should "refuse a single-column field when no column has a type it accepts" in {
185+
// `locations` still resolves, being a @SampleColumn; `color` is the one
186+
// constrained to integer/long/double, and this port holds neither.
187+
val result = ConfigGenerator.generate(classOf[ChoroplethMapOpDesc], textOnly)
188+
result.isRight shouldBe false
189+
result.swap.toOption.get should include("color accepts")
190+
result.swap.toOption.get should include("no column at port 0")
191+
}
192+
193+
it should "refuse a list field when no column has a type it accepts" in {
194+
// `valueColumns` takes integer/long/double; `nameColumn` is unconstrained
195+
// and would have been filled, which is what made the old fallback silent.
196+
val result = ConfigGenerator.generate(classOf[RadarChartOpDesc], textOnly)
197+
result.isRight shouldBe false
198+
result.swap.toOption.get should include("valueColumns accepts")
199+
result.swap.toOption.get should include("no column at port 0")
200+
}
201+
128202
private def variants(
129203
opClass: Class[_ <: org.apache.texera.amber.operator.LogicalOp],
130204
ports: Map[Int, Schema]

0 commit comments

Comments
 (0)