Skip to content

Commit b69a8d1

Browse files
committed
feat(amber): mark NoLoopVariable properties in the operator schema
1 parent 73463e7 commit b69a8d1

4 files changed

Lines changed: 96 additions & 7 deletions

File tree

‎common/workflow-core/src/main/scala/org/apache/texera/amber/core/state/StateReferencing.scala‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,20 +61,29 @@ object StateReferencing {
6161
}
6262

6363
/**
64-
* The entries of `descriptor`'s sidecar that fall under a property marked `NoLoopVariable`,
65-
* however deep: the plan is built from that property before the loop runs, so nothing binds
66-
* them.
64+
* The JSON names of `descriptorClass`'s properties marked `NoLoopVariable`, a trait's or a
65+
* superclass's included. The one list behind both the compiler's check
66+
* (`noLoopVariableReferences`) and the `noLoopVariable` keyword of the operator's schema
67+
* (`OperatorMetadataGenerator.generateOperatorJsonSchema`), so the two cannot drift apart.
6768
*/
68-
def noLoopVariableReferences(descriptor: StateReferencing): Map[String, String] = {
69-
val marked = objectMapper.getSerializationConfig
70-
.introspect(objectMapper.constructType(descriptor.getClass))
69+
def noLoopVariableProperties(descriptorClass: Class[_]): Set[String] =
70+
objectMapper.getSerializationConfig
71+
.introspect(objectMapper.constructType(descriptorClass))
7172
.findProperties()
7273
.asScala
7374
.filter(property =>
7475
Option(property.getField).exists(_.hasAnnotation(classOf[NoLoopVariable]))
7576
)
7677
.map(_.getName)
7778
.toSet
79+
80+
/**
81+
* The entries of `descriptor`'s sidecar that fall under a property marked `NoLoopVariable`,
82+
* however deep: the plan is built from that property before the loop runs, so nothing binds
83+
* them.
84+
*/
85+
def noLoopVariableReferences(descriptor: StateReferencing): Map[String, String] = {
86+
val marked = noLoopVariableProperties(descriptor.getClass)
7887
descriptor.stateReferences.filter {
7988
case (pointer, _) =>
8089
Try(JsonPointer.compile(pointer).getMatchingProperty).toOption.exists(marked.contains)

‎common/workflow-core/src/test/scala/org/apache/texera/amber/core/state/StateReferenceModuleSpec.scala‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,9 @@ import com.fasterxml.jackson.databind.exc.InvalidFormatException
2727
import com.fasterxml.jackson.databind.node.ObjectNode
2828
import com.fasterxml.jackson.module.scala.DefaultScalaModule
2929
import org.apache.texera.amber.core.state.StateReferencing.{
30-
noLoopVariableReferences,
3130
literalReferences,
31+
noLoopVariableProperties,
32+
noLoopVariableReferences,
3233
referencedVariable
3334
}
3435
import org.apache.texera.amber.core.tuple.AttributeType
@@ -207,6 +208,17 @@ class StateReferenceModuleSpec extends AnyFlatSpec {
207208
)
208209
}
209210

211+
"StateReferencing.noLoopVariableProperties" should "name each property marked @NoLoopVariable by its JSON name, a trait's included" in {
212+
assert(
213+
noLoopVariableProperties(classOf[PlanBuilding]) ==
214+
Set("result attribute", "a/b", "keys", "column")
215+
)
216+
}
217+
218+
it should "name nothing for a descriptor with no marked property" in {
219+
assert(noLoopVariableProperties(classOf[Bean]).isEmpty)
220+
}
221+
210222
"StateReferencing.noLoopVariableReferences" should "pick the references under a property marked @NoLoopVariable, renamed, escaped, nested or declared by a trait" in {
211223
val descriptor = new PlanBuilding
212224
descriptor.stateReferences = Map(

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/OperatorMetadataGenerator.scala‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,9 @@ case class AllOperatorMetadata(
8585
*/
8686
object OperatorMetadataGenerator {
8787

88+
/** The schema keyword, `true` on each property that cannot hold a loop variable. */
89+
final val NO_LOOP_VARIABLE_KEYWORD = "noLoopVariable"
90+
8891
val texeraSchemaGeneratorConfig: JsonSchemaConfig = html5EnabledSchema.copy(
8992
useOneOfForOption = false,
9093
useOneOfForNullables = false,
@@ -143,6 +146,15 @@ object OperatorMetadataGenerator {
143146
jsonSchema.get("required").asInstanceOf[ArrayNode].remove(operatorTypeIndex)
144147
// remove "title" for the operator - frontend uses userFriendlyName to show operator title
145148
jsonSchema.remove("title")
149+
// mark each property that cannot hold a loop variable (`NoLoopVariable`): the compiler rejects
150+
// a `$name` there inside a loop block, so the property panel need not offer one
151+
val properties = jsonSchema.get("properties").asInstanceOf[ObjectNode]
152+
StateReferencing.noLoopVariableProperties(opDescClass).foreach { name =>
153+
properties.get(name) match {
154+
case property: ObjectNode => property.put(NO_LOOP_VARIABLE_KEYWORD, true)
155+
case _ =>
156+
}
157+
}
146158
// let an operator add the part of its schema that its annotations cannot state
147159
opDescClass.getConstructor().newInstance() match {
148160
case customizer: JsonSchemaCustomizer => customizer.customizeJsonSchema(jsonSchema)

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/metadata/OperatorMetadataGeneratorSpec.scala‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,16 @@
1919

2020
package org.apache.texera.amber.operator.metadata
2121

22+
import com.fasterxml.jackson.databind.node.BooleanNode
2223
import org.apache.texera.amber.core.state.StateReferencing
2324
import org.apache.texera.amber.operator.LogicalOp
25+
import org.apache.texera.amber.operator.dictionary.DictionaryMatcherOpDesc
2426
import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc
27+
import org.apache.texera.amber.operator.limit.LimitOpDesc
28+
import org.apache.texera.amber.operator.projection.ProjectionOpDesc
29+
import org.apache.texera.amber.operator.sortPartitions.SortPartitionsOpDesc
30+
import org.apache.texera.amber.operator.source.scan.file.FileScanOpDesc
31+
import org.apache.texera.amber.operator.unneststring.UnnestStringOpDesc
2532
import org.scalatest.flatspec.AnyFlatSpec
2633
import org.scalatest.matchers.should.Matchers
2734

@@ -52,4 +59,53 @@ class OperatorMetadataGeneratorSpec extends AnyFlatSpec with Matchers {
5259
val required = schema.get("required").elements().asScala.map(_.asText()).toList
5360
required should not contain StateReferencing.SIDECAR_PROPERTY
5461
}
62+
63+
/** The properties of `opDescClass`'s schema whose `noLoopVariable` keyword is `true`. */
64+
private def markedNoLoopVariable(opDescClass: Class[_ <: LogicalOp]): Set[String] =
65+
OperatorMetadataGenerator
66+
.generateOperatorJsonSchema(opDescClass)
67+
.get("properties")
68+
.fields()
69+
.asScala
70+
.filter(_.getValue.get("noLoopVariable") == BooleanNode.TRUE)
71+
.map(_.getKey)
72+
.toSet
73+
74+
it should "mark each property that cannot hold a loop variable with 'noLoopVariable', by its JSON name" in {
75+
markedNoLoopVariable(classOf[ProjectionOpDesc]) shouldBe Set("isDrop", "attributes")
76+
// Renamed in JSON: the panel looks the property up by the name the schema gives it.
77+
markedNoLoopVariable(classOf[DictionaryMatcherOpDesc]) shouldBe Set("result attribute")
78+
markedNoLoopVariable(classOf[UnnestStringOpDesc]) shouldBe Set("Result attribute")
79+
// `attributeName` is declared and marked by the TextSourceOpDesc trait.
80+
markedNoLoopVariable(classOf[FileScanOpDesc]) shouldBe Set("attributeName", "outputFileName")
81+
markedNoLoopVariable(classOf[SortPartitionsOpDesc]) shouldBe
82+
Set("sortAttributeName", "domainMin", "domainMax")
83+
}
84+
85+
it should "leave 'noLoopVariable' off a property the executor reads after the loop state wrote it" in {
86+
val properties = Seq(
87+
classOf[SpecializedFilterOpDesc] -> "predicates",
88+
classOf[LimitOpDesc] -> "limit",
89+
classOf[DictionaryMatcherOpDesc] -> "Dictionary",
90+
classOf[UnnestStringOpDesc] -> "Delimiter"
91+
)
92+
properties.foreach {
93+
case (opDescClass, name) =>
94+
val schema = OperatorMetadataGenerator.generateOperatorJsonSchema(opDescClass)
95+
withClue(s"${opDescClass.getSimpleName}.$name: ") {
96+
schema.get("properties").has(name) shouldBe true
97+
schema.get("properties").get(name).has("noLoopVariable") shouldBe false
98+
}
99+
}
100+
}
101+
102+
it should "generate every operator's schema, marking exactly the properties the compiler rejects a loop variable in" in {
103+
// One list behind both: what the schema marks is exactly what the compiler checks.
104+
OperatorMetadataGenerator.operatorTypeMap.keys.foreach { opDescClass =>
105+
withClue(s"${opDescClass.getSimpleName}: ") {
106+
markedNoLoopVariable(opDescClass) shouldBe
107+
StateReferencing.noLoopVariableProperties(opDescClass)
108+
}
109+
}
110+
}
55111
}

0 commit comments

Comments
 (0)