Skip to content

Commit 927c5b2

Browse files
authored
[Go SDK] Fix generic registration of anonymous struct types (#40649)
Generic registration registers every struct type that a function or method takes or returns, with both the runtime type registry and the schema registry. runtime.RegisterType panics for unnamed types, so generic registration could not be used for a function with an anonymous struct parameter, and it stopped before the caller and struct wrappers were set up. registerMethodTypes now adds a struct type to the runtime type registry only when it has a type key, and still adds every struct type to the schema registry, which maps a row schema back to the user's Go type on the worker. graphx already encodes types without a type key field by field. This also enables the castFn registration in the BigQuery integration test that was waiting on this issue. Fixes #21789
1 parent 4550ac8 commit 927c5b2

5 files changed

Lines changed: 161 additions & 10 deletions

File tree

‎CHANGES.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@
141141
* (Python) `KeyedModelHandler` and `MaybeKeyedModelHandler` now apply the `element_size_fn` and `batch_length_fn` of the wrapped `ModelHandler` to examples, instead of to `(key, example)` pairs ([#40644](https://github.com/apache/beam/issues/40644)).
142142
* (Python) DataFrame API: `drop_duplicates`, `duplicated` and `nunique` no longer fail on rows with NaN values or on categorical columns with unused categories, and `nunique(dropna=False)` now counts NaN ([#40570](https://github.com/apache/beam/issues/40570)).
143143
* (Java) BigQuery Storage Write API `TableRow` conversion now honors `allowMissingRequiredFields` when building protocol-buffer messages, so callers that pass `true` with required descriptors get partial messages instead of a conversion-time `UninitializedMessageException` ([#40189](https://github.com/apache/beam/issues/40189)).
144+
* (Go) Fixed generic registration (for example `register.Function1x1` or `register.DoFn1x1`) panicking on functions and methods that take or return an anonymous struct. Unnamed struct types are now only registered with the schema registry, since they have no type key for the runtime type registry ([#21789](https://github.com/apache/beam/issues/21789)).
144145
* Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
145146

146147
## Security Fixes

‎sdks/go/pkg/beam/register/register.go‎

Lines changed: 12 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎sdks/go/pkg/beam/register/register.tmpl‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -567,19 +567,27 @@ func registerDoFnTypes(doFn any) {
567567
registerMethodTypes(reflect.ValueOf(doFn).MethodByName("ProcessElement").Type())
568568
}
569569

570+
// registerStructType registers t with the schema registry, and with the
571+
// runtime type registry when t has a type key. Unnamed types, such as anonymous
572+
// structs, have no type key, and graphx encodes them field by field instead.
573+
func registerStructType(t reflect.Type) {
574+
if _, ok := runtime.TypeKey(t); ok {
575+
runtime.RegisterType(t)
576+
}
577+
schema.RegisterType(t)
578+
}
579+
570580
func registerMethodTypes(fn reflect.Type) {
571581
for i := 0; i < fn.NumIn(); i++ {
572582
in := reflectx.SkipPtr(fn.In(i))
573583
if in.Kind() == reflect.Struct {
574-
runtime.RegisterType(in)
575-
schema.RegisterType(in)
584+
registerStructType(in)
576585
}
577586
}
578587
for i := 0; i < fn.NumOut(); i++ {
579588
out := reflectx.SkipPtr(fn.Out(i))
580589
if out.Kind() == reflect.Struct {
581-
runtime.RegisterType(out)
582-
schema.RegisterType(out)
590+
registerStructType(out)
583591
}
584592
}
585593
}

‎sdks/go/pkg/beam/register/register_test.go‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,141 @@ func TestFunction(t *testing.T) {
351351
}
352352
}
353353

354+
// anonymousStruct is a type alias, so the struct type stays unnamed and has no
355+
// type key to register it under in the runtime type registry.
356+
type anonymousStruct = struct {
357+
Counter *int64 `beam:"counter"`
358+
RandData *struct {
359+
Flip *bool `beam:"flip"`
360+
Word *string `beam:"word"`
361+
} `beam:"rand_data"`
362+
Meta AnonymousStructMeta `beam:"meta"`
363+
}
364+
365+
// AnonymousStructMeta is only used as a field of anonymousStruct.
366+
type AnonymousStructMeta struct {
367+
Source string
368+
}
369+
370+
type AnonymousStructReturn struct {
371+
Count int64
372+
}
373+
374+
func anonymousStructFunction(a *anonymousStruct) AnonymousStructReturn {
375+
return AnonymousStructReturn{Count: *a.Counter}
376+
}
377+
378+
// anonymousStructResult is unnamed too, with a different shape from
379+
// anonymousStruct, so that its schema registration comes from its own test.
380+
type anonymousStructResult = struct {
381+
Word string `json:"word"`
382+
Count int64 `json:"count"`
383+
}
384+
385+
func anonymousStructResultFunction(word string) anonymousStructResult {
386+
return anonymousStructResult{Word: word, Count: int64(len(word))}
387+
}
388+
389+
type AnonymousStructDoFn1x1 struct{}
390+
391+
func (fn *AnonymousStructDoFn1x1) ProcessElement(a *anonymousStruct) AnonymousStructReturn {
392+
return AnonymousStructReturn{Count: *a.Counter}
393+
}
394+
395+
func registerWithoutPanic(t *testing.T, name string, register func()) {
396+
t.Helper()
397+
defer func() {
398+
if r := recover(); r != nil {
399+
t.Fatalf("%v panicked: %v", name, r)
400+
}
401+
}()
402+
register()
403+
}
404+
405+
func checkTypeRegistered(t *testing.T, rt reflect.Type) {
406+
t.Helper()
407+
key, ok := runtime.TypeKey(rt)
408+
if !ok {
409+
t.Fatalf("runtime.TypeKey(%v) failed, want a registrable type", rt)
410+
}
411+
if _, ok := runtime.LookupType(key); !ok {
412+
t.Errorf("runtime.LookupType(%q) = false, want true", key)
413+
}
414+
checkSchemaRegistered(t, rt)
415+
}
416+
417+
func checkSchemaRegistered(t *testing.T, rt reflect.Type) {
418+
t.Helper()
419+
// Need to call FromType so that the registry will reconcile its registrations
420+
if _, err := schema.FromType(rt); err != nil {
421+
t.Errorf("schema.FromType(%v) failed: %v", rt, err)
422+
}
423+
if !schema.Registered(rt) {
424+
t.Errorf("schema.Registered(%v) = false, want true", rt)
425+
}
426+
}
427+
428+
func TestFunction_AnonymousStructParameter(t *testing.T) {
429+
registerWithoutPanic(t, "Function1x1", func() {
430+
Function1x1[*anonymousStruct, AnonymousStructReturn](anonymousStructFunction)
431+
})
432+
433+
// The unnamed parameter type has no type key, but it is still registered
434+
// with the schema registry, along with the named struct type it contains.
435+
checkSchemaRegistered(t, reflect.TypeOf(anonymousStruct{}))
436+
checkSchemaRegistered(t, reflect.TypeOf(AnonymousStructMeta{}))
437+
checkTypeRegistered(t, reflect.TypeOf(AnonymousStructReturn{}))
438+
439+
f := reflectx.MakeFunc(anonymousStructFunction)
440+
if _, ok := f.(*caller1x1[*anonymousStruct, AnonymousStructReturn]); !ok {
441+
t.Fatalf("reflectx.MakeFunc(anonymousStructFunction) = %T, want *caller1x1", f)
442+
}
443+
counter := int64(7)
444+
if got, want := f.Call([]any{&anonymousStruct{Counter: &counter}})[0].(AnonymousStructReturn).Count, counter; got != want {
445+
t.Errorf("Registered function call: got %v, want %v", got, want)
446+
}
447+
}
448+
449+
func TestFunction_AnonymousStructReturn(t *testing.T) {
450+
registerWithoutPanic(t, "Function1x1", func() {
451+
Function1x1[string, anonymousStructResult](anonymousStructResultFunction)
452+
})
453+
454+
checkSchemaRegistered(t, reflect.TypeOf(anonymousStructResult{}))
455+
456+
f := reflectx.MakeFunc(anonymousStructResultFunction)
457+
if _, ok := f.(*caller1x1[string, anonymousStructResult]); !ok {
458+
t.Fatalf("reflectx.MakeFunc(anonymousStructResultFunction) = %T, want *caller1x1", f)
459+
}
460+
want := anonymousStructResult{Word: "beam", Count: 4}
461+
if got := f.Call([]any{"beam"})[0].(anonymousStructResult); got != want {
462+
t.Errorf("Registered function call: got %v, want %v", got, want)
463+
}
464+
}
465+
466+
func TestRegister_AnonymousStructParameter(t *testing.T) {
467+
registerWithoutPanic(t, "DoFn1x1", func() {
468+
DoFn1x1[*anonymousStruct, AnonymousStructReturn](&AnonymousStructDoFn1x1{})
469+
})
470+
471+
checkTypeRegistered(t, reflect.TypeOf(AnonymousStructDoFn1x1{}))
472+
checkTypeRegistered(t, reflect.TypeOf(AnonymousStructReturn{}))
473+
checkSchemaRegistered(t, reflect.TypeOf(anonymousStruct{}))
474+
475+
m, ok := reflectx.WrapMethods(&AnonymousStructDoFn1x1{})
476+
if !ok {
477+
t.Fatalf("reflectx.WrapMethods(AnonymousStructDoFn1x1{}), no registered entry found")
478+
}
479+
pe, ok := m["ProcessElement"]
480+
if !ok {
481+
t.Fatalf("reflectx.WrapMethods(AnonymousStructDoFn1x1{}), no registered ProcessElement method found")
482+
}
483+
counter := int64(7)
484+
if got, want := pe.Call([]any{&anonymousStruct{Counter: &counter}})[0].(AnonymousStructReturn).Count, counter; got != want {
485+
t.Errorf("Wrapped ProcessElement call: got %v, want %v", got, want)
486+
}
487+
}
488+
354489
type elementProcessor struct {
355490
inFV exec.FullValue
356491
}

‎sdks/go/test/integration/io/xlang/bigquery/bigquery_test.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,7 @@ import (
3838
func init() {
3939
register.DoFn2x0[[]byte, func(TestRow)](&CreateTestRowsFn{})
4040
register.Emitter1[TestRow]()
41-
// TODO(https://github.com/apache/beam/issues/21789): Uncomment once this register no longer panics.
42-
//register.Function1x1(castFn)
41+
register.Function1x1(castFn)
4342
}
4443

4544
var expansionAddr string // Populate with expansion address labelled "gcpio".

0 commit comments

Comments
 (0)