Skip to content

Commit 2a3d887

Browse files
authored
[BEAM-3304, BEAM-12513] Trigger changes and Windowing. (#15644)
1 parent 935abae commit 2a3d887

8 files changed

Lines changed: 430 additions & 91 deletions

File tree

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one or more
2+
// contributor license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright ownership.
4+
// The ASF licenses this file to You under the Apache License, Version 2.0
5+
// (the "License"); you may not use this file except in compliance with
6+
// the License. You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
package snippets
17+
18+
import (
19+
"time"
20+
21+
"github.com/apache/beam/sdks/v2/go/pkg/beam"
22+
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime"
23+
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window"
24+
)
25+
26+
func settingWindows(s beam.Scope, items beam.PCollection) {
27+
// [START setting_fixed_windows]
28+
fixedWindowedItems := beam.WindowInto(s,
29+
window.NewFixedWindows(60*time.Second),
30+
items)
31+
// [END setting_fixed_windows]
32+
33+
// [START setting_sliding_windows]
34+
slidingWindowedItems := beam.WindowInto(s,
35+
window.NewSlidingWindows(5*time.Second, 30*time.Second),
36+
items)
37+
// [END setting_sliding_windows]
38+
39+
// [START setting_session_windows]
40+
sessionWindowedItems := beam.WindowInto(s,
41+
window.NewSessions(600*time.Second),
42+
items)
43+
// [END setting_session_windows]
44+
45+
// [START setting_global_window]
46+
globalWindowedItems := beam.WindowInto(s,
47+
window.NewGlobalWindows(),
48+
items)
49+
// [END setting_global_window]
50+
51+
// [START setting_allowed_lateness]
52+
windowedItems := beam.WindowInto(s,
53+
window.NewFixedWindows(1*time.Minute), items,
54+
beam.AllowedLateness(2*24*time.Hour), // 2 days
55+
)
56+
// [END setting_allowed_lateness]
57+
58+
_ = []beam.PCollection{
59+
fixedWindowedItems,
60+
slidingWindowedItems,
61+
sessionWindowedItems,
62+
globalWindowedItems,
63+
windowedItems,
64+
}
65+
}
66+
67+
// LogEntry is a dummy type for documentation purposes.
68+
type LogEntry int
69+
70+
func extractEventTime(LogEntry) time.Time {
71+
// Note: Returning time.Now() is always going to be processing time
72+
// not EventTime. For true event time, one needs to extract the
73+
// time from the element itself.
74+
return time.Now()
75+
}
76+
77+
// [START setting_timestamp]
78+
79+
// AddTimestampDoFn extracts an event time from a LogEntry.
80+
func AddTimestampDoFn(element LogEntry, emit func(beam.EventTime, LogEntry)) {
81+
et := extractEventTime(element)
82+
// Defining an emitter with beam.EventTime as the first parameter
83+
// allows the DoFn to set the event time for the emitted element.
84+
emit(mtime.FromTime(et), element)
85+
}
86+
87+
// [END setting_timestamp]
88+
89+
func timestampedCollection(s beam.Scope, unstampedLogs beam.PCollection) {
90+
// [START setting_timestamp_pipeline]
91+
stampedLogs := beam.ParDo(s, AddTimestampDoFn, unstampedLogs)
92+
// [END setting_timestamp_pipeline]
93+
_ = stampedLogs
94+
}

‎sdks/go/examples/snippets/09triggers.go‎

Lines changed: 62 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -23,33 +23,76 @@
2323
package snippets
2424

2525
import (
26+
"time"
27+
2628
"github.com/apache/beam/sdks/v2/go/pkg/beam"
2729
"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window"
28-
"github.com/apache/beam/sdks/v2/go/pkg/beam/testing/teststream"
29-
"time"
3030
)
3131

32-
func generateStream(s beam.Scope) beam.PCollection {
33-
con := teststream.NewConfig()
34-
con.AddElements(1000, 1.0, 2.0, 3.0)
35-
con.AdvanceWatermark(11000)
36-
return teststream.Create(s, con)
37-
}
38-
39-
func TriggerAfterEndOfWindow(s beam.Scope) {
40-
pCollection := generateStream(s)
41-
windowSize := 10 * time.Second
32+
func TriggerAfterEndOfWindow(s beam.Scope, pCollection beam.PCollection) {
4233
// [START after_window_trigger]
43-
trigger := window.TriggerAfterEndOfWindow().EarlyFiring(window.TriggerAfterProcessingTime(60000)).LateFiring(window.TriggerRepeat(window.TriggerAfterCount(1)))
34+
trigger := window.TriggerAfterEndOfWindow().
35+
EarlyFiring(window.TriggerAfterProcessingTime().
36+
PlusDelay(60 * time.Second)).
37+
LateFiring(window.TriggerRepeat(window.TriggerAfterCount(1)))
4438
// [END after_window_trigger]
45-
beam.WindowInto(s, window.NewFixedWindows(windowSize), pCollection, beam.Trigger(trigger), beam.PanesDiscard())
39+
beam.WindowInto(s, window.NewFixedWindows(10*time.Second), pCollection, beam.Trigger(trigger), beam.PanesDiscard())
4640
}
4741

48-
func TriggerAlways(s beam.Scope) {
49-
pCollection := generateStream(s)
42+
func TriggerAlways(s beam.Scope, pCollection beam.PCollection) {
5043
// [START always_trigger]
51-
windowSize := 10 * time.Second
52-
trigger := window.TriggerAlways()
53-
beam.WindowInto(s, window.NewFixedWindows(windowSize), pCollection, beam.Trigger(trigger), beam.PanesDiscard())
44+
beam.WindowInto(s, window.NewFixedWindows(10*time.Second), pCollection,
45+
beam.Trigger(window.TriggerAlways()),
46+
beam.PanesDiscard(),
47+
)
5448
// [END always_trigger]
5549
}
50+
51+
func ComplexTriggers(s beam.Scope, pcollection beam.PCollection) {
52+
// [START setting_a_trigger]
53+
windowedItems := beam.WindowInto(s,
54+
window.NewFixedWindows(1*time.Minute), pcollection,
55+
beam.Trigger(window.TriggerAfterProcessingTime().
56+
PlusDelay(1*time.Minute)),
57+
beam.AllowedLateness(30*time.Minute),
58+
beam.PanesDiscard(),
59+
)
60+
// [END setting_a_trigger]
61+
62+
// [START setting_allowed_lateness]
63+
allowedToBeLateItems := beam.WindowInto(s,
64+
window.NewFixedWindows(1*time.Minute), pcollection,
65+
beam.Trigger(window.TriggerAfterProcessingTime().
66+
PlusDelay(1*time.Minute)),
67+
beam.AllowedLateness(30*time.Minute),
68+
)
69+
// [END setting_allowed_lateness]
70+
71+
// [START model_composite_triggers]
72+
compositeTriggerItems := beam.WindowInto(s,
73+
window.NewFixedWindows(1*time.Minute), pcollection,
74+
beam.Trigger(window.TriggerAfterEndOfWindow().
75+
LateFiring(window.TriggerAfterProcessingTime().
76+
PlusDelay(10*time.Minute))),
77+
beam.AllowedLateness(2*24*time.Hour),
78+
)
79+
// [END model_composite_triggers]
80+
81+
// TODO(BEAM-3304) AfterAny is not yet implemented.
82+
// Implement so the following compiles when no longer commented out.
83+
84+
// [START other_composite_trigger]
85+
// beam.Trigger(
86+
// window.TriggerAfterAny(
87+
// window.TriggerAfterCount(100),
88+
// window.TriggerAfterProcessingTime().
89+
// PlusDelay(1*time.Minute)),
90+
// )
91+
// [END other_composite_trigger]
92+
93+
_ = []beam.PCollection{
94+
windowedItems,
95+
allowedToBeLateItems,
96+
compositeTriggerItems,
97+
}
98+
}

‎sdks/go/pkg/beam/core/graph/window/trigger.go‎

Lines changed: 109 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,54 @@
1515

1616
package window
1717

18-
import "fmt"
18+
import (
19+
"fmt"
20+
"time"
21+
)
1922

23+
// Trigger describes when to emit new aggregations.
24+
// Fields are exported for use by the framework, and not intended
25+
// to be set by end users.
26+
//
27+
// This API is experimental and subject to change.
2028
type Trigger struct {
21-
Kind string
22-
SubTriggers []Trigger
23-
Delay int64 // in milliseconds
24-
ElementCount int32
25-
EarlyTrigger *Trigger
26-
LateTrigger *Trigger
29+
Kind string
30+
SubTriggers []Trigger // Repeat, OrFinally, Any, All
31+
TimestampTransforms []TimestampTransform // AfterProcessingTime
32+
ElementCount int32 // ElementCount
33+
EarlyTrigger *Trigger // AfterEndOfWindow
34+
LateTrigger *Trigger // AfterEndOfWindow
35+
}
36+
37+
// TimestampTransform describes how an after processing time trigger
38+
// time is transformed to determine when to fire an aggregation.const
39+
// The base timestamp is always the when the first element of the pane
40+
// is received.
41+
//
42+
// A series of these transforms will be applied in order emit at regular intervals.
43+
type TimestampTransform interface {
44+
timestampTransform()
45+
}
46+
47+
// DelayTransform takes the timestamp and adds the given delay to it.
48+
type DelayTransform struct {
49+
Delay int64 // in milliseconds
2750
}
2851

52+
func (DelayTransform) timestampTransform() {}
53+
54+
// AlignToTransform takes the timestamp and transforms it to the lowest
55+
// multiple of the period starting from the offset.
56+
//
57+
// Eg. A period of 20 with an offset of 45 would have alignments at 5,25,45,65 etc.
58+
// Timestamps would be transformed as follows: 0 to 5 would be transformed to 5,
59+
// 6 to 25 would be transformed to 25, 26 to 45 would be transformed to 45, and so on.
60+
type AlignToTransform struct {
61+
Period, Offset int64 // in milliseconds
62+
}
63+
64+
func (AlignToTransform) timestampTransform() {}
65+
2966
const (
3067
DefaultTrigger string = "Trigger_Default_"
3168
AlwaysTrigger string = "Trigger_Always_"
@@ -46,49 +83,101 @@ func TriggerDefault() Trigger {
4683
return Trigger{Kind: DefaultTrigger}
4784
}
4885

49-
// TriggerAlways constructs an always trigger that keeps firing immediately after an element is processed.
86+
// TriggerAlways constructs a trigger that fires immediately
87+
// whenever an element is received.
88+
//
5089
// Equivalent to window.TriggerRepeat(window.TriggerAfterCount(1))
5190
func TriggerAlways() Trigger {
5291
return Trigger{Kind: AlwaysTrigger}
5392
}
5493

55-
// TriggerAfterCount constructs an element count trigger that fires after atleast `count` number of elements are processed.
94+
// TriggerAfterCount constructs a trigger that fires after
95+
// at least `count` number of elements are processed.
5696
func TriggerAfterCount(count int32) Trigger {
5797
return Trigger{Kind: ElementCountTrigger, ElementCount: count}
5898
}
5999

60-
// TriggerAfterProcessingTime constructs an after processing time trigger that fires after 'delay' milliseconds of processing time have passed.
61-
func TriggerAfterProcessingTime(delay int64) Trigger {
62-
return Trigger{Kind: AfterProcessingTimeTrigger, Delay: delay}
100+
// TriggerAfterProcessingTime constructs a trigger that fires relative to
101+
// when input first arrives.
102+
//
103+
// Must be configured with calls to PlusDelay, or AlignedTo. May be
104+
// configured with additional delay.
105+
func TriggerAfterProcessingTime() Trigger {
106+
return Trigger{Kind: AfterProcessingTimeTrigger}
63107
}
64108

65-
// TriggerRepeat constructs a repeat trigger that fires a trigger repeatedly once the condition has been met.
109+
// PlusDelay configures an AfterProcessingTime trigger to fire after a specified delay,
110+
// no smaller than a millisecond.
111+
func (tr Trigger) PlusDelay(delay time.Duration) Trigger {
112+
if tr.Kind != AfterProcessingTimeTrigger {
113+
panic(fmt.Errorf("can't apply processing delay to %s, want: AfterProcessingTimeTrigger", tr.Kind))
114+
}
115+
if delay < time.Millisecond {
116+
panic(fmt.Errorf("can't apply processing delay of less than a millisecond. Got: %v", delay))
117+
}
118+
tr.TimestampTransforms = append(tr.TimestampTransforms, DelayTransform{Delay: int64(delay / time.Millisecond)})
119+
return tr
120+
}
121+
122+
// AlignedTo configures an AfterProcessingTime trigger to fire
123+
// at the smallest multiple of period since the offset greater than the first element timestamp.
124+
//
125+
// * Period may not be smaller than a millisecond.
126+
// * Offset may be a zero time (time.Time{}).
127+
func (tr Trigger) AlignedTo(period time.Duration, offset time.Time) Trigger {
128+
if tr.Kind != AfterProcessingTimeTrigger {
129+
panic(fmt.Errorf("can't apply processing delay to %s, want: AfterProcessingTimeTrigger", tr.Kind))
130+
}
131+
if period < time.Millisecond {
132+
panic(fmt.Errorf("can't apply an alignment period of less than a millisecond. Got: %v", period))
133+
}
134+
offsetMillis := int64(0)
135+
if !offset.IsZero() {
136+
// TODO: Change to call UnixMilli() once we move to only supporting a go version > 1.17.
137+
offsetMillis = offset.Unix()*1e3 + int64(offset.Nanosecond())/1e6
138+
}
139+
tr.TimestampTransforms = append(tr.TimestampTransforms, AlignToTransform{
140+
Period: int64(period / time.Millisecond),
141+
Offset: offsetMillis,
142+
})
143+
return tr
144+
}
145+
146+
// TriggerRepeat constructs a trigger that fires a trigger repeatedly
147+
// once the condition has been met.
148+
//
66149
// Ex: window.TriggerRepeat(window.TriggerAfterCount(1)) is same as window.TriggerAlways().
67150
func TriggerRepeat(tr Trigger) Trigger {
68151
return Trigger{Kind: RepeatTrigger, SubTriggers: []Trigger{tr}}
69152
}
70153

71-
// TriggerAfterEndOfWindow constructs an end of window trigger that is configurable for early firing trigger(before the end of window)
72-
// and late firing trigger(after the end of window).
73-
// Default Options are: Default Trigger for EarlyFiring and No LateFiring. Override it with EarlyFiring and LateFiring methods on this trigger.
154+
// TriggerAfterEndOfWindow constructs a trigger that is configurable for early firing
155+
// (before the end of window) and late firing (after the end of window).
156+
//
157+
// Default Options are: Default Trigger for EarlyFiring and No LateFiring.
158+
// Override it with EarlyFiring and LateFiring methods on this trigger.
74159
func TriggerAfterEndOfWindow() Trigger {
75160
defaultEarly := TriggerDefault()
76161
return Trigger{Kind: AfterEndOfWindowTrigger, EarlyTrigger: &defaultEarly, LateTrigger: nil}
77162
}
78163

79-
// EarlyFiring configures AfterEndOfWindow trigger with an early firing trigger.
164+
// EarlyFiring configures an AfterEndOfWindow trigger with an implicitly
165+
// repeated trigger that applies before the end of the window.
80166
func (tr Trigger) EarlyFiring(early Trigger) Trigger {
81167
if tr.Kind != AfterEndOfWindowTrigger {
82-
panic(fmt.Errorf("can't apply early firing to %s, got: %s, want: AfterEndOfWindowTrigger", tr.Kind, tr.Kind))
168+
panic(fmt.Errorf("can't apply early firing to %s, want: AfterEndOfWindowTrigger", tr.Kind))
83169
}
84170
tr.EarlyTrigger = &early
85171
return tr
86172
}
87173

88-
// LateFiring configures AfterEndOfWindow trigger with a late firing trigger
174+
// LateFiring configures an AfterEndOfWindow trigger with an implicitly
175+
// repeated trigger that applies after the end of the window.
176+
//
177+
// Not setting a late firing trigger means elements are discarded.
89178
func (tr Trigger) LateFiring(late Trigger) Trigger {
90179
if tr.Kind != AfterEndOfWindowTrigger {
91-
panic(fmt.Errorf("can't apply late firing to %s, got: %s, want: AfterEndOfWindowTrigger", tr.Kind, tr.Kind))
180+
panic(fmt.Errorf("can't apply late firing to %s, want: AfterEndOfWindowTrigger", tr.Kind))
92181
}
93182
tr.LateTrigger = &late
94183
return tr

0 commit comments

Comments
 (0)