Skip to content

Commit e0701a4

Browse files
sushanbsushanb
authored
feat(bigtable): cap increase in conn for dynamic chan pool (#14144)
1 parent e57b013 commit e0701a4

3 files changed

Lines changed: 81 additions & 4 deletions

File tree

‎bigtable/internal/option/option.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,9 @@ type DynamicChannelPoolConfig struct {
240240
CheckInterval time.Duration // How often to check if scaling is needed.
241241
MaxRemoveConns int // Maximum number of connections to remove at once.
242242
ContinuousDownscaleRunsThreshold int // Continous downscale signals for downscale to actually occur
243+
MaxScaleUpPercentage int // MaxScaleUpPercentage limits the maximum number of connections added during a single
244+
// scale-up event, expressed as a percentage of the current pool size.
245+
// E.g., 30 means a maximum increase of 30%.
243246
}
244247

245248
// DefaultDynamicChannelPoolConfig is default settings for dynamic channel pool
@@ -254,6 +257,7 @@ func DefaultDynamicChannelPoolConfig() DynamicChannelPoolConfig {
254257
CheckInterval: 30 * time.Second,
255258
MaxRemoveConns: 2, // Only Cap for removals
256259
ContinuousDownscaleRunsThreshold: 3,
260+
MaxScaleUpPercentage: 30,
257261
}
258262
}
259263

‎bigtable/internal/transport/dynamic_scale_monitor.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ func NewDynamicScaleMonitor(config btopt.DynamicChannelPoolConfig, pool *Bigtabl
4646
config.ContinuousDownscaleRunsThreshold = 3
4747
}
4848

49+
// Fallback to a default max scale-up percentage of 30% if not specified or set to 0.
50+
if config.MaxScaleUpPercentage <= 0 {
51+
config.MaxScaleUpPercentage = 30
52+
}
53+
4954
perConnTargetLoad := math.Floor(config.AvgLoadLowThreshold+config.AvgLoadHighThreshold) / 2.0
5055
if perConnTargetLoad < 1.0 {
5156
perConnTargetLoad = 1.0 // targetLoad is at least 1 per channel
@@ -168,6 +173,10 @@ func ValidateDynamicConfig(config btopt.DynamicChannelPoolConfig, connPoolSize i
168173
if config.ContinuousDownscaleRunsThreshold < 0 {
169174
return fmt.Errorf("bigtable_connpool: DynamicChannelPoolConfig.ContinuousDownscaleRunsThreshold cannot be negative")
170175
}
176+
177+
if config.MaxScaleUpPercentage < 0 || config.MaxScaleUpPercentage > 100 {
178+
return fmt.Errorf("bigtable_connpool: DynamicChannelPoolConfig.MaxScaleUpPercentage must be between 0 and 100")
179+
}
171180
return nil
172181
}
173182

@@ -177,6 +186,16 @@ func ValidateDynamicConfig(config btopt.DynamicChannelPoolConfig, connPoolSize i
177186
func (dsm *DynamicScaleMonitor) scaleUp(currentLoadSum int32, currentConnsCount int) {
178187
desiredConns := int(math.Ceil(float64(currentLoadSum) / dsm.perConnTargetLoad))
179188
addCount := desiredConns - currentConnsCount
189+
190+
// Cap the addition based on the configured MaxScaleUpPercentage
191+
scaleUpFactor := float64(dsm.config.MaxScaleUpPercentage) / 100.0
192+
// we guarantee currentConnsCount > 0
193+
maxAddCount := int(math.Ceil(float64(currentConnsCount) * scaleUpFactor))
194+
195+
if addCount > maxAddCount {
196+
addCount = maxAddCount
197+
}
198+
180199
if addCount > 0 {
181200
btopt.Debugf(dsm.pool.logger, "bigtable_connpool: Scaling up: CurrentSize=%d, Adding=%d, TargetLoadPerConn=%.2f\n", currentConnsCount, addCount, dsm.perConnTargetLoad)
182201
if dsm.pool.addConnections(addCount, dsm.config.MaxConns) {

‎bigtable/internal/transport/dynamic_scale_monitor_test.go‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ func TestDynamicChannelScaling(t *testing.T) {
3838
CheckInterval: 10 * time.Second, // Not directly used by calling evaluateAndScale
3939
MaxRemoveConns: 3,
4040
ContinuousDownscaleRunsThreshold: 3,
41+
MaxScaleUpPercentage: 30,
4142
}
4243
tests := []struct {
4344
name string
@@ -51,10 +52,14 @@ func TestDynamicChannelScaling(t *testing.T) {
5152
name: "ScaleUp",
5253
initialSize: 3,
5354
setLoad: func(conns []*connEntry) {
54-
setConnLoads(conns, 12, 0) // Avg load 12 > 10
55+
// Target load per conn is 6.5.
56+
// Total load = 12 * 3 = 36. Desired = ceil(36 / 6.5) = 6.
57+
// try to add 3 conns
58+
setConnLoads(conns, 12, 0)
5559
},
5660
// Total load = 3 * 12 = 36. Desired = ceil(36 / 6.5) = 6
57-
wantSize: 6,
61+
// capped by 30% of 3, ceil (0.9) => 1
62+
wantSize: 4,
5863
evaluateCalls: 1,
5964
},
6065
{
@@ -122,10 +127,11 @@ func TestDynamicChannelScaling(t *testing.T) {
122127
name: "ScaleUpAddAtLeastOne",
123128
initialSize: 2,
124129
setLoad: func(conns []*connEntry) {
130+
// 10*2= 20 total load, conn count = ceil(20/6.5) = 4
125131
setConnLoads(conns, 10, 0) // Avg load 10, right at threshold.
126132
},
127-
// Total load = 20. Desired = ceil(20 / 6.5) = 4. Add 2.
128-
wantSize: 4,
133+
// Total load = 20. Desired = ceil(20 / 6.5) = 4. Add 2 but capped by 30%
134+
wantSize: 3,
129135
evaluateCalls: 1,
130136
},
131137
{
@@ -137,6 +143,54 @@ func TestDynamicChannelScaling(t *testing.T) {
137143
wantSize: 6,
138144
evaluateCalls: 2,
139145
},
146+
{
147+
name: "ScaleUpCappedByMaxScaleUpPercentage",
148+
initialSize: 10,
149+
configOpt: func(cfg *btopt.DynamicChannelPoolConfig) {
150+
cfg.MaxConns = 50 // Increase max conns so it doesn't artificially cap the test
151+
cfg.MaxScaleUpPercentage = 20 // 20% of 10 = 2 max additions allowed
152+
},
153+
setLoad: func(conns []*connEntry) {
154+
// Target load per conn is 6.5.
155+
// Total load = 10 * 20 = 200. Desired = ceil(200 / 6.5) = 31.
156+
// Normally it would try to add 21 connections.
157+
setConnLoads(conns, 20, 0)
158+
},
159+
// Capped at adding 2 connections
160+
wantSize: 12,
161+
evaluateCalls: 1,
162+
},
163+
{
164+
name: "ScaleUpNotCappedByMaxScaleUpPercentage",
165+
initialSize: 10,
166+
configOpt: func(cfg *btopt.DynamicChannelPoolConfig) {
167+
cfg.MaxConns = 50
168+
cfg.MaxScaleUpPercentage = 100 // 100% of 10 = 10
169+
},
170+
setLoad: func(conns []*connEntry) {
171+
// Total load = 10 * 10 = 100. Desired = ceil(100 / 6.5) = 16.
172+
setConnLoads(conns, 10, 0)
173+
},
174+
// max allowed 10, so 6 is fine
175+
wantSize: 16,
176+
evaluateCalls: 1,
177+
},
178+
{
179+
name: "ScaleUpCeilFractionalCap",
180+
initialSize: 4,
181+
configOpt: func(cfg *btopt.DynamicChannelPoolConfig) {
182+
cfg.MaxConns = 20
183+
cfg.MaxScaleUpPercentage = 30 // 30% of 4 = 1.2 -> ceil(1.2) = 2 allowed
184+
},
185+
setLoad: func(conns []*connEntry) {
186+
// Total load = 4 * 20 = 80. Desired = ceil(80 / 6.5) = 13.
187+
// Wants to add 9 connections.
188+
setConnLoads(conns, 20, 0)
189+
},
190+
// Capped at adding 2 connections due to math.Ceil(1.2)
191+
wantSize: 6,
192+
evaluateCalls: 1,
193+
},
140194
}
141195

142196
for _, tc := range tests {

0 commit comments

Comments
 (0)