Skip to content
This repository was archived by the owner on Mar 26, 2026. It is now read-only.

Commit e5f6fdb

Browse files
danieljbrucedanieljbruce
authored
feat: Autoscaler (#1077)
* work in progress - broken * Add autoscaling options to interface * Work in progress on function for setting metadata * Fixed some tests that were broken due to the chang * Cluster.ts in test file * Update cluster creation to accept new parameters * Fix for failing test * Enhance system tests for cluster * Add another test * condense tests * Added more tests * Test improvements * Change update mask to get manual scaling right * Added validation for cluster creation configs * Refactor cluster object in tests * Another cluster id refactor * Remove TODO that no longer applies * Code reorganization for test cases * Fix the test so that it breaks for good reason * update mask push fix * Add nodes in all calls to update and create cluste * Add license headers * correct copyright year * Add singlequote * Add a validation * Update test descriptions * refactor to use one check metadata function * Fix tests as specifying nodes is required anyway * should added to test case descriptions * PR updates * validation error * Remove call to get metadata * PR updates
1 parent 033bfc8 commit e5f6fdb

9 files changed

Lines changed: 667 additions & 50 deletions

File tree

‎src/cluster.ts‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ const pumpify = require('pumpify');
2222
import {google} from '../protos/protos';
2323
import {Bigtable} from '.';
2424
import {Instance} from './instance';
25+
import {ClusterUtils} from './utils/cluster';
2526

2627
import {
2728
Backup,
@@ -75,7 +76,10 @@ export type GetClustersCallback = (
7576
apiResponse?: google.bigtable.admin.v2.IListClustersResponse
7677
) => void;
7778
export interface SetClusterMetadataOptions {
78-
nodes: number;
79+
nodes?: number;
80+
minServeNodes?: number;
81+
maxServeNodes?: number;
82+
cpuUtilizationPercent?: number;
7983
}
8084
export type SetClusterMetadataCallback = GenericOperationCallback<
8185
Operation | null | undefined
@@ -84,8 +88,11 @@ export interface BasicClusterConfig {
8488
encryption?: google.bigtable.admin.v2.Cluster.IEncryptionConfig;
8589
key?: string;
8690
location: string;
87-
nodes: number;
91+
nodes?: number;
8892
storage?: string;
93+
minServeNodes?: number;
94+
maxServeNodes?: number;
95+
cpuUtilizationPercent?: number;
8996
}
9097

9198
export interface CreateBackupConfig extends ModifiableBackupFields {
@@ -690,29 +697,23 @@ Please use the format 'my-cluster' or '${instance.name}/clusters/my-cluster'.`);
690697
gaxOptionsOrCallback?: CallOptions | SetClusterMetadataCallback,
691698
cb?: SetClusterMetadataCallback
692699
): void | Promise<SetClusterMetadataResponse> {
700+
ClusterUtils.validateClusterMetadata(metadata);
693701
const callback =
694702
typeof gaxOptionsOrCallback === 'function' ? gaxOptionsOrCallback : cb!;
695703
const gaxOptions =
696704
typeof gaxOptionsOrCallback === 'object'
697705
? gaxOptionsOrCallback
698706
: ({} as CallOptions);
699-
700-
const reqOpts: ICluster = Object.assign(
701-
{},
702-
{
703-
name: this.name,
704-
serveNodes: metadata.nodes,
705-
},
706-
metadata
707+
const reqOpts = ClusterUtils.getRequestFromMetadata(
708+
metadata,
709+
this?.metadata?.location,
710+
this.name
707711
);
708-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
709-
delete (reqOpts as any).nodes;
710-
711712
this.bigtable.request<Operation>(
712713
{
713714
client: 'BigtableInstanceAdminClient',
714-
method: 'updateCluster',
715-
reqOpts,
715+
method: 'partialUpdateCluster',
716+
reqOpts: reqOpts,
716717
gaxOpts: gaxOptions,
717718
},
718719
(err, resp) => {

‎src/index.ts‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import {ServiceError} from 'google-gax';
3434
import * as v2 from './v2';
3535
import {PassThrough, Duplex} from 'stream';
3636
import grpcGcpModule = require('grpc-gcp');
37+
import {ClusterUtils} from './utils/cluster';
3738

3839
// eslint-disable-next-line @typescript-eslint/no-var-requires
3940
const streamEvents = require('stream-events');
@@ -610,12 +611,15 @@ export class Bigtable {
610611
'A cluster was provided with both `encryption` and `key` defined.'
611612
);
612613
}
613-
614-
clusters[cluster.id!] = {
615-
location: Cluster.getLocation_(this.projectId, cluster.location!),
616-
serveNodes: cluster.nodes,
614+
ClusterUtils.validateClusterMetadata(cluster);
615+
clusters[cluster.id!] = ClusterUtils.getClusterBaseConfig(
616+
cluster,
617+
Cluster.getLocation_(this.projectId, cluster.location!),
618+
undefined
619+
);
620+
Object.assign(clusters[cluster.id!], {
617621
defaultStorageType: Cluster.getStorageType_(cluster.storage!),
618-
};
622+
});
619623

620624
if (cluster.key) {
621625
clusters[cluster.id!].encryptionConfig = {

‎src/instance.ts‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ import {ServiceError} from 'google-gax';
6767
import {Bigtable} from '.';
6868
import {google} from '../protos/protos';
6969
import {Backup, RestoreTableCallback, RestoreTableResponse} from './backup';
70+
import {ClusterUtils} from './utils/cluster';
7071

7172
export interface ClusterInfo extends BasicClusterConfig {
7273
id: string;
@@ -391,9 +392,15 @@ Please use the format 'my-instance' or '${bigtable.projectName}/instances/my-ins
391392
parent: this.name,
392393
clusterId: id,
393394
} as google.bigtable.admin.v2.CreateClusterRequest;
394-
395+
ClusterUtils.validateClusterMetadata(options);
395396
if (!is.empty(options)) {
396-
reqOpts.cluster = {};
397+
reqOpts.cluster = ClusterUtils.getClusterBaseConfig(
398+
options,
399+
options.location
400+
? Cluster.getLocation_(this.bigtable.projectId, options.location)
401+
: undefined,
402+
undefined
403+
);
397404
}
398405

399406
if (
@@ -415,17 +422,6 @@ Please use the format 'my-instance' or '${bigtable.projectName}/instances/my-ins
415422
reqOpts.cluster!.encryptionConfig = options.encryption;
416423
}
417424

418-
if (options.location) {
419-
reqOpts.cluster!.location = Cluster.getLocation_(
420-
this.bigtable.projectId,
421-
options.location
422-
);
423-
}
424-
425-
if (options.nodes) {
426-
reqOpts.cluster!.serveNodes = options.nodes;
427-
}
428-
429425
if (options.storage) {
430426
const storageType = Cluster.getStorageType_(options.storage);
431427
reqOpts.cluster!.defaultStorageType = storageType;

‎src/utils/cluster.ts‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
// Copyright 2022 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import * as protos from '../../protos/protos';
16+
import {
17+
BasicClusterConfig,
18+
ICluster,
19+
SetClusterMetadataOptions,
20+
} from '../cluster';
21+
import {google} from '../../protos/protos';
22+
23+
export class ClusterUtils {
24+
static noConfigError =
25+
'Must specify either serve_nodes or all of the autoscaling configurations (min_serve_nodes, max_serve_nodes, and cpu_utilization_percent).';
26+
static allConfigError =
27+
'Cannot specify both serve_nodes and autoscaling configurations (min_serve_nodes, max_serve_nodes, and cpu_utilization_percent).';
28+
static incompleteConfigError =
29+
'All of autoscaling configurations must be specified at the same time (min_serve_nodes, max_serve_nodes, and cpu_utilization_percent).';
30+
31+
static validateClusterMetadata(
32+
metadata: SetClusterMetadataOptions | BasicClusterConfig
33+
): void {
34+
if (metadata.nodes) {
35+
if (
36+
metadata.minServeNodes ||
37+
metadata.maxServeNodes ||
38+
metadata.cpuUtilizationPercent
39+
) {
40+
throw new Error(this.allConfigError);
41+
}
42+
} else {
43+
if (
44+
metadata.minServeNodes ||
45+
metadata.maxServeNodes ||
46+
metadata.cpuUtilizationPercent
47+
) {
48+
if (
49+
!(
50+
metadata.minServeNodes &&
51+
metadata.maxServeNodes &&
52+
metadata.cpuUtilizationPercent
53+
)
54+
) {
55+
throw new Error(this.incompleteConfigError);
56+
}
57+
} else {
58+
throw new Error(this.noConfigError);
59+
}
60+
}
61+
}
62+
static getUpdateMask(metadata: SetClusterMetadataOptions): string[] {
63+
const updateMask: string[] = [];
64+
if (metadata.nodes) {
65+
updateMask.push('serve_nodes');
66+
if (
67+
!(
68+
metadata.minServeNodes ||
69+
metadata.maxServeNodes ||
70+
metadata.cpuUtilizationPercent
71+
)
72+
) {
73+
updateMask.push('cluster_config.cluster_autoscaling_config');
74+
}
75+
}
76+
if (metadata.minServeNodes) {
77+
updateMask.push(
78+
'cluster_config.cluster_autoscaling_config.autoscaling_limits.min_serve_nodes'
79+
);
80+
}
81+
if (metadata.maxServeNodes) {
82+
updateMask.push(
83+
'cluster_config.cluster_autoscaling_config.autoscaling_limits.max_serve_nodes'
84+
);
85+
}
86+
if (metadata.cpuUtilizationPercent) {
87+
updateMask.push(
88+
'cluster_config.cluster_autoscaling_config.autoscaling_targets.cpu_utilization_percent'
89+
);
90+
}
91+
return updateMask;
92+
}
93+
94+
static getClusterBaseConfig(
95+
metadata: SetClusterMetadataOptions | BasicClusterConfig,
96+
location: string | undefined | null,
97+
name: string | undefined
98+
): google.bigtable.admin.v2.ICluster {
99+
let clusterConfig;
100+
if (
101+
metadata.cpuUtilizationPercent ||
102+
metadata.minServeNodes ||
103+
metadata.maxServeNodes
104+
) {
105+
clusterConfig = {
106+
clusterAutoscalingConfig: {
107+
autoscalingTargets: {
108+
cpuUtilizationPercent: metadata.cpuUtilizationPercent,
109+
},
110+
autoscalingLimits: {
111+
minServeNodes: metadata.minServeNodes,
112+
maxServeNodes: metadata.maxServeNodes,
113+
},
114+
},
115+
};
116+
}
117+
return Object.assign(
118+
{},
119+
name ? {name} : null,
120+
location ? {location} : null,
121+
clusterConfig ? {clusterConfig} : null,
122+
metadata.nodes ? {serveNodes: metadata.nodes} : null
123+
);
124+
}
125+
126+
static getClusterFromMetadata(
127+
metadata: SetClusterMetadataOptions,
128+
location: string | undefined | null,
129+
name: string
130+
): google.bigtable.admin.v2.ICluster {
131+
const cluster: ICluster | SetClusterMetadataOptions = Object.assign(
132+
{},
133+
this.getClusterBaseConfig(metadata, location, name),
134+
metadata
135+
);
136+
delete (cluster as SetClusterMetadataOptions).nodes;
137+
delete (cluster as SetClusterMetadataOptions).minServeNodes;
138+
delete (cluster as SetClusterMetadataOptions).maxServeNodes;
139+
delete (cluster as SetClusterMetadataOptions).cpuUtilizationPercent;
140+
return cluster as ICluster;
141+
}
142+
143+
static getRequestFromMetadata(
144+
metadata: SetClusterMetadataOptions,
145+
location: string | undefined | null,
146+
name: string
147+
): protos.google.bigtable.admin.v2.IPartialUpdateClusterRequest {
148+
return {
149+
cluster: this.getClusterFromMetadata(metadata, location, name),
150+
updateMask: {paths: this.getUpdateMask(metadata)},
151+
};
152+
}
153+
}

0 commit comments

Comments
 (0)