Skip to content
This repository was archived by the owner on Nov 17, 2023. It is now read-only.

Commit 1a537af

Browse files
ChaiBapchyaChaiBapchya
committed
update autoscaling lambda with G4 details
1 parent 9e5fcfa commit 1a537af

File tree

2 files changed

+24
-14
lines changed

2 files changed

+24
-14
lines changed

‎services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/environment.yml‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@ test:
77
IAM_JENKINS_RESTRICTED_SLAVE_ROLE: arn:aws:iam::REDACTED:role/jenkins_restricted_slave_role
88
SECRETS_MANAGER_ARN: arn:aws:secretsmanager:us-west-2:REDACTED:secret:REDACTED
99
JENKINS_PRIV_TUNNEL: jenkins-priv.mxnet-ci-dev.amazon-ml.com:48593
10-
LAUNCH_TEMPLATES: '{"mxnetlinux-cpu":{"id":"lt-06a15945813ad44f2","version":"14"},"restricted-mxnetlinux-cpu":{"id":"lt-0dc74292f7d647ac6","version":"9"},"mxnetlinux-gpu":{"id":"lt-0c22f238c0edb58ab","version":"19"},"mxnetlinux-gpu-p3":{"id":"lt-00c83ee5d7aeaf4ab","version":"12"},"restricted-mxnetlinux-gpu-p3":{"id":"lt-0f893d7f3f2660c1c","version":"3"},"mxnetlinux-gpu-p3-8xlarge":{"id":"lt-0277305ae5f49782b","version":"6"},"mxnetwindows-cpu":{"id":"lt-09dff2fff6b5586f0","version":"11"},"mxnetwindows-gpu":{"id":"lt-0ce229129d0d3be27","version":"15"},"utility":{"id":"lt-028ee0bc3cef79942","version":"3"},"restricted-utility":{"id":"lt-05d66be1f50c9b3fc","version":"1"}, "restricted-mxnetlinux-gpu":{"id":"lt-0c246487c1570d396","version":"5"}}'
11-
EXECUTORS_PER_LABEL: '{"mxnetlinux-cpu":3,"restricted-mxnetlinux-cpu":3,"mxnetlinux-gpu":1,"mxnetlinux-gpu-p3":1,"restricted-mxnetlinux-gpu-p3":1,"mxnetlinux-gpu-p3-8xlarge":1,"mxnetwindows-cpu":4,"mxnetwindows-gpu":1,"utility":30,"restricted-utility":30, "restricted-mxnetlinux-gpu": 1}'
12-
WARM_POOL_SIZE: '{"mxnetlinux-cpu":1,"restricted-mxnetlinux-cpu":0,"mxnetlinux-gpu":0,"mxnetlinux-gpu-p3":0,"restricted-mxnetlinux-gpu-p3":0,"mxnetlinux-gpu-p3-8xlarge":0,"mxnetwindows-cpu":1,"mxnetwindows-gpu":0,"utility":1,"restricted-utility":1, "restricted-mxnetlinux-gpu": 0}'
13-
MINIMUM_QUEUE_TIMES_SEC: '{"mxnetlinux-cpu":30,"restricted-mxnetlinux-cpu":30,"mxnetlinux-gpu":30,"mxnetlinux-gpu-p3":30,"restricted-mxnetlinux-gpu-p3":30,"mxnetlinux-gpu-p3-8xlarge":30,"mxnetwindows-cpu":30,"mxnetwindows-gpu":30,"utility":3,"restricted-utility":3, "restricted-mxnetlinux-gpu": 30}'
14-
CCACHE_EFS_DNS: '{"mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","restricted-mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","mxnetlinux-gpu":"NONE","mxnetlinux-gpu-p3":"NONE","restricted-mxnetlinux-gpu-p3":"NONE","mxnetlinux-gpu-p3-8xlarge":"NONE","mxnetwindows-cpu":"NONE","mxnetwindows-gpu":"NONE","utility":"NONE","restricted-utility":"NONE", "restricted-mxnetlinux-gpu": "NONE"}'
15-
MAXIMUM_STARTUP_TIME_SEC: '{"mxnetlinux-cpu":300,"restricted-mxnetlinux-cpu":300,"mxnetlinux-gpu":300,"mxnetlinux-gpu-p3":300,"restricted-mxnetlinux-gpu-p3":300,"mxnetlinux-gpu-p3-8xlarge":300,"mxnetwindows-cpu":1800,"mxnetwindows-gpu":1800,"utility":300,"restricted-utility":300, "restricted-mxnetlinux-gpu":300}'
16-
MANAGED_JENKINS_NODE_LABELS: '["mxnetlinux-cpu","restricted-mxnetlinux-cpu","mxnetlinux-gpu", "restricted-mxnetlinux-gpu", "mxnetwindows-cpu","mxnetwindows-gpu","mxnetlinux-gpu-p3","restricted-mxnetlinux-gpu-p3","mxnetlinux-gpu-p3-8xlarge","utility","restricted-utility"]'
10+
LAUNCH_TEMPLATES: '{"mxnetlinux-cpu":{"id":"lt-06a15945813ad44f2","version":"14"},"restricted-mxnetlinux-cpu":{"id":"lt-0dc74292f7d647ac6","version":"9"},"mxnetlinux-gpu":{"id":"lt-0c22f238c0edb58ab","version":"19"},"mxnetlinux-gpu-g4":{"id":"lt-0f830794cba5041e2","version":"1"},"mxnetlinux-gpu-p3":{"id":"lt-00c83ee5d7aeaf4ab","version":"12"},"restricted-mxnetlinux-gpu-p3":{"id":"lt-0f893d7f3f2660c1c","version":"3"},"mxnetlinux-gpu-p3-8xlarge":{"id":"lt-0277305ae5f49782b","version":"6"},"mxnetwindows-cpu":{"id":"lt-09dff2fff6b5586f0","version":"11"},"mxnetwindows-gpu":{"id":"lt-0ce229129d0d3be27","version":"15"},"utility":{"id":"lt-028ee0bc3cef79942","version":"3"},"restricted-utility":{"id":"lt-05d66be1f50c9b3fc","version":"1"}, "restricted-mxnetlinux-gpu":{"id":"lt-0c246487c1570d396","version":"5"}}'
11+
EXECUTORS_PER_LABEL: '{"mxnetlinux-cpu":3,"restricted-mxnetlinux-cpu":3,"mxnetlinux-gpu":1,"mxnetlinux-gpu-g4":1,"mxnetlinux-gpu-p3":1,"restricted-mxnetlinux-gpu-p3":1,"mxnetlinux-gpu-p3-8xlarge":1,"mxnetwindows-cpu":4,"mxnetwindows-gpu":1,"utility":30,"restricted-utility":30, "restricted-mxnetlinux-gpu": 1}'
12+
WARM_POOL_SIZE: '{"mxnetlinux-cpu":1,"restricted-mxnetlinux-cpu":0,"mxnetlinux-gpu":0,"mxnetlinux-gpu-g4":0,"mxnetlinux-gpu-p3":0,"restricted-mxnetlinux-gpu-p3":0,"mxnetlinux-gpu-p3-8xlarge":0,"mxnetwindows-cpu":1,"mxnetwindows-gpu":0,"utility":1,"restricted-utility":1, "restricted-mxnetlinux-gpu": 0}'
13+
MINIMUM_QUEUE_TIMES_SEC: '{"mxnetlinux-cpu":30,"restricted-mxnetlinux-cpu":30,"mxnetlinux-gpu":30,"mxnetlinux-gpu-g4":30,"mxnetlinux-gpu-p3":30,"restricted-mxnetlinux-gpu-p3":30,"mxnetlinux-gpu-p3-8xlarge":30,"mxnetwindows-cpu":30,"mxnetwindows-gpu":30,"utility":3,"restricted-utility":3, "restricted-mxnetlinux-gpu": 30}'
14+
CCACHE_EFS_DNS: '{"mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","restricted-mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","mxnetlinux-gpu":"NONE","mxnetlinux-gpu-g4":"NONE","mxnetlinux-gpu-p3":"NONE","restricted-mxnetlinux-gpu-p3":"NONE","mxnetlinux-gpu-p3-8xlarge":"NONE","mxnetwindows-cpu":"NONE","mxnetwindows-gpu":"NONE","utility":"NONE","restricted-utility":"NONE", "restricted-mxnetlinux-gpu": "NONE"}'
15+
MAXIMUM_STARTUP_TIME_SEC: '{"mxnetlinux-cpu":300,"restricted-mxnetlinux-cpu":300,"mxnetlinux-gpu":300,"mxnetlinux-gpu-g4":300,"mxnetlinux-gpu-p3":300,"restricted-mxnetlinux-gpu-p3":300,"mxnetlinux-gpu-p3-8xlarge":300,"mxnetwindows-cpu":1800,"mxnetwindows-gpu":1800,"utility":300,"restricted-utility":300, "restricted-mxnetlinux-gpu":300}'
16+
MANAGED_JENKINS_NODE_LABELS: '["mxnetlinux-cpu","restricted-mxnetlinux-cpu","mxnetlinux-gpu", "mxnetlinux-gpu-g4", "restricted-mxnetlinux-gpu", "mxnetwindows-cpu","mxnetwindows-gpu","mxnetlinux-gpu-p3","restricted-mxnetlinux-gpu-p3","mxnetlinux-gpu-p3-8xlarge","utility","restricted-utility"]'
1717
IGNORED_JENKINS_NODE_LABELS: '["mxnetlinux","mxnetwindows","master"]'
1818
IGNORED_JENKINS_NODE_NAMES: '["master"]'
1919
LOGGING_LEVEL: DEBUG
@@ -31,13 +31,13 @@ prod:
3131
IAM_JENKINS_RESTRICTED_SLAVE_ROLE: arn:aws:iam::REDACTED:role/jenkins_restricted_slave_role
3232
SECRETS_MANAGER_ARN: arn:aws:secretsmanager:us-west-2:REDACTED:secret:REDACTED
3333
JENKINS_PRIV_TUNNEL: jenkins-priv.mxnet-ci.amazon-ml.com:48593
34-
LAUNCH_TEMPLATES: '{"mxnetlinux-cpu":{"id":"lt-059ca0af3b73fdd43","version":"6"},"restricted-mxnetlinux-cpu":{"id":"lt-0752e01a2f18939a4","version":"4"},"mxnetlinux-gpu":{"id":"lt-083414b180618edd0","version":"8"},"mxnetlinux-gpu-p3":{"id":"lt-09e887362d145072b","version":"8"},"mxnetlinux-gpu-p3-8xlarge":{"id":"lt-03458db28b362cf92","version":"5"},"mxnetwindows-cpu":{"id":"lt-02d3dfef15faf1298","version":"6"},"mxnetwindows-gpu":{"id":"lt-0eb15ef80c9a69ef1","version":"6"},"utility":{"id":"lt-0b64c7b2c90e53235","version":"1"},"restricted-utility":{"id":"lt-0b8c0a9d4ee3ea089","version":"1"},"restricted-mxnetlinux-gpu-p3":{"id":"lt-0a98711a162486aa5","version":"1"}}'
35-
EXECUTORS_PER_LABEL: '{"mxnetlinux-cpu":3,"restricted-mxnetlinux-cpu":3,"mxnetlinux-gpu":1,"restricted-mxnetlinux-gpu":0,"mxnetlinux-gpu-p3":1,"mxnetlinux-gpu-p3-8xlarge":1,"mxnetwindows-cpu":4,"restricted-mxnetlinux-gpu-p3":1,"mxnetwindows-gpu":1,"utility":30,"restricted-utility":30}'
36-
WARM_POOL_SIZE: '{"mxnetlinux-cpu":1,"restricted-mxnetlinux-cpu":0,"mxnetlinux-gpu":0,"mxnetlinux-gpu-p3":0,"mxnetlinux-gpu-p3-8xlarge":0,"mxnetwindows-cpu":1,"mxnetwindows-gpu":0,"restricted-mxnetlinux-gpu-p3":0,"utility":1,"restricted-utility":1}'
37-
MINIMUM_QUEUE_TIMES_SEC: '{"mxnetlinux-cpu":30,"restricted-mxnetlinux-cpu":30,"mxnetlinux-gpu":30,"mxnetlinux-gpu-p3":30,"mxnetlinux-gpu-p3-8xlarge":30,"restricted-mxnetlinux-gpu-p3":30,"mxnetwindows-cpu":30,"mxnetwindows-gpu":30,"utility":3,"restricted-utility":3}'
38-
CCACHE_EFS_DNS: '{"mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","restricted-mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","mxnetlinux-gpu":"NONE","mxnetlinux-gpu-p3":"NONE","restricted-mxnetlinux-gpu-p3":"NONE","mxnetlinux-gpu-p3-8xlarge":"NONE","mxnetwindows-cpu":"NONE","mxnetwindows-gpu":"NONE","utility":"NONE","restricted-utility":"NONE"}'
39-
MAXIMUM_STARTUP_TIME_SEC: '{"mxnetlinux-cpu":300,"restricted-mxnetlinux-cpu":300,"mxnetlinux-gpu":300,"mxnetlinux-gpu-p3":300,"restricted-mxnetlinux-gpu-p3":300,"mxnetlinux-gpu-p3-8xlarge":300,"mxnetwindows-cpu":1800,"mxnetwindows-gpu":1800,"utility":300,"restricted-utility":300}'
40-
MANAGED_JENKINS_NODE_LABELS: '["mxnetlinux-cpu","restricted-mxnetlinux-cpu","mxnetlinux-gpu","mxnetwindows-cpu","mxnetwindows-gpu","mxnetlinux-gpu-p3","restricted-mxnetlinux-gpu-p3","mxnetlinux-gpu-p3-8xlarge","utility","restricted-utility"]'
34+
LAUNCH_TEMPLATES: '{"mxnetlinux-cpu":{"id":"lt-059ca0af3b73fdd43","version":"6"},"restricted-mxnetlinux-cpu":{"id":"lt-0752e01a2f18939a4","version":"4"},"mxnetlinux-gpu":{"id":"lt-083414b180618edd0","version":"8"},"mxnetlinux-gpu-g4":{"id":"lt-0ebf575cc5a56ebf4","version":"1"},"mxnetlinux-gpu-p3":{"id":"lt-09e887362d145072b","version":"8"},"mxnetlinux-gpu-p3-8xlarge":{"id":"lt-03458db28b362cf92","version":"5"},"mxnetwindows-cpu":{"id":"lt-02d3dfef15faf1298","version":"6"},"mxnetwindows-gpu":{"id":"lt-0eb15ef80c9a69ef1","version":"6"},"utility":{"id":"lt-0b64c7b2c90e53235","version":"1"},"restricted-utility":{"id":"lt-0b8c0a9d4ee3ea089","version":"1"},"restricted-mxnetlinux-gpu-p3":{"id":"lt-0a98711a162486aa5","version":"1"}}'
35+
EXECUTORS_PER_LABEL: '{"mxnetlinux-cpu":3,"restricted-mxnetlinux-cpu":3,"mxnetlinux-gpu":1,"mxnetlinux-gpu-g4":1,"restricted-mxnetlinux-gpu":0,"mxnetlinux-gpu-p3":1,"mxnetlinux-gpu-p3-8xlarge":1,"mxnetwindows-cpu":4,"restricted-mxnetlinux-gpu-p3":1,"mxnetwindows-gpu":1,"utility":30,"restricted-utility":30}'
36+
WARM_POOL_SIZE: '{"mxnetlinux-cpu":1,"restricted-mxnetlinux-cpu":0,"mxnetlinux-gpu":0,"mxnetlinux-gpu-g4":0,"mxnetlinux-gpu-p3":0,"mxnetlinux-gpu-p3-8xlarge":0,"mxnetwindows-cpu":1,"mxnetwindows-gpu":0,"restricted-mxnetlinux-gpu-p3":0,"utility":1,"restricted-utility":1}'
37+
MINIMUM_QUEUE_TIMES_SEC: '{"mxnetlinux-cpu":30,"restricted-mxnetlinux-cpu":30,"mxnetlinux-gpu":30,"mxnetlinux-gpu-g4":30,"mxnetlinux-gpu-p3":30,"mxnetlinux-gpu-p3-8xlarge":30,"restricted-mxnetlinux-gpu-p3":30,"mxnetwindows-cpu":30,"mxnetwindows-gpu":30,"utility":3,"restricted-utility":3}'
38+
CCACHE_EFS_DNS: '{"mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","restricted-mxnetlinux-cpu":"fs-REDACTED.efs.us-west-2.amazonaws.com","mxnetlinux-gpu":"NONE","mxnetlinux-gpu-g4":"NONE","mxnetlinux-gpu-p3":"NONE","restricted-mxnetlinux-gpu-p3":"NONE","mxnetlinux-gpu-p3-8xlarge":"NONE","mxnetwindows-cpu":"NONE","mxnetwindows-gpu":"NONE","utility":"NONE","restricted-utility":"NONE"}'
39+
MAXIMUM_STARTUP_TIME_SEC: '{"mxnetlinux-cpu":300,"restricted-mxnetlinux-cpu":300,"mxnetlinux-gpu":300,"mxnetlinux-gpu-g4":300,"mxnetlinux-gpu-p3":300,"restricted-mxnetlinux-gpu-p3":300,"mxnetlinux-gpu-p3-8xlarge":300,"mxnetwindows-cpu":1800,"mxnetwindows-gpu":1800,"utility":300,"restricted-utility":300}'
40+
MANAGED_JENKINS_NODE_LABELS: '["mxnetlinux-cpu","restricted-mxnetlinux-cpu","mxnetlinux-gpu", "mxnetlinux-gpu-g4", "mxnetwindows-cpu","mxnetwindows-gpu","mxnetlinux-gpu-p3","restricted-mxnetlinux-gpu-p3","mxnetlinux-gpu-p3-8xlarge","utility","restricted-utility"]'
4141
IGNORED_JENKINS_NODE_LABELS: '["mxnetlinux","mxnetwindows","master"]'
4242
IGNORED_JENKINS_NODE_NAMES: '["master"]'
4343
LOGGING_LEVEL: DEBUG

‎services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/handler.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -828,6 +828,7 @@ def format_linux(label, target_instance_name):
828828
linux_types = ['mxnetlinux-cpu',
829829
'restricted-mxnetlinux-cpu',
830830
'mxnetlinux-gpu',
831+
'mxnetlinux-gpu-g4',
831832
'mxnetlinux-gpu-p3',
832833
'restricted-mxnetlinux-gpu-p3',
833834
'restricted-mxnetlinux-gpu',
@@ -1261,6 +1262,15 @@ def _get_slave_configuration():
12611262
'tunnel': _get_jenkins_private_tunnel_address(),
12621263
'job_name_restriction_regex': '^(?!restricted-).+' # Run only unrestricted jobs
12631264
},
1265+
'mxnetlinux-gpu-g4': {
1266+
'num_executors': _get_nb_executors_per_label()['mxnetlinux-gpu-g4'], # Number of executors
1267+
'node_description': '[AUTOSCALING] MXNet slave running Ubuntu 18.04 on a g4dn.4xlarge',
1268+
'remote_fs': '/home/jenkins_slave', # Remote workspace location
1269+
'labels': 'mxnetlinux-gpu-g4', # Space separated labels string
1270+
'exclusive': True, # Only run jobs assigned to it
1271+
'tunnel': _get_jenkins_private_tunnel_address(),
1272+
'job_name_restriction_regex': '^(?!restricted-).+' # Run only unrestricted jobs
1273+
},
12641274
'restricted-mxnetlinux-gpu': {
12651275
'num_executors': _get_nb_executors_per_label()['restricted-mxnetlinux-gpu'], # Number of executors
12661276
'node_description': '[AUTOSCALING] MXNet slave running Ubuntu 16.04 on a g3.8xlarge',

0 commit comments

Comments
 (0)