Skip to content

Commit 4ea36f7

Browse files
feat: Parallel builds implementation (aws#2337)
* chore: bump lambda-builder version v1.1.0 and sam-cli version v1.2.0. * after updating version, run a command to update requirements/reproducible-linux.txt file * De-duplicated builds * De-duplicated builds (updated unit tests) * De-duplicated builds (add actual build functionality and related unit tests) * De-duplicated builds (build graph should be created inside build folder) * De-duplicated builds (update tests with os specific separator) * De-duplicated builds (update tests with os specific separator) * De-duplicated builds (update tests with os specific separator) * De-duplicated builds (update tests with os specific separator) * De-duplicated builds (added integration tests) * De-duplicated builds - Changed to add functions to use dedicated method - Added extra checks when function name or handler name is called - Removed creation of tmp directory from one of the unit test - Added validation for dedup integ tests to validate functions build only once * De-duplicated builds - Changed to identify each build as unique for custom Makefile builds - Added unit tests related to this change - Added integration tests related to this change * De-duplicated builds - Update integration tests to run each couple of functions because of timeout * De-duplicated builds - Update is_building_specific_resource function name and add documentation for it - Updated formatting of some files * De-duplicated builds - Update documentation and match parameter names for specific build function * implement cache build * change attribute name from checksum to source_md5 * update logic for rm cache directory * update unit test for build graph * refactor build definition and update unit test * update unit test * bug fix for the cached flag property * add integration test for cache build * Update documentation and log message * add cached build for layers * feat: parallel builds (utils implementation) * feat: parallel builds * update comment for layer cached build * merge with cached builds * small refactoring * minor: update parallel text * feat: build strategy * feat: build strategy * fix unit tests * update unit tests * fix unit tests and add unit test for CacheBuildStrategy * delete unwanted changes * fix CacheBuildStrategyTest after merge * black reformat * feat: parallel builds (utils implementation) * feat: parallel builds * minor: update parallel text * feat: parallel builds * - Use partial instead of custom Task class - Update to use build strategy for Parallel builds * update unit tests * refactoring for CachedBuildStrategy unit test * merge changes from cached build * fix integration test * merge changes from cached build * fix integration test import * fix unittest in test_app_builder * update constructor of build graph to python 3 style and black reformat * update code based on review * update unit test * update unit test for build_strategy * update unit test * feat: parallel builds (utils implementation) * feat: parallel builds * minor: update parallel text * feat: parallel builds * - Use partial instead of custom Task class - Update to use build strategy for Parallel builds * update unit tests * merge changes from cached build * merge with cached builds * merge changes from cached build * merge changes with cache build * black changes Co-authored-by: Mingkun He <[email protected]> Co-authored-by: mingkun2020 <[email protected]>
1 parent bd33224 commit 4ea36f7

11 files changed

Lines changed: 431 additions & 9 deletions

File tree

‎samcli/commands/build/command.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,12 @@
9292
help="If your functions depend on packages that have natively compiled dependencies, use this flag "
9393
"to build your function inside an AWS Lambda-like Docker container",
9494
)
95+
@click.option(
96+
"--parallel",
97+
"-p",
98+
is_flag=True,
99+
help="Use this flag to run builds of each function/layer in parallel",
100+
)
95101
@click.option(
96102
"--manifest",
97103
"-m",
@@ -122,6 +128,7 @@ def cli(
122128
cache_dir,
123129
use_container,
124130
cached,
131+
parallel,
125132
manifest,
126133
docker_network,
127134
skip_pull_image,
@@ -142,6 +149,7 @@ def cli(
142149
True,
143150
use_container,
144151
cached,
152+
parallel,
145153
manifest,
146154
docker_network,
147155
skip_pull_image,
@@ -159,6 +167,7 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
159167
clean,
160168
use_container,
161169
cached,
170+
parallel,
162171
manifest_path,
163172
docker_network,
164173
skip_pull_image,
@@ -215,6 +224,7 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
215224
manifest_path_override=ctx.manifest_path_override,
216225
container_manager=ctx.container_manager,
217226
mode=ctx.mode,
227+
parallel=parallel
218228
)
219229
except FunctionNotFound as ex:
220230
raise UserException(str(ex), wrapped_from=ex.__class__.__name__) from ex

‎samcli/lib/build/app_builder.py‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,22 @@
22
Builds the application
33
"""
44

5-
import os
65
import io
76
import json
87
import logging
8+
import os
99
import pathlib
1010

1111
import docker
12+
from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version
1213
from aws_lambda_builders.builder import LambdaBuilder
1314
from aws_lambda_builders.exceptions import LambdaBuilderError
14-
from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version
1515

1616
import samcli.lib.utils.osutils as osutils
17-
from samcli.lib.utils.colors import Colored
18-
from samcli.lib.providers.sam_base_provider import SamBaseProvider
1917
from samcli.lib.build.build_graph import FunctionBuildDefinition, LayerBuildDefinition, BuildGraph
20-
from samcli.lib.build.build_strategy import DefaultBuildStrategy, CachedBuildStrategy
18+
from samcli.lib.build.build_strategy import DefaultBuildStrategy, CachedBuildStrategy, ParallelBuildStrategy
19+
from samcli.lib.providers.sam_base_provider import SamBaseProvider
20+
from samcli.lib.utils.colors import Colored
2121
from samcli.local.docker.lambda_build_container import LambdaBuildContainer
2222
from .workflow_config import get_workflow_config, get_layer_subfolder, supports_build_in_container
2323

@@ -126,7 +126,20 @@ def build(self):
126126
build_graph = self._get_build_graph()
127127
build_strategy = DefaultBuildStrategy(build_graph, self._build_dir, self._build_function, self._build_layer)
128128

129-
if self._cached:
129+
if self._parallel:
130+
if self._cached:
131+
build_strategy = ParallelBuildStrategy(
132+
build_graph,
133+
CachedBuildStrategy(build_graph,
134+
build_strategy,
135+
self._base_dir,
136+
self._build_dir,
137+
self._cache_dir,
138+
self._is_building_specific_resource)
139+
)
140+
else:
141+
build_strategy = ParallelBuildStrategy(build_graph, build_strategy)
142+
elif self._cached:
130143
build_strategy = CachedBuildStrategy(build_graph,
131144
build_strategy,
132145
self._base_dir,

‎samcli/lib/build/build_strategy.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
from samcli.commands.build.exceptions import MissingBuildMethodException
99
from samcli.lib.utils import osutils
10+
from samcli.lib.utils.async_utils import AsyncContext
1011
from samcli.lib.utils.hash import dir_checksum
1112

1213
LOG = logging.getLogger(__name__)
@@ -240,3 +241,52 @@ def _clean_redundant_cached(self):
240241
for cache_dir in pathlib.Path(self._cache_dir).iterdir():
241242
if cache_dir.name not in uuids:
242243
shutil.rmtree(pathlib.Path(self._cache_dir, cache_dir.name))
244+
245+
246+
class ParallelBuildStrategy(BuildStrategy):
247+
"""
248+
Parallel implementation of Build Strategy
249+
This strategy runs each build in parallel.
250+
For actual build implementation it calls delegate implementation (could be one of the other Build Strategy)
251+
"""
252+
253+
def __init__(self, build_graph, delegate_build_strategy, async_context=AsyncContext()):
254+
super().__init__(build_graph)
255+
self._delegate_build_strategy = delegate_build_strategy
256+
self._async_context = async_context
257+
258+
def build(self):
259+
"""
260+
Runs all build and collects results from async context
261+
"""
262+
result = {}
263+
with self, self._delegate_build_strategy:
264+
# ignore result
265+
super().build()
266+
# wait for other executions to complete
267+
268+
async_results = self._async_context.run_async()
269+
for async_result in async_results:
270+
result.update(async_result)
271+
272+
return result
273+
274+
def build_single_function_definition(self, build_definition):
275+
"""
276+
Passes single function build into async context, no actual result returned from this function
277+
"""
278+
self._async_context.add_async_task(
279+
self._delegate_build_strategy.build_single_function_definition,
280+
build_definition
281+
)
282+
return {}
283+
284+
def build_single_layer_definition(self, layer_definition):
285+
"""
286+
Passes single layer build into async context, no actual result returned from this function
287+
"""
288+
self._async_context.add_async_task(
289+
self._delegate_build_strategy.build_single_layer_definition,
290+
layer_definition
291+
)
292+
return {}

‎samcli/lib/utils/async_utils.py‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
"""
2+
Contains asyncio related methods and helpers
3+
"""
4+
import asyncio
5+
import logging
6+
from functools import partial
7+
8+
LOG = logging.getLogger(__name__)
9+
10+
11+
async def _run_given_tasks_async(tasks, event_loop=asyncio.get_event_loop()):
12+
"""
13+
Given list of Task objects, this method executes all tasks in the given event loop (or default one)
14+
and returns list of the results.
15+
The list of the results are in the same order as the list of the input.
16+
17+
Ex; If we say we are going to execute
18+
Task1, Task2 and Task3; and their results are Result1, Result2 and Result3.
19+
If the input of the task array is; [Task1, Task2, Task3]
20+
The result of this operation would be; [Result1, Result2, Result3]
21+
22+
If any of the tasks throws an exception, this method raise the exception to the caller
23+
24+
Parameters
25+
----------
26+
tasks : list of Task
27+
The list of tasks that will be executed
28+
29+
event_loop: EventLoop
30+
The EventLoop instance that will be used for execution. If nothing is provided, this will be the default one
31+
32+
Returns
33+
--------
34+
results : list of results that is returned by the list of Tasks in order. Raises an exception if the underlying
35+
task is thrown an exception during its execution
36+
"""
37+
async_tasks = []
38+
results = []
39+
40+
LOG.debug("Async execution started")
41+
42+
for task in tasks:
43+
# loop in all tasks and pass them to the executor
44+
LOG.debug("Invoking function %s", str(task))
45+
async_tasks.append(event_loop.run_in_executor(None, task))
46+
47+
LOG.debug("Waiting for async results")
48+
49+
for result in await asyncio.gather(*async_tasks, return_exceptions=True):
50+
# for each task, wait for them to complete
51+
if isinstance(result, Exception):
52+
LOG.debug("Exception raised during the execution")
53+
# if the result is a type of Exception, stop the event loop and raise it back to caller
54+
raise result
55+
results.append(result)
56+
57+
LOG.debug("Async execution completed")
58+
59+
# flush all loggers which is printed during async execution
60+
if logging.root and logging.root.handlers:
61+
for handler in logging.root.handlers:
62+
handler.flush()
63+
64+
return results
65+
66+
67+
def run_given_tasks_async(tasks, event_loop=asyncio.get_event_loop()):
68+
"""
69+
Runs the given list of tasks in the given (or default) event loop.
70+
This function will wait for execution to be completed
71+
72+
See _run_given_tasks_async for details
73+
74+
Parameters
75+
----------
76+
tasks: list of Task definitions that will be executed
77+
78+
event_loop: EventLoop instance that will be used to execute these tasks
79+
80+
Returns
81+
-------
82+
List of results from the given Task list. Raises the exception if any of the underlying functions throw one
83+
"""
84+
return event_loop.run_until_complete(_run_given_tasks_async(tasks, event_loop))
85+
86+
87+
class AsyncContext:
88+
"""
89+
A helper class to hold list of tasks, and manages their execution
90+
"""
91+
92+
def __init__(self):
93+
self._async_tasks = []
94+
95+
def add_async_task(self, function, *args):
96+
"""
97+
Add a function definition and its args to the the async context, which will be executed later
98+
99+
Parameters
100+
----------
101+
function: Reference to the function definition which will be executed in this AsyncContext
102+
103+
args: Parameters of the function which will be executed
104+
"""
105+
self._async_tasks.append(partial(function, *args))
106+
107+
def run_async(self):
108+
"""
109+
Will run all collected functions in async context, and return their results in order
110+
111+
Returns
112+
-------
113+
List of result of the executions in order
114+
"""
115+
event_loop = asyncio.new_event_loop()
116+
return run_given_tasks_async(self._async_tasks, event_loop)

‎tests/integration/buildcmd/build_integ_base.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ def get_command_list(
6767
debug=False,
6868
cached=False,
6969
cache_dir=None,
70+
parallel=False,
7071
):
7172

7273
command_list = [self.cmd, "build"]
@@ -100,6 +101,9 @@ def get_command_list(
100101
if cache_dir:
101102
command_list += ["-cd", cache_dir]
102103

104+
if parallel:
105+
command_list += ["--parallel"]
106+
103107
return command_list
104108

105109
def verify_docker_container_cleanedup(self, runtime):

‎tests/integration/buildcmd/test_build_cmd.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1133,3 +1133,55 @@ def test_cache_build(self, use_continer, code_uri, function1_handler, function2_
11331133
self._verify_build_and_invoke_functions(
11341134
expected_messages, command_result, self._make_parameter_override_arg(overrides)
11351135
)
1136+
1137+
1138+
@skipIf(
1139+
((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE),
1140+
"Skip build tests on windows when running in CI unless overridden",
1141+
)
1142+
class TestParallelBuilds(DedupBuildIntegBase):
1143+
template = "dedup-functions-template.yaml"
1144+
1145+
@parameterized.expand(
1146+
[
1147+
# in process
1148+
(
1149+
False,
1150+
"Dotnetcore3.1",
1151+
"HelloWorld::HelloWorld.FirstFunction::FunctionHandler",
1152+
"HelloWorld::HelloWorld.SecondFunction::FunctionHandler",
1153+
"dotnetcore3.1",
1154+
),
1155+
(False, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"),
1156+
(False, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs12.x"),
1157+
(False, "Python", "main.first_function_handler", "main.second_function_handler", "python3.8"),
1158+
(False, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"),
1159+
# container
1160+
(True, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"),
1161+
(True, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs12.x"),
1162+
(True, "Python", "main.first_function_handler", "main.second_function_handler", "python3.8"),
1163+
(True, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"),
1164+
]
1165+
)
1166+
@pytest.mark.flaky(reruns=3)
1167+
def test_dedup_build(self, use_continer, code_uri, function1_handler, function2_handler, runtime):
1168+
"""
1169+
Build template above and verify that each function call returns as expected
1170+
"""
1171+
overrides = {
1172+
"FunctionCodeUri": code_uri,
1173+
"Function1Handler": function1_handler,
1174+
"Function2Handler": function2_handler,
1175+
"FunctionRuntime": runtime,
1176+
}
1177+
cmdlist = self.get_command_list(use_container=use_continer, parameter_overrides=overrides, parallel=True)
1178+
1179+
LOG.info("Running Command: %s", cmdlist)
1180+
# Built using `native` python-pip builder for a python project.
1181+
command_result = run_command(cmdlist, cwd=self.working_dir)
1182+
1183+
expected_messages = ["World", "Mars"]
1184+
1185+
self._verify_build_and_invoke_functions(
1186+
expected_messages, command_result, self._make_parameter_override_arg(overrides)
1187+
)

‎tests/unit/commands/buildcmd/test_command.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ def test_must_succeed_build(self, os_mock, move_template_mock, ApplicationBuilde
4444
"clean",
4545
"use_container",
4646
"cached",
47+
"parallel",
4748
"manifest_path",
4849
"docker_network",
4950
"skip_pull",
@@ -61,6 +62,7 @@ def test_must_succeed_build(self, os_mock, move_template_mock, ApplicationBuilde
6162
manifest_path_override=ctx_mock.manifest_path_override,
6263
container_manager=ctx_mock.container_manager,
6364
mode=ctx_mock.mode,
65+
parallel="parallel",
6466
)
6567
builder_mock.build.assert_called_once()
6668
builder_mock.update_template.assert_called_once_with(
@@ -103,6 +105,7 @@ def test_must_catch_known_exceptions(self, exception, wrapped_exception, Applica
103105
"clean",
104106
"use_container",
105107
"cached",
108+
"parallel",
106109
"manifest_path",
107110
"docker_network",
108111
"skip_pull",
@@ -131,6 +134,7 @@ def test_must_catch_function_not_found_exception(self, ApplicationBuilderMock, B
131134
"clean",
132135
"use_container",
133136
"cached",
137+
"parallel",
134138
"manifest_path",
135139
"docker_network",
136140
"skip_pull",

‎tests/unit/commands/samconfig/test_samconfig.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ def test_build(self, do_cli_mock):
133133
True,
134134
True,
135135
False,
136+
False,
136137
"requirements.txt",
137138
"mynetwork",
138139
True,

0 commit comments

Comments
 (0)