Skip to content

Commit b142160

Browse files
authored
feat: Cached and Parallel build implementation (aws#2351) (aws#2357)
This reverts commit 74c2ba4.
1 parent 754a0f0 commit b142160

15 files changed

Lines changed: 1600 additions & 196 deletions

File tree

‎samcli/commands/build/build_context.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ def __init__(
2929
template_file,
3030
base_dir,
3131
build_dir,
32+
cache_dir,
33+
cached,
3234
mode,
3335
manifest_path=None,
3436
clean=False,
@@ -42,13 +44,15 @@ def __init__(
4244
self._template_file = template_file
4345
self._base_dir = base_dir
4446
self._build_dir = build_dir
47+
self._cache_dir = cache_dir
4548
self._manifest_path = manifest_path
4649
self._clean = clean
4750
self._use_container = use_container
4851
self._parameter_overrides = parameter_overrides
4952
self._docker_network = docker_network
5053
self._skip_pull_image = skip_pull_image
5154
self._mode = mode
55+
self._cached = cached
5256

5357
self._function_provider = None
5458
self._layer_provider = None
@@ -68,6 +72,11 @@ def __enter__(self):
6872

6973
self._build_dir = self._setup_build_dir(self._build_dir, self._clean)
7074

75+
if self._cached:
76+
cache_path = pathlib.Path(self._cache_dir)
77+
cache_path.mkdir(mode=self._BUILD_DIR_PERMISSIONS, parents=True, exist_ok=True)
78+
self._cache_dir = str(cache_path.resolve())
79+
7180
if self._use_container:
7281
self._container_manager = ContainerManager(
7382
docker_network_id=self._docker_network, skip_pull_image=self._skip_pull_image
@@ -119,6 +128,14 @@ def build_dir(self):
119128
def base_dir(self):
120129
return self._base_dir
121130

131+
@property
132+
def cache_dir(self):
133+
return self._cache_dir
134+
135+
@property
136+
def cached(self):
137+
return self._cached
138+
122139
@property
123140
def use_container(self):
124141
return self._use_container

‎samcli/commands/build/command.py‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
LOG = logging.getLogger(__name__)
2121

2222
DEFAULT_BUILD_DIR = os.path.join(".aws-sam", "build")
23+
DEFAULT_CACHE_DIR = os.path.join(".aws-sam", "cache")
2324

2425
HELP_TEXT = """
2526
Use this command to build your AWS Lambda Functions source code to generate artifacts that target AWS Lambda's
@@ -68,6 +69,13 @@
6869
type=click.Path(file_okay=False, dir_okay=True, writable=True), # Must be a directory
6970
help="Path to a folder where the built artifacts will be stored. This directory will be first removed before starting a build.",
7071
)
72+
@click.option(
73+
"--cache-dir",
74+
"-cd",
75+
default=DEFAULT_CACHE_DIR,
76+
type=click.Path(file_okay=False, dir_okay=True, writable=True), # Must be a directory
77+
help="Path to a folder where the cache artifacts will be stored.",
78+
)
7179
@click.option(
7280
"--base-dir",
7381
"-s",
@@ -84,13 +92,25 @@
8492
help="If your functions depend on packages that have natively compiled dependencies, use this flag "
8593
"to build your function inside an AWS Lambda-like Docker container",
8694
)
95+
@click.option(
96+
"--parallel",
97+
"-p",
98+
is_flag=True,
99+
help="Use this flag to run builds of each function/layer in parallel",
100+
)
87101
@click.option(
88102
"--manifest",
89103
"-m",
90104
default=None,
91105
type=click.Path(),
92106
help="Path to a custom dependency manifest (ex: package.json) to use instead of the default one",
93107
)
108+
@click.option(
109+
"--cached",
110+
"-c",
111+
is_flag=True,
112+
help="Use this flag to enable cached build",
113+
)
94114
@template_option_without_build
95115
@parameter_override_option
96116
@docker_common_options
@@ -105,7 +125,10 @@ def cli(
105125
template_file,
106126
base_dir,
107127
build_dir,
128+
cache_dir,
108129
use_container,
130+
cached,
131+
parallel,
109132
manifest,
110133
docker_network,
111134
skip_pull_image,
@@ -122,8 +145,11 @@ def cli(
122145
template_file,
123146
base_dir,
124147
build_dir,
148+
cache_dir,
125149
True,
126150
use_container,
151+
cached,
152+
parallel,
127153
manifest,
128154
docker_network,
129155
skip_pull_image,
@@ -137,8 +163,11 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
137163
template,
138164
base_dir,
139165
build_dir,
166+
cache_dir,
140167
clean,
141168
use_container,
169+
cached,
170+
parallel,
142171
manifest_path,
143172
docker_network,
144173
skip_pull_image,
@@ -164,7 +193,8 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
164193
from samcli.lib.build.build_graph import InvalidBuildGraphException
165194

166195
LOG.debug("'build' command is called")
167-
196+
if cached:
197+
LOG.info("Starting Build use cache")
168198
if use_container:
169199
LOG.info("Starting Build inside a container")
170200

@@ -173,6 +203,8 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
173203
template,
174204
base_dir,
175205
build_dir,
206+
cache_dir,
207+
cached,
176208
clean=clean,
177209
manifest_path=manifest_path,
178210
use_container=use_container,
@@ -186,10 +218,13 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
186218
ctx.resources_to_build,
187219
ctx.build_dir,
188220
ctx.base_dir,
221+
ctx.cache_dir,
222+
ctx.cached,
189223
ctx.is_building_specific_resource,
190224
manifest_path_override=ctx.manifest_path_override,
191225
container_manager=ctx.container_manager,
192226
mode=ctx.mode,
227+
parallel=parallel
193228
)
194229
except FunctionNotFound as ex:
195230
raise UserException(str(ex), wrapped_from=ex.__class__.__name__) from ex

‎samcli/lib/build/app_builder.py‎

Lines changed: 60 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,22 @@
22
Builds the application
33
"""
44

5-
import os
65
import io
76
import json
87
import logging
8+
import os
99
import pathlib
1010

1111
import docker
12+
from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version
1213
from aws_lambda_builders.builder import LambdaBuilder
1314
from aws_lambda_builders.exceptions import LambdaBuilderError
14-
from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version
1515

1616
import samcli.lib.utils.osutils as osutils
17-
from samcli.lib.utils.colors import Colored
18-
from samcli.commands.build.exceptions import MissingBuildMethodException
17+
from samcli.lib.build.build_graph import FunctionBuildDefinition, LayerBuildDefinition, BuildGraph
18+
from samcli.lib.build.build_strategy import DefaultBuildStrategy, CachedBuildStrategy, ParallelBuildStrategy
1919
from samcli.lib.providers.sam_base_provider import SamBaseProvider
20-
from samcli.lib.build.build_graph import BuildDefinition, BuildGraph
20+
from samcli.lib.utils.colors import Colored
2121
from samcli.local.docker.lambda_build_container import LambdaBuildContainer
2222
from .workflow_config import get_workflow_config, get_layer_subfolder, supports_build_in_container
2323

@@ -54,31 +54,37 @@ class ApplicationBuilder:
5454
converting source code into artifacts that can be run on AWS Lambda
5555
"""
5656

57-
def __init__(
58-
self,
59-
resources_to_build,
60-
build_dir,
61-
base_dir,
62-
is_building_specific_resource=False,
63-
manifest_path_override=None,
64-
container_manager=None,
65-
parallel=False,
66-
mode=None,
67-
):
57+
def __init__(self,
58+
resources_to_build,
59+
build_dir,
60+
base_dir,
61+
cache_dir,
62+
cached=False,
63+
is_building_specific_resource=False,
64+
manifest_path_override=None,
65+
container_manager=None,
66+
parallel=False,
67+
mode=None):
6868
"""
6969
Initialize the class
7070
7171
Parameters
7272
----------
73-
functions_to_build: Iterator
74-
Iterator that can vend out functions available in the SAM template
73+
resources_to_build: Iterator
74+
Iterator that can vend out resources available in the SAM template
7575
7676
build_dir : str
7777
Path to the directory where we will be storing built artifacts
7878
7979
base_dir : str
8080
Path to a folder. Use this folder as the root to resolve relative source code paths against
8181
82+
cache_dir : str
83+
Path to a the directory where we will be caching built artifacts
84+
85+
cached:
86+
Optional. Set to True to build each function with cache to improve performance
87+
8288
is_building_specific_resource : boolean
8389
Whether customer requested to build a specific resource alone in isolation,
8490
by specifying function_identifier to the build command.
@@ -96,6 +102,8 @@ def __init__(
96102
self._resources_to_build = resources_to_build
97103
self._build_dir = build_dir
98104
self._base_dir = base_dir
105+
self._cache_dir = cache_dir
106+
self._cached = cached
99107
self._manifest_path_override = manifest_path_override
100108
self._is_building_specific_resource = is_building_specific_resource
101109

@@ -115,72 +123,51 @@ def build(self):
115123
dict
116124
Returns the path to where each resource was built as a map of resource's LogicalId to the path string
117125
"""
118-
119-
result = self._build_functions()
120-
121-
for layer in self._resources_to_build.layers:
122-
LOG.info("Building layer '%s'", layer.name)
123-
if layer.build_method is None:
124-
raise MissingBuildMethodException(
125-
f"Layer {layer.name} cannot be build without BuildMethod. Please provide BuildMethod in Metadata."
126+
build_graph = self._get_build_graph()
127+
build_strategy = DefaultBuildStrategy(build_graph, self._build_dir, self._build_function, self._build_layer)
128+
129+
if self._parallel:
130+
if self._cached:
131+
build_strategy = ParallelBuildStrategy(
132+
build_graph,
133+
CachedBuildStrategy(build_graph,
134+
build_strategy,
135+
self._base_dir,
136+
self._build_dir,
137+
self._cache_dir,
138+
self._is_building_specific_resource)
126139
)
127-
result[layer.name] = self._build_layer(
128-
layer.name, layer.codeuri, layer.build_method, layer.compatible_runtimes
129-
)
130-
131-
return result
140+
else:
141+
build_strategy = ParallelBuildStrategy(build_graph, build_strategy)
142+
elif self._cached:
143+
build_strategy = CachedBuildStrategy(build_graph,
144+
build_strategy,
145+
self._base_dir,
146+
self._build_dir,
147+
self._cache_dir,
148+
self._is_building_specific_resource)
149+
150+
return build_strategy.build()
132151

133152
def _get_build_graph(self):
134153
"""
135-
Converts list of functions into a build graph, where we can iterate on each unique build and trigger build
154+
Converts list of functions and layers into a build graph, where we can iterate on each unique build and trigger
155+
build
136156
:return: BuildGraph, which represents list of unique build definitions
137157
"""
138158
build_graph = BuildGraph(self._build_dir)
139159
functions = self._resources_to_build.functions
160+
layers = self._resources_to_build.layers
140161
for function in functions:
141-
build_details = BuildDefinition(function.runtime, function.codeuri, function.metadata)
142-
build_graph.put_build_definition(build_details, function)
162+
function_build_details = FunctionBuildDefinition(function.runtime, function.codeuri, function.metadata)
163+
build_graph.put_function_build_definition(function_build_details, function)
143164

144-
build_graph.clean_redundant_functions_and_update(not self._is_building_specific_resource)
145-
return build_graph
165+
for layer in layers:
166+
layer_build_details = LayerBuildDefinition(layer.name, layer.codeuri, layer.build_method, layer.compatible_runtimes)
167+
build_graph.put_layer_build_definition(layer_build_details, layer)
146168

147-
def _build_functions(self):
148-
"""
149-
Iterates through build graph and runs each unique build and copies outcome to the corresponding function folder
150-
"""
151-
build_graph = self._get_build_graph()
152-
function_build_results = {}
153-
154-
for build_definition in build_graph.get_build_definitions():
155-
LOG.info("Building codeuri: %s runtime: %s metadata: %s functions: %s",
156-
build_definition.codeuri,
157-
build_definition.runtime,
158-
build_definition.metadata,
159-
[function.name for function in build_definition.functions])
160-
161-
# build into one of the functions from this build definition
162-
single_function_name = build_definition.get_function_name()
163-
single_build_dir = str(pathlib.Path(self._build_dir, single_function_name))
164-
165-
LOG.debug("Building to following folder %s", single_build_dir)
166-
self._build_function(build_definition.get_function_name(),
167-
build_definition.codeuri,
168-
build_definition.runtime,
169-
build_definition.get_handler_name(),
170-
single_build_dir,
171-
build_definition.metadata)
172-
function_build_results[single_function_name] = single_build_dir
173-
174-
# copy results to other functions
175-
for function in build_definition.functions:
176-
if function.name is not single_function_name:
177-
# artifacts directory will be created by the builder
178-
artifacts_dir = str(pathlib.Path(self._build_dir, function.name))
179-
LOG.debug("Copying artifacts from %s to %s", single_build_dir, artifacts_dir)
180-
osutils.copytree(single_build_dir, artifacts_dir)
181-
function_build_results[function.name] = artifacts_dir
182-
183-
return function_build_results
169+
build_graph.clean_redundant_definitions_and_update(not self._is_building_specific_resource)
170+
return build_graph
184171

185172
def update_template(self, template_dict, original_template_path, built_artifacts):
186173
"""

0 commit comments

Comments
 (0)