Skip to content

Commit 74c2ba4

Browse files
mndevecihawflau
andauthored
Revert cached parallel builds (aws#2351)
* Revert "feat: Parallel builds implementation (aws#2337)" This reverts commit 4ea36f7. * Revert "feat: Cache build (aws#2295)" This reverts commit bd33224. Co-authored-by: Wing Fung Lau <[email protected]>
1 parent 294bbaa commit 74c2ba4

15 files changed

Lines changed: 196 additions & 1600 deletions

File tree

‎samcli/commands/build/build_context.py‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@ def __init__(
2929
template_file,
3030
base_dir,
3131
build_dir,
32-
cache_dir,
33-
cached,
3432
mode,
3533
manifest_path=None,
3634
clean=False,
@@ -44,15 +42,13 @@ def __init__(
4442
self._template_file = template_file
4543
self._base_dir = base_dir
4644
self._build_dir = build_dir
47-
self._cache_dir = cache_dir
4845
self._manifest_path = manifest_path
4946
self._clean = clean
5047
self._use_container = use_container
5148
self._parameter_overrides = parameter_overrides
5249
self._docker_network = docker_network
5350
self._skip_pull_image = skip_pull_image
5451
self._mode = mode
55-
self._cached = cached
5652

5753
self._function_provider = None
5854
self._layer_provider = None
@@ -72,11 +68,6 @@ def __enter__(self):
7268

7369
self._build_dir = self._setup_build_dir(self._build_dir, self._clean)
7470

75-
if self._cached:
76-
cache_path = pathlib.Path(self._cache_dir)
77-
cache_path.mkdir(mode=self._BUILD_DIR_PERMISSIONS, parents=True, exist_ok=True)
78-
self._cache_dir = str(cache_path.resolve())
79-
8071
if self._use_container:
8172
self._container_manager = ContainerManager(
8273
docker_network_id=self._docker_network, skip_pull_image=self._skip_pull_image
@@ -128,14 +119,6 @@ def build_dir(self):
128119
def base_dir(self):
129120
return self._base_dir
130121

131-
@property
132-
def cache_dir(self):
133-
return self._cache_dir
134-
135-
@property
136-
def cached(self):
137-
return self._cached
138-
139122
@property
140123
def use_container(self):
141124
return self._use_container

‎samcli/commands/build/command.py‎

Lines changed: 1 addition & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
LOG = logging.getLogger(__name__)
2121

2222
DEFAULT_BUILD_DIR = os.path.join(".aws-sam", "build")
23-
DEFAULT_CACHE_DIR = os.path.join(".aws-sam", "cache")
2423

2524
HELP_TEXT = """
2625
Use this command to build your AWS Lambda Functions source code to generate artifacts that target AWS Lambda's
@@ -69,13 +68,6 @@
6968
type=click.Path(file_okay=False, dir_okay=True, writable=True), # Must be a directory
7069
help="Path to a folder where the built artifacts will be stored. This directory will be first removed before starting a build.",
7170
)
72-
@click.option(
73-
"--cache-dir",
74-
"-cd",
75-
default=DEFAULT_CACHE_DIR,
76-
type=click.Path(file_okay=False, dir_okay=True, writable=True), # Must be a directory
77-
help="Path to a folder where the cache artifacts will be stored.",
78-
)
7971
@click.option(
8072
"--base-dir",
8173
"-s",
@@ -92,25 +84,13 @@
9284
help="If your functions depend on packages that have natively compiled dependencies, use this flag "
9385
"to build your function inside an AWS Lambda-like Docker container",
9486
)
95-
@click.option(
96-
"--parallel",
97-
"-p",
98-
is_flag=True,
99-
help="Use this flag to run builds of each function/layer in parallel",
100-
)
10187
@click.option(
10288
"--manifest",
10389
"-m",
10490
default=None,
10591
type=click.Path(),
10692
help="Path to a custom dependency manifest (ex: package.json) to use instead of the default one",
10793
)
108-
@click.option(
109-
"--cached",
110-
"-c",
111-
is_flag=True,
112-
help="Use this flag to enable cached build",
113-
)
11494
@template_option_without_build
11595
@parameter_override_option
11696
@docker_common_options
@@ -125,10 +105,7 @@ def cli(
125105
template_file,
126106
base_dir,
127107
build_dir,
128-
cache_dir,
129108
use_container,
130-
cached,
131-
parallel,
132109
manifest,
133110
docker_network,
134111
skip_pull_image,
@@ -145,11 +122,8 @@ def cli(
145122
template_file,
146123
base_dir,
147124
build_dir,
148-
cache_dir,
149125
True,
150126
use_container,
151-
cached,
152-
parallel,
153127
manifest,
154128
docker_network,
155129
skip_pull_image,
@@ -163,11 +137,8 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
163137
template,
164138
base_dir,
165139
build_dir,
166-
cache_dir,
167140
clean,
168141
use_container,
169-
cached,
170-
parallel,
171142
manifest_path,
172143
docker_network,
173144
skip_pull_image,
@@ -193,8 +164,7 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
193164
from samcli.lib.build.build_graph import InvalidBuildGraphException
194165

195166
LOG.debug("'build' command is called")
196-
if cached:
197-
LOG.info("Starting Build use cache")
167+
198168
if use_container:
199169
LOG.info("Starting Build inside a container")
200170

@@ -203,8 +173,6 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
203173
template,
204174
base_dir,
205175
build_dir,
206-
cache_dir,
207-
cached,
208176
clean=clean,
209177
manifest_path=manifest_path,
210178
use_container=use_container,
@@ -218,13 +186,10 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
218186
ctx.resources_to_build,
219187
ctx.build_dir,
220188
ctx.base_dir,
221-
ctx.cache_dir,
222-
ctx.cached,
223189
ctx.is_building_specific_resource,
224190
manifest_path_override=ctx.manifest_path_override,
225191
container_manager=ctx.container_manager,
226192
mode=ctx.mode,
227-
parallel=parallel
228193
)
229194
except FunctionNotFound as ex:
230195
raise UserException(str(ex), wrapped_from=ex.__class__.__name__) from ex

‎samcli/lib/build/app_builder.py‎

Lines changed: 73 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,22 @@
22
Builds the application
33
"""
44

5+
import os
56
import io
67
import json
78
import logging
8-
import os
99
import pathlib
1010

1111
import docker
12-
from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version
1312
from aws_lambda_builders.builder import LambdaBuilder
1413
from aws_lambda_builders.exceptions import LambdaBuilderError
14+
from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version
1515

1616
import samcli.lib.utils.osutils as osutils
17-
from samcli.lib.build.build_graph import FunctionBuildDefinition, LayerBuildDefinition, BuildGraph
18-
from samcli.lib.build.build_strategy import DefaultBuildStrategy, CachedBuildStrategy, ParallelBuildStrategy
19-
from samcli.lib.providers.sam_base_provider import SamBaseProvider
2017
from samcli.lib.utils.colors import Colored
18+
from samcli.commands.build.exceptions import MissingBuildMethodException
19+
from samcli.lib.providers.sam_base_provider import SamBaseProvider
20+
from samcli.lib.build.build_graph import BuildDefinition, BuildGraph
2121
from samcli.local.docker.lambda_build_container import LambdaBuildContainer
2222
from .workflow_config import get_workflow_config, get_layer_subfolder, supports_build_in_container
2323

@@ -54,37 +54,31 @@ class ApplicationBuilder:
5454
converting source code into artifacts that can be run on AWS Lambda
5555
"""
5656

57-
def __init__(self,
58-
resources_to_build,
59-
build_dir,
60-
base_dir,
61-
cache_dir,
62-
cached=False,
63-
is_building_specific_resource=False,
64-
manifest_path_override=None,
65-
container_manager=None,
66-
parallel=False,
67-
mode=None):
57+
def __init__(
58+
self,
59+
resources_to_build,
60+
build_dir,
61+
base_dir,
62+
is_building_specific_resource=False,
63+
manifest_path_override=None,
64+
container_manager=None,
65+
parallel=False,
66+
mode=None,
67+
):
6868
"""
6969
Initialize the class
7070
7171
Parameters
7272
----------
73-
resources_to_build: Iterator
74-
Iterator that can vend out resources available in the SAM template
73+
functions_to_build: Iterator
74+
Iterator that can vend out functions available in the SAM template
7575
7676
build_dir : str
7777
Path to the directory where we will be storing built artifacts
7878
7979
base_dir : str
8080
Path to a folder. Use this folder as the root to resolve relative source code paths against
8181
82-
cache_dir : str
83-
Path to a the directory where we will be caching built artifacts
84-
85-
cached:
86-
Optional. Set to True to build each function with cache to improve performance
87-
8882
is_building_specific_resource : boolean
8983
Whether customer requested to build a specific resource alone in isolation,
9084
by specifying function_identifier to the build command.
@@ -102,8 +96,6 @@ def __init__(self,
10296
self._resources_to_build = resources_to_build
10397
self._build_dir = build_dir
10498
self._base_dir = base_dir
105-
self._cache_dir = cache_dir
106-
self._cached = cached
10799
self._manifest_path_override = manifest_path_override
108100
self._is_building_specific_resource = is_building_specific_resource
109101

@@ -123,52 +115,73 @@ def build(self):
123115
dict
124116
Returns the path to where each resource was built as a map of resource's LogicalId to the path string
125117
"""
126-
build_graph = self._get_build_graph()
127-
build_strategy = DefaultBuildStrategy(build_graph, self._build_dir, self._build_function, self._build_layer)
128-
129-
if self._parallel:
130-
if self._cached:
131-
build_strategy = ParallelBuildStrategy(
132-
build_graph,
133-
CachedBuildStrategy(build_graph,
134-
build_strategy,
135-
self._base_dir,
136-
self._build_dir,
137-
self._cache_dir,
138-
self._is_building_specific_resource)
118+
119+
result = self._build_functions()
120+
121+
for layer in self._resources_to_build.layers:
122+
LOG.info("Building layer '%s'", layer.name)
123+
if layer.build_method is None:
124+
raise MissingBuildMethodException(
125+
f"Layer {layer.name} cannot be build without BuildMethod. Please provide BuildMethod in Metadata."
139126
)
140-
else:
141-
build_strategy = ParallelBuildStrategy(build_graph, build_strategy)
142-
elif self._cached:
143-
build_strategy = CachedBuildStrategy(build_graph,
144-
build_strategy,
145-
self._base_dir,
146-
self._build_dir,
147-
self._cache_dir,
148-
self._is_building_specific_resource)
149-
150-
return build_strategy.build()
127+
result[layer.name] = self._build_layer(
128+
layer.name, layer.codeuri, layer.build_method, layer.compatible_runtimes
129+
)
130+
131+
return result
151132

152133
def _get_build_graph(self):
153134
"""
154-
Converts list of functions and layers into a build graph, where we can iterate on each unique build and trigger
155-
build
135+
Converts list of functions into a build graph, where we can iterate on each unique build and trigger build
156136
:return: BuildGraph, which represents list of unique build definitions
157137
"""
158138
build_graph = BuildGraph(self._build_dir)
159139
functions = self._resources_to_build.functions
160-
layers = self._resources_to_build.layers
161140
for function in functions:
162-
function_build_details = FunctionBuildDefinition(function.runtime, function.codeuri, function.metadata)
163-
build_graph.put_function_build_definition(function_build_details, function)
164-
165-
for layer in layers:
166-
layer_build_details = LayerBuildDefinition(layer.name, layer.codeuri, layer.build_method, layer.compatible_runtimes)
167-
build_graph.put_layer_build_definition(layer_build_details, layer)
141+
build_details = BuildDefinition(function.runtime, function.codeuri, function.metadata)
142+
build_graph.put_build_definition(build_details, function)
168143

169-
build_graph.clean_redundant_definitions_and_update(not self._is_building_specific_resource)
144+
build_graph.clean_redundant_functions_and_update(not self._is_building_specific_resource)
170145
return build_graph
171146

147+
def _build_functions(self):
148+
"""
149+
Iterates through build graph and runs each unique build and copies outcome to the corresponding function folder
150+
"""
151+
build_graph = self._get_build_graph()
152+
function_build_results = {}
153+
154+
for build_definition in build_graph.get_build_definitions():
155+
LOG.info("Building codeuri: %s runtime: %s metadata: %s functions: %s",
156+
build_definition.codeuri,
157+
build_definition.runtime,
158+
build_definition.metadata,
159+
[function.name for function in build_definition.functions])
160+
161+
# build into one of the functions from this build definition
162+
single_function_name = build_definition.get_function_name()
163+
single_build_dir = str(pathlib.Path(self._build_dir, single_function_name))
164+
165+
LOG.debug("Building to following folder %s", single_build_dir)
166+
self._build_function(build_definition.get_function_name(),
167+
build_definition.codeuri,
168+
build_definition.runtime,
169+
build_definition.get_handler_name(),
170+
single_build_dir,
171+
build_definition.metadata)
172+
function_build_results[single_function_name] = single_build_dir
173+
174+
# copy results to other functions
175+
for function in build_definition.functions:
176+
if function.name is not single_function_name:
177+
# artifacts directory will be created by the builder
178+
artifacts_dir = str(pathlib.Path(self._build_dir, function.name))
179+
LOG.debug("Copying artifacts from %s to %s", single_build_dir, artifacts_dir)
180+
osutils.copytree(single_build_dir, artifacts_dir)
181+
function_build_results[function.name] = artifacts_dir
182+
183+
return function_build_results
184+
172185
def update_template(self, template_dict, original_template_path, built_artifacts):
173186
"""
174187
Given the path to built artifacts, update the template to point appropriate resource CodeUris to the artifacts

0 commit comments

Comments
 (0)