Skip to content

Commit a372e82

Browse files
diegomarquezpblakeli0JoeWang1127
authored
feat: bake gapic-generator-java into the hermetic build docker image (#3067)
Makes the gapic-generator-java jar a prepared binary in the Docker image whose location is now assumed by the scripts. Developers need now to prepare this well-know location (specified in `library_generation/DEVELOPMENT.md`. --------- Co-authored-by: Blake Li <[email protected]> Co-authored-by: Joe Wang <[email protected]>
1 parent 67342ea commit a372e82

File tree

16 files changed

+336
-199
lines changed

16 files changed

+336
-199
lines changed

‎.cloudbuild/library_generation/library_generation.Dockerfile‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,26 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
# install gapic-generator-java in a separate layer so we don't overload the image
16+
# with the transferred source code and jars
17+
FROM gcr.io/cloud-devrel-public-resources/java21 AS ggj-build
18+
19+
WORKDIR /sdk-platform-java
20+
COPY . .
21+
# {x-version-update-start:gapic-generator-java:current}
22+
ENV DOCKER_GAPIC_GENERATOR_VERSION="2.44.1-SNAPSHOT"
23+
# {x-version-update-end:gapic-generator-java:current}
24+
25+
RUN mvn install -DskipTests -Dclirr.skip -Dcheckstyle.skip
26+
RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \
27+
"./gapic-generator-java.jar"
28+
1529
# build from the root of this repo:
1630
FROM gcr.io/cloud-devrel-public-resources/python
1731

1832
SHELL [ "/bin/bash", "-c" ]
1933

34+
2035
ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
2136
ARG PROTOC_VERSION=25.4
2237
ARG GRPC_VERSION=1.66.0
@@ -47,7 +62,15 @@ RUN source /src/utils/utilities.sh \
4762
ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
4863
ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
4964

50-
# use python 3.11 (the base image has several python versions; here we define the default one)
65+
66+
# Here we transfer gapic-generator-java from the previous stage.
67+
# Note that the destination is a well-known location that will be assumed at runtime
68+
# We hard-code the location string to avoid making it configurable (via ARG) as
69+
# well as to avoid it making it overridable at runtime (via ENV).
70+
COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
71+
RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
72+
73+
# use python 3.11 (the base image has several python versions; here we define the default one)
5174
RUN rm $(which python3)
5275
RUN ln -s $(which python3.11) /usr/local/bin/python
5376
RUN ln -s $(which python3.11) /usr/local/bin/python3

‎.github/workflows/ci.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,17 @@ jobs:
232232
- name: Showcase golden tests
233233
working-directory: showcase
234234
run: |
235+
# The golden test directly calls
236+
# library_generation/generate_library.sh, which expects the jar to be
237+
# located in its well-known location. More info in
238+
# library_generation/DEVELOPMENT.md
239+
# Here we prepare the jar in such location
240+
generator_version=$(grep "gapic-generator-java:" "../versions.txt" \
241+
| cut -d: -f3) # the 3rd field is the snapshot version
242+
mkdir -p "${HOME}/.library_generation"
243+
cp \
244+
"${HOME}/.m2/repository/com/google/api/gapic-generator-java/${generator_version}/gapic-generator-java-${generator_version}.jar" \
245+
"${HOME}/.library_generation/gapic-generator-java.jar"
235246
mvn test \
236247
-P enable-golden-tests \
237248
--batch-mode \

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,5 @@ target/
2424
**/*egg-info/
2525
**/build/
2626
**/dist/
27+
library_generation/**/*.jar
28+

‎.kokoro/presubmit/downstream-compatibility-spring.sh‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ git clone "https://github.com/GoogleCloudPlatform/spring-cloud-gcp.git" --depth=
3434
update_all_poms_dependency "spring-cloud-gcp" "gapic-generator-java-bom" "${GAPIC_GENERATOR_VERSION}"
3535

3636
# Install spring-cloud-gcp modules
37-
pushd spring-cloud-gcp/spring-cloud-generator
38-
../mvnw \
37+
pushd spring-cloud-gcp
38+
./mvnw \
3939
-U \
4040
--batch-mode \
4141
--no-transfer-progress \
@@ -47,10 +47,16 @@ pushd spring-cloud-gcp/spring-cloud-generator
4747

4848

4949
# Generate showcase autoconfig
50+
pushd spring-cloud-generator
51+
# The script is not executable for non-owners. Here we manually chmod it.
52+
# TODO(diegomarquezp): remove this line after
53+
# https://github.com/GoogleCloudPlatform/spring-cloud-gcp/pull/3183 is merged and released.
54+
chmod 755 ./scripts/generate-showcase.sh
5055
./scripts/generate-showcase.sh
5156
pushd showcase/showcase-spring-starter
5257
mvn verify
5358
popd # showcase/showcase-spring-starter
5459

55-
popd # spring-cloud-gcp/spring-cloud-generator
60+
popd # spring-cloud-generator
61+
popd # spring-cloud-gcp
5662
popd # gapic-generator-java/target

‎library_generation/DEVELOPMENT.md‎

Lines changed: 38 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@
44

55
# Linting
66

7-
When contributing, ensure your changes to python code have a valid
8-
format.
7+
When contributing, ensure your changes to python code have a valid format.
98

109
```
1110
python -m pip install black
@@ -30,9 +29,9 @@ python -m unittest test/integration_tests.py
3029
# Running the unit tests
3130

3231
The unit tests of the hermetic build scripts are contained in several scripts,
33-
corresponding to a specific component. Every unit test script ends with
34-
`unit_tests.py`. To avoid them specifying them
35-
individually, we can use the following command:
32+
corresponding to a specific component.
33+
Every unit test script ends with `unit_tests.py`.
34+
To avoid them specifying them individually, we can use the following command:
3635

3736
```bash
3837
python -m unittest discover -s test/ -p "*unit_tests.py"
@@ -45,35 +44,41 @@ python -m unittest discover -s test/ -p "*unit_tests.py"
4544
# Running the scripts in your local environment
4645

4746
Although the scripts are designed to be run in a Docker container, you can also
48-
run them directly. This section explains how to run the entrypoint script
47+
run them directly.
48+
This section explains how to run the entrypoint script
4949
(`library_generation/cli/entry_point.py`).
5050

51-
## Installing prerequisites
51+
## Assumptions made by the scripts
52+
### The Hermetic Build's well-known folder
53+
Located in `${HOME}/.library_generation`, this folder is assumed by the scripts
54+
to contain the generator JAR.
55+
Please note that this is a recent feature and only this jar is expected to be
56+
there.
57+
Developers must make sure this folder is properly configured before running the
58+
scripts locally.
59+
Note that this relies on the `HOME` en var which is always defined as per
60+
[POSIX env var definition](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html).
5261

53-
In order to run the generation scripts directly, there are a few tools we
54-
need to install beforehand.
62+
#### Put the gapic-generator-java jar in its well-known location
5563

56-
### Install synthtool
64+
Run `cd sdk-platform-java && mvn install -DskipTests -Dclirr.skip
65+
-Dcheckstyle.skip`.
66+
This will generate a jar located in
67+
`~/.m2/repository/com/google/api/gapic-generator-java/{version}/gapic-generator-java-{version}.jar`
5768

58-
It requires python 3.x to be installed.
59-
You will need to specify a committish of the synthtool repo in order to have
60-
your generation results matching exactly what the docker image would produce.
61-
You can achieve this by inspecting `SYNTHTOOL_COMMITISH` in
62-
`.cloudbuild/library_generation/library_generation.Dockerfile`.
69+
Then `mv` the jar into the well-known location of the jar.
70+
The generation scripts will assume the jar is there.
6371

64-
```bash
65-
# obtained from .cloudbuild/library_generation/library_generation.Dockerfile
66-
export SYNTHTOOL_COMMITTISH=6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
72+
```shell
73+
mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar"
6774
```
6875

69-
```bash
70-
git clone https://github.com/googleapis/synthtool
71-
cd synthtool
72-
git checkout "${SYNTHTOOL_COMMITTISH}"
73-
python -m pip install --require-hashes -r requirements.txt
74-
python -m pip install --no-deps -e .
75-
python -m synthtool --version
76-
```
76+
77+
78+
## Installing prerequisites
79+
80+
In order to run the generation scripts directly, there are a few tools we
81+
need to install beforehand.
7782

7883
### Install the owl-bot CLI
7984

@@ -93,6 +98,7 @@ owl-bot copy-code --version
9398
The key step is `npm link`, which will make the command available in you current
9499
shell session.
95100

101+
96102
## Running the script
97103
The entrypoint script (`library_generation/cli/entry_point.py`) allows you to
98104
update the target repository with the latest changes starting from the
@@ -132,9 +138,9 @@ This will create an `image-id` file at the root of the repo with the hash ID of
132138
the image.
133139

134140
## Run the docker image
135-
The docker image will perform changes on its internal `/workspace` folder, to which you
136-
need to map a folder on your host machine (i.e. map your downloaded repo to this
137-
folder).
141+
The docker image will perform changes on its internal `/workspace` folder,
142+
to which you need to map a folder on your host machine (i.e. map your downloaded
143+
repo to this folder).
138144

139145
To run the docker container on the google-cloud-java repo, you must run:
140146
```bash
@@ -151,9 +157,9 @@ docker run -u "$(id -u)":"$(id -g)" -v/path/to/google-cloud-java:/workspace $(c
151157

152158
## Debug the created containers
153159
If you are working on changing the way the containers are created, you may want
154-
to inspect the containers to check the setup. It would be convenient in such
155-
case to have a text editor/viewer available. You can achieve this by modifying
156-
the Dockerfile as follows:
160+
to inspect the containers to check the setup.
161+
It would be convenient in such case to have a text editor/viewer available.
162+
You can achieve this by modifying the Dockerfile as follows:
157163

158164
```docker
159165
# install OS tools

‎library_generation/gapic-generator-java-wrapper‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/usr/bin/env bash
2-
32
set -e
3+
wrapper_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
4+
source "${wrapper_dir}/utils/utilities.sh"
45

56
# Wrap gapic-generator-java.jar because protoc requires the plugin to be executable.
6-
exec java -classpath "gapic-generator-java-${gapic_generator_version}.jar" com.google.api.generator.Main
7+
exec java -classpath "$(get_gapic_generator_location)" com.google.api.generator.Main

‎library_generation/generate_composed_library.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,6 @@ def __construct_tooling_arg(config: GenerationConfig) -> List[str]:
131131
:return: arguments containing tooling versions
132132
"""
133133
arguments = []
134-
arguments += util.create_argument("gapic_generator_version", config)
135134
arguments += util.create_argument("grpc_version", config)
136135
arguments += util.create_argument("protoc_version", config)
137136

‎library_generation/generate_library.sh‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,6 @@ case $key in
1414
destination_path="$2"
1515
shift
1616
;;
17-
--gapic_generator_version)
18-
gapic_generator_version="$2"
19-
# export this variable so that it can be used in gapic-generator-java-wrapper.sh
20-
export gapic_generator_version
21-
shift
22-
;;
2317
--protoc_version)
2418
protoc_version="$2"
2519
shift
@@ -77,17 +71,12 @@ script_dir=$(dirname "$(readlink -f "$0")")
7771
source "${script_dir}"/utils/utilities.sh
7872
output_folder="$(get_output_folder)"
7973

80-
if [ -z "${gapic_generator_version}" ]; then
81-
echo 'missing required argument --gapic_generator_version'
82-
exit 1
83-
fi
84-
8574
if [ -z "${protoc_version}" ]; then
86-
protoc_version=$(get_protoc_version "${gapic_generator_version}")
75+
protoc_version=$(get_protoc_version)
8776
fi
8877

8978
if [ -z "${grpc_version}" ]; then
90-
grpc_version=$(get_grpc_version "${gapic_generator_version}")
79+
grpc_version=$(get_grpc_version)
9180
fi
9281

9382
if [ -z "${proto_only}" ]; then
@@ -185,7 +174,7 @@ esac
185174
# download gapic-generator-java, protobuf and grpc plugin.
186175
# the download_tools function will create the environment variables "protoc_path"
187176
# and "grpc_path", to be used in the protoc calls below.
188-
download_tools "${gapic_generator_version}" "${protoc_version}" "${grpc_version}" "${os_architecture}"
177+
download_tools "${protoc_version}" "${grpc_version}" "${os_architecture}"
189178
##################### Section 1 #####################
190179
# generate grpc-*/
191180
#####################################################

‎library_generation/test/generate_library_unit_tests.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
import subprocess
2+
import unittest
3+
import os
4+
from library_generation.utils.utilities import (
5+
run_process_and_print_output as bash_call,
6+
run_process_and_get_output_string as get_bash_call_output,
7+
)
8+
9+
script_dir = os.path.dirname(os.path.realpath(__file__))
10+
11+
12+
class GenerateLibraryUnitTests(unittest.TestCase):
13+
"""
14+
Confirms the correct behavior of `library_generation/utils/utilities.sh`.
15+
16+
Note that there is an already existing, shell-based, test suite for
17+
generate_library.sh, but these tests will soon be transferred to this one as
18+
an effort to unify the implementation of the Hermetic Build scripts as
19+
python-only. New tests for `utilities.sh` should be added in this file.
20+
"""
21+
22+
TEST_ARCHITECTURE = "linux-x86_64"
23+
24+
def setUp(self):
25+
# we create a simulated home folder that has a fake generator jar
26+
# in its well-known location
27+
self.simulated_home = get_bash_call_output("mktemp -d")
28+
bash_call(f"mkdir {self.simulated_home}/.library_generation")
29+
bash_call(
30+
f"touch {self.simulated_home}/.library_generation/gapic-generator-java.jar"
31+
)
32+
33+
# We create a per-test directory where all output files will be created into.
34+
# Each folder will be deleted after its corresponding test finishes.
35+
test_dir = get_bash_call_output("mktemp -d")
36+
self.output_folder = self._run_command_and_get_sdout(
37+
"get_output_folder",
38+
cwd=test_dir,
39+
)
40+
bash_call(f"mkdir {self.output_folder}")
41+
42+
def tearDown(self):
43+
bash_call(f"rm -rdf {self.simulated_home}")
44+
45+
def _run_command(self, command, **kwargs):
46+
env = os.environ.copy()
47+
env["HOME"] = self.simulated_home
48+
if "cwd" not in kwargs:
49+
kwargs["cwd"] = self.output_folder
50+
return bash_call(
51+
[
52+
"bash",
53+
"-exc",
54+
f"source {script_dir}/../utils/utilities.sh " + f"&& {command}",
55+
],
56+
exit_on_fail=False,
57+
env=env,
58+
**kwargs,
59+
)
60+
61+
def _run_command_and_get_sdout(self, command, **kwargs):
62+
return self._run_command(
63+
command, stderr=subprocess.PIPE, **kwargs
64+
).stdout.decode()[:-1]
65+
66+
def test_get_grpc_version_with_no_env_var_fails(self):
67+
# the absence of DOCKER_GRPC_VERSION will make this function to fail
68+
result = self._run_command("get_grpc_version")
69+
self.assertEquals(1, result.returncode)
70+
self.assertRegex(result.stdout.decode(), "DOCKER_GRPC_VERSION is not set")
71+
72+
def test_get_protoc_version_with_no_env_var_fails(self):
73+
# the absence of DOCKER_PROTOC_VERSION will make this function to fail
74+
result = self._run_command("get_protoc_version")
75+
self.assertEquals(1, result.returncode)
76+
self.assertRegex(result.stdout.decode(), "DOCKER_PROTOC_VERSION is not set")
77+
78+
def test_download_tools_without_baked_generator_fails(self):
79+
# This test has the same structure as
80+
# download_tools_succeed_with_baked_protoc, but meant for
81+
# gapic-generator-java.
82+
83+
test_protoc_version = "1.64.0"
84+
test_grpc_version = "1.64.0"
85+
jar_location = (
86+
f"{self.simulated_home}/.library_generation/gapic-generator-java.jar"
87+
)
88+
# we expect the function to fail because the generator jar is not found in
89+
# its well-known location. To achieve this, we temporarily remove the fake
90+
# generator jar
91+
bash_call(f"rm {jar_location}")
92+
result = self._run_command(
93+
f"download_tools {test_protoc_version} {test_grpc_version} {self.TEST_ARCHITECTURE}"
94+
)
95+
self.assertEquals(1, result.returncode)
96+
self.assertRegex(result.stdout.decode(), "Please configure your environment")

0 commit comments

Comments
 (0)