Skip to content

Commit 47b632a

Browse files
JoeWang1127JoeWang1127
authored
feat: add generate_repo.py (#2431)
In this PR: - Add `generate_repo.py` to generate libraries defined in a configuration file. - Regenerate `.repo-metadata.json` - Generate `.OwlBot.yaml` and `owlbot.py` for new clients - Invoke `generate_composed_library.py` to generate multiple versions of a GAPIC library. - Apply repo-level post processing using python - Regenerate root `pom.xml` - Regenerate `gapic-libraries-bom/pom.xml` - Format python script using `black`. - Add unit tests for utility functions. - Add an integration test using python, replacing shell integration tests. - Add a python lint check in CI. This is [step 2](https://docs.google.com/document/d/1JiCcG3X7lnxaJErKe0ES_JkyU7ECb40nf2Xez3gWvuo/edit?pli=1&tab=t.0#bookmark=id.h3n0hcp3ch2m) of milestone 2 of hermetic build project.
1 parent 564802b commit 47b632a

49 files changed

Lines changed: 2085 additions & 1845 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/verify_library_generation.yaml‎

Lines changed: 21 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,7 @@ jobs:
1313
strategy:
1414
matrix:
1515
java: [ 11 ]
16-
os: [ ubuntu-22.04, macos-12 ]
17-
post_processing: [ 'true', 'false' ]
18-
runs-on: ${{ matrix.os }}
16+
runs-on: ubuntu-22.04
1917
steps:
2018
- uses: actions/checkout@v3
2119
- uses: actions/setup-java@v3
@@ -45,35 +43,13 @@ jobs:
4543
pushd library_generation
4644
pip install -r requirements.in
4745
popd
48-
49-
- name: install utils (macos)
50-
if: matrix.os == 'macos-12'
51-
shell: bash
52-
run: |
53-
brew update --preinstall
54-
# we need the `realpath` command to be available
55-
brew install coreutils
56-
- name: install docker (ubuntu)
57-
if: matrix.os == 'ubuntu-22.04'
58-
shell: bash
59-
run: |
60-
set -x
61-
# install docker
62-
sudo apt install containerd -y
63-
sudo apt install -y docker.io docker-compose
64-
65-
# launch docker
66-
sudo systemctl start docker
6746
- name: Run integration tests
68-
# we don't run ITs with postprocessing on macos because one of its dependencies "synthtool" is designed to run on linux only
69-
if: matrix.os == 'ubuntu-22.04' || matrix.post_processing == 'false'
7047
shell: bash
7148
run: |
49+
set -x
7250
git config --global user.email "[email protected]"
7351
git config --global user.name "Github Workflow"
74-
library_generation/test/generate_library_integration_test.sh \
75-
--googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git \
76-
--enable_postprocessing "${{ matrix.post_processing }}"
52+
python -m unittest library_generation/test/integration_tests.py
7753
unit_tests:
7854
strategy:
7955
matrix:
@@ -106,7 +82,7 @@ jobs:
10682
run: |
10783
set -x
10884
python -m unittest library_generation/test/unit_tests.py
109-
lint:
85+
lint-shell:
11086
runs-on: ubuntu-22.04
11187
steps:
11288
- uses: actions/checkout@v3
@@ -116,3 +92,20 @@ jobs:
11692
scandir: 'library_generation'
11793
format: tty
11894
severity: error
95+
lint-python:
96+
runs-on: ubuntu-22.04
97+
steps:
98+
- uses: actions/checkout@v3
99+
- name: install python dependencies
100+
shell: bash
101+
run: |
102+
set -ex
103+
pushd library_generation
104+
pip install -r requirements.in
105+
popd
106+
- name: Lint
107+
shell: bash
108+
run: |
109+
# exclude generated golden files
110+
# exclude owlbot until further refaction
111+
black --check library_generation --exclude "(library_generation/owlbot)|(library_generation/test/resources/goldens)"

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,12 @@ target/
1919

2020
# Python
2121
**/__pycache__/
22+
.venv
2223

2324
# library generation
2425
output/
2526
library_generation/output/
27+
library_generation/test/output
28+
library_generation/test/googleapis
29+
library_generation/test/resources/integration/golden
2630
showcase/scripts/output/

‎library_generation/__init__.py‎

Whitespace-only changes.

‎library_generation/generate_composed_library.py‎

Lines changed: 147 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,18 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2024 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
116
"""
217
This script allows generation of libraries that are composed of more than one
318
service version. It is achieved by calling `generate_library.sh` without
@@ -12,130 +27,146 @@
1227
- A "grafeas" folder found in the googleapis/googleapis repository
1328
Note: googleapis repo is found in https://github.com/googleapis/googleapis.
1429
"""
15-
16-
import click
17-
import utilities as util
1830
import os
19-
import sys
20-
import subprocess
21-
import json
22-
from model.GenerationConfig import GenerationConfig
23-
from model.LibraryConfig import LibraryConfig
24-
from model.ClientInputs import parse as parse_build_file
31+
from pathlib import Path
32+
from typing import List
33+
import library_generation.utilities as util
34+
from library_generation.model.generation_config import GenerationConfig
35+
from library_generation.model.gapic_config import GapicConfig
36+
from library_generation.model.gapic_inputs import GapicInputs
37+
from library_generation.model.library_config import LibraryConfig
38+
from library_generation.model.gapic_inputs import parse as parse_build_file
2539

2640
script_dir = os.path.dirname(os.path.realpath(__file__))
2741

28-
"""
29-
Main function in charge of generating libraries composed of more than one
30-
service or service version.
31-
Arguments
32-
- config: a GenerationConfig object representing a parsed configuration
33-
yaml
34-
- library: a LibraryConfig object contained inside config, passed here for
35-
convenience and to prevent all libraries to be processed
36-
- enable_postprocessing: true if postprocessing should be done on the generated
37-
libraries
38-
- repository_path: path to the repository where the generated files will be
39-
sent. If not specified, it will default to the one defined in the configuration yaml
40-
and will be downloaded. The versions file will be inferred from this folder
41-
"""
42+
4243
def generate_composed_library(
4344
config: GenerationConfig,
45+
library_path: str,
4446
library: LibraryConfig,
45-
repository_path: str,
46-
enable_postprocessing: bool = True,
47+
output_folder: str,
48+
versions_file: str,
4749
) -> None:
48-
output_folder = util.sh_util('get_output_folder')
49-
50-
print(f'output_folder: {output_folder}')
51-
print('library: ', library)
52-
os.makedirs(output_folder, exist_ok=True)
53-
54-
googleapis_commitish = config.googleapis_commitish
55-
if library.googleapis_commitish is not None:
56-
googleapis_commitish = library.googleapis_commitish
57-
print('using library-specific googleapis commitish: ' + googleapis_commitish)
58-
else:
59-
print('using common googleapis_commitish')
60-
61-
print('removing old googleapis folders and files')
62-
util.delete_if_exists(f'{output_folder}/google')
63-
util.delete_if_exists(f'{output_folder}/grafeas')
64-
65-
print('downloading googleapis')
66-
util.sh_util(f'download_googleapis_files_and_folders "{output_folder}" "{googleapis_commitish}"')
67-
68-
is_monorepo = len(config.libraries) > 1
50+
"""
51+
Generate libraries composed of more than one service or service version
52+
:param config: a GenerationConfig object representing a parsed configuration
53+
yaml
54+
:param library_path: the path to which the generated file goes
55+
:param library: a LibraryConfig object contained inside config, passed here
56+
for convenience and to prevent all libraries to be processed
57+
:param output_folder:
58+
:param versions_file:
59+
:return None
60+
"""
61+
util.pull_api_definition(
62+
config=config, library=library, output_folder=output_folder
63+
)
64+
65+
is_monorepo = util.check_monorepo(config=config)
66+
base_arguments = __construct_tooling_arg(config=config)
67+
owlbot_cli_source_folder = util.sh_util("mktemp -d")
68+
os.makedirs(f"{library_path}", exist_ok=True)
69+
for gapic in library.gapic_configs:
70+
build_file_folder = Path(f"{output_folder}/{gapic.proto_path}").resolve()
71+
print(f"build_file_folder: {build_file_folder}")
72+
gapic_inputs = parse_build_file(build_file_folder, gapic.proto_path)
73+
# generate prerequisite files (.repo-metadata.json, .OwlBot.yaml,
74+
# owlbot.py) here because transport is parsed from BUILD.bazel,
75+
# which lives in a versioned proto_path.
76+
util.generate_prerequisite_files(
77+
library=library,
78+
proto_path=util.remove_version_from(gapic.proto_path),
79+
transport=gapic_inputs.transport,
80+
library_path=library_path,
81+
)
82+
service_version = gapic.proto_path.split("/")[-1]
83+
temp_destination_path = f"java-{library.api_shortname}-{service_version}"
84+
effective_arguments = __construct_effective_arg(
85+
base_arguments=base_arguments,
86+
gapic=gapic,
87+
gapic_inputs=gapic_inputs,
88+
temp_destination_path=temp_destination_path,
89+
)
90+
print("arguments: ")
91+
print(effective_arguments)
92+
print(f"Generating library from {gapic.proto_path} to {library_path}")
93+
util.run_process_and_print_output(
94+
["bash", f"{script_dir}/generate_library.sh", *effective_arguments],
95+
"Library generation",
96+
)
97+
98+
util.sh_util(
99+
f'build_owlbot_cli_source_folder "{library_path}"'
100+
+ f' "{owlbot_cli_source_folder}" "{output_folder}/{temp_destination_path}"'
101+
+ f' "{gapic.proto_path}"',
102+
cwd=output_folder,
103+
)
69104

70-
base_arguments = []
71-
base_arguments += util.create_argument('gapic_generator_version', config)
72-
base_arguments += util.create_argument('grpc_version', config)
73-
base_arguments += util.create_argument('protobuf_version', config)
74-
75-
library_name = f'java-{library.api_shortname}'
76-
library_path = None
77-
78-
versions_file = ''
79-
if is_monorepo:
80-
print('this is a monorepo library')
81-
destination_path = config.destination_path + '/' + library_name
82-
library_folder = destination_path.split('/')[-1]
83-
if repository_path is None:
84-
print(f'sparse_cloning monorepo with {library_name}')
85-
repository_path = f'{output_folder}/{config.destination_path}'
86-
clone_out = util.sh_util(f'sparse_clone "https://github.com/googleapis/{MONOREPO_NAME}.git" "{library_folder} google-cloud-pom-parent google-cloud-jar-parent versions.txt .github"', cwd=output_folder)
87-
print(clone_out)
88-
library_path = f'{repository_path}/{library_name}'
89-
versions_file = f'{repository_path}/versions.txt'
90-
else:
91-
print('this is a HW library')
92-
destination_path = library_name
93-
if repository_path is None:
94-
repository_path = f'{output_folder}/{destination_path}'
95-
util.delete_if_exists(f'{output_folder}/{destination_path}')
96-
clone_out = util.sh_util(f'git clone "https://github.com/googleapis/{destination_path}.git"', cwd=output_folder)
97-
print(clone_out)
98-
library_path = f'{repository_path}'
99-
versions_file = f'{repository_path}/versions.txt'
100-
101-
owlbot_cli_source_folder = util.sh_util('mktemp -d')
102-
for gapic in library.gapic_configs:
103-
104-
effective_arguments = list(base_arguments)
105-
effective_arguments += util.create_argument('proto_path', gapic)
106-
107-
build_file_folder = f'{output_folder}/{gapic.proto_path}'
108-
print(f'build_file_folder: {build_file_folder}')
109-
client_inputs = parse_build_file(build_file_folder, gapic.proto_path)
110-
effective_arguments += [
111-
'--proto_only', client_inputs.proto_only,
112-
'--gapic_additional_protos', client_inputs.additional_protos,
113-
'--transport', client_inputs.transport,
114-
'--rest_numeric_enums', client_inputs.rest_numeric_enum,
115-
'--gapic_yaml', client_inputs.gapic_yaml,
116-
'--service_config', client_inputs.service_config,
117-
'--service_yaml', client_inputs.service_yaml,
118-
'--include_samples', client_inputs.include_samples,
119-
]
120-
service_version = gapic.proto_path.split('/')[-1]
121-
temp_destination_path = f'java-{library.api_shortname}-{service_version}'
122-
effective_arguments += [ '--destination_path', temp_destination_path ]
123-
print('arguments: ')
124-
print(effective_arguments)
125-
print(f'Generating library from {gapic.proto_path} to {destination_path}...')
126-
util.run_process_and_print_output(['bash', '-x', f'{script_dir}/generate_library.sh',
127-
*effective_arguments], 'Library generation')
128-
129-
130-
if enable_postprocessing:
131-
util.sh_util(f'build_owlbot_cli_source_folder "{library_path}"'
132-
+ f' "{owlbot_cli_source_folder}" "{output_folder}/{temp_destination_path}"'
133-
+ f' "{gapic.proto_path}"',
134-
cwd=output_folder)
135-
136-
if enable_postprocessing:
137105
# call postprocess library
138-
util.run_process_and_print_output([f'{script_dir}/postprocess_library.sh',
139-
f'{library_path}', '', versions_file, owlbot_cli_source_folder,
140-
config.owlbot_cli_image, config.synthtool_commitish, str(is_monorepo).lower()], 'Library postprocessing')
106+
util.run_process_and_print_output(
107+
[
108+
f"{script_dir}/postprocess_library.sh",
109+
f"{library_path}",
110+
"",
111+
versions_file,
112+
owlbot_cli_source_folder,
113+
config.owlbot_cli_image,
114+
config.synthtool_commitish,
115+
str(is_monorepo).lower(),
116+
],
117+
"Library postprocessing",
118+
)
119+
120+
121+
def __construct_tooling_arg(config: GenerationConfig) -> List[str]:
122+
"""
123+
Construct arguments of tooling versions used in generate_library.sh
124+
:param config: the generation config
125+
:return: arguments containing tooling versions
126+
"""
127+
arguments = []
128+
arguments += util.create_argument("gapic_generator_version", config)
129+
arguments += util.create_argument("grpc_version", config)
130+
arguments += util.create_argument("protobuf_version", config)
131+
132+
return arguments
133+
134+
135+
def __construct_effective_arg(
136+
base_arguments: List[str],
137+
gapic: GapicConfig,
138+
gapic_inputs: GapicInputs,
139+
temp_destination_path: str,
140+
) -> List[str]:
141+
"""
142+
Construct arguments consist attributes of a GAPIC library which used in
143+
generate_library.sh
144+
:param base_arguments: arguments consist of tooling versions
145+
:param gapic: an object of GapicConfig
146+
:param gapic_inputs: an object of GapicInput
147+
:param temp_destination_path: the path to which the generated library goes
148+
:return: arguments containing attributes to generate a GAPIC library
149+
"""
150+
arguments = list(base_arguments)
151+
arguments += util.create_argument("proto_path", gapic)
152+
arguments += [
153+
"--proto_only",
154+
gapic_inputs.proto_only,
155+
"--gapic_additional_protos",
156+
gapic_inputs.additional_protos,
157+
"--transport",
158+
gapic_inputs.transport,
159+
"--rest_numeric_enums",
160+
gapic_inputs.rest_numeric_enum,
161+
"--gapic_yaml",
162+
gapic_inputs.gapic_yaml,
163+
"--service_config",
164+
gapic_inputs.service_config,
165+
"--service_yaml",
166+
gapic_inputs.service_yaml,
167+
"--include_samples",
168+
gapic_inputs.include_samples,
169+
]
170+
arguments += ["--destination_path", temp_destination_path]
141171

172+
return arguments

0 commit comments

Comments
 (0)