Skip to content

Commit 3bca9a4

Browse files
JoeWang1127JoeWang1127
authored
feat: populate .repo-metadata.json from highest version (#2890)
In this PR: - populate `.repo-metadata.json` from highest stable version of proto paths. The `.repo-metadata.json` is only generated once per library and the first item in proto paths is used, so the proto paths with the highest stable version should be the first item after sorting the proto paths in the library. The method used to compare two proto paths, `a` and `b`: 1. if both of `a` and `b` don't have a version, the one with lower depth is smaller. 2. if either `a` or `b` has a version, the one with the version is smaller. 3. if either `a` or `b` has a stable version, but not both, the one with the stable version is smaller, e.g., `google/spanner/v2` is smaller than `google/spanner/v3beta`. 4. if both `a` and `b` have a non-stable version, the one with higher version is smaller, e.g., `google/spanner/v2beta1` is smaller than `google/spanner/v2alpha2`. 5. if both `a` and `b` have a stable version, the one with lower depth is smaller, e.g., `google/spanner/v1` is smaller than `google/spanner/admin/v1`, `google/spanner/v1` is smaller than `google/spanner/admin/v2` 6. if both `a` and `b` have a stable version and the depth is the same, the one with higher version is smaller, e.g., `google/spanner/v2` is smaller than `google/spanner/v1`. Test the change: 1. build the image locally: ``` docker build -f .cloudbuild/library_generation/library_generation.Dockerfile -t local:latest . ``` 2. build the generator locally: ``` mvn -V -B -ntp clean install -DskipTests ``` 3. run the image: ``` docker run --rm -u "$(id -u):$(id -g)" -v "$(pwd):/workspace" -v "$HOME/.m2:/home/.m2" local:latest ``` Result: 1. no change in common protos 2. `transport` changed to `both` in `java-iam/.repo-metadata.json`. This is expected since the `transport` in [`google/iam/v2/BUILD.bazel`](https://github.com/googleapis/googleapis/blob/master/google/iam/v2/BUILD.bazel#L94) is `grpc+rest`. Without this change, the `transport` is parsed from `google/iam/v2/BUILD.bazel`, which doesn't have a `java_gapic_library` target, thus the value is `grpc` by default.
1 parent 634166e commit 3bca9a4

File tree

5 files changed

+230
-3
lines changed

5 files changed

+230
-3
lines changed

‎sdk-platform-java/library_generation/generate_composed_library.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ def generate_composed_library(
6666
base_arguments = __construct_tooling_arg(config=config)
6767
owlbot_cli_source_folder = util.sh_util("mktemp -d")
6868
os.makedirs(f"{library_path}", exist_ok=True)
69-
for gapic in library.gapic_configs:
69+
for gapic in library.get_sorted_gapic_configs():
7070
build_file_folder = Path(f"{output_folder}/{gapic.proto_path}").resolve()
7171
print(f"build_file_folder: {build_file_folder}")
7272
gapic_inputs = parse_build_file(build_file_folder, gapic.proto_path)

‎sdk-platform-java/library_generation/model/gapic_config.py‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@
1212
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
15+
import re
16+
from typing import Optional
17+
18+
ALPHA_VERSION = "alpha"
19+
BETA_VERSION = "beta"
1520

1621

1722
class GapicConfig:
@@ -22,3 +27,73 @@ class GapicConfig:
2227

2328
def __init__(self, proto_path: str):
2429
self.proto_path = proto_path
30+
self.version = self.__parse_version()
31+
32+
def is_stable(self):
33+
return (
34+
self.version
35+
and (ALPHA_VERSION not in self.version)
36+
and (BETA_VERSION not in self.version)
37+
)
38+
39+
def get_version(self):
40+
return self.version
41+
42+
def __parse_version(self) -> Optional[str]:
43+
version_regex = re.compile(r"^v[1-9]+(p[1-9]+)*(alpha|beta)?.*")
44+
for directory in self.proto_path.split("/"):
45+
if version_regex.search(directory):
46+
return directory
47+
return None
48+
49+
def __eq__(self, other) -> bool:
50+
if not isinstance(other, GapicConfig):
51+
return False
52+
return self.proto_path == other.proto_path
53+
54+
def __lt__(self, other) -> bool:
55+
if not isinstance(other, GapicConfig):
56+
raise ValueError(
57+
f"Type {type(other)} can't be comparable " f"with GapicConfig."
58+
)
59+
60+
self_version = self.get_version()
61+
other_version = other.get_version()
62+
self_dirs = self.proto_path.split("/")
63+
other_dirs = other.proto_path.split("/")
64+
# Case 1: if both of the configs don't have a version in proto_path,
65+
# the one with lower depth is smaller.
66+
if (not self_version) and (not other_version):
67+
return len(self_dirs) < len(other_dirs)
68+
# Case 2: if only one config has a version in proto_path, it is smaller
69+
# than the other one.
70+
if self_version and (not other_version):
71+
return True
72+
if (not self_version) and other_version:
73+
return False
74+
# Two configs both have a version in proto_path.
75+
self_stable = self.is_stable()
76+
other_stable = other.is_stable()
77+
# Case 3, if only config has a stable version in proto_path, it is
78+
# smaller than the other one.
79+
if self_stable and (not other_stable):
80+
return True
81+
if (not self_stable) and other_stable:
82+
return False
83+
# Case 4, if two configs have a non-stable version in proto_path,
84+
# the one with higher version is smaller.
85+
# Note that using string comparison may lead unexpected result,
86+
# e.g., v1beta10 is smaller than v1beta2.
87+
# In reality, however, there's unlikely that a library has >10 beta
88+
# versions.
89+
if (not self_stable) and (not other_stable):
90+
return self_version > other_version
91+
# Two configs both have a stable version in proto_path.
92+
# Case 5, if two configs have different depth in proto_path, the one
93+
# with lower depth is smaller.
94+
if len(self_dirs) != len(other_dirs):
95+
return len(self_dirs) < len(other_dirs)
96+
# Case 6, the config with higher stable version is smaller.
97+
self_num = int(self_version[1:])
98+
other_num = int(other_version[1:])
99+
return self_num > other_num

‎sdk-platform-java/library_generation/model/library_config.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# limitations under the License.
1515
from hashlib import sha1
1616

17-
from typing import List, Optional
17+
from typing import Optional
1818
from library_generation.model.gapic_config import GapicConfig
1919

2020

@@ -29,7 +29,7 @@ def __init__(
2929
api_description: str,
3030
name_pretty: str,
3131
product_documentation: str,
32-
gapic_configs: List[GapicConfig],
32+
gapic_configs: list[GapicConfig],
3333
library_type: Optional[str] = None,
3434
release_level: Optional[str] = None,
3535
api_id: Optional[str] = None,
@@ -84,6 +84,9 @@ def get_library_name(self) -> str:
8484
"""
8585
return self.library_name if self.library_name else self.api_shortname
8686

87+
def get_sorted_gapic_configs(self) -> list[GapicConfig]:
88+
return sorted(self.gapic_configs)
89+
8790
def __eq__(self, other):
8891
return (
8992
self.api_shortname == other.api_shortname

‎sdk-platform-java/library_generation/test/model/gapic_config_unit_tests.py‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
import unittest
15+
16+
from library_generation.model.gapic_config import GapicConfig
17+
18+
19+
class GapicConfigTest(unittest.TestCase):
20+
def test_get_version_returns_none(self):
21+
self.assertIsNone(GapicConfig(proto_path="example/dir1/dir2").get_version())
22+
23+
def test_get_version_returns_non_stable_version(self):
24+
self.assertEqual(
25+
"v2p2beta1",
26+
GapicConfig(proto_path="example/dir1/dir2/v2p2beta1").get_version(),
27+
)
28+
self.assertEqual(
29+
"v2beta1",
30+
GapicConfig(proto_path="example/dir1/dir2/v2beta1").get_version(),
31+
)
32+
33+
def test_get_version_returns_stable_version(self):
34+
self.assertEqual(
35+
"v20",
36+
GapicConfig(proto_path="example/dir1/dir2/v20").get_version(),
37+
)
38+
39+
def test_is_stable_with_no_version_returns_false(self):
40+
self.assertFalse(
41+
GapicConfig(proto_path="example/dir1/dir2/non_version").is_stable(),
42+
)
43+
44+
def test_is_stable_with_non_stable_version_returns_false(self):
45+
self.assertFalse(
46+
GapicConfig(proto_path="example/dir1/dir2/v20alpha").is_stable(),
47+
)
48+
self.assertFalse(
49+
GapicConfig(proto_path="example/dir1/dir2/v20beta2").is_stable(),
50+
)
51+
52+
def test_is_stable_with_stable_version_returns_true(self):
53+
self.assertTrue(
54+
GapicConfig(proto_path="example/dir1/dir2/v30").is_stable(),
55+
)
56+
57+
def test_compare_configs_without_a_version(self):
58+
config_len_3 = GapicConfig(proto_path="example/dir1/dir2")
59+
config_len_4 = GapicConfig(proto_path="example/dir1/dir2/dir3")
60+
self.assertLess(
61+
config_len_3,
62+
config_len_4,
63+
"config_len_3 should be smaller since it has a lower depth.",
64+
)
65+
66+
def test_compare_configs_only_one_has_a_stable_version(self):
67+
versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v1")
68+
non_versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3")
69+
self.assertLess(
70+
versioned_config,
71+
non_versioned_config,
72+
"versioned_config should be smaller since it has a version.",
73+
)
74+
75+
def test_compare_configs_only_one_has_a_non_stable_version(self):
76+
non_stable_versioned_config = GapicConfig(
77+
proto_path="example/dir1/dir2/dir3/dir4/v1beta"
78+
)
79+
non_versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3")
80+
self.assertLess(
81+
non_stable_versioned_config,
82+
non_versioned_config,
83+
"non_stable_versioned_config should be smaller since it has a version.",
84+
)
85+
86+
def test_compare_configs_one_has_non_stable_and_one_has_stable_version(self):
87+
stable_versioned_config = GapicConfig(
88+
proto_path="example/dir1/dir2/dir3/dir4/v1"
89+
)
90+
non_stable_versioned_config = GapicConfig(proto_path="example/dir1/dir2/v2beta")
91+
self.assertLess(
92+
stable_versioned_config,
93+
non_stable_versioned_config,
94+
"stable_versioned_config should be smaller since it has a stable version.",
95+
)
96+
97+
def test_compare_configs_two_have_non_stable_version(self):
98+
v3p2beta = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v3p2beta")
99+
v2p4beta = GapicConfig(proto_path="example/dir1/dir2/v2p4beta")
100+
self.assertLess(
101+
v3p2beta,
102+
v2p4beta,
103+
"v3p2beta should be smaller since it has a higher version.",
104+
)
105+
106+
def test_compare_configs_two_have_stable_version_different_depth(self):
107+
v3 = GapicConfig(proto_path="example/dir1/dir2/v3")
108+
v4 = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v4")
109+
self.assertLess(
110+
v3,
111+
v4,
112+
"v3 should be smaller since it has lower depth",
113+
)
114+
115+
def test_compare_configs_two_have_stable_version_same_depth(self):
116+
v4 = GapicConfig(proto_path="example/dir1/dir2/v4")
117+
v3 = GapicConfig(proto_path="example/dir1/dir2/v3")
118+
self.assertLess(
119+
v4,
120+
v3,
121+
"v4 should be smaller since it has a higher version",
122+
)

‎sdk-platform-java/library_generation/test/model/library_config_unit_tests.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
# limitations under the License.
1414
import unittest
1515

16+
from library_generation.model.gapic_config import GapicConfig
1617
from library_generation.model.library_config import LibraryConfig
1718

1819

@@ -37,3 +38,29 @@ def test_get_library_returns_api_shortname(self):
3738
gapic_configs=list(),
3839
)
3940
self.assertEqual("secret", library.get_library_name())
41+
42+
def test_get_sorted_gapic_configs_returns_correct_order(self):
43+
v1beta1 = GapicConfig(proto_path="google/spanner/v1beta1")
44+
v1 = GapicConfig(proto_path="google/spanner/v1")
45+
v1alpha1 = GapicConfig(proto_path="google/spanner/v1alpha")
46+
v2 = GapicConfig(proto_path="google/spanner/v2")
47+
admin_v2 = GapicConfig(proto_path="google/spanner/admin/v2")
48+
non_versioned = GapicConfig(proto_path="google/spanner/type")
49+
library = LibraryConfig(
50+
api_shortname="secret",
51+
name_pretty="",
52+
product_documentation="",
53+
api_description="",
54+
gapic_configs=[v1alpha1, v1, v2, admin_v2, non_versioned, v1beta1],
55+
)
56+
self.assertEqual(
57+
[
58+
v2,
59+
v1,
60+
admin_v2,
61+
v1beta1,
62+
v1alpha1,
63+
non_versioned,
64+
],
65+
library.get_sorted_gapic_configs(),
66+
)

0 commit comments

Comments
 (0)