Skip to content

Commit f7a5161

Browse files
JoeWang1127JoeWang1127
authored
feat: generate proto-only repository (#2720)
In this PR: - Enable hermetic build script to generate `java-common-protos` and `java-iam` in this repo. - Change `libraries_bom_version` to optional. - Refactor unit tests. - Remove `java-common-protos/codecov.yaml` and `java-iam/codecov.yaml` as they are no longer needed.
1 parent 6643536 commit f7a5161

13 files changed

+254
-195
lines changed

‎java-common-protos/codecov.yaml

-4
This file was deleted.

‎java-common-protos/owlbot.py

+1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
"SECURITY.md",
4545
"java.header",
4646
"license-checks.xml",
47+
"README.md",
4748
"renovate.json",
4849
".gitignore"
4950
])

‎java-iam/codecov.yaml

-4
This file was deleted.

‎library_generation/generate_repo.py

+1-3
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ def generate_from_yaml(
4848

4949
for library_path, library in repo_config.libraries.items():
5050
print(f"generating library {library.get_library_name()}")
51-
5251
generate_composed_library(
5352
config_path=config_path,
5453
config=config,
@@ -58,8 +57,7 @@ def generate_from_yaml(
5857
versions_file=repo_config.versions_file,
5958
)
6059

61-
# we skip monorepo_postprocessing if not in a monorepo
62-
if not config.is_monorepo():
60+
if not config.is_monorepo() or config.contains_common_protos():
6361
return
6462

6563
monorepo_postprocessing(

‎library_generation/model/gapic_inputs.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ def __init__(
5656
self,
5757
proto_only="true",
5858
additional_protos="google/cloud/common_resources.proto",
59-
transport="",
59+
transport="grpc",
6060
rest_numeric_enum="",
6161
gapic_yaml="",
6262
service_config="",

‎library_generation/model/generation_config.py

+17-6
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
REPO_LEVEL_PARAMETER = "Repo level parameter"
2121
LIBRARY_LEVEL_PARAMETER = "Library level parameter"
2222
GAPIC_LEVEL_PARAMETER = "GAPIC level parameter"
23+
COMMON_PROTOS_LIBRARY_NAME = "common-protos"
2324

2425

2526
class GenerationConfig:
@@ -31,9 +32,9 @@ def __init__(
3132
self,
3233
gapic_generator_version: str,
3334
googleapis_commitish: str,
34-
libraries_bom_version: str,
3535
template_excludes: list[str],
3636
libraries: list[LibraryConfig],
37+
libraries_bom_version: Optional[str] = None,
3738
grpc_version: Optional[str] = None,
3839
protoc_version: Optional[str] = None,
3940
):
@@ -44,6 +45,9 @@ def __init__(
4445
self.libraries = libraries
4546
self.grpc_version = grpc_version
4647
self.protoc_version = protoc_version
48+
# explicit set to None so that we can compute the
49+
# value in getter.
50+
self.__contains_common_protos = None
4751
self.__validate()
4852

4953
def get_proto_path_to_library_name(self) -> dict[str, str]:
@@ -61,6 +65,15 @@ def get_proto_path_to_library_name(self) -> dict[str, str]:
6165
def is_monorepo(self) -> bool:
6266
return len(self.libraries) > 1
6367

68+
def contains_common_protos(self) -> bool:
69+
if self.__contains_common_protos is None:
70+
self.__contains_common_protos = False
71+
for library in self.libraries:
72+
if library.get_library_name() == COMMON_PROTOS_LIBRARY_NAME:
73+
self.__contains_common_protos = True
74+
break
75+
return self.__contains_common_protos
76+
6477
def __validate(self) -> None:
6578
seen_library_names = dict()
6679
for library in self.libraries:
@@ -133,15 +146,13 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
133146
gapic_generator_version=__required(
134147
config, "gapic_generator_version", REPO_LEVEL_PARAMETER
135148
),
136-
grpc_version=__optional(config, "grpc_version", None),
137-
protoc_version=__optional(config, "protoc_version", None),
138149
googleapis_commitish=__required(
139150
config, "googleapis_commitish", REPO_LEVEL_PARAMETER
140151
),
141-
libraries_bom_version=__required(
142-
config, "libraries_bom_version", REPO_LEVEL_PARAMETER
143-
),
144152
template_excludes=__required(config, "template_excludes", REPO_LEVEL_PARAMETER),
153+
grpc_version=__optional(config, "grpc_version", None),
154+
protoc_version=__optional(config, "protoc_version", None),
155+
libraries_bom_version=__optional(config, "libraries_bom_version", None),
145156
libraries=parsed_libraries,
146157
)
147158

‎library_generation/test/cli/entry_point_unit_tests.py

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
class EntryPointTest(unittest.TestCase):
2424
def test_entry_point_without_config_raise_file_exception(self):
25+
os.chdir(script_dir)
2526
runner = CliRunner()
2627
# noinspection PyTypeChecker
2728
result = runner.invoke(generate, ["--repository-path=."])

‎library_generation/test/model/gapic_inputs_unit_tests.py

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import os
2+
import unittest
3+
from pathlib import Path
4+
5+
from parameterized import parameterized
6+
from library_generation.model.gapic_inputs import parse
7+
8+
script_dir = os.path.dirname(os.path.realpath(__file__))
9+
resources_dir = os.path.join(script_dir, "..", "resources")
10+
build_file = Path(os.path.join(resources_dir, "misc")).resolve()
11+
12+
13+
class UtilitiesTest(unittest.TestCase):
14+
@parameterized.expand(
15+
[
16+
("BUILD_no_additional_protos.bazel", " "),
17+
("BUILD_common_resources.bazel", " google/cloud/common_resources.proto"),
18+
("BUILD_comment_common_resources.bazel", " "),
19+
("BUILD_locations.bazel", " google/cloud/location/locations.proto"),
20+
("BUILD_comment_locations.bazel", " "),
21+
("BUILD_iam_policy.bazel", " google/iam/v1/iam_policy.proto"),
22+
("BUILD_comment_iam_policy.bazel", " "),
23+
(
24+
"BUILD_iam_locations.bazel",
25+
" google/cloud/location/locations.proto google/iam/v1/iam_policy.proto",
26+
),
27+
]
28+
)
29+
def test_gapic_inputs_parse_additional_protos(self, build_name, expected):
30+
parsed = parse(build_file, "", build_name)
31+
self.assertEqual(
32+
expected,
33+
parsed.additional_protos,
34+
)
35+
36+
def test_gapic_inputs_parse_grpc_only_succeeds(self):
37+
parsed = parse(build_file, "", "BUILD_grpc.bazel")
38+
self.assertEqual("grpc", parsed.transport)
39+
40+
def test_gapic_inputs_parse_grpc_rest_succeeds(self):
41+
parsed = parse(build_file, "", "BUILD_grpc_rest.bazel")
42+
self.assertEqual("grpc+rest", parsed.transport)
43+
44+
def test_gapic_inputs_parse_rest_succeeds(self):
45+
parsed = parse(build_file, "", "BUILD_rest.bazel")
46+
self.assertEqual("rest", parsed.transport)
47+
48+
def test_gapic_inputs_parse_empty_include_samples_succeeds(self):
49+
parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel")
50+
self.assertEqual("false", parsed.include_samples)
51+
52+
def test_gapic_inputs_parse_include_samples_false_succeeds(self):
53+
parsed = parse(build_file, "", "BUILD_include_samples_false.bazel")
54+
self.assertEqual("false", parsed.include_samples)
55+
56+
def test_gapic_inputs_parse_include_samples_true_succeeds(self):
57+
parsed = parse(build_file, "", "BUILD_include_samples_true.bazel")
58+
self.assertEqual("true", parsed.include_samples)
59+
60+
def test_gapic_inputs_parse_empty_rest_numeric_enums_succeeds(self):
61+
parsed = parse(build_file, "", "BUILD_rest_numeric_enums_empty.bazel")
62+
self.assertEqual("false", parsed.rest_numeric_enum)
63+
64+
def test_gapic_inputs_parse_rest_numeric_enums_false_succeeds(self):
65+
parsed = parse(build_file, "", "BUILD_rest_numeric_enums_false.bazel")
66+
self.assertEqual("false", parsed.rest_numeric_enum)
67+
68+
def test_gapic_inputs_parse_rest_numeric_enums_true_succeeds(self):
69+
parsed = parse(build_file, "", "BUILD_rest_numeric_enums_true.bazel")
70+
self.assertEqual("true", parsed.rest_numeric_enum)
71+
72+
def test_gapic_inputs_parse_no_gapic_library_returns_proto_only_true(self):
73+
# include_samples_empty only has a gradle assembly rule
74+
parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel")
75+
self.assertEqual("true", parsed.proto_only)
76+
77+
def test_gapic_inputs_parse_with_gapic_library_returns_proto_only_false(self):
78+
# rest.bazel has a java_gapic_library rule
79+
parsed = parse(build_file, "", "BUILD_rest.bazel")
80+
self.assertEqual("false", parsed.proto_only)
81+
82+
def test_gapic_inputs_parse_gapic_yaml_succeeds(self):
83+
parsed = parse(build_file, "test/versioned/path", "BUILD_gapic_yaml.bazel")
84+
self.assertEqual("test/versioned/path/test_gapic_yaml.yaml", parsed.gapic_yaml)
85+
86+
def test_gapic_inputs_parse_no_gapic_yaml_returns_empty_string(self):
87+
parsed = parse(build_file, "test/versioned/path", "BUILD_no_gapic_yaml.bazel")
88+
self.assertEqual("", parsed.gapic_yaml)
89+
90+
def test_gapic_inputs_parse_service_config_succeeds(self):
91+
parsed = parse(build_file, "test/versioned/path", "BUILD_service_config.bazel")
92+
self.assertEqual(
93+
"test/versioned/path/test_service_config.json", parsed.service_config
94+
)
95+
96+
def test_gapic_inputs_parse_service_yaml_relative_target(self):
97+
parsed = parse(
98+
build_file,
99+
"google/cloud/compute/v1",
100+
"BUILD_service_config_relative_target.bazel",
101+
)
102+
self.assertEqual(
103+
"google/cloud/compute/v1/compute_grpc_service_config.json",
104+
parsed.service_config,
105+
)
106+
107+
def test_gapic_inputs_parse_no_service_config_returns_empty_string(self):
108+
parsed = parse(
109+
build_file, "test/versioned/path", "BUILD_no_service_config.bazel"
110+
)
111+
self.assertEqual("", parsed.service_config)
112+
113+
def test_gapic_inputs_parse_service_yaml_succeeds(self):
114+
parsed = parse(build_file, "test/versioned/path", "BUILD_service_yaml.bazel")
115+
self.assertEqual(
116+
"test/versioned/path/test_service_yaml.yaml", parsed.service_yaml
117+
)
118+
119+
def test_gapic_inputs_parse_service_yaml_absolute_target(self):
120+
parsed = parse(build_file, "", "BUILD_service_yaml_absolute_target.bazel")
121+
self.assertEqual(
122+
"google/cloud/videointelligence/videointelligence_v1p3beta1.yaml",
123+
parsed.service_yaml,
124+
)
125+
126+
def test_gapic_inputs_parse_no_service_yaml_returns_empty_string(self):
127+
parsed = parse(build_file, "test/versioned/path", "BUILD_no_service_yaml.bazel")
128+
self.assertEqual("", parsed.service_yaml)
129+
130+
def test_gapic_inputs_parse_proto_only_returns_grpc(self):
131+
parsed = parse(build_file, "test/versioned/path", "BUILD_proto_only.bazel")
132+
self.assertEqual("grpc", parsed.transport)

‎library_generation/test/model/generation_config_unit_test.py renamed to ‎library_generation/test/model/generation_config_unit_tests.py

+27-26
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,13 @@
3535
product_documentation="",
3636
gapic_configs=[],
3737
)
38+
common_protos_library = LibraryConfig(
39+
api_shortname="common-protos",
40+
api_description="",
41+
name_pretty="",
42+
product_documentation="",
43+
gapic_configs=[],
44+
)
3845

3946

4047
class GenerationConfigTest(unittest.TestCase):
@@ -123,6 +130,26 @@ def test_is_monorepo_with_two_libraries_returns_true(self):
123130
)
124131
self.assertTrue(config.is_monorepo())
125132

133+
def test_contains_common_protos_with_common_protos_returns_true(self):
134+
config = GenerationConfig(
135+
gapic_generator_version="",
136+
googleapis_commitish="",
137+
libraries_bom_version="",
138+
template_excludes=[],
139+
libraries=[library_1, library_2, common_protos_library],
140+
)
141+
self.assertTrue(config.contains_common_protos())
142+
143+
def test_contains_common_protos_without_common_protos_returns_false(self):
144+
config = GenerationConfig(
145+
gapic_generator_version="",
146+
googleapis_commitish="",
147+
libraries_bom_version="",
148+
template_excludes=[],
149+
libraries=[library_1, library_2],
150+
)
151+
self.assertFalse(config.contains_common_protos())
152+
126153
def test_validate_with_duplicate_library_name_raise_exception(self):
127154
self.assertRaisesRegex(
128155
ValueError,
@@ -131,8 +158,6 @@ def test_validate_with_duplicate_library_name_raise_exception(self):
131158
gapic_generator_version="",
132159
googleapis_commitish="",
133160
libraries_bom_version="",
134-
owlbot_cli_image="",
135-
synthtool_commitish="",
136161
template_excludes=[],
137162
libraries=[
138163
LibraryConfig(
@@ -169,30 +194,6 @@ def test_from_yaml_without_googleapis_commitish_raise_exception(self):
169194
f"{test_config_dir}/config_without_googleapis.yaml",
170195
)
171196

172-
def test_from_yaml_without_libraries_bom_version_raise_exception(self):
173-
self.assertRaisesRegex(
174-
ValueError,
175-
"Repo level parameter, libraries_bom_version",
176-
from_yaml,
177-
f"{test_config_dir}/config_without_libraries_bom_version.yaml",
178-
)
179-
180-
def test_from_yaml_without_owlbot_cli_image_raise_exception(self):
181-
self.assertRaisesRegex(
182-
ValueError,
183-
"Repo level parameter, owlbot_cli_image",
184-
from_yaml,
185-
f"{test_config_dir}/config_without_owlbot.yaml",
186-
)
187-
188-
def test_from_yaml_without_synthtool_commitish_raise_exception(self):
189-
self.assertRaisesRegex(
190-
ValueError,
191-
"Repo level parameter, synthtool_commitish",
192-
from_yaml,
193-
f"{test_config_dir}/config_without_synthtool.yaml",
194-
)
195-
196197
def test_from_yaml_without_template_excludes_raise_exception(self):
197198
self.assertRaisesRegex(
198199
ValueError,

‎library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"api_shortname": "baremetalsolution",
3+
"name_pretty": "Bare Metal Solution",
4+
"product_documentation": "https://cloud.google.com/bare-metal/docs",
5+
"api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.",
6+
"client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview",
7+
"release_level": "preview",
8+
"transport": "grpc",
9+
"language": "java",
10+
"repo": "googleapis/sdk-platform-java",
11+
"repo_short": "java-bare-metal-solution",
12+
"distribution_name": "com.google.cloud:google-cloud-bare-metal-solution",
13+
"library_type": "OTHER",
14+
"requires_billing": true,
15+
"rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest",
16+
"rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc"
17+
}

‎library_generation/test/resources/misc/BUILD_proto_only.bazel

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
java_gapic_assembly_gradle_pkg(
2+
name = "google-api-java",
3+
transport = "grpc+rest",
4+
deps = [
5+
"annotations_proto",
6+
"auth_proto",
7+
"backend_proto",
8+
"billing_proto",
9+
"client_proto",
10+
"config_change_proto",
11+
"consumer_proto",
12+
"context_proto",
13+
"control_proto",
14+
"distribution_proto",
15+
],
16+
)

0 commit comments

Comments
 (0)