feat: generate showcase using docker image (#3568)

Fixes #2470

### Approach
#### In `showcase/`
* Revamp the generation and verification scripts used by
`-Penable-golden-tests` and `-Pupdate` to build and use the Hermetic
Build Docker image

### Effects
 * New `samples/` folder added in `showcase`
 * Reformatted all of grpc, proto and gapic libraries in `showcase/`

---------

Co-authored-by: cloud-java-bot <[email protected]>

Author: diegomarquezp

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -98,14 +98,6 @@ RUN source /src/library_generation/utils/utilities.sh \
 # similar to protoc, we indicate grpc is available in the container via env vars
 ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java.exe"
 
-# Here we transfer gapic-generator-java from the previous stage.
-# Note that the destination is a well-known location that will be assumed at runtime
-# We hard-code the location string to avoid making it configurable (via ARG) as
-# well as to avoid it making it overridable at runtime (via ENV).
-COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
-RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
-ENV GAPIC_GENERATOR_LOCATION="${HOME}/.library_generation/gapic-generator-java.jar"
-
 RUN python -m pip install --upgrade pip
 
 # install main scripts as a python package
@@ -125,6 +117,14 @@ RUN owl-bot copy-code --version
 RUN chmod o+rx $(which owl-bot)
 RUN apk del -r npm && apk cache clean
 
+# Here we transfer gapic-generator-java from the previous stage.
+# Note that the destination is a well-known location that will be assumed at runtime
+# We hard-code the location string to avoid making it configurable (via ARG) as
+# well as to avoid it making it overridable at runtime (via ENV).
+COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
+RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
+ENV GAPIC_GENERATOR_LOCATION="${HOME}/.library_generation/gapic-generator-java.jar"
+
 # download the Java formatter
 ADD https://maven-central.storage-download.googleapis.com/maven2/com/google/googlejavaformat/google-java-format/${JAVA_FORMAT_VERSION}/google-java-format-${JAVA_FORMAT_VERSION}-all-deps.jar \
   "${HOME}"/.library_generation/google-java-format.jar

.github/scripts/action.yaml

@@ -30,6 +30,9 @@ inputs:
   image_tag:
     description: the tag of hermetic build image
     required: false
+  showcase_mode:
+    description: true if we need to download the showcase api definitions
+    required: false
   token:
     description: Personal Access Token
     required: true
@@ -72,9 +75,11 @@ runs:
       bash hermetic_library_generation.sh \
         --target_branch "${BASE_REF}" \
         --current_branch "${HEAD_REF}" \
+        --showcase_mode "${SHOWCASE_MODE}" \
         --image_tag "${IMAGE_TAG}"
     env:
       BASE_REF: ${{ inputs.base_ref }}
       HEAD_REF: ${{ inputs.head_ref }}
       IMAGE_TAG: ${{ inputs.image_tag }}
+      SHOWCASE_MODE: ${{ inputs.showcase_mode }}
       GH_TOKEN: ${{ inputs.token }}

.github/scripts/hermetic_library_generation.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -ex
 # This script should be run at the root of the repository.
 # This script is used to, when a pull request changes the generation
 # configuration (generation_config.yaml by default) or Dockerfile:
@@ -25,6 +25,8 @@ set -e
 # 3. [optional] image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation.
 # 4. [optional] generation_config, the path to the generation configuration,
 # the default value is generation_config.yaml in the repository root.
+# 5. [optional] showcase_mode, true if we wish to download the showcase api
+# definitions, which are necessary for generating the showcase library.
 while [[ $# -gt 0 ]]; do
 key="$1"
 case "${key}" in
@@ -44,6 +46,10 @@ case "${key}" in
     generation_config="$2"
     shift
     ;;
+  --showcase_mode)
+    showcase_mode="$2"
+    shift
+    ;;
   *)
     echo "Invalid option: [$1]"
     exit 1
@@ -62,6 +68,10 @@ if [ -z "${current_branch}" ]; then
   exit 1
 fi
 
+if [ -z "${download_showcase}" ]; then
+  download_showcase="false"
+fi
+
 if [ -z "${generation_config}" ]; then
   generation_config=generation_config.yaml
   echo "Use default generation config: ${generation_config}"
@@ -89,6 +99,12 @@ pushd "${api_def_dir}"
 git checkout "${googleapis_commitish}"
 popd
 
+# we also setup showcase
+if [[ "${showcase_mode}" == "true" ]]; then
+  source java-showcase/scripts/showcase_utilities.sh
+  append_showcase_to_api_defs "${api_def_dir}"
+fi
+
 # get changed library list.
 changed_libraries=$(python hermetic_build/common/cli/get_changed_libraries.py create \
   --baseline-generation-config-path="${baseline_generation_config}" \
@@ -98,7 +114,6 @@ echo "Changed libraries are: ${changed_libraries:-"No changed library"}."
 # run hermetic code generation docker image.
 docker run \
   --rm \
-  --quiet \
   -u "$(id -u):$(id -g)" \
   -v "$(pwd):${workspace_name}" \
   -v "${api_def_dir}:${workspace_name}/googleapis" \

.github/snippet-bot.yml

@@ -3,6 +3,6 @@ alwaysCreateStatusCheck: false
 ignoreFiles:
   - src/test/**
   - test/**
-  - showcase/**
+  - java-showcase/**
   - library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml
   - library_generation/owlbot/synthtool/gcp/snippets.py

.github/workflows/ci.yaml

@@ -229,18 +229,18 @@ jobs:
         run: |
           mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip
       - name: Java Linter
-        working-directory: showcase
+        working-directory: java-showcase
         run: |
           mvn -B -ntp fmt:check
       - name: Showcase golden tests
-        working-directory: showcase
+        working-directory: java-showcase
         run: |
           mvn test \
             -P enable-golden-tests \
             --batch-mode \
             --no-transfer-progress
       - name: Parse showcase version
-        working-directory: showcase/gapic-showcase
+        working-directory: java-showcase/gapic-showcase
         run: echo "SHOWCASE_VERSION=$(mvn help:evaluate -Dexpression=gapic-showcase.version -q -DforceStdout)" >> "$GITHUB_ENV"
       - name: Install showcase server
         run: |
@@ -252,15 +252,15 @@ jobs:
           ./gapic-showcase run &
           cd -
       - name: Showcase integration tests
-        working-directory: showcase
+        working-directory: java-showcase
         run: |
           mvn verify \
             -P enable-integration-tests \
             --batch-mode \
             --no-transfer-progress
       # The `slf4j2_logback` profile brings logging dependency and compiles logging tests, require env var to be set
       - name: Showcase integration tests - Logging SLF4J 2.x
-        working-directory: showcase
+        working-directory: java-showcase
         run: |
           mvn clean verify -P '!showcase,enable-integration-tests,loggingTestBase,slf4j2_logback' \
             --batch-mode \
@@ -270,7 +270,7 @@ jobs:
           GOOGLE_SDK_JAVA_LOGGING: true
     # The `slf4j1_logback` profile brings logging dependency and compiles logging tests, require env var to be set
       - name: Showcase integration tests - Logging SLF4J 1.x
-        working-directory: showcase
+        working-directory: java-showcase
         run: |
           mvn clean verify -P '!showcase,enable-integration-tests,loggingTestBase,slf4j1_logback' \
             --batch-mode \
@@ -281,7 +281,7 @@ jobs:
     # The `disabledLogging` profile tests logging disabled when logging dependency present,
     # do not set env var for this step
       - name: Showcase integration tests - Logging disabed
-        working-directory: showcase
+        working-directory: java-showcase
         run: |
           mvn clean verify -P '!showcase,enable-integration-tests,loggingTestBase,disabledLogging'  \
             --batch-mode \
@@ -303,6 +303,7 @@ jobs:
       - name: Install sdk-platform-java and showcase to local maven repository
         run: |
           mvn install -B -ntp -T 1C -DskipTests
+          # change to `cd java-showcase` once https://github.com/googleapis/sdk-platform-java/pull/3568/ is merged
           cd showcase
           mvn install -B -ntp -T 1C -DskipTests
           SHOWCASE_CLIENT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
@@ -313,7 +314,7 @@ jobs:
         run: mvn install -B -ntp -T 1C -DskipTests
         # Showcase golden test ensures that src changes are already reflected in the PR.
       - name: Clirr check
-        working-directory: showcase
+        working-directory: java-showcase
         run: |
           mvn versions:set -B -ntp -DnewVersion=local
           mvn clirr:check -B -ntp -Dclirr.skip=false -DcomparisonVersion=$SHOWCASE_CLIENT_VERSION

.github/workflows/hermetic_library_generation.yaml

@@ -62,3 +62,4 @@ jobs:
         head_ref: ${{ github.head_ref }}
         image_tag: ${{ env.GENERATOR_VERSION }}
         token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
+        showcase_mode: true

.github/workflows/sonar.yaml

@@ -33,10 +33,10 @@ jobs:
       - name: Install modules to local maven
         run: |
           mvn install -T 1C -DskipTests -ntp -B
-          cd showcase
+          cd java-showcase
           mvn install -T 1C -DskipTests -ntp -B
       - name: Parse showcase version
-        working-directory: showcase/gapic-showcase
+        working-directory: java-showcase/gapic-showcase
         run: echo "SHOWCASE_VERSION=$(mvn help:evaluate -Dexpression=gapic-showcase.version -q -DforceStdout)" >> "$GITHUB_ENV"
       - name: Install showcase server
         run: |

.kokoro/presubmit/common_test.sh

@@ -22,7 +22,7 @@ cd target
 function test_find_all_poms_with_versioned_dependency {
   mkdir -p test_find_all_poms_with_dependency
   pushd test_find_all_poms_with_dependency
-  cp ../../showcase/gapic-showcase/pom.xml pom.xml
+  cp ../../java-showcase/gapic-showcase/pom.xml pom.xml
 
   find_all_poms_with_versioned_dependency 'truth'
   if [ "${#POMS[@]}" != 1 ]; then
@@ -45,7 +45,7 @@ function test_find_all_poms_with_versioned_dependency {
 function test_update_pom_dependency {
   mkdir -p test_update_pom_dependency
   pushd test_update_pom_dependency
-  cp ../../showcase/gapic-showcase/pom.xml pom.xml
+  cp ../../java-showcase/gapic-showcase/pom.xml pom.xml
 
   update_pom_dependency . truth "99.88.77"
 
@@ -66,7 +66,7 @@ EOF
 function test_parse_pom_version {
   mkdir -p test_parse_pom_version
   pushd test_parse_pom_version
-  cp ../../showcase/gapic-showcase/pom.xml pom.xml
+  cp ../../java-showcase/gapic-showcase/pom.xml pom.xml
 
   VERSION=$(parse_pom_version .)
   if [ "$VERSION" != "0.0.1-SNAPSHOT" ]; then

.kokoro/presubmit/showcase-native.sh

@@ -38,7 +38,7 @@ mvn install --projects '!gapic-generator-java' \
 SHARED_DEPS_VERSION=$(parse_pom_version java-shared-dependencies)
 
 # Run showcase integration tests in GraalVM
-pushd showcase/gapic-showcase
+pushd java-showcase/gapic-showcase
 SHOWCASE_VERSION=$(mvn help:evaluate -Dexpression=gapic-showcase.version -q -DforceStdout)
 popd
 # Start showcase server
@@ -52,7 +52,7 @@ tar -xf showcase-*
 popd
 
 # Run showcase tests with `native` profile
-pushd showcase
+pushd java-showcase
 mvn test -Pnative,-showcase \
   -Denforcer.skip=true \
   -Dcheckstyle.skip \

generation_config.yaml

@@ -3,6 +3,15 @@ googleapis_commitish: fbbbf5023815f9a662c85aa8af8f3b72467fcb6f
 # java-{library.library_name} or java-{library.api-shortname} when
 # library.library_name is not defined.
 libraries:
+- api_shortname: showcase
+  excluded_poms: gapic-showcase-bom
+  name_pretty: Showcase
+  api_description: Showcase module
+  product_documentation: https://cloud.google.com/dummy
+  distribution_name: com.google.cloud:gapic-showcase
+  library_type: OTHER
+  GAPICs:
+  - proto_path: schema/google/showcase/v1beta1
 - api_shortname: common-protos
   name_pretty: Common Protos
   product_documentation: https://github.com/googleapis/api-common-protos

hermetic_build/common/tests/resources/misc/BUILD_additional_protos_in_gapic.bazel

@@ -0,0 +1,6 @@
+java_gapic_library(
+    deps = [
+        "//google/cloud/location:location_java_proto",
+        "//google/iam/v1:iam_java_proto"
+   ],
+)

hermetic_build/common/tests/resources/misc/BUILD_target_with_nested_parenthesis.bazel

@@ -0,0 +1,13 @@
+proto_library_with_info(
+  # this comment should not indicate an end of a pattern match )
+  deps = [
+    "//google/cloud/location:location_proto",
+  ]
+)
+
+java_gapic_library(
+  # this comment should not indicate an end of a pattern match )
+  deps = [
+    "//google/iam/v1:iam_java_proto"
+  ],
+)

java-showcase/.OwlBot-hermetic.yaml

@@ -0,0 +1,35 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+deep-remove-regex:
+- "/grpc-gapic-showcase-v1beta1/src"
+- "/proto-gapic-showcase-v1beta1/src"
+- "/gapic-showcase/src"
+- "/samples/snippets/generated"
+
+deep-preserve-regex:
+- "/gapic-showcase/src/test"
+
+deep-copy-regex:
+- source: "/schema/google/showcase/(v.*)/.*-java/proto-.*/src"
+  dest: "/owl-bot-staging/showcase/$1/proto-gapic-showcase-$1/src"
+- source: "/schema/google/showcase/(v.*)/.*-java/grpc-.*/src"
+  dest: "/owl-bot-staging/showcase/$1/grpc-gapic-showcase-$1/src"
+- source: "/schema/google/showcase/(v.*)/.*-java/gapic-.*/src"
+  dest: "/owl-bot-staging/showcase/$1/gapic-showcase/src"
+- source: "/schema/google/showcase/(v.*)/.*-java/samples/snippets/generated"
+  dest: "/owl-bot-staging/showcase/$1/samples/snippets/generated"
+
+api-name: showcase

java-showcase/.repo-metadata.json

@@ -0,0 +1,16 @@
+{
+  "api_shortname": "showcase",
+  "name_pretty": "Showcase",
+  "product_documentation": "https://cloud.google.com/dummy",
+  "api_description": "Showcase module",
+  "client_documentation": "https://cloud.google.com/java/docs/reference/gapic-showcase/latest/overview",
+  "release_level": "preview",
+  "transport": "both",
+  "language": "java",
+  "repo": "googleapis/sdk-platform-java",
+  "repo_short": "java-showcase",
+  "distribution_name": "com.google.cloud:gapic-showcase",
+  "library_type": "OTHER",
+  "requires_billing": true,
+  "excluded_poms": "gapic-showcase-bom"
+}

showcase/README.md renamed to java-showcase/README.md

@@ -64,28 +64,32 @@ gapic-showcase run
 ## Running the Integration Tests
 
 Open a new terminal window in the root project directory.
+This step does not require Docker.
 
 ```shell
 cd showcase
-mvn verify -P enable-integration-tests -P enable-golden-tests
+mvn verify -P enable-integration-tests
 ```
 
-Note:
+## Running the Golden tests
+
+**NOTE** This requires Docker to be installed in your machine.
+Open a new terminal window in the root project directory.
+
+```shell
+cd java-showcase
+mvn verify -P enable-golden-tests
+```
 
-* `-P enable-golden-tests` is optional. These tests do not require a local server.
 
 ## Update the Golden Showcase Files
 
+**NOTE** This requires Docker to be installed in your machine.
 Open a new terminal window in the root project directory.
-Note that the underlying scripts will modify your `$HOME` folder by creating a
-`.library_generation` folder.
-This is the well-known location of the artifacts the generation scripts use.
-For more details see the
-[development guide](https://github.com/googleapis/sdk-platform-java/blob/main/hermetic_build/DEVELOPMENT.md#the-hermetic-builds-well-known-folder)
 
 ```shell
 # In repository's root directory
 mvn clean install -DskipTests
-cd showcase
+cd java-showcase
 mvn compile -P update
 ```