Add resource Dataproc batch (#11750) (#8306)

[upstream:a0b7099e82cb2d112518628de9032ab92754838b]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

Diff for: .changelog/11750.txt

@@ -0,0 +1,6 @@
+```release-note:new-resource
+`google_dataproc_batch`
+```
+```release-note:enhancement
+dataproc: switched to the v1 API for `google_dataproc_autoscaling_policy` resource (beta)
+```

Diff for: google-beta/acctest/bootstrap_test_utils.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"maps"
 	"os"
 	"strings"
 	"testing"
@@ -912,7 +913,25 @@ func BootstrapSharedCaPoolInLocation(t *testing.T, location string) string {
 	return poolName
 }
 
+func BootstrapSubnetForDataprocBatches(t *testing.T, subnetName string, networkName string) string {
+	subnetOptions := map[string]interface{}{
+		"privateIpGoogleAccess": true,
+	}
+	return BootstrapSubnetWithOverrides(t, subnetName, networkName, subnetOptions)
+}
+
 func BootstrapSubnet(t *testing.T, subnetName string, networkName string) string {
+	return BootstrapSubnetWithOverrides(t, subnetName, networkName, make(map[string]interface{}))
+}
+
+func BootstrapSubnetWithFirewallForDataprocBatches(t *testing.T, testId string, subnetName string) string {
+	networkName := BootstrapSharedTestNetwork(t, testId)
+	subnetworkName := BootstrapSubnetForDataprocBatches(t, subnetName, networkName)
+	BootstrapFirewallForDataprocSharedNetwork(t, subnetName, networkName)
+	return subnetworkName
+}
+
+func BootstrapSubnetWithOverrides(t *testing.T, subnetName string, networkName string, subnetOptions map[string]interface{}) string {
 	projectID := envvar.GetTestProjectFromEnv()
 	region := envvar.GetTestRegionFromEnv()
 
@@ -934,20 +953,24 @@ func BootstrapSubnet(t *testing.T, subnetName string, networkName string) string
 		networkUrl := fmt.Sprintf("%sprojects/%s/global/networks/%s", config.ComputeBasePath, projectID, networkName)
 		url := fmt.Sprintf("%sprojects/%s/regions/%s/subnetworks", config.ComputeBasePath, projectID, region)
 
-		subnetObj := map[string]interface{}{
+		defaultSubnetObj := map[string]interface{}{
 			"name":        subnetName,
 			"region ":     region,
 			"network":     networkUrl,
 			"ipCidrRange": "10.77.0.0/20",
 		}
 
+		if len(subnetOptions) != 0 {
+			maps.Copy(defaultSubnetObj, subnetOptions)
+		}
+
 		res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{
 			Config:    config,
 			Method:    "POST",
 			Project:   projectID,
 			RawURL:    url,
 			UserAgent: config.UserAgent,
-			Body:      subnetObj,
+			Body:      defaultSubnetObj,
 			Timeout:   4 * time.Minute,
 		})
 

Diff for: google-beta/provider/provider_mmv1_resources.go

@@ -486,9 +486,9 @@ var handwrittenIAMDatasources = map[string]*schema.Resource{
 }
 
 // Resources
-// Generated resources: 529
+// Generated resources: 530
 // Generated IAM resources: 291
-// Total generated resources: 820
+// Total generated resources: 821
 var generatedResources = map[string]*schema.Resource{
 	"google_folder_access_approval_settings":                                     accessapproval.ResourceAccessApprovalFolderSettings(),
 	"google_organization_access_approval_settings":                               accessapproval.ResourceAccessApprovalOrganizationSettings(),
@@ -864,6 +864,7 @@ var generatedResources = map[string]*schema.Resource{
 	"google_dataproc_autoscaling_policy_iam_binding":                             tpgiamresource.ResourceIamBinding(dataproc.DataprocAutoscalingPolicyIamSchema, dataproc.DataprocAutoscalingPolicyIamUpdaterProducer, dataproc.DataprocAutoscalingPolicyIdParseFunc),
 	"google_dataproc_autoscaling_policy_iam_member":                              tpgiamresource.ResourceIamMember(dataproc.DataprocAutoscalingPolicyIamSchema, dataproc.DataprocAutoscalingPolicyIamUpdaterProducer, dataproc.DataprocAutoscalingPolicyIdParseFunc),
 	"google_dataproc_autoscaling_policy_iam_policy":                              tpgiamresource.ResourceIamPolicy(dataproc.DataprocAutoscalingPolicyIamSchema, dataproc.DataprocAutoscalingPolicyIamUpdaterProducer, dataproc.DataprocAutoscalingPolicyIdParseFunc),
+	"google_dataproc_batch":                                                      dataproc.ResourceDataprocBatch(),
 	"google_dataproc_metastore_federation":                                       dataprocmetastore.ResourceDataprocMetastoreFederation(),
 	"google_dataproc_metastore_federation_iam_binding":                           tpgiamresource.ResourceIamBinding(dataprocmetastore.DataprocMetastoreFederationIamSchema, dataprocmetastore.DataprocMetastoreFederationIamUpdaterProducer, dataprocmetastore.DataprocMetastoreFederationIdParseFunc),
 	"google_dataproc_metastore_federation_iam_member":                            tpgiamresource.ResourceIamMember(dataprocmetastore.DataprocMetastoreFederationIamSchema, dataprocmetastore.DataprocMetastoreFederationIamUpdaterProducer, dataprocmetastore.DataprocMetastoreFederationIdParseFunc),

Diff for: google-beta/services/dataproc/dataproc_operation.go

@@ -0,0 +1,92 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+// ----------------------------------------------------------------------------
+//
+//     ***     AUTO GENERATED CODE    ***    Type: MMv1     ***
+//
+// ----------------------------------------------------------------------------
+//
+//     This file is automatically generated by Magic Modules and manual
+//     changes will be clobbered when the file is regenerated.
+//
+//     Please read more about how to change this file in
+//     .github/CONTRIBUTING.md.
+//
+// ----------------------------------------------------------------------------
+
+package dataproc
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/hashicorp/terraform-provider-google-beta/google-beta/tpgresource"
+	transport_tpg "github.com/hashicorp/terraform-provider-google-beta/google-beta/transport"
+)
+
+type DataprocOperationWaiter struct {
+	Config    *transport_tpg.Config
+	UserAgent string
+	Project   string
+	tpgresource.CommonOperationWaiter
+}
+
+func (w *DataprocOperationWaiter) QueryOp() (interface{}, error) {
+	if w == nil {
+		return nil, fmt.Errorf("Cannot query operation, it's unset or nil.")
+	}
+	// Returns the proper get.
+	url := fmt.Sprintf("%s%s", w.Config.DataprocBasePath, w.CommonOperationWaiter.Op.Name)
+
+	return transport_tpg.SendRequest(transport_tpg.SendRequestOptions{
+		Config:    w.Config,
+		Method:    "GET",
+		Project:   w.Project,
+		RawURL:    url,
+		UserAgent: w.UserAgent,
+	})
+}
+
+func createDataprocWaiter(config *transport_tpg.Config, op map[string]interface{}, project, activity, userAgent string) (*DataprocOperationWaiter, error) {
+	w := &DataprocOperationWaiter{
+		Config:    config,
+		UserAgent: userAgent,
+		Project:   project,
+	}
+	if err := w.CommonOperationWaiter.SetOp(op); err != nil {
+		return nil, err
+	}
+	return w, nil
+}
+
+// nolint: deadcode,unused
+func DataprocOperationWaitTimeWithResponse(config *transport_tpg.Config, op map[string]interface{}, response *map[string]interface{}, project, activity, userAgent string, timeout time.Duration) error {
+	w, err := createDataprocWaiter(config, op, project, activity, userAgent)
+	if err != nil {
+		return err
+	}
+	if err := tpgresource.OperationWait(w, activity, timeout, config.PollInterval); err != nil {
+		return err
+	}
+	rawResponse := []byte(w.CommonOperationWaiter.Op.Response)
+	if len(rawResponse) == 0 {
+		return errors.New("`resource` not set in operation response")
+	}
+	return json.Unmarshal(rawResponse, response)
+}
+
+func DataprocOperationWaitTime(config *transport_tpg.Config, op map[string]interface{}, project, activity, userAgent string, timeout time.Duration) error {
+	if val, ok := op["name"]; !ok || val == "" {
+		// This was a synchronous call - there is no operation to wait for.
+		return nil
+	}
+	w, err := createDataprocWaiter(config, op, project, activity, userAgent)
+	if err != nil {
+		// If w is nil, the op was synchronous.
+		return err
+	}
+	return tpgresource.OperationWait(w, activity, timeout, config.PollInterval)
+}

Diff for: google-beta/services/dataproc/resource_dataproc_autoscaling_policy_sweeper.go

@@ -64,7 +64,7 @@ func testSweepDataprocAutoscalingPolicy(region string) error {
 		},
 	}
 
-	listTemplate := strings.Split("https://dataproc.googleapis.com/v1beta2/projects/{{project}}/locations/{{location}}/autoscalingPolicies", "?")[0]
+	listTemplate := strings.Split("https://dataproc.googleapis.com/v1/projects/{{project}}/locations/{{location}}/autoscalingPolicies", "?")[0]
 	listUrl, err := tpgresource.ReplaceVars(d, config, listTemplate)
 	if err != nil {
 		log.Printf("[INFO][SWEEPER_LOG] error preparing sweeper list url: %s", err)
@@ -112,7 +112,7 @@ func testSweepDataprocAutoscalingPolicy(region string) error {
 			continue
 		}
 
-		deleteTemplate := "https://dataproc.googleapis.com/v1beta2/projects/{{project}}/locations/{{location}}/autoscalingPolicies/{{policy_id}}"
+		deleteTemplate := "https://dataproc.googleapis.com/v1/projects/{{project}}/locations/{{location}}/autoscalingPolicies/{{policy_id}}"
 		deleteUrl, err := tpgresource.ReplaceVars(d, config, deleteTemplate)
 		if err != nil {
 			log.Printf("[INFO][SWEEPER_LOG] error preparing delete url: %s", err)