Skip to content

Commit 59c92f2

Browse files
modular-magicianmodular-magician
authored
Allow passing node service_account when autopilot enabled (#6733) (#13024)
fixes #9505 Signed-off-by: Modular Magician <[email protected]> Signed-off-by: Modular Magician <[email protected]>
1 parent bc240c0 commit 59c92f2

File tree

4 files changed

+125
-38
lines changed

4 files changed

+125
-38
lines changed

‎.changelog/6733.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:bug
2+
container: fixed a bug where `cluster_autoscaling.auto_provisioning_defaults.service_account` can not be set when `enable_autopilot = true` for `google_container_cluster`
3+
```

‎google/resource_container_cluster.go

+43-26
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,13 @@ var (
6767
forceNewClusterNodeConfigFields = []string{
6868
"workload_metadata_config",
6969
}
70+
71+
suppressDiffForAutopilot = schema.SchemaDiffSuppressFunc(func(k, oldValue, newValue string, d *schema.ResourceData) bool {
72+
if v, _ := d.Get("enable_autopilot").(bool); v {
73+
return true
74+
}
75+
return false
76+
})
7077
)
7178

7279
// This uses the node pool nodeConfig schema but sets
@@ -326,21 +333,24 @@ func resourceContainerCluster() *schema.Resource {
326333
MaxItems: 1,
327334
// This field is Optional + Computed because we automatically set the
328335
// enabled value to false if the block is not returned in API responses.
329-
Optional: true,
330-
Computed: true,
331-
Description: `Per-cluster configuration of Node Auto-Provisioning with Cluster Autoscaler to automatically adjust the size of the cluster and create/delete node pools based on the current needs of the cluster's workload. See the guide to using Node Auto-Provisioning for more details.`,
332-
ConflictsWith: []string{"enable_autopilot"},
336+
Optional: true,
337+
Computed: true,
338+
Description: `Per-cluster configuration of Node Auto-Provisioning with Cluster Autoscaler to automatically adjust the size of the cluster and create/delete node pools based on the current needs of the cluster's workload. See the guide to using Node Auto-Provisioning for more details.`,
333339
Elem: &schema.Resource{
334340
Schema: map[string]*schema.Schema{
335341
"enabled": {
336-
Type: schema.TypeBool,
337-
Required: true,
338-
Description: `Whether node auto-provisioning is enabled. Resource limits for cpu and memory must be defined to enable node auto-provisioning.`,
342+
Type: schema.TypeBool,
343+
Optional: true,
344+
Computed: true,
345+
ConflictsWith: []string{"enable_autopilot"},
346+
Description: `Whether node auto-provisioning is enabled. Resource limits for cpu and memory must be defined to enable node auto-provisioning.`,
339347
},
340348
"resource_limits": {
341-
Type: schema.TypeList,
342-
Optional: true,
343-
Description: `Global constraints for machine resources in the cluster. Configuring the cpu and memory types is required if node auto-provisioning is enabled. These limits will apply to node pool autoscaling in addition to node auto-provisioning.`,
349+
Type: schema.TypeList,
350+
Optional: true,
351+
ConflictsWith: []string{"enable_autopilot"},
352+
DiffSuppressFunc: suppressDiffForAutopilot,
353+
Description: `Global constraints for machine resources in the cluster. Configuring the cpu and memory types is required if node auto-provisioning is enabled. These limits will apply to node pool autoscaling in addition to node auto-provisioning.`,
344354
Elem: &schema.Resource{
345355
Schema: map[string]*schema.Schema{
346356
"resource_type": {
@@ -384,25 +394,28 @@ func resourceContainerCluster() *schema.Resource {
384394
Description: `The Google Cloud Platform Service Account to be used by the node VMs.`,
385395
},
386396
"disk_size": {
387-
Type: schema.TypeInt,
388-
Optional: true,
389-
Default: 100,
390-
Description: `Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB.`,
391-
ValidateFunc: validation.IntAtLeast(10),
397+
Type: schema.TypeInt,
398+
Optional: true,
399+
Default: 100,
400+
Description: `Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB.`,
401+
DiffSuppressFunc: suppressDiffForAutopilot,
402+
ValidateFunc: validation.IntAtLeast(10),
392403
},
393404
"disk_type": {
394-
Type: schema.TypeString,
395-
Optional: true,
396-
Default: "pd-standard",
397-
Description: `Type of the disk attached to each node.`,
398-
ValidateFunc: validation.StringInSlice([]string{"pd-standard", "pd-ssd", "pd-balanced"}, false),
405+
Type: schema.TypeString,
406+
Optional: true,
407+
Default: "pd-standard",
408+
Description: `Type of the disk attached to each node.`,
409+
DiffSuppressFunc: suppressDiffForAutopilot,
410+
ValidateFunc: validation.StringInSlice([]string{"pd-standard", "pd-ssd", "pd-balanced"}, false),
399411
},
400412
"image_type": {
401-
Type: schema.TypeString,
402-
Optional: true,
403-
Default: "COS_CONTAINERD",
404-
Description: `The default image type used by NAP once a new node pool is being created.`,
405-
ValidateFunc: validation.StringInSlice([]string{"COS_CONTAINERD", "COS", "UBUNTU_CONTAINERD", "UBUNTU"}, false),
413+
Type: schema.TypeString,
414+
Optional: true,
415+
Default: "COS_CONTAINERD",
416+
Description: `The default image type used by NAP once a new node pool is being created.`,
417+
DiffSuppressFunc: suppressDiffForAutopilot,
418+
ValidateFunc: validation.StringInSlice([]string{"COS_CONTAINERD", "COS", "UBUNTU_CONTAINERD", "UBUNTU"}, false),
406419
},
407420
"boot_disk_kms_key": {
408421
Type: schema.TypeString,
@@ -3257,8 +3270,12 @@ func expandMaintenancePolicy(d *schema.ResourceData, meta interface{}) *containe
32573270

32583271
func expandClusterAutoscaling(configured interface{}, d *schema.ResourceData) *container.ClusterAutoscaling {
32593272
l, ok := configured.([]interface{})
3273+
enableAutopilot := false
3274+
if v, ok := d.GetOk("enable_autopilot"); ok && v == true {
3275+
enableAutopilot = true
3276+
}
32603277
if !ok || l == nil || len(l) == 0 || l[0] == nil {
3261-
if v, ok := d.GetOk("enable_autopilot"); ok && v == true {
3278+
if enableAutopilot {
32623279
return nil
32633280
}
32643281
return &container.ClusterAutoscaling{

‎google/resource_container_cluster_test.go

+73-7
Original file line numberDiff line numberDiff line change
@@ -1895,6 +1895,7 @@ func TestAccContainerCluster_withShieldedNodes(t *testing.T) {
18951895
func TestAccContainerCluster_withAutopilot(t *testing.T) {
18961896
t.Parallel()
18971897

1898+
pid := getTestProjectFromEnv()
18981899
containerNetName := fmt.Sprintf("tf-test-container-net-%s", randString(t, 10))
18991900
clusterName := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10))
19001901

@@ -1904,7 +1905,42 @@ func TestAccContainerCluster_withAutopilot(t *testing.T) {
19041905
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
19051906
Steps: []resource.TestStep{
19061907
{
1907-
Config: testAccContainerCluster_withAutopilot(containerNetName, clusterName, "us-central1", true, false),
1908+
Config: testAccContainerCluster_withAutopilot(pid, containerNetName, clusterName, "us-central1", true, false, ""),
1909+
},
1910+
{
1911+
ResourceName: "google_container_cluster.with_autopilot",
1912+
ImportState: true,
1913+
ImportStateVerify: true,
1914+
ImportStateVerifyIgnore: []string{"min_master_version"},
1915+
},
1916+
},
1917+
})
1918+
}
1919+
1920+
func TestAccContainerClusterCustomServiceAccount_withAutopilot(t *testing.T) {
1921+
t.Parallel()
1922+
1923+
pid := getTestProjectFromEnv()
1924+
containerNetName := fmt.Sprintf("tf-test-container-net-%s", randString(t, 10))
1925+
clusterName := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10))
1926+
serviceAccountName := fmt.Sprintf("tf-test-sa-%s", randString(t, 10))
1927+
1928+
vcrTest(t, resource.TestCase{
1929+
PreCheck: func() { testAccPreCheck(t) },
1930+
Providers: testAccProviders,
1931+
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
1932+
Steps: []resource.TestStep{
1933+
{
1934+
Config: testAccContainerCluster_withAutopilot(pid, containerNetName, clusterName, "us-central1", true, false, serviceAccountName),
1935+
Check: resource.ComposeTestCheckFunc(
1936+
resource.TestCheckResourceAttr("google_container_cluster.with_autopilot",
1937+
"cluster_autoscaling.0.enabled", "true"),
1938+
resource.TestCheckResourceAttr("google_container_cluster.with_autopilot",
1939+
"cluster_autoscaling.0.auto_provisioning_defaults.0.service_account",
1940+
fmt.Sprintf("%s@%s.iam.gserviceaccount.com", serviceAccountName, pid)),
1941+
resource.TestCheckResourceAttr("google_container_cluster.with_autopilot",
1942+
"cluster_autoscaling.0.auto_provisioning_defaults.0.oauth_scopes.0", "https://www.googleapis.com/auth/cloud-platform"),
1943+
),
19081944
},
19091945
{
19101946
ResourceName: "google_container_cluster.with_autopilot",
@@ -1919,6 +1955,7 @@ func TestAccContainerCluster_withAutopilot(t *testing.T) {
19191955
func TestAccContainerCluster_errorAutopilotLocation(t *testing.T) {
19201956
t.Parallel()
19211957

1958+
pid := getTestProjectFromEnv()
19221959
containerNetName := fmt.Sprintf("tf-test-container-net-%s", randString(t, 10))
19231960
clusterName := fmt.Sprintf("tf-test-cluster-%s", randString(t, 10))
19241961

@@ -1928,7 +1965,7 @@ func TestAccContainerCluster_errorAutopilotLocation(t *testing.T) {
19281965
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
19291966
Steps: []resource.TestStep{
19301967
{
1931-
Config: testAccContainerCluster_withAutopilot(containerNetName, clusterName, "us-central1-a", true, false),
1968+
Config: testAccContainerCluster_withAutopilot(pid, containerNetName, clusterName, "us-central1-a", true, false, ""),
19321969
ExpectError: regexp.MustCompile(`Autopilot clusters must be regional clusters.`),
19331970
},
19341971
},
@@ -2657,8 +2694,8 @@ func testAccCheckContainerClusterDestroyProducer(t *testing.T) func(s *terraform
26572694
}
26582695

26592696
attributes := rs.Primary.Attributes
2660-
_, err := config.NewContainerClient(config.userAgent).Projects.Zones.Clusters.Get(
2661-
config.Project, attributes["location"], attributes["name"]).Do()
2697+
_, err := config.NewContainerClient(config.userAgent).Projects.Locations.Clusters.Get(
2698+
fmt.Sprintf("projects/%s/locations/%s/clusters/%s", config.Project, attributes["location"], attributes["name"])).Do()
26622699
if err == nil {
26632700
return fmt.Errorf("Cluster still exists")
26642701
}
@@ -5223,8 +5260,36 @@ resource "google_container_cluster" "primary" {
52235260
`, name)
52245261
}
52255262

5226-
func testAccContainerCluster_withAutopilot(containerNetName string, clusterName string, location string, enabled bool, withNetworkTag bool) string {
5227-
config := fmt.Sprintf(`
5263+
func testAccContainerCluster_withAutopilot(projectID string, containerNetName string, clusterName string, location string, enabled bool, withNetworkTag bool, serviceAccount string) string {
5264+
config := ""
5265+
clusterAutoscaling := ""
5266+
if serviceAccount != "" {
5267+
config += fmt.Sprintf(`
5268+
resource "google_service_account" "service_account" {
5269+
account_id = "%[1]s"
5270+
project = "%[2]s"
5271+
display_name = "Service Account"
5272+
}
5273+
5274+
resource "google_project_iam_binding" "project" {
5275+
project = "%[2]s"
5276+
role = "roles/container.nodeServiceAccount"
5277+
members = [
5278+
"serviceAccount:%[1]s@%[2]s.iam.gserviceaccount.com",
5279+
]
5280+
}`, serviceAccount, projectID)
5281+
5282+
clusterAutoscaling = fmt.Sprintf(`
5283+
cluster_autoscaling {
5284+
auto_provisioning_defaults {
5285+
service_account = "%s@%s.iam.gserviceaccount.com"
5286+
oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"]
5287+
}
5288+
}`, serviceAccount, projectID)
5289+
}
5290+
5291+
config += fmt.Sprintf(`
5292+
52285293
resource "google_compute_network" "container_network" {
52295294
name = "%s"
52305295
auto_create_subnetworks = false
@@ -5271,9 +5336,10 @@ resource "google_container_cluster" "with_autopilot" {
52715336
disabled = false
52725337
}
52735338
}
5339+
%s
52745340
vertical_pod_autoscaling {
52755341
enabled = true
5276-
}`, containerNetName, clusterName, location, enabled)
5342+
}`, containerNetName, clusterName, location, enabled, clusterAutoscaling)
52775343
if withNetworkTag {
52785344
config += `
52795345
node_pool_auto_config {

‎website/docs/r/container_cluster.html.markdown

+6-5
Original file line numberDiff line numberDiff line change
@@ -470,15 +470,16 @@ addons_config {
470470

471471
<a name="nested_cluster_autoscaling"></a>The `cluster_autoscaling` block supports:
472472

473-
* `enabled` - (Required) Whether node auto-provisioning is enabled. Resource
474-
limits for `cpu` and `memory` must be defined to enable node auto-provisioning.
473+
* `enabled` - (Optional) Whether node auto-provisioning is enabled. Must be supplied for GKE Standard clusters, `true` is implied
474+
for autopilot clusters. Resource limits for `cpu` and `memory` must be defined to enable node auto-provisioning for GKE Standard.
475475

476476
* `resource_limits` - (Optional) Global constraints for machine resources in the
477477
cluster. Configuring the `cpu` and `memory` types is required if node
478478
auto-provisioning is enabled. These limits will apply to node pool autoscaling
479479
in addition to node auto-provisioning. Structure is [documented below](#nested_resource_limits).
480480

481-
* `auto_provisioning_defaults` - (Optional) Contains defaults for a node pool created by NAP.
481+
* `auto_provisioning_defaults` - (Optional) Contains defaults for a node pool created by NAP. A subset of fields also apply to
482+
GKE Autopilot clusters.
482483
Structure is [documented below](#nested_auto_provisioning_defaults).
483484

484485
* `autoscaling_profile` - (Optional, [Beta](https://terraform.io/docs/providers/google/provider_versions.html)) Configuration
@@ -503,11 +504,11 @@ Minimum CPU platform to be used for NAP created node pools. The instance may be
503504
specified or newer CPU platform. Applicable values are the friendly names of CPU platforms, such
504505
as "Intel Haswell" or "Intel Sandy Bridge".
505506

506-
* `oauth_scopes` - (Optional) Scopes that are used by NAP when creating node pools. Use the "https://www.googleapis.com/auth/cloud-platform" scope to grant access to all APIs. It is recommended that you set `service_account` to a non-default service account and grant IAM roles to that service account for only the resources that it needs.
507+
* `oauth_scopes` - (Optional) Scopes that are used by NAP and GKE Autopilot when creating node pools. Use the "https://www.googleapis.com/auth/cloud-platform" scope to grant access to all APIs. It is recommended that you set `service_account` to a non-default service account and grant IAM roles to that service account for only the resources that it needs.
507508

508509
-> `monitoring.write` is always enabled regardless of user input. `monitoring` and `logging.write` may also be enabled depending on the values for `monitoring_service` and `logging_service`.
509510

510-
* `service_account` - (Optional) The Google Cloud Platform Service Account to be used by the node VMs.
511+
* `service_account` - (Optional) The Google Cloud Platform Service Account to be used by the node VMs created by GKE Autopilot or NAP.
511512

512513
* `boot_disk_kms_key` - (Optional) The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption
513514

0 commit comments

Comments
 (0)