Skip to content

Commit 913c1b1

Browse files
modular-magicianemilymye
modular-magician
authored and
emilymye
committed
Add batching for IAM binding/member/audit config changes (#4207)
Signed-off-by: Modular Magician <[email protected]>
1 parent 64375af commit 913c1b1

9 files changed

+177
-53
lines changed

‎google/config.go

+9-4
Original file line numberDiff line numberDiff line change
@@ -158,9 +158,8 @@ type Config struct {
158158
ServiceManagementBasePath string
159159
clientServiceMan *servicemanagement.APIService
160160

161-
ServiceUsageBasePath string
162-
clientServiceUsage *serviceusage.Service
163-
requestBatcherServiceUsage *RequestBatcher
161+
ServiceUsageBasePath string
162+
clientServiceUsage *serviceusage.Service
164163

165164
BigQueryBasePath string
166165
clientBigQuery *bigquery.Service
@@ -187,6 +186,9 @@ type Config struct {
187186
// we expose those directly instead of providing the `Service` object
188187
// as a factory.
189188
clientBigtableProjectsInstances *bigtableadmin.ProjectsInstancesService
189+
190+
requestBatcherServiceUsage *RequestBatcher
191+
requestBatcherIam *RequestBatcher
190192
}
191193

192194
var defaultClientScopes = []string{
@@ -399,7 +401,6 @@ func (c *Config) LoadAndValidate() error {
399401
}
400402
c.clientServiceUsage.UserAgent = userAgent
401403
c.clientServiceUsage.BasePath = serviceUsageClientBasePath
402-
c.requestBatcherServiceUsage = NewRequestBatcher("Service Usage", context, c.BatchingConfig)
403404

404405
cloudBillingClientBasePath := removeBasePathVersion(c.CloudBillingBasePath)
405406
log.Printf("[INFO] Instantiating Google Cloud Billing client for path %s", cloudBillingClientBasePath)
@@ -544,6 +545,10 @@ func (c *Config) LoadAndValidate() error {
544545
c.clientStorageTransfer.BasePath = storageTransferClientBasePath
545546

546547
c.Region = GetRegionFromRegionSelfLink(c.Region)
548+
549+
c.requestBatcherServiceUsage = NewRequestBatcher("Service Usage", context, c.BatchingConfig)
550+
c.requestBatcherIam = NewRequestBatcher("IAM", context, c.BatchingConfig)
551+
547552
return nil
548553
}
549554

‎google/iam_batching.go

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package google
2+
3+
import (
4+
"fmt"
5+
"google.golang.org/api/cloudresourcemanager/v1"
6+
"time"
7+
)
8+
9+
const (
10+
batchKeyTmplModifyIamPolicy = "%s modifyIamPolicy"
11+
12+
IamBatchingEnabled = true
13+
IamBatchingDisabled = false
14+
)
15+
16+
func BatchRequestModifyIamPolicy(updater ResourceIamUpdater, modify iamPolicyModifyFunc, config *Config, reqDesc string) error {
17+
batchKey := fmt.Sprintf(batchKeyTmplModifyIamPolicy, updater.GetMutexKey())
18+
19+
request := &BatchRequest{
20+
ResourceName: updater.GetResourceId(),
21+
Body: []iamPolicyModifyFunc{modify},
22+
CombineF: combineBatchIamPolicyModifiers,
23+
SendF: sendBatchModifyIamPolicy(updater),
24+
DebugId: reqDesc,
25+
}
26+
27+
_, err := config.requestBatcherIam.SendRequestWithTimeout(batchKey, request, time.Minute*30)
28+
return err
29+
}
30+
31+
func combineBatchIamPolicyModifiers(currV interface{}, toAddV interface{}) (interface{}, error) {
32+
currModifiers, ok := currV.([]iamPolicyModifyFunc)
33+
if !ok {
34+
return nil, fmt.Errorf("provider error in batch combiner: expected data to be type []iamPolicyModifyFunc, got %v with type %T", currV, currV)
35+
}
36+
37+
newModifiers, ok := toAddV.([]iamPolicyModifyFunc)
38+
if !ok {
39+
return nil, fmt.Errorf("provider error in batch combiner: expected data to be type []iamPolicyModifyFunc, got %v with type %T", currV, currV)
40+
}
41+
42+
return append(currModifiers, newModifiers...), nil
43+
}
44+
45+
func sendBatchModifyIamPolicy(updater ResourceIamUpdater) batcherSendFunc {
46+
return func(resourceName string, body interface{}) (interface{}, error) {
47+
modifiers, ok := body.([]iamPolicyModifyFunc)
48+
if !ok {
49+
return nil, fmt.Errorf("provider error: expected data to be type []iamPolicyModifyFunc, got %v with type %T", body, body)
50+
}
51+
return nil, iamPolicyReadModifyWrite(updater, func(policy *cloudresourcemanager.Policy) error {
52+
for _, modifyF := range modifiers {
53+
if err := modifyF(policy); err != nil {
54+
return err
55+
}
56+
}
57+
return nil
58+
})
59+
}
60+
}

‎google/provider.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -327,9 +327,9 @@ func ResourceMapWithErrors() (map[string]*schema.Resource, error) {
327327
"google_organization_policy": resourceGoogleOrganizationPolicy(),
328328
"google_project": resourceGoogleProject(),
329329
"google_project_iam_policy": resourceGoogleProjectIamPolicy(),
330-
"google_project_iam_binding": ResourceIamBinding(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc),
331-
"google_project_iam_member": ResourceIamMember(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc),
332-
"google_project_iam_audit_config": ResourceIamAuditConfig(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc),
330+
"google_project_iam_binding": ResourceIamBindingWithBatching(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc, IamBatchingEnabled),
331+
"google_project_iam_member": ResourceIamMemberWithBatching(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc, IamBatchingEnabled),
332+
"google_project_iam_audit_config": ResourceIamAuditConfigWithBatching(IamProjectSchema, NewProjectIamUpdater, ProjectIdParseFunc, IamBatchingEnabled),
333333
"google_project_service": resourceGoogleProjectService(),
334334
"google_project_iam_custom_role": resourceGoogleProjectIamCustomRole(),
335335
"google_project_organization_policy": resourceGoogleProjectOrganizationPolicy(),

‎google/resource_google_project_service.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ func resourceGoogleProjectServiceCreate(d *schema.ResourceData, meta interface{}
7676
}
7777

7878
srv := d.Get("service").(string)
79-
err = globalBatchEnableServices([]string{srv}, project, d, config)
79+
err = BatchRequestEnableServices([]string{srv}, project, d, config)
8080
if err != nil {
8181
return err
8282
}

‎google/resource_google_project_services.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ func setServiceUsageProjectEnabledServices(services []string, project string, d
142142
}
143143
}
144144

145-
if err := globalBatchEnableServices(toEnable, project, d, config); err != nil {
145+
if err := BatchRequestEnableServices(toEnable, project, d, config); err != nil {
146146
return fmt.Errorf("unable to enable Project Services %s (%+v): %s", project, services, err)
147147
}
148148

@@ -223,7 +223,7 @@ func listCurrentlyEnabledServices(project string, config *Config, timeout time.D
223223
return apiServices, nil
224224
}
225225

226-
// Enables services. WARNING: Use globalBatchEnableServices for better batching if possible.
226+
// Enables services. WARNING: Use BatchRequestEnableServices for better batching if possible.
227227
func enableServiceUsageProjectServices(services []string, project string, config *Config, timeout time.Duration) error {
228228
// ServiceUsage does not allow more than 20 services to be enabled per
229229
// batchEnable API call. See

‎google/resource_iam_audit_config.go

+41-18
Original file line numberDiff line numberDiff line change
@@ -39,35 +39,46 @@ var iamAuditConfigSchema = map[string]*schema.Schema{
3939
}
4040

4141
func ResourceIamAuditConfig(parentSpecificSchema map[string]*schema.Schema, newUpdaterFunc newResourceIamUpdaterFunc, resourceIdParser resourceIdParserFunc) *schema.Resource {
42+
return ResourceIamAuditConfigWithBatching(parentSpecificSchema, newUpdaterFunc, resourceIdParser, IamBatchingDisabled)
43+
}
44+
45+
func ResourceIamAuditConfigWithBatching(parentSpecificSchema map[string]*schema.Schema, newUpdaterFunc newResourceIamUpdaterFunc, resourceIdParser resourceIdParserFunc, enableBatching bool) *schema.Resource {
4246
return &schema.Resource{
43-
Create: resourceIamAuditConfigCreate(newUpdaterFunc),
47+
Create: resourceIamAuditConfigCreate(newUpdaterFunc, enableBatching),
4448
Read: resourceIamAuditConfigRead(newUpdaterFunc),
45-
Update: resourceIamAuditConfigUpdate(newUpdaterFunc),
46-
Delete: resourceIamAuditConfigDelete(newUpdaterFunc),
49+
Update: resourceIamAuditConfigUpdate(newUpdaterFunc, enableBatching),
50+
Delete: resourceIamAuditConfigDelete(newUpdaterFunc, enableBatching),
4751
Schema: mergeSchemas(iamAuditConfigSchema, parentSpecificSchema),
4852
Importer: &schema.ResourceImporter{
4953
State: iamAuditConfigImport(resourceIdParser),
5054
},
5155
}
5256
}
5357

54-
func resourceIamAuditConfigCreate(newUpdaterFunc newResourceIamUpdaterFunc) schema.CreateFunc {
58+
func resourceIamAuditConfigCreate(newUpdaterFunc newResourceIamUpdaterFunc, enableBatching bool) schema.CreateFunc {
5559
return func(d *schema.ResourceData, meta interface{}) error {
5660
config := meta.(*Config)
5761
updater, err := newUpdaterFunc(d, config)
5862
if err != nil {
5963
return err
6064
}
6165

62-
p := getResourceIamAuditConfig(d)
63-
err = iamPolicyReadModifyWrite(updater, func(ep *cloudresourcemanager.Policy) error {
64-
ep.AuditConfigs = mergeAuditConfigs(append(ep.AuditConfigs, p))
66+
ac := getResourceIamAuditConfig(d)
67+
modifyF := func(ep *cloudresourcemanager.Policy) error {
68+
ep.AuditConfigs = mergeAuditConfigs(append(ep.AuditConfigs, ac))
6569
return nil
66-
})
70+
}
71+
72+
if enableBatching {
73+
err = BatchRequestModifyIamPolicy(updater, modifyF, config, fmt.Sprintf(
74+
"Add audit config for service %s on resource %q", ac.Service, updater.DescribeResource()))
75+
} else {
76+
err = iamPolicyReadModifyWrite(updater, modifyF)
77+
}
6778
if err != nil {
6879
return err
6980
}
70-
d.SetId(updater.GetResourceId() + "/audit_config/" + p.Service)
81+
d.SetId(updater.GetResourceId() + "/audit_config/" + ac.Service)
7182
return resourceIamAuditConfigRead(newUpdaterFunc)(d, meta)
7283
}
7384
}
@@ -139,7 +150,7 @@ func iamAuditConfigImport(resourceIdParser resourceIdParserFunc) schema.StateFun
139150
}
140151
}
141152

142-
func resourceIamAuditConfigUpdate(newUpdaterFunc newResourceIamUpdaterFunc) schema.UpdateFunc {
153+
func resourceIamAuditConfigUpdate(newUpdaterFunc newResourceIamUpdaterFunc, enableBatching bool) schema.UpdateFunc {
143154
return func(d *schema.ResourceData, meta interface{}) error {
144155
config := meta.(*Config)
145156
updater, err := newUpdaterFunc(d, config)
@@ -148,11 +159,17 @@ func resourceIamAuditConfigUpdate(newUpdaterFunc newResourceIamUpdaterFunc) sche
148159
}
149160

150161
ac := getResourceIamAuditConfig(d)
151-
err = iamPolicyReadModifyWrite(updater, func(p *cloudresourcemanager.Policy) error {
152-
cleaned := removeAllAuditConfigsWithService(p.AuditConfigs, ac.Service)
153-
p.AuditConfigs = append(cleaned, ac)
162+
modifyF := func(ep *cloudresourcemanager.Policy) error {
163+
cleaned := removeAllAuditConfigsWithService(ep.AuditConfigs, ac.Service)
164+
ep.AuditConfigs = append(cleaned, ac)
154165
return nil
155-
})
166+
}
167+
if enableBatching {
168+
err = BatchRequestModifyIamPolicy(updater, modifyF, config, fmt.Sprintf(
169+
"Overwrite audit config for service %s on resource %q", ac.Service, updater.DescribeResource()))
170+
} else {
171+
err = iamPolicyReadModifyWrite(updater, modifyF)
172+
}
156173
if err != nil {
157174
return err
158175
}
@@ -161,7 +178,7 @@ func resourceIamAuditConfigUpdate(newUpdaterFunc newResourceIamUpdaterFunc) sche
161178
}
162179
}
163180

164-
func resourceIamAuditConfigDelete(newUpdaterFunc newResourceIamUpdaterFunc) schema.DeleteFunc {
181+
func resourceIamAuditConfigDelete(newUpdaterFunc newResourceIamUpdaterFunc, enableBatching bool) schema.DeleteFunc {
165182
return func(d *schema.ResourceData, meta interface{}) error {
166183
config := meta.(*Config)
167184
updater, err := newUpdaterFunc(d, config)
@@ -170,10 +187,16 @@ func resourceIamAuditConfigDelete(newUpdaterFunc newResourceIamUpdaterFunc) sche
170187
}
171188

172189
ac := getResourceIamAuditConfig(d)
173-
err = iamPolicyReadModifyWrite(updater, func(p *cloudresourcemanager.Policy) error {
174-
p.AuditConfigs = removeAllAuditConfigsWithService(p.AuditConfigs, ac.Service)
190+
modifyF := func(ep *cloudresourcemanager.Policy) error {
191+
ep.AuditConfigs = removeAllAuditConfigsWithService(ep.AuditConfigs, ac.Service)
175192
return nil
176-
})
193+
}
194+
if enableBatching {
195+
err = BatchRequestModifyIamPolicy(updater, modifyF, config, fmt.Sprintf(
196+
"Delete audit config for service %s on resource %q", ac.Service, updater.DescribeResource()))
197+
} else {
198+
err = iamPolicyReadModifyWrite(updater, modifyF)
199+
}
177200
if err != nil {
178201
return handleNotFoundError(err, d, fmt.Sprintf("Resource %s with IAM audit config %q", updater.DescribeResource(), d.Id()))
179202
}

‎google/resource_iam_binding.go

+28-9
Original file line numberDiff line numberDiff line change
@@ -34,19 +34,24 @@ var iamBindingSchema = map[string]*schema.Schema{
3434
}
3535

3636
func ResourceIamBinding(parentSpecificSchema map[string]*schema.Schema, newUpdaterFunc newResourceIamUpdaterFunc, resourceIdParser resourceIdParserFunc) *schema.Resource {
37+
return ResourceIamBindingWithBatching(parentSpecificSchema, newUpdaterFunc, resourceIdParser, IamBatchingDisabled)
38+
}
39+
40+
// Resource that batches requests to the same IAM policy across multiple IAM fine-grained resources
41+
func ResourceIamBindingWithBatching(parentSpecificSchema map[string]*schema.Schema, newUpdaterFunc newResourceIamUpdaterFunc, resourceIdParser resourceIdParserFunc, enableBatching bool) *schema.Resource {
3742
return &schema.Resource{
38-
Create: resourceIamBindingCreateUpdate(newUpdaterFunc),
43+
Create: resourceIamBindingCreateUpdate(newUpdaterFunc, enableBatching),
3944
Read: resourceIamBindingRead(newUpdaterFunc),
40-
Update: resourceIamBindingCreateUpdate(newUpdaterFunc),
41-
Delete: resourceIamBindingDelete(newUpdaterFunc),
45+
Update: resourceIamBindingCreateUpdate(newUpdaterFunc, enableBatching),
46+
Delete: resourceIamBindingDelete(newUpdaterFunc, enableBatching),
4247
Schema: mergeSchemas(iamBindingSchema, parentSpecificSchema),
4348
Importer: &schema.ResourceImporter{
4449
State: iamBindingImport(resourceIdParser),
4550
},
4651
}
4752
}
4853

49-
func resourceIamBindingCreateUpdate(newUpdaterFunc newResourceIamUpdaterFunc) func(*schema.ResourceData, interface{}) error {
54+
func resourceIamBindingCreateUpdate(newUpdaterFunc newResourceIamUpdaterFunc, enableBatching bool) func(*schema.ResourceData, interface{}) error {
5055
return func(d *schema.ResourceData, meta interface{}) error {
5156
config := meta.(*Config)
5257
updater, err := newUpdaterFunc(d, config)
@@ -55,11 +60,18 @@ func resourceIamBindingCreateUpdate(newUpdaterFunc newResourceIamUpdaterFunc) fu
5560
}
5661

5762
binding := getResourceIamBinding(d)
58-
err = iamPolicyReadModifyWrite(updater, func(ep *cloudresourcemanager.Policy) error {
63+
modifyF := func(ep *cloudresourcemanager.Policy) error {
5964
cleaned := removeAllBindingsWithRole(ep.Bindings, binding.Role)
6065
ep.Bindings = append(cleaned, binding)
6166
return nil
62-
})
67+
}
68+
69+
if enableBatching {
70+
err = BatchRequestModifyIamPolicy(updater, modifyF, config, fmt.Sprintf(
71+
"Set IAM Binding for role %q on %q", binding.Role, updater.DescribeResource()))
72+
} else {
73+
err = iamPolicyReadModifyWrite(updater, modifyF)
74+
}
6375
if err != nil {
6476
return err
6577
}
@@ -142,7 +154,7 @@ func iamBindingImport(resourceIdParser resourceIdParserFunc) schema.StateFunc {
142154
}
143155
}
144156

145-
func resourceIamBindingDelete(newUpdaterFunc newResourceIamUpdaterFunc) schema.DeleteFunc {
157+
func resourceIamBindingDelete(newUpdaterFunc newResourceIamUpdaterFunc, enableBatching bool) schema.DeleteFunc {
146158
return func(d *schema.ResourceData, meta interface{}) error {
147159
config := meta.(*Config)
148160
updater, err := newUpdaterFunc(d, config)
@@ -151,10 +163,17 @@ func resourceIamBindingDelete(newUpdaterFunc newResourceIamUpdaterFunc) schema.D
151163
}
152164

153165
binding := getResourceIamBinding(d)
154-
err = iamPolicyReadModifyWrite(updater, func(p *cloudresourcemanager.Policy) error {
166+
modifyF := func(p *cloudresourcemanager.Policy) error {
155167
p.Bindings = removeAllBindingsWithRole(p.Bindings, binding.Role)
156168
return nil
157-
})
169+
}
170+
171+
if enableBatching {
172+
err = BatchRequestModifyIamPolicy(updater, modifyF, config, fmt.Sprintf(
173+
"Delete IAM Binding for role %q on %q", binding.Role, updater.DescribeResource()))
174+
} else {
175+
err = iamPolicyReadModifyWrite(updater, modifyF)
176+
}
158177
if err != nil {
159178
return handleNotFoundError(err, d, fmt.Sprintf("Resource %q for IAM binding with role %q", updater.DescribeResource(), binding.Role))
160179
}

0 commit comments

Comments
 (0)