Add gRPC Logging (#5139) (#10721)

Co-authored-by: upodroid <[email protected]>
Signed-off-by: Modular Magician <[email protected]>

Co-authored-by: upodroid <[email protected]>

Author: modular-magician

.changelog/5139.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+provider: gRPC requests and response logging have been enabled
+```

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/gammazero/deque v0.0.0-20180920172122-f6adf94963e4 // indirect
 	github.com/gammazero/workerpool v0.0.0-20181230203049-86a96b5d5d92
 	github.com/golangci/golangci-lint v1.40.1
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/hashicorp/errwrap v1.0.0
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-multierror v1.1.1
@@ -18,6 +19,7 @@ require (
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.5.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/hashstructure v1.1.0
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/afero v1.2.2 // indirect
 	github.com/zclconf/go-cty v1.5.1 // indirect
 	golang.org/x/mod v0.5.0 // indirect

go.sum

@@ -286,6 +286,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -425,6 +426,8 @@ github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:Fecb
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
@@ -547,6 +550,7 @@ github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT
 github.com/keybase/go-crypto v0.0.0-20161004153544-93f5b35093ba/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/errcheck v1.6.0 h1:YTDO4pNy7AUN/021p+JGHycQyYNIyMoenM1YDVK6RlY=
 github.com/kisielk/errcheck v1.6.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
@@ -1223,6 +1227,7 @@ golang.org/x/tools v0.0.0-20201230224404-63754364767c/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210104081019-d8d6ddbec6ee/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.2-0.20210512205948-8287d5da45e4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=

google/bigtable_client_factory.go

@@ -10,6 +10,7 @@ import (
 )
 
 type BigtableClientFactory struct {
+	gRPCLoggingOptions  []option.ClientOption
 	UserAgent           string
 	TokenSource         oauth2.TokenSource
 	BillingProject      string
@@ -27,6 +28,8 @@ func (s BigtableClientFactory) NewInstanceAdminClient(project string) (*bigtable
 	}
 
 	opts = append(opts, option.WithTokenSource(s.TokenSource), option.WithUserAgent(s.UserAgent))
+	opts = append(opts, s.gRPCLoggingOptions...)
+
 	return bigtable.NewInstanceAdminClient(context.Background(), project, opts...)
 }
 
@@ -41,5 +44,7 @@ func (s BigtableClientFactory) NewAdminClient(project, instance string) (*bigtab
 	}
 
 	opts = append(opts, option.WithTokenSource(s.TokenSource), option.WithUserAgent(s.UserAgent))
+	opts = append(opts, s.gRPCLoggingOptions...)
+
 	return bigtable.NewAdminClient(context.Background(), project, instance, opts...)
 }

google/config.go

@@ -1,16 +1,21 @@
 package google
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 	"net/http"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
+	grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/logging"
+	"github.com/sirupsen/logrus"
 	"google.golang.org/api/option"
 
 	"golang.org/x/oauth2"
@@ -46,12 +51,86 @@ import (
 	"google.golang.org/api/storage/v1"
 	"google.golang.org/api/storagetransfer/v1"
 	"google.golang.org/api/transport"
+	"google.golang.org/grpc"
 )
 
 type providerMeta struct {
 	ModuleName string `cty:"module_name"`
 }
 
+type Formatter struct {
+	TimestampFormat string
+	LogFormat       string
+}
+
+// Borrowed logic from https://github.com/sirupsen/logrus/blob/master/json_formatter.go and https://github.com/t-tomalak/logrus-easy-formatter/blob/master/formatter.go
+func (f *Formatter) Format(entry *logrus.Entry) ([]byte, error) {
+	if !logging.IsDebugOrHigher() {
+		return nil, nil // Suppress logs if TF_LOG is not DEBUG or TRACE
+	}
+	output := f.LogFormat
+	entry.Level = logrus.DebugLevel // Force Entries to be Debug
+
+	timestampFormat := f.TimestampFormat
+
+	output = strings.Replace(output, "%time%", entry.Time.Format(timestampFormat), 1)
+
+	output = strings.Replace(output, "%msg%", entry.Message, 1)
+
+	level := strings.ToUpper(entry.Level.String())
+	output = strings.Replace(output, "%lvl%", level, 1)
+
+	var gRPCMessageFlag bool
+	for k, val := range entry.Data {
+		switch v := val.(type) {
+		case string:
+			output = strings.Replace(output, "%"+k+"%", v, 1)
+		case int:
+			s := strconv.Itoa(v)
+			output = strings.Replace(output, "%"+k+"%", s, 1)
+		case bool:
+			s := strconv.FormatBool(v)
+			output = strings.Replace(output, "%"+k+"%", s, 1)
+		}
+
+		if k != "system" {
+			gRPCMessageFlag = true
+		}
+	}
+
+	if gRPCMessageFlag {
+		data := make(logrus.Fields, len(entry.Data)+4)
+		for k, v := range entry.Data {
+			switch v := v.(type) {
+			case error:
+				// Otherwise errors are ignored by `encoding/json`
+				// https://github.com/sirupsen/logrus/issues/137
+				data[k] = v.Error()
+			default:
+				data[k] = v
+			}
+		}
+
+		var b *bytes.Buffer
+		if entry.Buffer != nil {
+			b = entry.Buffer
+		} else {
+			b = &bytes.Buffer{}
+		}
+
+		encoder := json.NewEncoder(b)
+		encoder.SetIndent("", "  ")
+		if err := encoder.Encode(data); err != nil {
+			return nil, fmt.Errorf("failed to marshal fields to JSON, %w", err)
+		}
+
+		finalOutput := append([]byte(output), b.Bytes()...)
+		return finalOutput, nil
+	}
+
+	return []byte(output), nil
+}
+
 // Config is the configuration structure used to instantiate the Google
 // provider.
 type Config struct {
@@ -72,9 +151,10 @@ type Config struct {
 	// It controls the interval at which we poll for successful operations
 	PollInterval time.Duration
 
-	client    *http.Client
-	context   context.Context
-	userAgent string
+	client             *http.Client
+	context            context.Context
+	userAgent          string
+	gRPCLoggingOptions []option.ClientOption
 
 	tokenSource oauth2.TokenSource
 
@@ -396,6 +476,25 @@ func (c *Config) LoadAndValidate(ctx context.Context) error {
 	c.requestBatcherIam = NewRequestBatcher("IAM", ctx, c.BatchingConfig)
 	c.PollInterval = 10 * time.Second
 
+	// gRPC Logging setup
+	logger := logrus.StandardLogger()
+
+	logrus.SetLevel(logrus.DebugLevel)
+	logrus.SetFormatter(&Formatter{
+		TimestampFormat: "2006/01/02 15:04:05",
+		LogFormat:       "%time% [%lvl%] %msg% \n",
+	})
+
+	alwaysLoggingDeciderClient := func(ctx context.Context, fullMethodName string) bool { return true }
+	grpc_logrus.ReplaceGrpcLogger(logrus.NewEntry(logger))
+
+	c.gRPCLoggingOptions = append(
+		c.gRPCLoggingOptions, option.WithGRPCDialOption(grpc.WithUnaryInterceptor(
+			grpc_logrus.PayloadUnaryClientInterceptor(logrus.NewEntry(logger), alwaysLoggingDeciderClient))),
+		option.WithGRPCDialOption(grpc.WithStreamInterceptor(
+			grpc_logrus.PayloadStreamClientInterceptor(logrus.NewEntry(logger), alwaysLoggingDeciderClient))),
+	)
+
 	return nil
 }
 
@@ -939,6 +1038,7 @@ func (c *Config) BigTableClientFactory(userAgent string) *BigtableClientFactory
 	bigtableClientFactory := &BigtableClientFactory{
 		UserAgent:           userAgent,
 		TokenSource:         c.tokenSource,
+		gRPCLoggingOptions:  c.gRPCLoggingOptions,
 		BillingProject:      c.BillingProject,
 		UserProjectOverride: c.UserProjectOverride,
 	}

website/docs/r/os_config_os_policy_assignment.html.markdown

@@ -703,28 +703,6 @@ The `disruption_budget` block supports:
   (Optional)
   Specifies the relative value defined as a percentage, which will be multiplied by a reference value.
 
-The `validate` block supports:
-    
-* `interpreter` -
-  (Required)
-  Required. The script interpreter to use. Possible values: INTERPRETER_UNSPECIFIED, NONE, SHELL, POWERSHELL
-    
-* `args` -
-  (Optional)
-  Optional arguments to pass to the source during execution.
-    
-* `file` -
-  (Optional)
-  Required. A deb package.
-    
-* `output_file_path` -
-  (Optional)
-  Only recorded for enforce Exec. Path to an output file (that is created by this Exec) whose content will be recorded in OSPolicyResourceCompliance after a successful run. Absence or failure to read this file will result in this ExecResource being non-compliant. Output file size is limited to 100K bytes.
-    
-* `script` -
-  (Optional)
-  An inline script. The size of the script is limited to 1024 characters.
-    
 - - -
 
 * `description` -
@@ -1016,6 +994,28 @@ The `remote` block supports:
   (Optional)
   SHA256 checksum of the remote file.
 
+The `enforce` block supports:
+    
+* `interpreter` -
+  (Required)
+  Required. The script interpreter to use. Possible values: INTERPRETER_UNSPECIFIED, NONE, SHELL, POWERSHELL
+    
+* `args` -
+  (Optional)
+  Optional arguments to pass to the source during execution.
+    
+* `file` -
+  (Optional)
+  Required. A deb package.
+    
+* `output_file_path` -
+  (Optional)
+  Only recorded for enforce Exec. Path to an output file (that is created by this Exec) whose content will be recorded in OSPolicyResourceCompliance after a successful run. Absence or failure to read this file will result in this ExecResource being non-compliant. Output file size is limited to 100K bytes.
+    
+* `script` -
+  (Optional)
+  An inline script. The size of the script is limited to 1024 characters.
+    
 ## Attributes Reference
 
 In addition to the arguments listed above, the following computed attributes are exported: