Add proper categorization for client connection closing error (#6844)

Author: timl3136

common/constants/constants.go

@@ -316,3 +316,6 @@ func GetTaskPriority(
 ) int {
 	return class | subClass
 }
+
+// GRPCConnectionClosingError is the error message returned when a gRPC client connection is closing
+const GRPCConnectionClosingError = "grpc: the client connection is closing"

common/metrics/defs.go

@@ -2085,6 +2085,7 @@ const (
 	CadenceErrQueryFailedCounter
 	CadenceErrLimitExceededCounter
 	CadenceErrContextTimeoutCounter
+	CadenceErrGRPCConnectionClosingCounter
 	CadenceErrRetryTaskCounter
 	CadenceErrBadBinaryCounter
 	CadenceErrClientVersionNotSupportedCounter
@@ -2814,6 +2815,7 @@ var MetricDefs = map[ServiceIdx]map[int]metricDefinition{
 		CadenceErrQueryFailedCounter:                                 {metricName: "cadence_errors_query_failed", metricType: Counter},
 		CadenceErrLimitExceededCounter:                               {metricName: "cadence_errors_limit_exceeded", metricType: Counter},
 		CadenceErrContextTimeoutCounter:                              {metricName: "cadence_errors_context_timeout", metricType: Counter},
+		CadenceErrGRPCConnectionClosingCounter:                       {metricName: "cadence_errors_grpc_connection_closing", metricType: Counter},
 		CadenceErrRetryTaskCounter:                                   {metricName: "cadence_errors_retry_task", metricType: Counter},
 		CadenceErrBadBinaryCounter:                                   {metricName: "cadence_errors_bad_binary", metricType: Counter},
 		CadenceErrClientVersionNotSupportedCounter:                   {metricName: "cadence_errors_client_version_not_supported", metricType: Counter},

service/frontend/wrappers/metered/metered.go

@@ -29,6 +29,7 @@ import (
 
 	"go.uber.org/yarpc/yarpcerrors"
 
+	"github.com/uber/cadence/common/constants"
 	"github.com/uber/cadence/common/log"
 	"github.com/uber/cadence/common/log/tag"
 	"github.com/uber/cadence/common/metrics"
@@ -80,12 +81,18 @@ func (h *apiHandler) handleErr(err error, scope metrics.Scope, logger log.Logger
 			scope.IncCounter(metrics.CadenceErrContextTimeoutCounter)
 			return err
 		}
+		if err.Code() == yarpcerrors.CodeCancelled {
+			scope.IncCounter(metrics.CadenceErrGRPCConnectionClosingCounter)
+			logger.Warn(constants.GRPCConnectionClosingError, tag.Error(err))
+			return err
+		}
 	}
 	if errors.Is(err, context.DeadlineExceeded) {
 		logger.Error("Frontend request timedout", tag.Error(err))
 		scope.IncCounter(metrics.CadenceErrContextTimeoutCounter)
 		return err
 	}
+
 	logger.Error("Uncategorized error", tag.Error(err))
 	scope.IncCounter(metrics.CadenceFailures)
 	return frontendInternalServiceError("cadence internal uncategorized error, msg: %v", err.Error())

service/frontend/wrappers/metered/metered_test.go

@@ -33,6 +33,7 @@ import (
 	"go.uber.org/yarpc/yarpcerrors"
 
 	"github.com/uber/cadence/common/cache"
+	"github.com/uber/cadence/common/constants"
 	"github.com/uber/cadence/common/dynamicconfig/dynamicproperties"
 	"github.com/uber/cadence/common/log/tag"
 	"github.com/uber/cadence/common/log/testlogger"
@@ -111,6 +112,18 @@ func TestHandleErr_InternalServiceError(t *testing.T) {
 	assert.Contains(t, err.Error(), "cadence internal error")
 }
 
+func TestHandleErr_ClientConnectionClosingError(t *testing.T) {
+	logger := testlogger.New(t)
+	testScope := tally.NewTestScope("test", nil)
+	metricsClient := metrics.NewClient(testScope, metrics.Frontend)
+	handler := &apiHandler{}
+
+	err := handler.handleErr(errors.New(constants.GRPCConnectionClosingError), metricsClient.Scope(0), logger)
+
+	assert.NotNil(t, err)
+	assert.Contains(t, err.Error(), constants.GRPCConnectionClosingError)
+}
+
 func TestHandleErr_UncategorizedError(t *testing.T) {
 	logger := testlogger.New(t)
 	testScope := tally.NewTestScope("test", nil)