Enable meta-tagging for redaction purposes

See https://protobuf.dev/news/2024-12-04/

PiperOrigin-RevId: 721507009

Author: tanx16

src/google/protobuf/BUILD.bazel

@@ -878,6 +878,7 @@ filegroup(
         "unittest_proto3_extensions.proto",
         "unittest_proto3_lite.proto",
         "unittest_proto3_optional.proto",
+        "unittest_redaction.proto",
         "unittest_retention.proto",
         "unittest_string_type.proto",
         "unittest_well_known_types.proto",
@@ -961,6 +962,7 @@ proto_library(
         "unittest_proto3_bad_macros.proto",
         "unittest_proto3_extensions.proto",
         "unittest_proto3_lite.proto",
+        "unittest_redaction.proto",
         "unittest_retention.proto",
         "unittest_string_type.proto",
         "unittest_string_view.proto",

src/google/protobuf/text_format.cc

@@ -3026,12 +3026,70 @@ void TextFormat::Printer::PrintUnknownFields(
   }
 }
 
+// Traverse the tree of field options and check if any of them are sensitive.
+// We check for sensitive enum values in the options and in the fields of the
+// message-type options, recursively.
+TextFormat::RedactionState TextFormat::IsOptionSensitive(
+    const Message& opts, const Reflection* reflection,
+    const FieldDescriptor* option) {
+  if (option->type() == FieldDescriptor::TYPE_ENUM) {
+    auto count =
+        option->is_repeated() ? reflection->FieldSize(opts, option) : 1;
+    for (auto i = 0; i < count; i++) {
+      int enum_val = option->is_repeated()
+                         ? reflection->GetRepeatedEnumValue(opts, option, i)
+                         : reflection->GetEnumValue(opts, option);
+      const EnumValueDescriptor* option_value =
+          option->enum_type()->FindValueByNumber(enum_val);
+      if (option_value->options().debug_redact()) {
+        return TextFormat::RedactionState{true, false};
+      }
+    }
+  } else if (option->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+    auto count =
+        option->is_repeated() ? reflection->FieldSize(opts, option) : 1;
+    for (auto i = 0; i < count; i++) {
+      const Message& sub_message =
+          option->is_repeated()
+              ? reflection->GetRepeatedMessage(opts, option, i)
+              : reflection->GetMessage(opts, option);
+      const Reflection* sub_reflection = sub_message.GetReflection();
+      std::vector<const FieldDescriptor*> message_fields;
+      sub_reflection->ListFields(sub_message, &message_fields);
+      for (const FieldDescriptor* message_field : message_fields) {
+        auto result = TextFormat::IsOptionSensitive(sub_message, sub_reflection,
+                                                    message_field);
+        if (result.redact) {
+          return result;
+        }
+      }
+    }
+  }
+  return TextFormat::RedactionState{false, false};
+}
+
+TextFormat::RedactionState TextFormat::GetRedactionState(
+    const FieldDescriptor* field) {
+  auto options = field->options();
+  auto state = TextFormat::RedactionState{options.debug_redact(), false};
+  std::vector<const FieldDescriptor*> field_options;
+  const Reflection* reflection = options.GetReflection();
+  reflection->ListFields(options, &field_options);
+  for (const FieldDescriptor* option : field_options) {
+    auto result = TextFormat::IsOptionSensitive(options, reflection, option);
+    state = TextFormat::RedactionState{state.redact || result.redact,
+                                       state.report || result.report};
+  }
+  return state;
+}
 bool TextFormat::Printer::TryRedactFieldValue(
     const Message& message, const FieldDescriptor* field,
     BaseTextGenerator* generator, bool insert_value_separator) const {
-  RedactionState redaction_state = field->options().debug_redact()
-                                       ? RedactionState{true, false}
-                                       : RedactionState{false, false};
+  TextFormat::RedactionState redaction_state =
+      field->file()->pool()->MemoizeProjection(
+          field, [](const FieldDescriptor* field) {
+            return TextFormat::GetRedactionState(field);
+          });
   if (redaction_state.redact) {
     if (redact_debug_string_) {
       IncrementRedactedFieldCounter();
@@ -3051,7 +3109,6 @@ bool TextFormat::Printer::TryRedactFieldValue(
   }
   return false;
 }
-
 }  // namespace protobuf
 }  // namespace google
 

src/google/protobuf/text_format.h

@@ -622,6 +622,12 @@ class PROTOBUF_EXPORT TextFormat {
     bool report;
   };
 
+  static TextFormat::RedactionState GetRedactionState(
+      const FieldDescriptor* field);
+
+  static TextFormat::RedactionState IsOptionSensitive(
+      const Message& opts, const Reflection* reflection,
+      const FieldDescriptor* option);
   // Data structure which is populated with the locations of each field
   // value parsed from the text.
   class PROTOBUF_EXPORT ParseInfoTree {

src/google/protobuf/text_format_unittest.cc

@@ -49,6 +49,7 @@
 #include "google/protobuf/unittest_mset.pb.h"
 #include "google/protobuf/unittest_mset_wire_format.pb.h"
 #include "google/protobuf/unittest_proto3.pb.h"
+#include "google/protobuf/unittest_redaction.pb.h"
 #include "utf8_validity.h"
 
 
@@ -2679,6 +2680,85 @@ TEST(TextFormatUnknownFieldTest, TestUnknownExtension) {
 }
 
 
+TEST(AbslStringifyTest, TextFormatIsUnchanged) {
+  unittest::TestAllTypes proto;
+  proto.set_optional_int32(1);
+  proto.set_optional_string("foo");
+
+  std::string text;
+  ASSERT_TRUE(TextFormat::PrintToString(proto, &text));
+  EXPECT_EQ(
+      "optional_int32: 1\n"
+      "optional_string: \"foo\"\n",
+      text);
+}
+
+TEST(AbslStringifyTest, StringifyHasRedactionMarker) {
+  unittest::TestAllTypes proto;
+  proto.set_optional_int32(1);
+  proto.set_optional_string("foo");
+
+  EXPECT_THAT(absl::StrCat(proto), testing::MatchesRegex(
+                                       "optional_int32: 1\n"
+                                       "optional_string: \"foo\"\n"));
+}
+
+
+TEST(AbslStringifyTest, StringifyMetaAnnotatedIsRedacted) {
+  unittest::TestRedactedMessage proto;
+  proto.set_meta_annotated("foo");
+  EXPECT_THAT(absl::StrCat(proto), testing::MatchesRegex(absl::Substitute(
+                                       "meta_annotated: $0\n",
+                                       value_replacement)));
+}
+
+TEST(AbslStringifyTest, StringifyRepeatedMetaAnnotatedIsRedacted) {
+  unittest::TestRedactedMessage proto;
+  proto.set_repeated_meta_annotated("foo");
+  EXPECT_THAT(absl::StrCat(proto), testing::MatchesRegex(absl::Substitute(
+                                       "repeated_meta_annotated: $0\n",
+                                       value_replacement)));
+}
+
+TEST(AbslStringifyTest, StringifyRepeatedMetaAnnotatedIsNotRedacted) {
+  unittest::TestRedactedMessage proto;
+  proto.set_unredacted_repeated_annotations("foo");
+  EXPECT_THAT(absl::StrCat(proto),
+              testing::MatchesRegex(
+                  "unredacted_repeated_annotations: \"foo\"\n"));
+}
+
+TEST(AbslStringifyTest, TextFormatMetaAnnotatedIsNotRedacted) {
+  unittest::TestRedactedMessage proto;
+  proto.set_meta_annotated("foo");
+  std::string text;
+  ASSERT_TRUE(TextFormat::PrintToString(proto, &text));
+  EXPECT_EQ("meta_annotated: \"foo\"\n", text);
+}
+TEST(AbslStringifyTest, StringifyDirectMessageEnumIsRedacted) {
+  unittest::TestRedactedMessage proto;
+  proto.set_test_direct_message_enum("foo");
+  EXPECT_THAT(absl::StrCat(proto), testing::MatchesRegex(absl::Substitute(
+                                       "test_direct_message_enum: $0\n",
+                                       value_replacement)));
+}
+TEST(AbslStringifyTest, StringifyNestedMessageEnumIsRedacted) {
+  unittest::TestRedactedMessage proto;
+  proto.set_test_nested_message_enum("foo");
+  EXPECT_THAT(absl::StrCat(proto), testing::MatchesRegex(absl::Substitute(
+                                       "test_nested_message_enum: $0\n",
+                                       value_replacement)));
+}
+
+TEST(AbslStringifyTest, StringifyRedactedOptionDoesNotRedact) {
+  unittest::TestRedactedMessage proto;
+  proto.set_test_redacted_message_enum("foo");
+  EXPECT_THAT(absl::StrCat(proto),
+              testing::MatchesRegex(
+                  "test_redacted_message_enum: \"foo\"\n"));
+}
+
+
 TEST(TextFormatFloatingPointTest, PreservesNegative0) {
   proto3_unittest::TestAllTypes in_message;
   in_message.set_optional_float(-0.0f);

src/google/protobuf/unittest_redaction.proto

@@ -0,0 +1,72 @@
+// Test proto for redaction
+edition = "2023";
+
+package protobuf_unittest;
+
+import "google/protobuf/any.proto";
+import "google/protobuf/descriptor.proto";
+
+option java_package = "com.google.protos";
+option java_outer_classname = "RedactionProto";
+option features.repeated_field_encoding = EXPANDED;
+option features.utf8_validation = NONE;
+
+extend .google.protobuf.FieldOptions {
+  MetaAnnotatedEnum meta_annotated_enum = 535801413;
+  repeated MetaAnnotatedEnum repeated_meta_annotated_enum = 535801414;
+  TestNestedMessageEnum test_nested_message_enum = 535801415;
+}
+
+message TestRedactedNestMessage {
+  string foo = 1;
+}
+
+message TestRepeatedRedactedNestMessage {
+  string bar = 1;
+}
+
+message TestMessageEnum {
+  repeated MetaAnnotatedEnum redactable_enum = 1;
+}
+
+message TestNestedMessageEnum {
+  repeated MetaAnnotatedEnum direct_enum = 1;
+  TestMessageEnum nested_enum = 2;
+  string redacted_string = 3 [debug_redact = true];
+}
+
+message TestRedactedMessage {
+  string text_field = 1 [deprecated = true];
+  string meta_annotated = 8 [(meta_annotated_enum) = TEST_REDACTABLE];
+  string repeated_meta_annotated = 9 [
+    (protobuf_unittest.repeated_meta_annotated_enum) = TEST_NO_REDACT,
+    (protobuf_unittest.repeated_meta_annotated_enum) = TEST_REDACTABLE
+  ];
+  string unredacted_repeated_annotations = 10 [
+    (protobuf_unittest.repeated_meta_annotated_enum) = TEST_NO_REDACT,
+    (protobuf_unittest.repeated_meta_annotated_enum) = TEST_NO_REDACT_AGAIN
+  ];
+  string unreported_non_meta_debug_redact_field = 17 [debug_redact = true];
+  google.protobuf.Any any_field = 18 [debug_redact = true];
+  string redactable_false = 19 [(meta_annotated_enum) = TEST_REDACTABLE_FALSE];
+  string test_direct_message_enum = 22
+      [(protobuf_unittest.test_nested_message_enum) = {
+        direct_enum: [ TEST_NO_REDACT, TEST_REDACTABLE ]
+      }];
+  string test_nested_message_enum = 23
+      [(protobuf_unittest.test_nested_message_enum) = {
+        nested_enum { redactable_enum: [ TEST_NO_REDACT, TEST_REDACTABLE ] }
+      }];
+  string test_redacted_message_enum = 24
+      [(protobuf_unittest.test_nested_message_enum) = {
+        redacted_string: "redacted_but_doesnt_redact"
+      }];
+}
+
+enum MetaAnnotatedEnum {
+  TEST_NULL = 0;
+  TEST_REDACTABLE = 1 [debug_redact = true];
+  TEST_NO_REDACT = 2;
+  TEST_NO_REDACT_AGAIN = 3;
+  TEST_REDACTABLE_FALSE = 4 [debug_redact = false];
+}