Send secret masking telemetry if opted in to it

src/Agent.Sdk/Agent.Sdk.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Security.Utilities.Core" Version="1.18.0" />
+    <PackageReference Include="Microsoft.Security.Utilities.Core" Version="1.19.0" />
     <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageReference Include="System.IO.FileSystem.AccessControl" Version="6.0.0-preview.5.21301.5" />
     <PackageReference Include="System.Management" Version="4.7.0" />

src/Agent.Sdk/Knob/AgentKnobs.cs

@@ -681,6 +681,13 @@ public class AgentKnobs
             new EnvironmentKnobSource("AZP_ENABLE_NEW_MASKER_AND_REGEXES"),
             new BuiltInDefaultKnobSource("false"));
 
+        public static readonly Knob SendSecretMaskerTelemetry = new Knob(
+            nameof(SendSecretMaskerTelemetry),
+            "If true, the agent will send telemetry about secret masking",
+            new RuntimeKnobSource("AZP_SEND_SECRET_MASKER_TELEMETRY"),
+            new EnvironmentKnobSource("AZP_SEND_SECRET_MASKER_TELEMETRY"),
+            new BuiltInDefaultKnobSource("false"));
+
         public static readonly Knob AddDockerInitOption = new Knob(
             nameof(AddDockerInitOption),
             "If true, the agent will create docker container with the --init option.",

src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs

@@ -2,12 +2,19 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Collections.Generic;
 
 namespace Agent.Sdk.SecretMasking
 {
     /// <summary>
-    /// Extended ISecretMasker interface that adds support for logging the origin of
-    /// regexes, encoders and literal secret values.
+    /// An action that publishes the given data corresonding to the given
+    /// feature to a telemetry channel.
+    /// </summary>
+    public delegate void PublishSecretMaskerTelemetryAction(string feature, Dictionary<string, string> data);
+
+    /// <summary>
+    /// Extended ISecretMasker interface that adds support for telemetry and
+    /// logging the origin of regexes, encoders and literal secret values.
     /// </summary>
     public interface ILoggedSecretMasker : IDisposable
     {
@@ -19,5 +26,34 @@ public interface ILoggedSecretMasker : IDisposable
         string MaskSecrets(string input);
         void RemoveShortSecretsFromDictionary();
         void SetTrace(ITraceWriter trace);
+
+        /// <summary>
+        /// Begin collecting data for secret masking telemetry.
+        /// </summary>
+        /// <remarks>
+        /// This is a no-op if <see cref="LegacySecretMasker"/> is being used,
+        /// only <see cref="OssSecretMasker"/> supports telemetry. Also, the
+        /// agent will only call this if a feature flag that opts in to secret
+        /// masking telemetry is enabled..
+        /// </remarks>
+        /// <param name="maxUniqueCorrelatingIds">
+        /// The maximum number of unique correlating IDs to collect.
+        /// </param>
+        void StartTelemetry(int maxUniqueCorrelatingIds);
+
+        /// <summary>
+        /// Stop collecting data for secret masking telemetry and publish the
+        /// telemetry events.
+        /// </summary>
+        /// <remarks>
+        /// This is a no-op if <see cref="LegacySecretMasker"/> is being used,
+        /// only <see cref="OssSecretMasker"/> supports telemetry.
+        /// <param name="maxCorrelatingIdsPerEvent">
+        /// The maximum number of correlating IDs to report in a single
+        /// telemetry event.
+        /// <param name="publishAction">
+        /// Callback to publish the telemetry data.
+        /// </param>
+        void StopAndPublishTelemetry(int maxCorrelatingIdsPerEvent, PublishSecretMaskerTelemetryAction publishAction);
     }
 }

src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs

@@ -1,4 +1,3 @@
-
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
@@ -143,6 +142,16 @@ public void Dispose()
             GC.SuppressFinalize(this);
         }
 
+        public void StartTelemetry(int maxDetections)
+        {
+            (_secretMasker as OssSecretMasker)?.StartTelemetry(maxDetections);
+        }
+
+        public void StopAndPublishTelemetry(int maxDetectionsPerEvent, PublishSecretMaskerTelemetryAction publishAction)
+        {
+            (_secretMasker as OssSecretMasker)?.StopAndPublishTelemetry(publishAction, maxDetectionsPerEvent);
+        }
+
         protected virtual void Dispose(bool disposing)
         {
             if (disposing)

src/Agent.Sdk/SecretMasking/OssSecretMasker.cs

@@ -1,27 +1,28 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Text.RegularExpressions;
+using System.Threading;
+
 using Microsoft.Security.Utilities;
 
 namespace Agent.Sdk.SecretMasking;
 
 public sealed class OssSecretMasker : IRawSecretMasker
 {
     private SecretMasker _secretMasker;
+    private Telemetry _telemetry;
 
-    public OssSecretMasker() : this(Array.Empty<RegexPattern>())
-    {
-    }
-
-    public OssSecretMasker(IEnumerable<RegexPattern> patterns)
+    public OssSecretMasker(IEnumerable<RegexPattern> patterns = null)
     {
-        _secretMasker = new SecretMasker(patterns, generateCorrelatingIds: true);
-        _secretMasker.DefaultRegexRedactionToken = "***";
+        _secretMasker = new SecretMasker(patterns,
+                                         generateCorrelatingIds: true,
+                                         defaultRegexRedactionToken: "***");
     }
 
-
     /// <summary>
     /// This property allows to set the minimum length of a secret for masking
     /// </summary>
@@ -31,9 +32,6 @@ public int MinSecretLength
         set => _secretMasker.MinimumSecretLength = value;
     }
 
-    /// <summary>
-    /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time.
-    /// </summary>
     public void AddRegex(string pattern)
     {
         // NOTE: This code path is used for regexes sent to the agent via
@@ -52,95 +50,189 @@ public void AddRegex(string pattern)
         _secretMasker.AddRegex(regexPattern);
     }
 
-    /// <summary>
-    /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time.
-    /// </summary>
     public void AddValue(string test)
     {
         _secretMasker.AddValue(test);
     }
 
-    /// <summary>
-    /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time.
-    /// </summary>
     public void AddValueEncoder(Func<string, string> encoder)
     {
-       _secretMasker.AddLiteralEncoder(x => encoder(x));
+        _secretMasker.AddLiteralEncoder(x => encoder(x));
     }
 
     public void Dispose()
     {
         _secretMasker?.Dispose();
         _secretMasker = null;
+        _telemetry = null;
     }
 
     public string MaskSecrets(string input)
     {
-        return _secretMasker.MaskSecrets(input);
+        _secretMasker.SyncObject.EnterReadLock();
+        try
+        {
+            _telemetry?.ProcessInput(input);
+            return _secretMasker.MaskSecrets(input, _telemetry?.ProcessDetection);
+        }
+        finally
+        {
+            _secretMasker.SyncObject.ExitReadLock();
+        }
     }
 
-    /// <summary>
-    /// Removes secrets from the dictionary shorter than the MinSecretLength property.
-    /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time.
-    /// </summary>
-    public void RemoveShortSecretsFromDictionary()
+    public void StartTelemetry(int maxUniqueCorrelatingIds)
     {
-        var filteredValueSecrets = new HashSet<SecretLiteral>();
-        var filteredRegexSecrets = new HashSet<RegexPattern>();
-
+        _secretMasker.SyncObject.EnterWriteLock();
         try
         {
-            _secretMasker.SyncObject.EnterReadLock();
+            _telemetry ??= new Telemetry(maxUniqueCorrelatingIds);
+        }
+        finally
+        {
+            _secretMasker.SyncObject.ExitWriteLock();
+        }
+    }
 
-            foreach (var secret in _secretMasker.EncodedSecretLiterals)
-            {
-                if (secret.Value.Length < MinSecretLength)
-                {
-                    filteredValueSecrets.Add(secret);
-                }
-            }
+    public void StopAndPublishTelemetry(PublishSecretMaskerTelemetryAction publishAction, int maxCorrelatingIdsPerEvent)
+    {
+        Telemetry telemetry;
 
-            foreach (var secret in _secretMasker.RegexPatterns)
-            {
-                if (secret.Pattern.Length < MinSecretLength)
-                {
-                    filteredRegexSecrets.Add(secret);
-                }
-            }
+        _secretMasker.SyncObject.EnterWriteLock();
+        try
+        {
+            telemetry = _telemetry;
+            _telemetry = null;
         }
         finally
         {
-            if (_secretMasker.SyncObject.IsReadLockHeld)
-            {
-                _secretMasker.SyncObject.ExitReadLock();
-            }
+            _secretMasker.SyncObject.ExitWriteLock();
         }
 
-        try
+        telemetry?.Publish(publishAction, _secretMasker.ElapsedMaskingTime, maxCorrelatingIdsPerEvent);
+    }
+
+    private sealed class Telemetry
+    {
+        // NOTE: Telemetry does not fit into the reader-writer lock model of the
+        // SecretMasker API because we *write* telemetry during *read*
+        // operations. We therefore use separate interlocked operations and a
+        // concurrent dictionary when writing to telemetry.
+
+        // Key=CrossCompanyCorrelatingId (C3ID), Value=Rule Moniker C3ID is a
+        // non-reversible seeded hash and only available when detection is made
+        // by a high-confidence rule that matches secrets with high entropy.
+        private readonly ConcurrentDictionary<string, string> _correlationData;
+        private readonly int _maxUniqueCorrelatingIds;
+        private long _charsScanned;
+        private long _stringsScanned;
+        private long _totalDetections;
+
+        public Telemetry(int maxDetections)
         {
-            _secretMasker.SyncObject.EnterWriteLock();
+            _correlationData = new ConcurrentDictionary<string, string>();
+            _maxUniqueCorrelatingIds = maxDetections;
+            ProcessDetection = ProcessDetectionImplementation;
+        }
 
-            foreach (var secret in filteredValueSecrets)
-            {
-                _secretMasker.EncodedSecretLiterals.Remove(secret);
-            }
+        public void ProcessInput(string input)
+        {
+            Interlocked.Add(ref _charsScanned, input.Length);
+            Interlocked.Increment(ref _stringsScanned);
+        }
 
-            foreach (var secret in filteredRegexSecrets)
-            {
-                _secretMasker.RegexPatterns.Remove(secret);
-            }
+        public Action<Detection> ProcessDetection { get; }
 
-            foreach (var secret in filteredValueSecrets)
+        private void ProcessDetectionImplementation(Detection detection)
+        {
+            Interlocked.Increment(ref _totalDetections);
+
+            // NOTE: We cannot prevent the concurrent dictionary from exceeding
+            // the maximum detection count when multiple threads add detections
+            // in parallel. The condition here is therefore a best effort to
+            // constrain the memory consumed by excess detections that will not
+            // be published. Furthermore, it is deliberate that we use <=
+            // instead of < here as it allows us to detect the case where the
+            // maximum number of events have been exceeded without adding any
+            // additional state.
+            if (_correlationData.Count <= _maxUniqueCorrelatingIds &&
+                detection.CrossCompanyCorrelatingId != null)
             {
-                _secretMasker.ExplicitlyAddedSecretLiterals.Remove(secret);
+                _correlationData.TryAdd(detection.CrossCompanyCorrelatingId, detection.Moniker);
             }
         }
-        finally
+
+        public void Publish(PublishSecretMaskerTelemetryAction publishAction, TimeSpan elapsedMaskingTime, int maxCorrelatingIdsPerEvent)
         {
-            if (_secretMasker.SyncObject.IsWriteLockHeld)
+            Dictionary<string, string> correlationData = null;
+            int uniqueCorrelatingIds = 0;
+            bool correlationDataIsIncomplete = false;
+
+            // Publish 'SecretMaskerCorrelation' events mapping unique C3IDs to
+            // rule moniker. No more than 'maxCorrelatingIdsPerEvent' are
+            // published in a single event.
+            foreach (var pair in _correlationData)
             {
-                _secretMasker.SyncObject.ExitWriteLock();
+                if (uniqueCorrelatingIds >= _maxUniqueCorrelatingIds)
+                {
+                    correlationDataIsIncomplete = true;
+                    break;
+                }
+
+                correlationData ??= new Dictionary<string, string>(maxCorrelatingIdsPerEvent);
+                correlationData.Add(pair.Key, pair.Value);
+                uniqueCorrelatingIds++;
+
+                if (correlationData.Count >= maxCorrelatingIdsPerEvent)
+                {
+                    publishAction("SecretMaskerCorrelation", correlationData);
+                    correlationData = null;
+                }
             }
+
+            if (correlationData != null)
+            {
+                publishAction("SecretMaskerCorrelation", correlationData);
+                correlationData = null;
+            }
+
+            // Send overall information in a 'SecretMasker' event.
+            var overallData = new Dictionary<string, string> {
+                // The version of Microsoft.Security.Utilities.Core used.
+                { "Version", SecretMasker.Version.ToString() },
+
+                // The total number number of characters scanned by the secret masker.
+                { "CharsScanned", _charsScanned.ToString(CultureInfo.InvariantCulture) },
+
+                // The total number of strings scanned by the secret masker.
+                { "StringsScanned", _stringsScanned.ToString(CultureInfo.InvariantCulture) },
+
+                // The total number of detections made by the secret masker.
+                // This includes duplicate detections and detections without
+                // correlating IDs such as those made by literal values.
+                { "TotalDetections", _totalDetections.ToString(CultureInfo.InvariantCulture) },
+
+                // The total amount of time spent masking secrets.
+                { "ElapsedMaskingTimeInMilliseconds", elapsedMaskingTime.TotalMilliseconds.ToString(CultureInfo.InvariantCulture) },
+
+                // Whether the 'maxUniqueCorrelatingIds' limit was exceeded and
+                // therefore the 'SecretMaskerDetectionCorrelation' events does
+                // not contain all unique correlating IDs detected.
+                { "CorrelationDataIsIncomplete", correlationDataIsIncomplete.ToString(CultureInfo.InvariantCulture) },
+
+                // The total number of unique correlating IDs reported in
+                // 'SecretMaskerCorrelation' events.
+                //
+                // NOTE: This may be less than the total number of unique
+                // correlating IDs if the maximum was exceeded. See above.
+                { "UniqueCorrelatingIds", uniqueCorrelatingIds.ToString(CultureInfo.InvariantCulture) },
+            };
+
+            publishAction("SecretMasker", overallData);
         }
     }
+
+    // This is a no-op for the OSS SecretMasker because it respects
+    // MinimumSecretLength immediately without requiring an extra API call.
+    void IRawSecretMasker.RemoveShortSecretsFromDictionary() { }
 }