Skip to content

Commit 18c009f

Browse files
authored
[Feat] Detector implementation for Azure API Management Direct Management Key (#3938)
* azuredirectmanagementapikey detector implementation and tests * add reference url * use sanehttpclient * better regex * add prefix regex for key pattern * some refactoring * cleanup some unnecessary logic for expiry formatting. code refactoring, remove unnecessary code. updated the description of detector. reduce the expiry to 5 seconds. * improvise the regex to increase confidence level. refactor variable name for better understanding. * remove \b from regex * implemented CustomFalsePositiveChecker interface. implemented host caching if host is not reachable to avoid unnecessary extra verification calls. * add delimiter to raw v2 * make protos
1 parent 08b7e3b commit 18c009f

File tree

6 files changed

+392
-6
lines changed

6 files changed

+392
-6
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
package azuredirectmanagementkey
2+
3+
import (
4+
"context"
5+
"crypto/hmac"
6+
"crypto/sha512"
7+
"encoding/base64"
8+
"errors"
9+
"fmt"
10+
"net/http"
11+
"strings"
12+
"time"
13+
14+
regexp "github.com/wasilibs/go-re2"
15+
16+
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
17+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
18+
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
19+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
20+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
21+
)
22+
23+
const RFC3339WithoutMicroseconds = "2006-01-02T15:04:05"
24+
25+
type Scanner struct {
26+
client *http.Client
27+
detectors.DefaultMultiPartCredentialProvider
28+
}
29+
30+
// Ensure the Scanner satisfies the interface at compile time.
31+
var _ detectors.Detector = (*Scanner)(nil)
32+
var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
33+
34+
var (
35+
defaultClient = common.SaneHttpClient()
36+
urlPat = regexp.MustCompile(`https://([a-z0-9][a-z0-9-]{0,48}[a-z0-9])\.management\.azure-api\.net`) // https://azure.github.io/PSRule.Rules.Azure/en/rules/Azure.APIM.Name/
37+
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", ".management.azure-api.net"}) + `([a-zA-Z0-9+\/]{83,85}[a-zA-Z0-9]==)`) // pattern for both Primary and secondary key
38+
39+
invalidHosts = simple.NewCache[struct{}]()
40+
noSuchHostErr = errors.New("no such host")
41+
)
42+
43+
// Keywords are used for efficiently pre-filtering chunks.
44+
// Use identifiers in the secret preferably, or the provider name.
45+
func (s Scanner) Keywords() []string {
46+
return []string{".management.azure-api.net"}
47+
}
48+
49+
// FromData will find and optionally verify Azure Management API keys in a given set of bytes.
50+
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
51+
logger := logContext.AddLogger(ctx).Logger().WithName("azuredirectmanagementkey")
52+
dataStr := string(data)
53+
54+
urlMatchesUnique := make(map[string]string)
55+
for _, urlMatch := range urlPat.FindAllStringSubmatch(dataStr, -1) {
56+
urlMatchesUnique[urlMatch[0]] = urlMatch[1] // urlMatch[0] is the full url, urlMatch[1] is the service name
57+
}
58+
keyMatchesUnique := make(map[string]struct{})
59+
for _, keyMatch := range keyPat.FindAllStringSubmatch(dataStr, -1) {
60+
keyMatchesUnique[strings.TrimSpace(keyMatch[1])] = struct{}{}
61+
}
62+
63+
EndpointLoop:
64+
for baseUrl, serviceName := range urlMatchesUnique {
65+
for key := range keyMatchesUnique {
66+
s1 := detectors.Result{
67+
DetectorType: detectorspb.DetectorType_AzureDirectManagementKey,
68+
Raw: []byte(baseUrl),
69+
RawV2: []byte(baseUrl + ":" + key),
70+
}
71+
72+
if verify {
73+
if invalidHosts.Exists(baseUrl) {
74+
logger.V(3).Info("Skipping invalid registry", "baseUrl", baseUrl)
75+
continue EndpointLoop
76+
}
77+
78+
client := s.client
79+
if client == nil {
80+
client = defaultClient
81+
}
82+
83+
isVerified, verificationErr := s.verifyMatch(ctx, client, baseUrl, serviceName, key)
84+
s1.Verified = isVerified
85+
if verificationErr != nil {
86+
if errors.Is(verificationErr, noSuchHostErr) {
87+
invalidHosts.Set(baseUrl, struct{}{})
88+
continue EndpointLoop
89+
}
90+
s1.SetVerificationError(verificationErr, baseUrl)
91+
}
92+
}
93+
94+
results = append(results, s1)
95+
}
96+
}
97+
98+
return results, nil
99+
}
100+
101+
func (s Scanner) Type() detectorspb.DetectorType {
102+
return detectorspb.DetectorType_AzureDirectManagementKey
103+
}
104+
105+
func (s Scanner) Description() string {
106+
return "Azure API Management provides a direct management REST API for performing operations on selected entities, such as users, groups, products, and subscriptions."
107+
}
108+
109+
func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {
110+
return false, ""
111+
}
112+
113+
func (s Scanner) verifyMatch(ctx context.Context, client *http.Client, baseUrl, serviceName, key string) (bool, error) {
114+
url := fmt.Sprintf(
115+
"%s/subscriptions/default/resourceGroups/default/providers/Microsoft.ApiManagement/service/%s/apis?api-version=2024-05-01",
116+
baseUrl, serviceName,
117+
)
118+
accessToken, err := generateAccessToken(key)
119+
if err != nil {
120+
return false, err
121+
}
122+
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
123+
if err != nil {
124+
return false, err
125+
}
126+
req.Header.Set("Content-Type", "application/json")
127+
req.Header.Set("Authorization", fmt.Sprintf("SharedAccessSignature %s", accessToken))
128+
resp, err := client.Do(req)
129+
if err != nil {
130+
return false, nil
131+
}
132+
defer resp.Body.Close()
133+
134+
switch resp.StatusCode {
135+
case http.StatusOK:
136+
return true, nil
137+
case http.StatusUnauthorized:
138+
return false, nil
139+
default:
140+
return false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
141+
}
142+
}
143+
144+
// https://learn.microsoft.com/en-us/rest/api/apimanagement/apimanagementrest/azure-api-management-rest-api-authentication
145+
func generateAccessToken(key string) (string, error) {
146+
expiry := time.Now().UTC().Add(5 * time.Second).Format(RFC3339WithoutMicroseconds) // expire in 5 seconds
147+
expiry = expiry + ".0000000Z" // 7 decimals microsecond's precision is must for access token
148+
149+
// Construct the string-to-sign
150+
stringToSign := fmt.Sprintf("integration\n%s", expiry)
151+
152+
// Generate HMAC-SHA512 signature
153+
h := hmac.New(sha512.New, []byte(key))
154+
h.Write([]byte(stringToSign))
155+
signature := h.Sum(nil)
156+
157+
// Base64 encode the signature
158+
encodedSignature := base64.StdEncoding.EncodeToString(signature)
159+
160+
// Create the access token
161+
accessToken := fmt.Sprintf("uid=integration&ex=%s&sn=%s", expiry, encodedSignature)
162+
return accessToken, nil
163+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
//go:build detectors
2+
// +build detectors
3+
4+
package azuredirectmanagementkey
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"testing"
10+
"time"
11+
12+
"github.com/google/go-cmp/cmp"
13+
"github.com/google/go-cmp/cmp/cmpopts"
14+
15+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
16+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
17+
18+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
19+
)
20+
21+
func TestAzureDirectManagementAPIKey_FromChunk(t *testing.T) {
22+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
23+
defer cancel()
24+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
25+
if err != nil {
26+
t.Fatalf("could not get test secrets from GCP: %s", err)
27+
}
28+
url := testSecrets.MustGetField("AZUREDIRECTMANAGEMENTAPI_URL")
29+
secret := testSecrets.MustGetField("AZUREDIRECTMANAGEMENTAPI_KEY")
30+
inactiveSecret := testSecrets.MustGetField("AZUREDIRECTMANAGEMENTAPI_KEY_INACTIVE")
31+
32+
type args struct {
33+
ctx context.Context
34+
data []byte
35+
verify bool
36+
}
37+
tests := []struct {
38+
name string
39+
s Scanner
40+
args args
41+
want []detectors.Result
42+
wantErr bool
43+
wantVerificationErr bool
44+
}{
45+
{
46+
name: "found, verified",
47+
s: Scanner{},
48+
args: args{
49+
ctx: ctx,
50+
data: []byte(fmt.Sprintf("You can find a azure management api url %s and key %s within", url, secret)),
51+
verify: true,
52+
},
53+
want: []detectors.Result{
54+
{
55+
DetectorType: detectorspb.DetectorType_AzureDirectManagementKey,
56+
Verified: true,
57+
},
58+
},
59+
wantErr: false,
60+
wantVerificationErr: false,
61+
},
62+
{
63+
name: "found, unverified",
64+
s: Scanner{},
65+
args: args{
66+
ctx: ctx,
67+
data: []byte(fmt.Sprintf("You can find a azure management api url %s and key %s within but not valid", url, inactiveSecret)), // the secret would satisfy the regex but not pass validation
68+
verify: true,
69+
},
70+
want: []detectors.Result{
71+
{
72+
DetectorType: detectorspb.DetectorType_AzureDirectManagementKey,
73+
Verified: false,
74+
},
75+
},
76+
wantErr: false,
77+
wantVerificationErr: false,
78+
},
79+
{
80+
name: "not found",
81+
s: Scanner{},
82+
args: args{
83+
ctx: ctx,
84+
data: []byte("You cannot find the secret within"),
85+
verify: true,
86+
},
87+
want: nil,
88+
wantErr: false,
89+
wantVerificationErr: false,
90+
},
91+
}
92+
for _, tt := range tests {
93+
t.Run(tt.name, func(t *testing.T) {
94+
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
95+
if (err != nil) != tt.wantErr {
96+
t.Errorf("AzureDirectManagementAPIKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
97+
return
98+
}
99+
for i := range got {
100+
if len(got[i].Raw) == 0 {
101+
t.Fatalf("no raw secret present: \n %+v", got[i])
102+
}
103+
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
104+
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
105+
}
106+
}
107+
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "Redacted", "verificationError")
108+
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
109+
t.Errorf("AzureDirectManagementAPIKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
110+
}
111+
})
112+
}
113+
}
114+
115+
func BenchmarkFromData(benchmark *testing.B) {
116+
ctx := context.Background()
117+
s := Scanner{}
118+
for name, data := range detectors.MustGetBenchmarkData() {
119+
benchmark.Run(name, func(b *testing.B) {
120+
b.ResetTimer()
121+
for n := 0; n < b.N; n++ {
122+
_, err := s.FromData(ctx, false, data)
123+
if err != nil {
124+
b.Fatal(err)
125+
}
126+
}
127+
})
128+
}
129+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package azuredirectmanagementkey
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
"github.com/google/go-cmp/cmp"
8+
9+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
10+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
11+
)
12+
13+
var (
14+
validPattern = `
15+
AZURE_MANGEMENT_API_KEY=UJh1Wn7txjls2GPK1YxO9+3tpqQffSfxb+97PmT8j3cSQoXvGa74lCKpBqPeppTHCharbaMeKqKs/H4gA/go1w==
16+
AZURE_MANAGEMENT_API_URL=https://trufflesecuritytest.management.azure-api.net
17+
`
18+
invalidPattern = `
19+
AZURE_MANGEMENT_API_KEY=UJh1Wn7txjls2GPK1YxO9+3tpqQffSfxb+97PmT8j3cSQoXvGa74lCKp
20+
AZURE_MANAGEMENT_API_URL=https://trufflesecuritytest.management.azure-api.net
21+
`
22+
)
23+
24+
func TestAzureDirectManagementAPIKey_Pattern(t *testing.T) {
25+
d := Scanner{}
26+
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
27+
28+
tests := []struct {
29+
name string
30+
input string
31+
want []string
32+
}{
33+
{
34+
name: "valid pattern",
35+
input: validPattern,
36+
want: []string{"https://trufflesecuritytest.management.azure-api.net:UJh1Wn7txjls2GPK1YxO9+3tpqQffSfxb+97PmT8j3cSQoXvGa74lCKpBqPeppTHCharbaMeKqKs/H4gA/go1w=="},
37+
},
38+
{
39+
name: "invalid pattern",
40+
input: invalidPattern,
41+
want: nil,
42+
},
43+
}
44+
45+
for _, test := range tests {
46+
t.Run(test.name, func(t *testing.T) {
47+
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
48+
if len(matchedDetectors) == 0 {
49+
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
50+
return
51+
}
52+
53+
results, err := d.FromData(context.Background(), false, []byte(test.input))
54+
if err != nil {
55+
t.Errorf("error = %v", err)
56+
return
57+
}
58+
59+
if len(results) != len(test.want) {
60+
if len(results) == 0 {
61+
t.Errorf("did not receive result")
62+
} else {
63+
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
64+
}
65+
return
66+
}
67+
68+
actual := make(map[string]struct{}, len(results))
69+
for _, r := range results {
70+
if len(r.RawV2) > 0 {
71+
actual[string(r.RawV2)] = struct{}{}
72+
} else {
73+
actual[string(r.Raw)] = struct{}{}
74+
}
75+
}
76+
expected := make(map[string]struct{}, len(test.want))
77+
for _, v := range test.want {
78+
expected[v] = struct{}{}
79+
}
80+
81+
if diff := cmp.Diff(expected, actual); diff != "" {
82+
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
83+
}
84+
})
85+
}
86+
}

pkg/engine/defaults/defaults.go

+2
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ import (
7777
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azureapimanagementsubscriptionkey"
7878
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azurecontainerregistry"
7979
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuredevopspersonalaccesstoken"
80+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuredirectmanagementkey"
8081
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresastoken"
8182
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchadminkey"
8283
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchquerykey"
@@ -916,6 +917,7 @@ func buildDetectorList() []detectors.Detector {
916917
&azure_cosmosdb.Scanner{},
917918
&azurecontainerregistry.Scanner{},
918919
&azuredevopspersonalaccesstoken.Scanner{},
920+
&azuredirectmanagementkey.Scanner{},
919921
// &azurefunctionkey.Scanner{}, // detector is throwing some FPs
920922
&azure_openai.Scanner{},
921923
&azuresastoken.Scanner{},

0 commit comments

Comments
 (0)