Skip to content

Commit 793c09d

Browse files
casey-trancasey-tran
authored
Postman source handle different JSON info for headers (#3995)
1 parent eacf176 commit 793c09d

File tree

6 files changed

+192
-34
lines changed

6 files changed

+192
-34
lines changed

‎pkg/output/plain.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,9 @@ func structToMap(obj any) (m map[string]map[string]any, err error) {
117117
}
118118
err = json.Unmarshal(data, &m)
119119
// Due to PostmanLocationType protobuf field being an enum, we want to be able to assign the string value of the enum to the field without needing to create another Protobuf field.
120-
// To have the "UNKNOWN_POSTMAN = 0" value be assigned correctly to the field, we need to check if the Postman workspace ID is filled since every secret in the Postman source
121-
// should have a valid workspace ID and the 0 value is considered nil for integers.
122-
if m["Postman"]["workspace_uuid"] != nil {
120+
// To have the "UNKNOWN_POSTMAN = 0" value be assigned correctly to the field, we need to check if the Postman workspace ID or collection ID is filled since every secret
121+
// in the Postman source should have a valid workspace ID or collection ID and the 0 value is considered nil for integers.
122+
if m["Postman"]["workspace_uuid"] != nil || m["Postman"]["collection_id"] != nil {
123123
if m["Postman"]["location_type"] == nil {
124124
m["Postman"]["location_type"] = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN.String()
125125
} else {

‎pkg/sources/postman/postman.go

+27-11
Original file line numberDiff line numberDiff line change
@@ -422,7 +422,7 @@ func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk,
422422
metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_SCRIPT
423423
}
424424

425-
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, data)), metadata)
425+
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, data)), metadata)
426426
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
427427
}
428428

@@ -520,7 +520,7 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m
520520
} else if strings.Contains(m.Type, COLLECTION_TYPE) {
521521
m.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_AUTHORIZATION
522522
}
523-
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, authData)), m)
523+
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, authData)), m)
524524
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
525525
}
526526

@@ -529,22 +529,30 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C
529529
originalType := metadata.Type
530530

531531
// Add in var procesisng for headers
532-
if r.Header != nil {
532+
if r.HeaderKeyValue != nil {
533533
vars := VariableData{
534-
KeyValues: r.Header,
534+
KeyValues: r.HeaderKeyValue,
535535
}
536536
metadata.Type = originalType + " > header"
537537
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_HEADER
538538
s.scanVariableData(ctx, chunksChan, metadata, vars)
539539
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
540540
}
541541

542+
if r.HeaderString != nil {
543+
metadata.Type = originalType + " > header"
544+
metadata.Link = metadata.Link + "?tab=headers"
545+
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_HEADER
546+
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, strings.Join(r.HeaderString, " "))), metadata)
547+
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
548+
}
549+
542550
if r.URL.Raw != "" {
543551
metadata.Type = originalType + " > request URL (no query parameters)"
544552
// Note: query parameters are handled separately
545553
u := fmt.Sprintf("%s://%s/%s", r.URL.Protocol, strings.Join(r.URL.Host, "."), strings.Join(r.URL.Path, "/"))
546554
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_URL
547-
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, u)), metadata)
555+
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, u)), metadata)
548556
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
549557
}
550558

@@ -595,13 +603,13 @@ func (s *Source) scanRequestBody(ctx context.Context, chunksChan chan *sources.C
595603
m.Type = originalType + " > raw"
596604
data := b.Raw
597605
m.LocationType = source_metadatapb.PostmanLocationType_REQUEST_BODY_RAW
598-
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, data)), m)
606+
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, data)), m)
599607
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
600608
case "graphql":
601609
m.Type = originalType + " > graphql"
602610
data := b.GraphQL.Query + " " + b.GraphQL.Variables
603611
m.LocationType = source_metadatapb.PostmanLocationType_REQUEST_BODY_GRAPHQL
604-
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, data)), m)
612+
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, data)), m)
605613
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
606614
}
607615
}
@@ -613,21 +621,29 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.
613621
}
614622
originalType := m.Type
615623

616-
if response.Header != nil {
624+
if response.HeaderKeyValue != nil {
617625
vars := VariableData{
618-
KeyValues: response.Header,
626+
KeyValues: response.HeaderKeyValue,
619627
}
620628
m.Type = originalType + " > response header"
621629
m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_HEADER
622630
s.scanVariableData(ctx, chunksChan, m, vars)
623631
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
624632
}
625633

634+
if response.HeaderString != nil {
635+
m.Type = originalType + " > response header"
636+
// TODO Note: for now, links to Postman responses do not include a more granular tab for the params/header/body, but when they do, we will need to update the metadata.Link info
637+
m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_HEADER
638+
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, strings.Join(response.HeaderString, " "))), m)
639+
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
640+
}
641+
626642
// Body in a response is just a string
627643
if response.Body != "" {
628644
m.Type = originalType + " > response body"
629645
m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_BODY
630-
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, response.Body)), m)
646+
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, response.Body)), m)
631647
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
632648
}
633649

@@ -660,7 +676,7 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.
660676
if valStr == "" {
661677
continue
662678
}
663-
values = append(values, s.buildSubstitueSet(m, valStr)...)
679+
values = append(values, s.buildSubstituteSet(m, valStr)...)
664680
}
665681

666682
m.FieldType = m.Type + " variables"

‎pkg/sources/postman/postman_client.go

+47-18
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,12 @@ type IDNameUUID struct {
4444
}
4545

4646
type KeyValue struct {
47-
Key string `json:"key"`
48-
Value interface{} `json:"value"`
49-
Enabled bool `json:"enabled,omitempty"`
50-
Type string `json:"type,omitempty"`
51-
SessionValue string `json:"sessionValue,omitempty"`
52-
Id string `json:"id,omitempty"`
47+
Key string `json:"key"`
48+
Value any `json:"value"`
49+
Enabled bool `json:"enabled,omitempty"`
50+
Type string `json:"type,omitempty"`
51+
SessionValue string `json:"sessionValue,omitempty"`
52+
Id string `json:"id,omitempty"`
5353
}
5454

5555
type VariableData struct {
@@ -136,12 +136,14 @@ type Script struct {
136136
}
137137

138138
type Request struct {
139-
Auth Auth `json:"auth,omitempty"`
140-
Method string `json:"method"`
141-
Header []KeyValue `json:"header,omitempty"`
142-
Body Body `json:"body,omitempty"` //Need to update with additional options
143-
URL URL `json:"url"`
144-
Description string `json:"description,omitempty"`
139+
Auth Auth `json:"auth,omitempty"`
140+
Method string `json:"method"`
141+
HeaderRaw json.RawMessage `json:"header,omitempty"`
142+
HeaderKeyValue []KeyValue
143+
HeaderString []string
144+
Body Body `json:"body,omitempty"` //Need to update with additional options
145+
URL URL `json:"url"`
146+
Description string `json:"description,omitempty"`
145147
}
146148

147149
type Body struct {
@@ -171,12 +173,14 @@ type URL struct {
171173
}
172174

173175
type Response struct {
174-
ID string `json:"id"`
175-
Name string `json:"name,omitempty"`
176-
OriginalRequest Request `json:"originalRequest,omitempty"`
177-
Header []KeyValue `json:"header,omitempty"`
178-
Body string `json:"body,omitempty"`
179-
UID string `json:"uid,omitempty"`
176+
ID string `json:"id"`
177+
Name string `json:"name,omitempty"`
178+
OriginalRequest Request `json:"originalRequest,omitempty"`
179+
HeaderRaw json.RawMessage `json:"header,omitempty"`
180+
HeaderKeyValue []KeyValue
181+
HeaderString []string
182+
Body string `json:"body,omitempty"`
183+
UID string `json:"uid,omitempty"`
180184
}
181185

182186
// A Client manages communication with the Postman API.
@@ -375,5 +379,30 @@ func (c *Client) GetCollection(ctx context.Context, collection_uuid string) (Col
375379
return Collection{}, fmt.Errorf("could not unmarshal JSON for collection (%s): %w", collection_uuid, err)
376380
}
377381

382+
// Loop used to deal with seeing whether a request/response header is a string or a key value pair
383+
for i := range obj.Collection.Items {
384+
if obj.Collection.Items[i].Request.HeaderRaw != nil {
385+
if err := json.Unmarshal(obj.Collection.Items[i].Request.HeaderRaw, &obj.Collection.Items[i].Request.HeaderKeyValue); err == nil {
386+
} else if err := json.Unmarshal(obj.Collection.Items[i].Request.HeaderRaw, &obj.Collection.Items[i].Request.HeaderString); err == nil {
387+
} else {
388+
return Collection{}, fmt.Errorf("could not unmarshal request header JSON for collection (%s): %w", collection_uuid, err)
389+
}
390+
}
391+
392+
for j := range obj.Collection.Items[i].Response {
393+
if err := json.Unmarshal(obj.Collection.Items[i].Response[j].OriginalRequest.HeaderRaw, &obj.Collection.Items[i].Response[j].OriginalRequest.HeaderKeyValue); err == nil {
394+
} else if err := json.Unmarshal(obj.Collection.Items[i].Response[j].OriginalRequest.HeaderRaw, &obj.Collection.Items[i].Response[j].OriginalRequest.HeaderString); err == nil {
395+
} else {
396+
return Collection{}, fmt.Errorf("could not unmarshal original request header in response JSON for collection (%s): %w", collection_uuid, err)
397+
}
398+
399+
if err := json.Unmarshal(obj.Collection.Items[i].Response[j].HeaderRaw, &obj.Collection.Items[i].Response[j].HeaderKeyValue); err == nil {
400+
} else if err := json.Unmarshal(obj.Collection.Items[i].Response[j].HeaderRaw, &obj.Collection.Items[i].Response[j].HeaderString); err == nil {
401+
} else {
402+
return Collection{}, fmt.Errorf("could not unmarshal response header JSON for collection (%s): %w", collection_uuid, err)
403+
}
404+
}
405+
}
406+
378407
return obj.Collection, nil
379408
}

‎pkg/sources/postman/postman_test.go

+113
Original file line numberDiff line numberDiff line change
@@ -333,3 +333,116 @@ func TestSource_ScanGeneralRateLimit(t *testing.T) {
333333
t.Errorf("Rate limiting not working as expected. Elapsed time: %v seconds, expected at least %v seconds", elapsed.Seconds(), (float64(numRequests)-1)/5)
334334
}
335335
}
336+
337+
func TestSource_UnmarshalMultipleHeaderTypes(t *testing.T) {
338+
defer gock.Off()
339+
// Mock a collection with request and response headers of KeyValue type
340+
gock.New("https://api.getpostman.com").
341+
Get("/collections/1234-abc1").
342+
Reply(200).
343+
BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",
344+
"updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-abc1"},
345+
"item":[{"name":"echo","id":"req-ues-t1","protocolProfileBehavior":{"disableBodyPruning":true},"request":{"method":"GET","header":[{"key":"Date","value":"Fri, 21 Mar 2025 17:38:58 GMT"}]},
346+
"response":[{"id":"res-pon-se1","name":"echo-response","originalRequest":{"method":"GET","header":[{"key":"Date","value":"Fri, 21 Mar 2025 17:38:58 GMT"}],
347+
"url":{"raw":"postman-echo.com/get","host":["postman-echo","com"],"path":["get"]}},"status":"OK","code":200,"_postman_previewlanguage":"json",
348+
"header":[{"key":"Date","value":"Fri, 21 Mar 2025 17:38:58 GMT"},{"key":"Content-Type","value":"application/json; charset=utf-8"},{"key":"Content-Length","value":"508"},
349+
{"key":"Connection","value":"keep-alive"},{"key":"Server","value":"nginx"},{"key":"ETag","value":"random-string"},
350+
{"key":"set-cookie","value":"sails.sid=long-string; Path=/; HttpOnly"}],"cookie":[], "responseTime":null,"body":"{response-body}","uid":"1234-res-pon-se1"}],"uid":"1234-req-ues-t1"}]}}`)
351+
// Mock a collection with request and response headers of string type
352+
gock.New("https://api.getpostman.com").
353+
Get("/collections/1234-def1").
354+
Reply(200).
355+
BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",
356+
"updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-def1"},
357+
"item":[{"name":"echo","id":"req-ues-t1","protocolProfileBehavior":{"disableBodyPruning":true},"request":{"method":"GET","header":["request-header-string"]},
358+
"response":[{"id":"res-pon-se1","name":"echo-response","originalRequest":{"method":"GET","header":["request-header-string"],
359+
"url":{"raw":"postman-echo.com/get","host":["postman-echo","com"],"path":["get"]}},"status":"OK","code":200,"_postman_previewlanguage":"json",
360+
"header":["response-header-string"],"cookie":[], "responseTime":null,"body":"{response-body}","uid":"1234-res-pon-se1"}],"uid":"1234-req-ues-t1"}]}}`)
361+
362+
ctx := context.Background()
363+
s, conn := createTestSource(&sourcespb.Postman{
364+
Credential: &sourcespb.Postman_Token{
365+
Token: "super-secret-token",
366+
},
367+
})
368+
err := s.Init(ctx, "test - postman", 0, 1, false, conn, 1)
369+
if err != nil {
370+
t.Fatalf("init error: %v", err)
371+
}
372+
gock.InterceptClient(s.client.HTTPClient)
373+
defer gock.RestoreClient(s.client.HTTPClient)
374+
375+
collectionIds := []string{"1234-abc1", "1234-def1"}
376+
for _, collectionId := range collectionIds {
377+
_, err := s.client.GetCollection(ctx, collectionId)
378+
if err != nil {
379+
t.Fatalf("failed to get collection: %v", err)
380+
}
381+
}
382+
}
383+
384+
// The purpose of the TestSource_HeadersScanning test is to check that at least one of the fields HeaderKeyValue or HeaderString are non-null after unmarshalling and that chunks can
385+
// be generated from them.
386+
func TestSource_HeadersScanning(t *testing.T) {
387+
defer gock.Off()
388+
// Mock a collection with request and response headers of KeyValue type
389+
gock.New("https://api.getpostman.com").
390+
Get("/collections/1234-abc1").
391+
Reply(200).
392+
BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",
393+
"updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-abc1"},
394+
"item":[{"name":"echo","id":"req-ues-t1", "request":{"method":"GET","header":[{"key":"token","value":"keyword1"}]},
395+
"response":[{"id":"res-pon-se1","name":"echo-response","originalRequest":{"method":"GET","header":[{"key":"token","value":"keyword1"}]},
396+
"header":[{"key":"token","value":"keyword1"}]}],"uid":"1234-req-ues-t1"}]}}`)
397+
// Mock a collection with request and response headers of string type
398+
gock.New("https://api.getpostman.com").
399+
Get("/collections/1234-def1").
400+
Reply(200).
401+
BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",
402+
"updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-def1"},
403+
"item":[{"name":"echo","id":"req-ues-t1","protocolProfileBehavior":{"disableBodyPruning":true},"request":{"method":"GET","header":["keyword1-request-header-string"]},
404+
"response":[{"id":"res-pon-se1","name":"echo-response","originalRequest":{"method":"GET","header":["keyword1-request-header-string"]},
405+
"header":["keyword1-response-header-string"]}],"uid":"1234-req-ues-t1"}]}}`)
406+
407+
ctx := context.Background()
408+
s, conn := createTestSource(&sourcespb.Postman{
409+
Credential: &sourcespb.Postman_Token{
410+
Token: "super-secret-token",
411+
},
412+
})
413+
414+
// Add detector keywords to trigger chunk generation
415+
s.DetectorKeywords = map[string]struct{}{
416+
"keyword1": {},
417+
}
418+
s.keywords = map[string]struct{}{
419+
"keyword1": {},
420+
}
421+
422+
err := s.Init(ctx, "test - postman", 0, 1, false, conn, 1)
423+
if err != nil {
424+
t.Fatalf("init error: %v", err)
425+
}
426+
gock.InterceptClient(s.client.HTTPClient)
427+
defer gock.RestoreClient(s.client.HTTPClient)
428+
429+
chunksChan := make(chan *sources.Chunk, 10)
430+
collectionIds := []string{"1234-abc1", "1234-def1"}
431+
432+
for _, collectionId := range collectionIds {
433+
collection, err := s.client.GetCollection(ctx, collectionId)
434+
if err != nil {
435+
t.Fatalf("failed to get collection: %v", err)
436+
}
437+
s.scanCollection(ctx, chunksChan, Metadata{CollectionInfo: collection.Info}, collection)
438+
}
439+
440+
close(chunksChan)
441+
chunksReceived := len(chunksChan)
442+
443+
if chunksReceived == 0 {
444+
t.Errorf("No chunks were generated from the mock data")
445+
} else {
446+
t.Logf("Generated %d chunks from the mock data", chunksReceived)
447+
}
448+
}

‎pkg/sources/postman/substitution.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ func (s *Source) formatAndInjectKeywords(data []string) string {
4747
return strings.Join(ret, "")
4848
}
4949

50-
func (s *Source) buildSubstitueSet(metadata Metadata, data string) []string {
50+
func (s *Source) buildSubstituteSet(metadata Metadata, data string) []string {
5151
var ret []string
5252
combos := make(map[string]struct{})
5353

‎pkg/sources/postman/substitution_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ func TestSource_BuildSubstituteSet(t *testing.T) {
8989
}
9090

9191
for _, tc := range testCases {
92-
result := s.buildSubstitueSet(metadata, tc.data)
92+
result := s.buildSubstituteSet(metadata, tc.data)
9393
if !reflect.DeepEqual(result, tc.expected) {
9494
t.Errorf("Expected substitution set: %v, got: %v", tc.expected, result)
9595
}

0 commit comments

Comments
 (0)