Skip to content

Commit 4d85d36

Browse files
MaineK00nshino
MaineK00n
and
shino
authored
feat!(detector): timeout can be set, default is no timeout (#2185)
* feat!(detector): timeout can be set, default is no timeout * fix(detector/cve): fix typo Co-authored-by: Shunichi Shinohara <[email protected]> * fix(detector): correct log msg --------- Co-authored-by: Shunichi Shinohara <[email protected]>
1 parent 28d1021 commit 4d85d36

File tree

11 files changed

+139
-39
lines changed

11 files changed

+139
-39
lines changed

‎config/vulnDictConf.go

+6
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,12 @@ type VulnDict struct {
3939
SQLite3Path string
4040

4141
DebugSQL bool
42+
43+
// Timeout for entire request (type: http only)
44+
TimeoutSec uint
45+
46+
// Timeout for each request (type: http only)
47+
TimeoutSecPerRequest uint
4248
}
4349

4450
// GetType returns type

‎detector/cti.go

+10-4
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,10 @@ func getCTIsViaHTTP(cveIDs []string, urlPrefix string) (responses []ctiResponse,
146146
}
147147
}
148148

149-
timeout := time.After(2 * 60 * time.Second)
149+
var timeout <-chan time.Time
150+
if config.Conf.Cti.TimeoutSec > 0 {
151+
timeout = time.After(time.Duration(config.Conf.Cti.TimeoutSec) * time.Second)
152+
}
150153
var errs []error
151154
for i := 0; i < nReq; i++ {
152155
select {
@@ -174,8 +177,11 @@ func httpGetCTI(url string, req ctiRequest, resChan chan<- ctiResponse, errChan
174177
var resp *http.Response
175178
count, retryMax := 0, 3
176179
f := func() (err error) {
177-
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
178-
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
180+
req := gorequest.New().Get(url)
181+
if config.Conf.Cti.TimeoutSecPerRequest > 0 {
182+
req = req.Timeout(time.Duration(config.Conf.Cti.TimeoutSecPerRequest) * time.Second)
183+
}
184+
resp, body, errs = req.End()
179185
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
180186
count++
181187
if count == retryMax {
@@ -186,7 +192,7 @@ func httpGetCTI(url string, req ctiRequest, resChan chan<- ctiResponse, errChan
186192
return nil
187193
}
188194
notify := func(err error, t time.Duration) {
189-
logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err)
195+
logging.Log.Warnf("Failed to HTTP GET. retrying in %f seconds. err: %+v", t.Seconds(), err)
190196
}
191197
if err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify); err != nil {
192198
errChan <- xerrors.Errorf("HTTP Error %w", err)

‎detector/cve_client.go

+15-5
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,10 @@ func (client goCveDictClient) fetchCveDetails(cveIDs []string) (cveDetails []cve
8080
}
8181
}
8282

83-
timeout := time.After(2 * 60 * time.Second)
83+
var timeout <-chan time.Time
84+
if config.Conf.CveDict.TimeoutSec > 0 {
85+
timeout = time.After(time.Duration(config.Conf.CveDict.TimeoutSec) * time.Second)
86+
}
8487
var errs []error
8588
for range cveIDs {
8689
select {
@@ -113,15 +116,19 @@ func httpGet(key, url string, resChan chan<- response, errChan chan<- error) {
113116
var errs []error
114117
var resp *http.Response
115118
f := func() (err error) {
116-
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
119+
req := gorequest.New().Get(url)
120+
if config.Conf.CveDict.TimeoutSecPerRequest > 0 {
121+
req = req.Timeout(time.Duration(config.Conf.CveDict.TimeoutSecPerRequest) * time.Second)
122+
}
123+
resp, body, errs = req.End()
117124
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
118125
return xerrors.Errorf("HTTP GET Error, url: %s, resp: %v, err: %+v",
119126
url, resp, errs)
120127
}
121128
return nil
122129
}
123130
notify := func(err error, t time.Duration) {
124-
logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err)
131+
logging.Log.Warnf("Failed to HTTP GET. retrying in %f seconds. err: %+v", t.Seconds(), err)
125132
}
126133
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
127134
if err != nil {
@@ -177,7 +184,10 @@ func httpPost(url string, query map[string]string) ([]cvemodels.CveDetail, error
177184
var errs []error
178185
var resp *http.Response
179186
f := func() (err error) {
180-
req := gorequest.New().Timeout(10 * time.Second).Post(url)
187+
req := gorequest.New().Post(url)
188+
if config.Conf.CveDict.TimeoutSecPerRequest > 0 {
189+
req = req.Timeout(time.Duration(config.Conf.CveDict.TimeoutSecPerRequest) * time.Second)
190+
}
181191
for key := range query {
182192
req = req.Send(fmt.Sprintf("%s=%s", key, query[key])).Type("json")
183193
}
@@ -188,7 +198,7 @@ func httpPost(url string, query map[string]string) ([]cvemodels.CveDetail, error
188198
return nil
189199
}
190200
notify := func(err error, t time.Duration) {
191-
logging.Log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %+v", t, err)
201+
logging.Log.Warnf("Failed to HTTP POST. retrying in %f seconds. err: %+v", t.Seconds(), err)
192202
}
193203
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
194204
if err != nil {

‎detector/exploitdb.go

+10-4
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,10 @@ func getExploitsViaHTTP(cveIDs []string, urlPrefix string) (
181181
}
182182
}
183183

184-
timeout := time.After(2 * 60 * time.Second)
184+
var timeout <-chan time.Time
185+
if config.Conf.Exploit.TimeoutSec > 0 {
186+
timeout = time.After(time.Duration(config.Conf.Exploit.TimeoutSec) * time.Second)
187+
}
185188
var errs []error
186189
for i := 0; i < nReq; i++ {
187190
select {
@@ -209,8 +212,11 @@ func httpGetExploit(url string, req exploitRequest, resChan chan<- exploitRespon
209212
var resp *http.Response
210213
count, retryMax := 0, 3
211214
f := func() (err error) {
212-
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
213-
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
215+
req := gorequest.New().Get(url)
216+
if config.Conf.Exploit.TimeoutSecPerRequest > 0 {
217+
req = req.Timeout(time.Duration(config.Conf.Exploit.TimeoutSecPerRequest) * time.Second)
218+
}
219+
resp, body, errs = req.End()
214220
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
215221
count++
216222
if count == retryMax {
@@ -221,7 +227,7 @@ func httpGetExploit(url string, req exploitRequest, resChan chan<- exploitRespon
221227
return nil
222228
}
223229
notify := func(err error, t time.Duration) {
224-
logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err)
230+
logging.Log.Warnf("Failed to HTTP GET. retrying in %f seconds. err: %+v", t.Seconds(), err)
225231
}
226232
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
227233
if err != nil {

‎detector/kevuln.go

+10-4
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,10 @@ func getKEVulnsViaHTTP(cveIDs []string, urlPrefix string) (
276276
}
277277
}
278278

279-
timeout := time.After(2 * 60 * time.Second)
279+
var timeout <-chan time.Time
280+
if config.Conf.KEVuln.TimeoutSec > 0 {
281+
timeout = time.After(time.Duration(config.Conf.KEVuln.TimeoutSec) * time.Second)
282+
}
280283
var errs []error
281284
for i := 0; i < nReq; i++ {
282285
select {
@@ -304,8 +307,11 @@ func httpGetKEVuln(url string, req kevulnRequest, resChan chan<- kevulnResponse,
304307
var resp *http.Response
305308
count, retryMax := 0, 3
306309
f := func() (err error) {
307-
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
308-
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
310+
req := gorequest.New().Get(url)
311+
if config.Conf.KEVuln.TimeoutSecPerRequest > 0 {
312+
req = req.Timeout(time.Duration(config.Conf.KEVuln.TimeoutSecPerRequest) * time.Second)
313+
}
314+
resp, body, errs = req.End()
309315
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
310316
count++
311317
if count == retryMax {
@@ -316,7 +322,7 @@ func httpGetKEVuln(url string, req kevulnRequest, resChan chan<- kevulnResponse,
316322
return nil
317323
}
318324
notify := func(err error, t time.Duration) {
319-
logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err)
325+
logging.Log.Warnf("Failed to HTTP GET. retrying in %f seconds. err: %+v", t.Seconds(), err)
320326
}
321327
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
322328
if err != nil {

‎detector/msf.go

+10-4
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,10 @@ func getMetasploitsViaHTTP(cveIDs []string, urlPrefix string) (
147147
}
148148
}
149149

150-
timeout := time.After(2 * 60 * time.Second)
150+
var timeout <-chan time.Time
151+
if config.Conf.Metasploit.TimeoutSec > 0 {
152+
timeout = time.After(time.Duration(config.Conf.Metasploit.TimeoutSec) * time.Second)
153+
}
151154
var errs []error
152155
for i := 0; i < nReq; i++ {
153156
select {
@@ -175,8 +178,11 @@ func httpGetMetasploit(url string, req metasploitRequest, resChan chan<- metaspl
175178
var resp *http.Response
176179
count, retryMax := 0, 3
177180
f := func() (err error) {
178-
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
179-
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
181+
req := gorequest.New().Get(url)
182+
if config.Conf.Metasploit.TimeoutSecPerRequest > 0 {
183+
req = req.Timeout(time.Duration(config.Conf.Metasploit.TimeoutSecPerRequest) * time.Second)
184+
}
185+
resp, body, errs = req.End()
180186
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
181187
count++
182188
if count == retryMax {
@@ -187,7 +193,7 @@ func httpGetMetasploit(url string, req metasploitRequest, resChan chan<- metaspl
187193
return nil
188194
}
189195
notify := func(err error, t time.Duration) {
190-
logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err)
196+
logging.Log.Warnf("Failed to HTTP GET. retrying in %f seconds. err: %+v", t.Seconds(), err)
191197
}
192198
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
193199
if err != nil {

‎gost/microsoft.go

+19-6
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/parnurzeal/gorequest"
1919
"golang.org/x/xerrors"
2020

21+
"github.com/future-architect/vuls/config"
2122
"github.com/future-architect/vuls/logging"
2223
"github.com/future-architect/vuls/models"
2324
"github.com/future-architect/vuls/util"
@@ -47,14 +48,18 @@ func (ms Microsoft) DetectCVEs(r *models.ScanResult, _ bool) (nCVEs int, err err
4748
var errs []error
4849
var resp *http.Response
4950
f := func() error {
50-
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Post(u).SendStruct(content).Type("json").EndBytes()
51+
req := gorequest.New().Post(u).SendStruct(content).Type("json")
52+
if config.Conf.Gost.TimeoutSecPerRequest > 0 {
53+
req = req.Timeout(time.Duration(config.Conf.Gost.TimeoutSecPerRequest) * time.Second)
54+
}
55+
resp, body, errs = req.EndBytes()
5156
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
5257
return xerrors.Errorf("HTTP POST error. url: %s, resp: %v, err: %+v", u, resp, errs)
5358
}
5459
return nil
5560
}
5661
notify := func(err error, t time.Duration) {
57-
logging.Log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %+v", t, err)
62+
logging.Log.Warnf("Failed to HTTP POST. retrying in %f seconds. err: %+v", t.Seconds(), err)
5863
}
5964
if err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify); err != nil {
6065
return 0, xerrors.Errorf("HTTP Error: %w", err)
@@ -88,14 +93,18 @@ func (ms Microsoft) DetectCVEs(r *models.ScanResult, _ bool) (nCVEs int, err err
8893
var errs []error
8994
var resp *http.Response
9095
f := func() error {
91-
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Post(u).SendStruct(content).Type("json").EndBytes()
96+
req := gorequest.New().Post(u).SendStruct(content).Type("json")
97+
if config.Conf.Gost.TimeoutSecPerRequest > 0 {
98+
req = req.Timeout(time.Duration(config.Conf.Gost.TimeoutSecPerRequest) * time.Second)
99+
}
100+
resp, body, errs = req.EndBytes()
92101
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
93102
return xerrors.Errorf("HTTP POST error. url: %s, resp: %v, err: %+v", u, resp, errs)
94103
}
95104
return nil
96105
}
97106
notify := func(err error, t time.Duration) {
98-
logging.Log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %+v", t, err)
107+
logging.Log.Warnf("Failed to HTTP POST. retrying in %f seconds. err: %+v", t.Seconds(), err)
99108
}
100109
if err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify); err != nil {
101110
return 0, xerrors.Errorf("HTTP Error: %w", err)
@@ -151,14 +160,18 @@ func (ms Microsoft) DetectCVEs(r *models.ScanResult, _ bool) (nCVEs int, err err
151160
var errs []error
152161
var resp *http.Response
153162
f := func() error {
154-
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Post(u).SendStruct(content).Type("json").EndBytes()
163+
req := gorequest.New().Post(u).SendStruct(content).Type("json")
164+
if config.Conf.Gost.TimeoutSecPerRequest > 0 {
165+
req = req.Timeout(time.Duration(config.Conf.Gost.TimeoutSecPerRequest) * time.Second)
166+
}
167+
resp, body, errs = req.EndBytes()
155168
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
156169
return xerrors.Errorf("HTTP POST error. url: %s, resp: %v, err: %+v", u, resp, errs)
157170
}
158171
return nil
159172
}
160173
notify := func(err error, t time.Duration) {
161-
logging.Log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %+v", t, err)
174+
logging.Log.Warnf("Failed to HTTP POST. retrying in %f seconds. err: %+v", t.Seconds(), err)
162175
}
163176
if err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify); err != nil {
164177
return 0, xerrors.Errorf("HTTP Error: %w", err)

‎gost/util.go

+17-7
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/parnurzeal/gorequest"
1414
"golang.org/x/xerrors"
1515

16+
"github.com/future-architect/vuls/config"
1617
"github.com/future-architect/vuls/logging"
1718
"github.com/future-architect/vuls/models"
1819
"github.com/future-architect/vuls/util"
@@ -59,7 +60,10 @@ func getCvesViaHTTP(cveIDs []string, urlPrefix string) (
5960
}
6061
}
6162

62-
timeout := time.After(2 * 60 * time.Second)
63+
var timeout <-chan time.Time
64+
if config.Conf.Gost.TimeoutSec > 0 {
65+
timeout = time.After(time.Duration(config.Conf.Gost.TimeoutSec) * time.Second)
66+
}
6367
var errs []error
6468
for i := 0; i < nReq; i++ {
6569
select {
@@ -68,11 +72,11 @@ func getCvesViaHTTP(cveIDs []string, urlPrefix string) (
6872
case err := <-errChan:
6973
errs = append(errs, err)
7074
case <-timeout:
71-
return nil, xerrors.New("Timeout Fetching OVAL")
75+
return nil, xerrors.New("Timeout Fetching Gost")
7276
}
7377
}
7478
if len(errs) != 0 {
75-
return nil, xerrors.Errorf("Failed to fetch OVAL. err: %w", errs)
79+
return nil, xerrors.Errorf("Failed to fetch Gost. err: %w", errs)
7680
}
7781
return
7882
}
@@ -124,7 +128,10 @@ func getCvesWithFixStateViaHTTP(r *models.ScanResult, urlPrefix, fixState string
124128
}
125129
}
126130

127-
timeout := time.After(2 * 60 * time.Second)
131+
var timeout <-chan time.Time
132+
if config.Conf.Gost.TimeoutSec > 0 {
133+
timeout = time.After(time.Duration(config.Conf.Gost.TimeoutSec) * time.Second)
134+
}
128135
var errs []error
129136
for i := 0; i < nReq; i++ {
130137
select {
@@ -148,8 +155,11 @@ func httpGet(url string, req request, resChan chan<- response, errChan chan<- er
148155
var resp *http.Response
149156
count, retryMax := 0, 3
150157
f := func() (err error) {
151-
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
152-
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
158+
req := gorequest.New().Get(url)
159+
if config.Conf.Gost.TimeoutSecPerRequest > 0 {
160+
req = req.Timeout(time.Duration(config.Conf.Gost.TimeoutSecPerRequest) * time.Second)
161+
}
162+
resp, body, errs = req.End()
153163
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
154164
count++
155165
if count == retryMax {
@@ -160,7 +170,7 @@ func httpGet(url string, req request, resChan chan<- response, errChan chan<- er
160170
return nil
161171
}
162172
notify := func(err error, t time.Duration) {
163-
logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err)
173+
logging.Log.Warnf("Failed to HTTP GET. retrying in %f seconds. err: %+v", t.Seconds(), err)
164174
}
165175
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
166176
if err != nil {

‎oval/oval.go

+11-2
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,12 @@ func (b Base) CheckIfOvalFetched(osFamily, release string) (bool, error) {
8484
if err != nil {
8585
return false, xerrors.Errorf("Failed to join URLPath. err: %w", err)
8686
}
87-
resp, body, errs := gorequest.New().Timeout(10 * time.Second).Get(url).End()
87+
88+
req := gorequest.New().Get(url)
89+
if config.Conf.OvalDict.TimeoutSecPerRequest > 0 {
90+
req = req.Timeout(time.Duration(config.Conf.OvalDict.TimeoutSecPerRequest) * time.Second)
91+
}
92+
resp, body, errs := req.End()
8893
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
8994
return false, xerrors.Errorf("HTTP GET error, url: %s, resp: %v, err: %+v", url, resp, errs)
9095
}
@@ -143,7 +148,11 @@ func (b Base) CheckIfOvalFresh(osFamily, release string) (ok bool, err error) {
143148
if err != nil {
144149
return false, xerrors.Errorf("Failed to join URLPath. err: %w", err)
145150
}
146-
resp, body, errs := gorequest.New().Timeout(10 * time.Second).Get(url).End()
151+
req := gorequest.New().Get(url)
152+
if config.Conf.OvalDict.TimeoutSecPerRequest > 0 {
153+
req = req.Timeout(time.Duration(config.Conf.OvalDict.TimeoutSecPerRequest) * time.Second)
154+
}
155+
resp, body, errs := req.End()
147156
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
148157
return false, xerrors.Errorf("HTTP GET error, url: %s, resp: %v, err: %+v", url, resp, errs)
149158
}

0 commit comments

Comments
 (0)