refactor: don't use global Config in private func (#1197)
* refactor: cve_client.go * refactor: don't use global Config in private func * remove import alias for config * refactor: dbclient * refactor: resultDir * refactor: resultsDir * refactor * refactor: gost * refactor: db client * refactor: cveDB * refactor: cvedb * refactor: exploitDB * refactor: remove detector/dbclient.go * refactor: writer * refactor: syslog writer * refactor: ips * refactor: ensureResultDir * refactor: proxy * fix(db): call CloseDB * add integration test * feat(report): sort array in json * sort func for json diff * add build-int to makefile * add int-rds-redis to makefile * fix: test case, makefile * fix makefile * show cve count after diff * make diff * diff -c * sort exploits in json for diff * sort metasploit, exploit
This commit is contained in:
Kota Kanbe
223
detector/exploitdb.go
Normal file
223
detector/exploitdb.go
Normal file
@@ -0,0 +1,223 @@
|
||||
// +build !scanner
|
||||
|
||||
package detector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/cenkalti/backoff"
|
||||
"github.com/future-architect/vuls/config"
|
||||
"github.com/future-architect/vuls/logging"
|
||||
"github.com/future-architect/vuls/models"
|
||||
"github.com/future-architect/vuls/util"
|
||||
"github.com/parnurzeal/gorequest"
|
||||
exploitdb "github.com/vulsio/go-exploitdb/db"
|
||||
exploitmodels "github.com/vulsio/go-exploitdb/models"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
// FillWithExploit fills exploit information that has in Exploit
|
||||
func FillWithExploit(r *models.ScanResult, cnf config.ExploitConf) (nExploitCve int, err error) {
|
||||
if cnf.IsFetchViaHTTP() {
|
||||
var cveIDs []string
|
||||
for cveID := range r.ScannedCves {
|
||||
cveIDs = append(cveIDs, cveID)
|
||||
}
|
||||
prefix, _ := util.URLPathJoin(cnf.GetURL(), "cves")
|
||||
responses, err := getCvesViaHTTP(cveIDs, prefix)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, res := range responses {
|
||||
exps := []*exploitmodels.Exploit{}
|
||||
if err := json.Unmarshal([]byte(res.json), &exps); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
exploits := ConvertToModels(exps)
|
||||
v, ok := r.ScannedCves[res.request.cveID]
|
||||
if ok {
|
||||
v.Exploits = exploits
|
||||
}
|
||||
r.ScannedCves[res.request.cveID] = v
|
||||
nExploitCve++
|
||||
}
|
||||
} else {
|
||||
|
||||
driver, locked, err := newExploitDB(&cnf)
|
||||
if locked {
|
||||
return 0, xerrors.Errorf("SQLite3 is locked: %s", cnf.GetSQLite3Path())
|
||||
} else if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() {
|
||||
if err := driver.CloseDB(); err != nil {
|
||||
logging.Log.Errorf("Failed to close DB. err: %+v")
|
||||
}
|
||||
}()
|
||||
|
||||
for cveID, vuln := range r.ScannedCves {
|
||||
if cveID == "" {
|
||||
continue
|
||||
}
|
||||
es := driver.GetExploitByCveID(cveID)
|
||||
if len(es) == 0 {
|
||||
continue
|
||||
}
|
||||
exploits := ConvertToModels(es)
|
||||
vuln.Exploits = exploits
|
||||
r.ScannedCves[cveID] = vuln
|
||||
nExploitCve++
|
||||
}
|
||||
}
|
||||
return nExploitCve, nil
|
||||
}
|
||||
|
||||
// ConvertToModels converts gost model to vuls model
|
||||
func ConvertToModels(es []*exploitmodels.Exploit) (exploits []models.Exploit) {
|
||||
for _, e := range es {
|
||||
var documentURL, shellURL *string
|
||||
if e.OffensiveSecurity != nil {
|
||||
os := e.OffensiveSecurity
|
||||
if os.Document != nil {
|
||||
documentURL = &os.Document.DocumentURL
|
||||
}
|
||||
if os.ShellCode != nil {
|
||||
shellURL = &os.ShellCode.ShellCodeURL
|
||||
}
|
||||
}
|
||||
exploit := models.Exploit{
|
||||
ExploitType: e.ExploitType,
|
||||
ID: e.ExploitUniqueID,
|
||||
URL: e.URL,
|
||||
Description: e.Description,
|
||||
DocumentURL: documentURL,
|
||||
ShellCodeURL: shellURL,
|
||||
}
|
||||
exploits = append(exploits, exploit)
|
||||
}
|
||||
return exploits
|
||||
}
|
||||
|
||||
type exploitResponse struct {
|
||||
request request
|
||||
json string
|
||||
}
|
||||
|
||||
func getCvesViaHTTP(cveIDs []string, urlPrefix string) (
|
||||
responses []exploitResponse, err error) {
|
||||
nReq := len(cveIDs)
|
||||
reqChan := make(chan request, nReq)
|
||||
resChan := make(chan exploitResponse, nReq)
|
||||
errChan := make(chan error, nReq)
|
||||
defer close(reqChan)
|
||||
defer close(resChan)
|
||||
defer close(errChan)
|
||||
|
||||
go func() {
|
||||
for _, cveID := range cveIDs {
|
||||
reqChan <- request{
|
||||
cveID: cveID,
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
concurrency := 10
|
||||
tasks := util.GenWorkers(concurrency)
|
||||
for i := 0; i < nReq; i++ {
|
||||
tasks <- func() {
|
||||
select {
|
||||
case req := <-reqChan:
|
||||
url, err := util.URLPathJoin(
|
||||
urlPrefix,
|
||||
req.cveID,
|
||||
)
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
} else {
|
||||
logging.Log.Debugf("HTTP Request to %s", url)
|
||||
httpGet(url, req, resChan, errChan)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
timeout := time.After(2 * 60 * time.Second)
|
||||
var errs []error
|
||||
for i := 0; i < nReq; i++ {
|
||||
select {
|
||||
case res := <-resChan:
|
||||
responses = append(responses, res)
|
||||
case err := <-errChan:
|
||||
errs = append(errs, err)
|
||||
case <-timeout:
|
||||
return nil, xerrors.New("Timeout Fetching OVAL")
|
||||
}
|
||||
}
|
||||
if len(errs) != 0 {
|
||||
return nil, xerrors.Errorf("Failed to fetch OVAL. err: %w", errs)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type request struct {
|
||||
osMajorVersion string
|
||||
packName string
|
||||
isSrcPack bool
|
||||
cveID string
|
||||
}
|
||||
|
||||
func httpGet(url string, req request, resChan chan<- exploitResponse, errChan chan<- error) {
|
||||
var body string
|
||||
var errs []error
|
||||
var resp *http.Response
|
||||
count, retryMax := 0, 3
|
||||
f := func() (err error) {
|
||||
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
|
||||
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
|
||||
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
|
||||
count++
|
||||
if count == retryMax {
|
||||
return nil
|
||||
}
|
||||
return xerrors.Errorf("HTTP GET error, url: %s, resp: %v, err: %+v", url, resp, errs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
notify := func(err error, t time.Duration) {
|
||||
logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err)
|
||||
}
|
||||
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
|
||||
if err != nil {
|
||||
errChan <- xerrors.Errorf("HTTP Error %w", err)
|
||||
return
|
||||
}
|
||||
if count == retryMax {
|
||||
errChan <- xerrors.New("Retry count exceeded")
|
||||
return
|
||||
}
|
||||
|
||||
resChan <- exploitResponse{
|
||||
request: req,
|
||||
json: body,
|
||||
}
|
||||
}
|
||||
|
||||
func newExploitDB(cnf config.VulnDictInterface) (driver exploitdb.DB, locked bool, err error) {
|
||||
if cnf.IsFetchViaHTTP() {
|
||||
return nil, false, nil
|
||||
}
|
||||
path := cnf.GetURL()
|
||||
if cnf.GetType() == "sqlite3" {
|
||||
path = cnf.GetSQLite3Path()
|
||||
}
|
||||
if driver, locked, err = exploitdb.NewDB(cnf.GetType(), path, cnf.GetDebugSQL()); err != nil {
|
||||
if locked {
|
||||
return nil, true, xerrors.Errorf("exploitDB is locked. err: %w", err)
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
return driver, false, nil
|
||||
}
|
||||
Reference in New Issue
Block a user