Refactoring
This commit is contained in:
Kota Kanbe
263
report/cve_client.go
Normal file
263
report/cve_client.go
Normal file
@@ -0,0 +1,263 @@
|
||||
/* Vuls - Vulnerability Scanner
|
||||
Copyright (C) 2016 Future Architect, Inc. Japan.
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package report
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/cenkalti/backoff"
|
||||
"github.com/parnurzeal/gorequest"
|
||||
|
||||
"github.com/future-architect/vuls/config"
|
||||
"github.com/future-architect/vuls/util"
|
||||
cveconfig "github.com/kotakanbe/go-cve-dictionary/config"
|
||||
cvedb "github.com/kotakanbe/go-cve-dictionary/db"
|
||||
cve "github.com/kotakanbe/go-cve-dictionary/models"
|
||||
)
|
||||
|
||||
// CveClient is api client of CVE disctionary service.
|
||||
var CveClient cvedictClient
|
||||
|
||||
type cvedictClient struct {
|
||||
// httpProxy string
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (api *cvedictClient) initialize() {
|
||||
api.baseURL = config.Conf.CveDBURL
|
||||
}
|
||||
|
||||
func (api cvedictClient) CheckHealth() (ok bool, err error) {
|
||||
if config.Conf.CveDBURL == "" || config.Conf.CveDBType == "mysql" || config.Conf.CveDBType == "postgres" {
|
||||
util.Log.Debugf("get cve-dictionary from %s", config.Conf.CveDBType)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
api.initialize()
|
||||
url := fmt.Sprintf("%s/health", api.baseURL)
|
||||
var errs []error
|
||||
var resp *http.Response
|
||||
resp, _, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
|
||||
// resp, _, errs = gorequest.New().Proxy(api.httpProxy).Get(url).End()
|
||||
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
|
||||
return false, fmt.Errorf("Failed to request to CVE server. url: %s, errs: %v", url, errs)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Key string
|
||||
CveDetail cve.CveDetail
|
||||
}
|
||||
|
||||
func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDetails, err error) {
|
||||
switch config.Conf.CveDBType {
|
||||
case "sqlite3", "mysql", "postgres":
|
||||
return api.FetchCveDetailsFromCveDB(cveIDs)
|
||||
}
|
||||
|
||||
api.baseURL = config.Conf.CveDBURL
|
||||
reqChan := make(chan string, len(cveIDs))
|
||||
resChan := make(chan response, len(cveIDs))
|
||||
errChan := make(chan error, len(cveIDs))
|
||||
defer close(reqChan)
|
||||
defer close(resChan)
|
||||
defer close(errChan)
|
||||
|
||||
go func() {
|
||||
for _, cveID := range cveIDs {
|
||||
reqChan <- cveID
|
||||
}
|
||||
}()
|
||||
|
||||
concurrency := 10
|
||||
tasks := util.GenWorkers(concurrency)
|
||||
for range cveIDs {
|
||||
tasks <- func() {
|
||||
select {
|
||||
case cveID := <-reqChan:
|
||||
url, err := util.URLPathJoin(api.baseURL, "cves", cveID)
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
} else {
|
||||
util.Log.Debugf("HTTP Request to %s", url)
|
||||
api.httpGet(cveID, url, resChan, errChan)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
timeout := time.After(2 * 60 * time.Second)
|
||||
var errs []error
|
||||
for range cveIDs {
|
||||
select {
|
||||
case res := <-resChan:
|
||||
if len(res.CveDetail.CveID) == 0 {
|
||||
cveDetails = append(cveDetails, cve.CveDetail{
|
||||
CveID: res.Key,
|
||||
})
|
||||
} else {
|
||||
cveDetails = append(cveDetails, res.CveDetail)
|
||||
}
|
||||
case err := <-errChan:
|
||||
errs = append(errs, err)
|
||||
case <-timeout:
|
||||
return []cve.CveDetail{}, fmt.Errorf("Timeout Fetching CVE")
|
||||
}
|
||||
}
|
||||
if len(errs) != 0 {
|
||||
return []cve.CveDetail{},
|
||||
fmt.Errorf("Failed to fetch CVE. err: %v", errs)
|
||||
}
|
||||
|
||||
//TODO
|
||||
// sort.Sort(cveDetails)
|
||||
return
|
||||
}
|
||||
|
||||
func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails cve.CveDetails, err error) {
|
||||
util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType)
|
||||
cveconfig.Conf.DBType = config.Conf.CveDBType
|
||||
if config.Conf.CveDBType == "sqlite3" {
|
||||
cveconfig.Conf.DBPath = config.Conf.CveDBPath
|
||||
} else {
|
||||
cveconfig.Conf.DBPath = config.Conf.CveDBURL
|
||||
}
|
||||
cveconfig.Conf.DebugSQL = config.Conf.DebugSQL
|
||||
if err := cvedb.OpenDB(); err != nil {
|
||||
return []cve.CveDetail{},
|
||||
fmt.Errorf("Failed to open DB. err: %s", err)
|
||||
}
|
||||
for _, cveID := range cveIDs {
|
||||
cveDetail := cvedb.Get(cveID)
|
||||
if len(cveDetail.CveID) == 0 {
|
||||
cveDetails = append(cveDetails, cve.CveDetail{
|
||||
CveID: cveID,
|
||||
})
|
||||
} else {
|
||||
cveDetails = append(cveDetails, cveDetail)
|
||||
}
|
||||
}
|
||||
|
||||
//TODO
|
||||
// order by CVE ID desc
|
||||
// sort.Sort(cveDetails)
|
||||
return
|
||||
}
|
||||
|
||||
func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errChan chan<- error) {
|
||||
var body string
|
||||
var errs []error
|
||||
var resp *http.Response
|
||||
f := func() (err error) {
|
||||
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
|
||||
resp, body, errs = gorequest.New().Get(url).End()
|
||||
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
|
||||
return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", errs, url, resp)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
notify := func(err error, t time.Duration) {
|
||||
util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err)
|
||||
}
|
||||
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
|
||||
if err != nil {
|
||||
errChan <- fmt.Errorf("HTTP Error %s", err)
|
||||
}
|
||||
cveDetail := cve.CveDetail{}
|
||||
if err := json.Unmarshal([]byte(body), &cveDetail); err != nil {
|
||||
errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err)
|
||||
}
|
||||
resChan <- response{
|
||||
key,
|
||||
cveDetail,
|
||||
}
|
||||
}
|
||||
|
||||
type responseGetCveDetailByCpeName struct {
|
||||
CpeName string
|
||||
CveDetails []cve.CveDetail
|
||||
}
|
||||
|
||||
func (api cvedictClient) FetchCveDetailsByCpeName(cpeName string) ([]cve.CveDetail, error) {
|
||||
switch config.Conf.CveDBType {
|
||||
case "sqlite3", "mysql", "postgres":
|
||||
return api.FetchCveDetailsByCpeNameFromDB(cpeName)
|
||||
}
|
||||
|
||||
api.baseURL = config.Conf.CveDBURL
|
||||
url, err := util.URLPathJoin(api.baseURL, "cpes")
|
||||
if err != nil {
|
||||
return []cve.CveDetail{}, err
|
||||
}
|
||||
|
||||
query := map[string]string{"name": cpeName}
|
||||
util.Log.Debugf("HTTP Request to %s, query: %#v", url, query)
|
||||
return api.httpPost(cpeName, url, query)
|
||||
}
|
||||
|
||||
func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]cve.CveDetail, error) {
|
||||
var body string
|
||||
var errs []error
|
||||
var resp *http.Response
|
||||
f := func() (err error) {
|
||||
req := gorequest.New().SetDebug(config.Conf.Debug).Post(url)
|
||||
for key := range query {
|
||||
req = req.Send(fmt.Sprintf("%s=%s", key, query[key])).Type("json")
|
||||
}
|
||||
resp, body, errs = req.End()
|
||||
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
|
||||
return fmt.Errorf("HTTP POST error: %v, url: %s, resp: %v", errs, url, resp)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
notify := func(err error, t time.Duration) {
|
||||
util.Log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %s", t, err)
|
||||
}
|
||||
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
|
||||
if err != nil {
|
||||
return []cve.CveDetail{}, fmt.Errorf("HTTP Error %s", err)
|
||||
}
|
||||
|
||||
cveDetails := []cve.CveDetail{}
|
||||
if err := json.Unmarshal([]byte(body), &cveDetails); err != nil {
|
||||
return []cve.CveDetail{},
|
||||
fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err)
|
||||
}
|
||||
return cveDetails, nil
|
||||
}
|
||||
|
||||
func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) ([]cve.CveDetail, error) {
|
||||
util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType)
|
||||
cveconfig.Conf.DBType = config.Conf.CveDBType
|
||||
if config.Conf.CveDBType == "sqlite3" {
|
||||
cveconfig.Conf.DBPath = config.Conf.CveDBPath
|
||||
} else {
|
||||
cveconfig.Conf.DBPath = config.Conf.CveDBURL
|
||||
}
|
||||
cveconfig.Conf.DebugSQL = config.Conf.DebugSQL
|
||||
|
||||
if err := cvedb.OpenDB(); err != nil {
|
||||
return []cve.CveDetail{},
|
||||
fmt.Errorf("Failed to open DB. err: %s", err)
|
||||
}
|
||||
return cvedb.GetByCpeName(cpeName), nil
|
||||
}
|
||||
204
report/report.go
Normal file
204
report/report.go
Normal file
@@ -0,0 +1,204 @@
|
||||
/* Vuls - Vulnerability Scanner
|
||||
Copyright (C) 2016 Future Architect, Inc. Japan.
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package report
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
c "github.com/future-architect/vuls/config"
|
||||
"github.com/future-architect/vuls/models"
|
||||
"github.com/future-architect/vuls/oval"
|
||||
"github.com/future-architect/vuls/util"
|
||||
"github.com/k0kubun/pp"
|
||||
)
|
||||
|
||||
// FillCveInfos fills CVE Detailed Information
|
||||
func FillCveInfos(rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
|
||||
var filled []models.ScanResult
|
||||
for _, r := range rs {
|
||||
if c.Conf.RefreshCve || needToRefreshCve(r) {
|
||||
if err := fillCveInfo(&r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Lang = c.Conf.Lang
|
||||
if err := overwriteJSONFile(dir, r); err != nil {
|
||||
return nil, fmt.Errorf("Failed to write JSON: %s", err)
|
||||
}
|
||||
filled = append(filled, r)
|
||||
} else {
|
||||
util.Log.Debugf("No need to refresh")
|
||||
filled = append(filled, r)
|
||||
}
|
||||
}
|
||||
|
||||
if c.Conf.Diff {
|
||||
previous, err := loadPrevious(filled)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
diff, err := diff(filled, previous)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filled = []models.ScanResult{}
|
||||
for _, r := range diff {
|
||||
if err := fillCveDetail(&r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filled = append(filled, r)
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range filled {
|
||||
pp.Printf("filled: %d\n", len(r.ScannedCves))
|
||||
}
|
||||
|
||||
filtered := []models.ScanResult{}
|
||||
for _, r := range filled {
|
||||
filtered = append(filtered, r.FilterByCvssOver(c.Conf.CvssScoreOver))
|
||||
}
|
||||
|
||||
for _, r := range filtered {
|
||||
pp.Printf("filtered: %d\n", len(r.ScannedCves))
|
||||
}
|
||||
|
||||
// TODO Sort
|
||||
return filtered, nil
|
||||
}
|
||||
|
||||
func fillCveInfo(r *models.ScanResult) error {
|
||||
util.Log.Debugf("need to refresh")
|
||||
if c.Conf.CveDBType == "sqlite3" && c.Conf.CveDBURL == "" {
|
||||
if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) {
|
||||
return fmt.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s",
|
||||
c.Conf.CveDBPath)
|
||||
}
|
||||
}
|
||||
|
||||
if err := fillCveInfoFromOvalDB(r); err != nil {
|
||||
return fmt.Errorf("Failed to fill OVAL information: %s", err)
|
||||
}
|
||||
|
||||
if err := fillCveInfoFromCveDB(r); err != nil {
|
||||
return fmt.Errorf("Failed to fill CVE information: %s", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fillCveDetail fetches NVD, JVN from CVE Database, and then set to fields.
|
||||
func fillCveDetail(r *models.ScanResult) error {
|
||||
var cveIDs []string
|
||||
for _, v := range r.ScannedCves {
|
||||
cveIDs = append(cveIDs, v.CveID)
|
||||
}
|
||||
|
||||
ds, err := CveClient.FetchCveDetails(cveIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, d := range ds {
|
||||
nvd := r.ConvertNvdToModel(d.CveID, d.Nvd)
|
||||
jvn := r.ConvertJvnToModel(d.CveID, d.Jvn)
|
||||
for cveID, vinfo := range r.ScannedCves {
|
||||
if vinfo.CveID == d.CveID {
|
||||
if vinfo.CveContents == nil {
|
||||
vinfo.CveContents = models.CveContents{}
|
||||
}
|
||||
for _, con := range []models.CveContent{*nvd, *jvn} {
|
||||
if !con.Empty() {
|
||||
vinfo.CveContents[con.Type] = con
|
||||
}
|
||||
}
|
||||
r.ScannedCves[cveID] = vinfo
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
//TODO Remove
|
||||
// sort.Slice(r.ScannedCves, func(i, j int) bool {
|
||||
// if r.ScannedCves[j].CveContents.CvssV2Score() == r.ScannedCves[i].CveContents.CvssV2Score() {
|
||||
// return r.ScannedCves[j].CveContents.CvssV2Score() < r.ScannedCves[i].CveContents.CvssV2Score()
|
||||
// }
|
||||
// return r.ScannedCves[j].CveContents.CvssV2Score() < r.ScannedCves[i].CveContents.CvssV2Score()
|
||||
// })
|
||||
return nil
|
||||
}
|
||||
|
||||
func fillCveInfoFromCveDB(r *models.ScanResult) error {
|
||||
sInfo := c.Conf.Servers[r.ServerName]
|
||||
if err := fillVulnByCpeNames(sInfo.CpeNames, r.ScannedCves); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fillCveDetail(r); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fillCveInfoFromOvalDB(r *models.ScanResult) error {
|
||||
var ovalClient oval.Client
|
||||
switch r.Family {
|
||||
case "debian":
|
||||
ovalClient = oval.NewDebian()
|
||||
case "ubuntu":
|
||||
ovalClient = oval.NewUbuntu()
|
||||
case "rhel":
|
||||
ovalClient = oval.NewRedhat()
|
||||
case "centos":
|
||||
ovalClient = oval.NewCentOS()
|
||||
case "amazon", "oraclelinux", "Raspbian", "FreeBSD":
|
||||
//TODO implement OracleLinux
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("Oval %s is not implemented yet", r.Family)
|
||||
}
|
||||
if err := ovalClient.FillCveInfoFromOvalDB(r); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fillVulnByCpeNames(cpeNames []string, scannedVulns models.VulnInfos) error {
|
||||
for _, name := range cpeNames {
|
||||
details, err := CveClient.FetchCveDetailsByCpeName(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, detail := range details {
|
||||
if val, ok := scannedVulns[detail.CveID]; ok {
|
||||
names := val.CpeNames
|
||||
names = util.AppendIfMissing(names, name)
|
||||
val.CpeNames = names
|
||||
val.Confidence = models.CpeNameMatch
|
||||
scannedVulns[detail.CveID] = val
|
||||
} else {
|
||||
v := models.VulnInfo{
|
||||
CveID: detail.CveID,
|
||||
CpeNames: []string{name},
|
||||
Confidence: models.CpeNameMatch,
|
||||
}
|
||||
//TODO
|
||||
// v.NilToEmpty()
|
||||
scannedVulns[detail.CveID] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
1
report/report_test.go
Normal file
1
report/report_test.go
Normal file
@@ -0,0 +1 @@
|
||||
package report
|
||||
261
report/util.go
261
report/util.go
@@ -19,11 +19,19 @@ package report
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/future-architect/vuls/config"
|
||||
"github.com/future-architect/vuls/models"
|
||||
"github.com/future-architect/vuls/util"
|
||||
"github.com/gosuri/uitable"
|
||||
)
|
||||
|
||||
@@ -516,3 +524,256 @@ func formatOneChangelog(p models.Package) string {
|
||||
buf = append(buf, packVer, delim.String(), clog)
|
||||
return strings.Join(buf, "\n")
|
||||
}
|
||||
|
||||
func needToRefreshCve(r models.ScanResult) bool {
|
||||
if r.Lang != config.Conf.Lang {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, cve := range r.ScannedCves {
|
||||
if 0 < len(cve.CveContents) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func overwriteJSONFile(dir string, r models.ScanResult) error {
|
||||
before := config.Conf.FormatJSON
|
||||
beforeDiff := config.Conf.Diff
|
||||
config.Conf.FormatJSON = true
|
||||
config.Conf.Diff = false
|
||||
w := LocalFileWriter{CurrentDir: dir}
|
||||
if err := w.Write(r); err != nil {
|
||||
return fmt.Errorf("Failed to write summary report: %s", err)
|
||||
}
|
||||
config.Conf.FormatJSON = before
|
||||
config.Conf.Diff = beforeDiff
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadPrevious(current models.ScanResults) (previous models.ScanResults, err error) {
|
||||
dirs, err := ListValidJSONDirs()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, result := range current {
|
||||
for _, dir := range dirs[1:] {
|
||||
var r models.ScanResult
|
||||
path := filepath.Join(dir, result.ServerName+".json")
|
||||
if r, err = loadOneServerScanResult(path); err != nil {
|
||||
continue
|
||||
}
|
||||
if r.Family == result.Family && r.Release == result.Release {
|
||||
previous = append(previous, r)
|
||||
util.Log.Infof("Privious json found: %s", path)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return previous, nil
|
||||
}
|
||||
|
||||
func diff(curResults, preResults models.ScanResults) (diffed models.ScanResults, err error) {
|
||||
for _, current := range curResults {
|
||||
found := false
|
||||
var previous models.ScanResult
|
||||
for _, r := range preResults {
|
||||
if current.ServerName == r.ServerName {
|
||||
found = true
|
||||
previous = r
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if found {
|
||||
current.ScannedCves = getDiffCves(previous, current)
|
||||
packages := models.Packages{}
|
||||
for _, s := range current.ScannedCves {
|
||||
for _, name := range s.PackageNames {
|
||||
p := current.Packages[name]
|
||||
packages[name] = p
|
||||
}
|
||||
}
|
||||
current.Packages = packages
|
||||
}
|
||||
|
||||
diffed = append(diffed, current)
|
||||
}
|
||||
return diffed, err
|
||||
}
|
||||
|
||||
func getDiffCves(previous, current models.ScanResult) models.VulnInfos {
|
||||
previousCveIDsSet := map[string]bool{}
|
||||
for _, previousVulnInfo := range previous.ScannedCves {
|
||||
previousCveIDsSet[previousVulnInfo.CveID] = true
|
||||
}
|
||||
|
||||
new := models.VulnInfos{}
|
||||
updated := models.VulnInfos{}
|
||||
for _, v := range current.ScannedCves {
|
||||
if previousCveIDsSet[v.CveID] {
|
||||
if isCveInfoUpdated(v.CveID, previous, current) {
|
||||
updated[v.CveID] = v
|
||||
}
|
||||
} else {
|
||||
new[v.CveID] = v
|
||||
}
|
||||
}
|
||||
|
||||
for cveID, vuln := range new {
|
||||
updated[cveID] = vuln
|
||||
}
|
||||
return updated
|
||||
}
|
||||
|
||||
func isCveInfoUpdated(cveID string, previous, current models.ScanResult) bool {
|
||||
cTypes := []models.CveContentType{
|
||||
models.NVD,
|
||||
models.JVN,
|
||||
models.NewCveContentType(current.Family),
|
||||
}
|
||||
|
||||
prevLastModified := map[models.CveContentType]time.Time{}
|
||||
for _, c := range previous.ScannedCves {
|
||||
if cveID == c.CveID {
|
||||
for _, cType := range cTypes {
|
||||
content, _ := c.CveContents[cType]
|
||||
prevLastModified[cType] = content.LastModified
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
curLastModified := map[models.CveContentType]time.Time{}
|
||||
for _, c := range current.ScannedCves {
|
||||
if cveID == c.CveID {
|
||||
for _, cType := range cTypes {
|
||||
content, _ := c.CveContents[cType]
|
||||
curLastModified[cType] = content.LastModified
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, cType := range cTypes {
|
||||
if equal := prevLastModified[cType].Equal(curLastModified[cType]); !equal {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// jsonDirPattern is file name pattern of JSON directory
|
||||
// 2016-11-16T10:43:28+09:00
|
||||
// 2016-11-16T10:43:28Z
|
||||
var jsonDirPattern = regexp.MustCompile(
|
||||
`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})$`)
|
||||
|
||||
// ListValidJSONDirs returns valid json directory as array
|
||||
// Returned array is sorted so that recent directories are at the head
|
||||
func ListValidJSONDirs() (dirs []string, err error) {
|
||||
var dirInfo []os.FileInfo
|
||||
if dirInfo, err = ioutil.ReadDir(config.Conf.ResultsDir); err != nil {
|
||||
err = fmt.Errorf("Failed to read %s: %s",
|
||||
config.Conf.ResultsDir, err)
|
||||
return
|
||||
}
|
||||
for _, d := range dirInfo {
|
||||
if d.IsDir() && jsonDirPattern.MatchString(d.Name()) {
|
||||
jsonDir := filepath.Join(config.Conf.ResultsDir, d.Name())
|
||||
dirs = append(dirs, jsonDir)
|
||||
}
|
||||
}
|
||||
sort.Slice(dirs, func(i, j int) bool {
|
||||
return dirs[j] < dirs[i]
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// JSONDir returns
|
||||
// If there is an arg, check if it is a valid format and return the corresponding path under results.
|
||||
// If arg passed via PIPE (such as history subcommand), return that path.
|
||||
// Otherwise, returns the path of the latest directory
|
||||
func JSONDir(args []string) (string, error) {
|
||||
var err error
|
||||
dirs := []string{}
|
||||
|
||||
if 0 < len(args) {
|
||||
if dirs, err = ListValidJSONDirs(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
path := filepath.Join(config.Conf.ResultsDir, args[0])
|
||||
for _, d := range dirs {
|
||||
ss := strings.Split(d, string(os.PathSeparator))
|
||||
timedir := ss[len(ss)-1]
|
||||
if timedir == args[0] {
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("Invalid path: %s", path)
|
||||
}
|
||||
|
||||
// PIPE
|
||||
if config.Conf.Pipe {
|
||||
bytes, err := ioutil.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Failed to read stdin: %s", err)
|
||||
}
|
||||
fields := strings.Fields(string(bytes))
|
||||
if 0 < len(fields) {
|
||||
return filepath.Join(config.Conf.ResultsDir, fields[0]), nil
|
||||
}
|
||||
return "", fmt.Errorf("Stdin is invalid: %s", string(bytes))
|
||||
}
|
||||
|
||||
// returns latest dir when no args or no PIPE
|
||||
if dirs, err = ListValidJSONDirs(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
return "", fmt.Errorf("No results under %s",
|
||||
config.Conf.ResultsDir)
|
||||
}
|
||||
return dirs[0], nil
|
||||
}
|
||||
|
||||
// LoadScanResults read JSON data
|
||||
func LoadScanResults(jsonDir string) (results models.ScanResults, err error) {
|
||||
var files []os.FileInfo
|
||||
if files, err = ioutil.ReadDir(jsonDir); err != nil {
|
||||
return nil, fmt.Errorf("Failed to read %s: %s", jsonDir, err)
|
||||
}
|
||||
for _, f := range files {
|
||||
if filepath.Ext(f.Name()) != ".json" || strings.HasSuffix(f.Name(), "_diff.json") {
|
||||
continue
|
||||
}
|
||||
|
||||
var r models.ScanResult
|
||||
path := filepath.Join(jsonDir, f.Name())
|
||||
if r, err = loadOneServerScanResult(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
results = append(results, r)
|
||||
}
|
||||
if len(results) == 0 {
|
||||
return nil, fmt.Errorf("There is no json file under %s", jsonDir)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// loadOneServerScanResult read JSON data of one server
|
||||
func loadOneServerScanResult(jsonFile string) (result models.ScanResult, err error) {
|
||||
var data []byte
|
||||
if data, err = ioutil.ReadFile(jsonFile); err != nil {
|
||||
err = fmt.Errorf("Failed to read %s: %s", jsonFile, err)
|
||||
return
|
||||
}
|
||||
if json.Unmarshal(data, &result) != nil {
|
||||
err = fmt.Errorf("Failed to parse %s: %s", jsonFile, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
327
report/util_test.go
Normal file
327
report/util_test.go
Normal file
@@ -0,0 +1,327 @@
|
||||
package report
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/future-architect/vuls/models"
|
||||
"github.com/k0kubun/pp"
|
||||
)
|
||||
|
||||
func TestIsCveInfoUpdated(t *testing.T) {
|
||||
f := "2006-01-02"
|
||||
old, _ := time.Parse(f, "2015-12-15")
|
||||
new, _ := time.Parse(f, "2015-12-16")
|
||||
|
||||
type In struct {
|
||||
cveID string
|
||||
cur models.ScanResult
|
||||
prev models.ScanResult
|
||||
}
|
||||
var tests = []struct {
|
||||
in In
|
||||
expected bool
|
||||
}{
|
||||
// NVD compare non-initialized times
|
||||
{
|
||||
in: In{
|
||||
cveID: "CVE-2017-0001",
|
||||
cur: models.ScanResult{
|
||||
ScannedCves: models.VulnInfos{
|
||||
"CVE-2017-0001": {
|
||||
CveID: "CVE-2017-0001",
|
||||
CveContents: models.NewCveContents(
|
||||
models.CveContent{
|
||||
Type: models.NVD,
|
||||
CveID: "CVE-2017-0001",
|
||||
LastModified: time.Time{},
|
||||
},
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
prev: models.ScanResult{
|
||||
ScannedCves: models.VulnInfos{
|
||||
"CVE-2017-0001": {
|
||||
CveID: "CVE-2017-0001",
|
||||
CveContents: models.NewCveContents(
|
||||
models.CveContent{
|
||||
Type: models.NVD,
|
||||
CveID: "CVE-2017-0001",
|
||||
LastModified: time.Time{},
|
||||
},
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
// JVN not updated
|
||||
{
|
||||
in: In{
|
||||
cveID: "CVE-2017-0002",
|
||||
cur: models.ScanResult{
|
||||
ScannedCves: models.VulnInfos{
|
||||
"CVE-2017-0002": {
|
||||
CveID: "CVE-2017-0002",
|
||||
CveContents: models.NewCveContents(
|
||||
models.CveContent{
|
||||
Type: models.NVD,
|
||||
CveID: "CVE-2017-0002",
|
||||
LastModified: old,
|
||||
},
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
prev: models.ScanResult{
|
||||
ScannedCves: models.VulnInfos{
|
||||
"CVE-2017-0002": {
|
||||
CveID: "CVE-2017-0002",
|
||||
CveContents: models.NewCveContents(
|
||||
models.CveContent{
|
||||
Type: models.NVD,
|
||||
CveID: "CVE-2017-0002",
|
||||
LastModified: old,
|
||||
},
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
// OVAL updated
|
||||
{
|
||||
in: In{
|
||||
cveID: "CVE-2017-0003",
|
||||
cur: models.ScanResult{
|
||||
Family: "ubuntu",
|
||||
ScannedCves: models.VulnInfos{
|
||||
"CVE-2017-0003": {
|
||||
CveID: "CVE-2017-0003",
|
||||
CveContents: models.NewCveContents(
|
||||
models.CveContent{
|
||||
Type: models.NVD,
|
||||
CveID: "CVE-2017-0002",
|
||||
LastModified: new,
|
||||
},
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
prev: models.ScanResult{
|
||||
Family: "ubuntu",
|
||||
ScannedCves: models.VulnInfos{
|
||||
"CVE-2017-0003": {
|
||||
CveID: "CVE-2017-0003",
|
||||
CveContents: models.NewCveContents(
|
||||
models.CveContent{
|
||||
Type: models.NVD,
|
||||
CveID: "CVE-2017-0002",
|
||||
LastModified: old,
|
||||
},
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
// OVAL newly detected
|
||||
{
|
||||
in: In{
|
||||
cveID: "CVE-2017-0004",
|
||||
cur: models.ScanResult{
|
||||
Family: "redhat",
|
||||
ScannedCves: models.VulnInfos{
|
||||
"CVE-2017-0004": {
|
||||
CveID: "CVE-2017-0004",
|
||||
CveContents: models.NewCveContents(
|
||||
models.CveContent{
|
||||
Type: models.NVD,
|
||||
CveID: "CVE-2017-0002",
|
||||
LastModified: old,
|
||||
},
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
prev: models.ScanResult{
|
||||
Family: "redhat",
|
||||
ScannedCves: models.VulnInfos{},
|
||||
},
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
}
|
||||
for i, tt := range tests {
|
||||
actual := isCveInfoUpdated(tt.in.cveID, tt.in.prev, tt.in.cur)
|
||||
if actual != tt.expected {
|
||||
t.Errorf("[%d] actual: %t, expected: %t", i, actual, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiff(t *testing.T) {
|
||||
atCurrent, _ := time.Parse("2006-01-02", "2014-12-31")
|
||||
atPrevious, _ := time.Parse("2006-01-02", "2014-11-31")
|
||||
var tests = []struct {
|
||||
inCurrent models.ScanResults
|
||||
inPrevious models.ScanResults
|
||||
out models.ScanResult
|
||||
}{
|
||||
{
|
||||
inCurrent: models.ScanResults{
|
||||
{
|
||||
ScannedAt: atCurrent,
|
||||
ServerName: "u16",
|
||||
Family: "ubuntu",
|
||||
Release: "16.04",
|
||||
ScannedCves: models.VulnInfos{
|
||||
"CVE-2012-6702": {
|
||||
CveID: "CVE-2012-6702",
|
||||
PackageNames: []string{"libexpat1"},
|
||||
DistroAdvisories: []models.DistroAdvisory{},
|
||||
CpeNames: []string{},
|
||||
},
|
||||
"CVE-2014-9761": {
|
||||
CveID: "CVE-2014-9761",
|
||||
PackageNames: []string{"libc-bin"},
|
||||
DistroAdvisories: []models.DistroAdvisory{},
|
||||
CpeNames: []string{},
|
||||
},
|
||||
},
|
||||
Packages: models.Packages{},
|
||||
Errors: []string{},
|
||||
Optional: [][]interface{}{},
|
||||
},
|
||||
},
|
||||
inPrevious: models.ScanResults{
|
||||
{
|
||||
ScannedAt: atPrevious,
|
||||
ServerName: "u16",
|
||||
Family: "ubuntu",
|
||||
Release: "16.04",
|
||||
ScannedCves: models.VulnInfos{
|
||||
"CVE-2012-6702": {
|
||||
CveID: "CVE-2012-6702",
|
||||
PackageNames: []string{"libexpat1"},
|
||||
DistroAdvisories: []models.DistroAdvisory{},
|
||||
CpeNames: []string{},
|
||||
},
|
||||
"CVE-2014-9761": {
|
||||
CveID: "CVE-2014-9761",
|
||||
PackageNames: []string{"libc-bin"},
|
||||
DistroAdvisories: []models.DistroAdvisory{},
|
||||
CpeNames: []string{},
|
||||
},
|
||||
},
|
||||
Packages: models.Packages{},
|
||||
Errors: []string{},
|
||||
Optional: [][]interface{}{},
|
||||
},
|
||||
},
|
||||
out: models.ScanResult{
|
||||
ScannedAt: atCurrent,
|
||||
ServerName: "u16",
|
||||
Family: "ubuntu",
|
||||
Release: "16.04",
|
||||
Packages: models.Packages{},
|
||||
ScannedCves: models.VulnInfos{},
|
||||
Errors: []string{},
|
||||
Optional: [][]interface{}{},
|
||||
},
|
||||
},
|
||||
{
|
||||
inCurrent: models.ScanResults{
|
||||
{
|
||||
ScannedAt: atCurrent,
|
||||
ServerName: "u16",
|
||||
Family: "ubuntu",
|
||||
Release: "16.04",
|
||||
ScannedCves: models.VulnInfos{
|
||||
"CVE-2016-6662": {
|
||||
CveID: "CVE-2016-6662",
|
||||
PackageNames: []string{"mysql-libs"},
|
||||
DistroAdvisories: []models.DistroAdvisory{},
|
||||
CpeNames: []string{},
|
||||
},
|
||||
},
|
||||
Packages: models.Packages{
|
||||
"mysql-libs": {
|
||||
Name: "mysql-libs",
|
||||
Version: "5.1.73",
|
||||
Release: "7.el6",
|
||||
NewVersion: "5.1.73",
|
||||
NewRelease: "8.el6_8",
|
||||
Repository: "",
|
||||
Changelog: models.Changelog{
|
||||
Contents: "",
|
||||
Method: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
inPrevious: models.ScanResults{
|
||||
{
|
||||
ScannedAt: atPrevious,
|
||||
ServerName: "u16",
|
||||
Family: "ubuntu",
|
||||
Release: "16.04",
|
||||
ScannedCves: models.VulnInfos{},
|
||||
},
|
||||
},
|
||||
out: models.ScanResult{
|
||||
ScannedAt: atCurrent,
|
||||
ServerName: "u16",
|
||||
Family: "ubuntu",
|
||||
Release: "16.04",
|
||||
ScannedCves: models.VulnInfos{
|
||||
"CVE-2016-6662": {
|
||||
CveID: "CVE-2016-6662",
|
||||
PackageNames: []string{"mysql-libs"},
|
||||
DistroAdvisories: []models.DistroAdvisory{},
|
||||
CpeNames: []string{},
|
||||
},
|
||||
},
|
||||
Packages: models.Packages{
|
||||
"mysql-libs": {
|
||||
Name: "mysql-libs",
|
||||
Version: "5.1.73",
|
||||
Release: "7.el6",
|
||||
NewVersion: "5.1.73",
|
||||
NewRelease: "8.el6_8",
|
||||
Repository: "",
|
||||
Changelog: models.Changelog{
|
||||
Contents: "",
|
||||
Method: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, tt := range tests {
|
||||
diff, _ := diff(tt.inCurrent, tt.inPrevious)
|
||||
for _, actual := range diff {
|
||||
if !reflect.DeepEqual(actual.ScannedCves, tt.out.ScannedCves) {
|
||||
h := pp.Sprint(actual.ScannedCves)
|
||||
x := pp.Sprint(tt.out.ScannedCves)
|
||||
t.Errorf("[%d] cves actual: \n %s \n expected: \n %s", i, h, x)
|
||||
}
|
||||
|
||||
for j := range tt.out.Packages {
|
||||
if !reflect.DeepEqual(tt.out.Packages[j], actual.Packages[j]) {
|
||||
h := pp.Sprint(tt.out.Packages[j])
|
||||
x := pp.Sprint(actual.Packages[j])
|
||||
t.Errorf("[%d] packages actual: \n %s \n expected: \n %s", i, x, h)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user