Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Check existence and last modified time of local OVAL database when reporting

Browse Source
This commit is contained in:
kota kanbe
2017-07-25 17:25:09 +09:00
parent ffbaa0a508
commit 26e447f11a
9 changed files with 147 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
Gopkg.lock generated
View File

@@ -4,8 +4,8 @@
[[projects]]
name = "github.com/Azure/azure-sdk-for-go"
packages = ["storage"]
revision = "59c277f1b488b81b1a5f944212f25b69bea8ece3"
version = "v10.1.0-beta"
revision = "5b6066bbd213e47c49a5fa2be2b29529bbcb6704"
version = "v10.1.1-beta"
[[projects]]
name = "github.com/Azure/go-autorest"
@@ -28,8 +28,8 @@
[[projects]]
name = "github.com/aws/aws-sdk-go"
packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/shareddefaults","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"]
revision = "6d7fc1a00fcae6bbb53550f4a0b98324fd7aa250"
version = "v1.10.12"
revision = "6ff7be1a127941b8dc9a408ac9c2dd6eb2167a5d"
version = "v1.10.15"
[[projects]]
name = "github.com/boltdb/bolt"
@@ -64,8 +64,8 @@
[[projects]]
name = "github.com/go-redis/redis"
packages = [".","internal","internal/consistenthash","internal/hashtag","internal/pool","internal/proto"]
revision = "da63fe7def48e378caf9539abf64b9b1e37bc01e"
version = "v6.5.3"
revision = "a005081ecd2d0d963a1a20efda049223205cf90a"
version = "v6.5.4"
[[projects]]
name = "github.com/go-sql-driver/mysql"
@@ -149,7 +149,7 @@
branch = "master"
name = "github.com/kotakanbe/goval-dictionary"
packages = ["config","db","db/rdb","log","models"]
revision = "adf0b39cd7fea8f4493f7b65e7316179634be95d"
revision = "2c949ba2967dcd35574f2a78a12551c5326de6a9"
[[projects]]
branch = "master"
@@ -239,7 +239,7 @@
branch = "master"
name = "github.com/sirupsen/logrus"
packages = ["."]
revision = "5ff5dd844dfeb4e23e27528f79f1f845bc8bb78f"
revision = "3eef8ce63d02f65d2da43214faf7bb19b0b2bb7a"
[[projects]]
branch = "master"
@@ -263,19 +263,19 @@
branch = "master"
name = "golang.org/x/crypto"
packages = ["curve25519","ed25519","ed25519/internal/edwards25519","ssh","ssh/agent","ssh/terminal"]
revision = "7f7c0c2d75ebb4e32a21396ce36e87b6dadc91c9"
revision = "6914964337150723782436d56b3f21610a74ce7b"
[[projects]]
branch = "master"
name = "golang.org/x/net"
packages = ["context","idna","publicsuffix"]
revision = "b3756b4b77d7b13260a0a2ec658753cf48922eac"
revision = "ab5485076ff3407ad2d02db054635913f017b0ed"
[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = ["unix"]
revision = "4cd6d1a821c7175768725b55ca82f14683a29ea4"
revision = "c4489faa6e5ab84c0ef40d6ee878f7a030281f0f"
[[projects]]
branch = "master"

2
commands/report.go
View File

@@ -410,7 +410,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
}
if c.Conf.OvalDBURL != "" {
err := oval.Base{}.CheckHealth()
err := oval.Base{}.CheckHTTPHealth()
if err != nil {
util.Log.Errorf("OVAL HTTP server is not running. err: %s", err)
util.Log.Errorf("Run goval-dictionary as server mode before reporting or run with --ovaldb-path option")

8
oval/debian.go
View File

@@ -25,7 +25,7 @@ func (o DebianBase) fillFromOvalDB(r *models.ScanResult) error {
} else {
ovalconf.Conf.DBPath = config.Conf.OvalDBURL
}
util.Log.Infof("open oval-dictionary db (%s): %s",
util.Log.Infof("Open oval-dictionary db (%s): %s",
ovalconf.Conf.DBType, ovalconf.Conf.DBPath)
ovallog.Initialize(config.Conf.LogDir)
@@ -68,7 +68,7 @@ func (o DebianBase) update(r *models.ScanResult, definition *ovalmodels.Definiti
ovalContent.Type = models.NewCveContentType(r.Family)
vinfo, ok := r.ScannedCves[definition.Debian.CveID]
if !ok {
util.Log.Infof("%s is newly detected by OVAL", definition.Debian.CveID)
util.Log.Debugf("%s is newly detected by OVAL", definition.Debian.CveID)
vinfo = models.VulnInfo{
CveID: definition.Debian.CveID,
Confidence: models.OvalMatch,
@@ -79,9 +79,9 @@ func (o DebianBase) update(r *models.ScanResult, definition *ovalmodels.Definiti
cveContents := vinfo.CveContents
ctype := models.NewCveContentType(r.Family)
if _, ok := vinfo.CveContents[ctype]; ok {
util.Log.Infof("%s will be updated by OVAL", definition.Debian.CveID)
util.Log.Debugf("%s will be updated by OVAL", definition.Debian.CveID)
} else {
util.Log.Infof("%s is also detected by OVAL", definition.Debian.CveID)
util.Log.Debugf("%s is also detected by OVAL", definition.Debian.CveID)
cveContents = models.CveContents{}
}
if vinfo.Confidence.Score < models.OvalMatch.Score {

88
oval/oval.go
View File

@@ -11,21 +11,27 @@ import (
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
ver "github.com/knqyf263/go-deb-version"
"github.com/kotakanbe/goval-dictionary/db"
ovallog "github.com/kotakanbe/goval-dictionary/log"
ovalmodels "github.com/kotakanbe/goval-dictionary/models"
"github.com/parnurzeal/gorequest"
)
// Client is the interface of OVAL client.
type Client interface {
CheckHealth() error
CheckHTTPHealth() error
FillWithOval(r *models.ScanResult) error
// CheckIfOvalFetched checks if oval entries are in DB by family, release.
CheckIfOvalFetched(string, string) (bool, error)
CheckIfOvalFresh(string, string) (bool, error)
}
// Base is a base struct
type Base struct{}
// CheckHealth do health check
func (b Base) CheckHealth() error {
// CheckHTTPHealth do health check
func (b Base) CheckHTTPHealth() error {
if !b.isFetchViaHTTP() {
return nil
}
@@ -43,6 +49,82 @@ func (b Base) CheckHealth() error {
return nil
}
// CheckIfOvalFetched checks if oval entries are in DB by family, release.
func (b Base) CheckIfOvalFetched(osFamily, release string) (fetched bool, err error) {
ovallog.Initialize(config.Conf.LogDir)
if !b.isFetchViaHTTP() {
var ovaldb db.DB
if ovaldb, err = db.NewDB(
osFamily,
config.Conf.OvalDBType,
config.Conf.OvalDBPath,
config.Conf.DebugSQL,
); err != nil {
return false, err
}
defer ovaldb.CloseDB()
count, err := ovaldb.CountDefs(osFamily, release)
if err != nil {
return false, fmt.Errorf("Failed to count OVAL defs: %s, %s, %v",
osFamily, release, err)
}
return 0 < count, nil
}
url, _ := util.URLPathJoin(config.Conf.OvalDBURL, "count", osFamily, release)
resp, body, errs := gorequest.New().Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
return false, fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v",
errs, url, resp)
}
count := 0
if err := json.Unmarshal([]byte(body), &count); err != nil {
return false, fmt.Errorf("Failed to Unmarshall. body: %s, err: %s",
body, err)
}
return 0 < count, nil
}
// CheckIfOvalFresh checks if oval entries are fresh enough
func (b Base) CheckIfOvalFresh(osFamily, release string) (ok bool, err error) {
ovallog.Initialize(config.Conf.LogDir)
var lastModified time.Time
if !b.isFetchViaHTTP() {
var ovaldb db.DB
if ovaldb, err = db.NewDB(
osFamily,
config.Conf.OvalDBType,
config.Conf.OvalDBPath,
config.Conf.DebugSQL,
); err != nil {
return false, err
}
defer ovaldb.CloseDB()
lastModified = ovaldb.GetLastModified(osFamily, release)
} else {
url, _ := util.URLPathJoin(config.Conf.OvalDBURL, "lastmodified", osFamily, release)
resp, body, errs := gorequest.New().Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
return false, fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v",
errs, url, resp)
}
if err := json.Unmarshal([]byte(body), &lastModified); err != nil {
return false, fmt.Errorf("Failed to Unmarshall. body: %s, err: %s",
body, err)
}
}
since := time.Now()
since = since.AddDate(0, 0, -3)
if lastModified.Before(since) {
util.Log.Warnf("%s-%s OVAL is old, last modified is %s. It's recommended to update OVAL to improve scanning accuracy. To update OVAL database, see https://github.com/kotakanbe/goval-dictionary#usage",
osFamily, release, lastModified)
return false, nil
}
return true, nil
}
func (b Base) isFetchViaHTTP() bool {
// Default value of OvalDBType is sqlite3
if config.Conf.OvalDBURL != "" && config.Conf.OvalDBType == "sqlite3" {

8
oval/redhat.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
ver "github.com/knqyf263/go-deb-version"
ver "github.com/knqyf263/go-rpm-version"
ovalconf "github.com/kotakanbe/goval-dictionary/config"
db "github.com/kotakanbe/goval-dictionary/db"
ovallog "github.com/kotakanbe/goval-dictionary/log"
@@ -63,7 +63,7 @@ func (o RedHatBase) getDefsByPackNameFromOvalDB(osRelease string,
} else {
ovalconf.Conf.DBPath = config.Conf.OvalDBURL
}
util.Log.Infof("open oval-dictionary db (%s): %s",
util.Log.Infof("Open oval-dictionary db (%s): %s",
ovalconf.Conf.DBType, ovalconf.Conf.DBPath)
ovallog.Initialize(config.Conf.LogDir)
@@ -84,9 +84,9 @@ func (o RedHatBase) getDefsByPackNameFromOvalDB(osRelease string,
return nil, fmt.Errorf("Failed to get RedHat OVAL info by package name: %v", err)
}
for _, def := range definitions {
current, _ := ver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release))
current := ver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release))
for _, p := range def.AffectedPacks {
affected, _ := ver.NewVersion(p.Version)
affected := ver.NewVersion(p.Version)
if pack.Name != p.Name || !current.LessThan(affected) {
continue
}

2
report/cve_client.go
View File

@@ -147,7 +147,7 @@ func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails [
return []*cve.CveDetail{}, fmt.Errorf("Failed to New DB. err: %s", err)
}
log.Infof("Opening DB (%s).", driver.Name())
util.Log.Infof("Opening DB (%s).", driver.Name())
if err := driver.OpenDB(
cveconfig.Conf.DBType,
cveconfig.Conf.DBPath,

35
report/report.go
View File

@@ -79,12 +79,12 @@ func FillCveInfos(rs []models.ScanResult, dir string) ([]models.ScanResult, erro
func fillCveInfo(r *models.ScanResult) error {
util.Log.Debugf("need to refresh")
util.Log.Debugf("Fill CVE detailed information with OVAL")
util.Log.Infof("Fill CVE detailed information with OVAL")
if err := fillWithOval(r); err != nil {
return fmt.Errorf("Failed to fill OVAL information: %s", err)
}
util.Log.Debugf("Fill CVE detailed information with CVE-DB")
util.Log.Infof("Fill CVE detailed information with CVE-DB")
if err := fillWithCveDB(r); err != nil {
return fmt.Errorf("Failed to fill CVE information: %s", err)
}
@@ -139,23 +139,50 @@ func fillWithCveDB(r *models.ScanResult) error {
return nil
}
func fillWithOval(r *models.ScanResult) error {
func fillWithOval(r *models.ScanResult) (err error) {
var ovalClient oval.Client
var ovalFamily string
// TODO
switch r.Family {
case c.Debian:
ovalClient = oval.NewDebian()
ovalFamily = c.Debian
case c.Ubuntu:
ovalClient = oval.NewUbuntu()
ovalFamily = c.Ubuntu
case c.RedHat:
ovalClient = oval.NewRedhat()
ovalFamily = c.RedHat
case c.CentOS:
ovalClient = oval.NewCentOS()
//use RedHat's OVAL
ovalFamily = c.RedHat
//TODO implement OracleLinux
// case c.Oracle:
// ovalClient = oval.New()
// ovalFamily = c.Oracle
case c.Amazon, c.Oracle, c.Raspbian, c.FreeBSD:
//TODO implement OracleLinux
return nil
default:
return fmt.Errorf("Oval %s is not implemented yet", r.Family)
}
ok, err := ovalClient.CheckIfOvalFetched(ovalFamily, r.Release)
if err != nil {
return err
}
if !ok {
util.Log.Warnf("OVAL is emtpy: %s-%s. It's recommended to use OVAL to improve scanning accuracy. To fetch OVAL database, see https://github.com/kotakanbe/goval-dictionary#usage", r.Family, r.Release)
return nil
}
_, err = ovalClient.CheckIfOvalFresh(ovalFamily, r.Release)
if err != nil {
return err
}
util.Log.Infof("OVAL is fresh: %s-%s ", r.Family, r.Release)
if err := ovalClient.FillWithOval(r); err != nil {
return err
}

4
report/tui.go
View File

@@ -47,6 +47,10 @@ func RunTui(results models.ScanResults) subcommands.ExitStatus {
// g, err := gocui.NewGui(gocui.OutputNormal)
g := gocui.NewGui()
if err := g.Init(); err != nil {
log.Errorf("%s", err)
return subcommands.ExitFailure
}
defer g.Close()
g.SetLayout(layout)

12
scan/redhat.go
View File

@@ -254,12 +254,6 @@ func (o *redhat) scanPackages() error {
return nil
}
//TODO Cache changelogs to bolt
//TODO --with-changelog
if err := o.fillChangelogs(updatable); err != nil {
return nil
}
var vinfos models.VulnInfos
if vinfos, err = o.scanUnsecurePackages(updatable); err != nil {
o.log.Errorf("Failed to scan vulnerable packages")
@@ -379,6 +373,12 @@ func (o *redhat) parseUpdatablePacksLine(line string) (models.Package, error) {
}
func (o *redhat) scanUnsecurePackages(updatable models.Packages) (models.VulnInfos, error) {
//TODO Cache changelogs to bolt
//TODO --with-changelog
if err := o.fillChangelogs(updatable); err != nil {
return nil, err
}
if o.Distro.Family != config.CentOS {
// Amazon, RHEL, Oracle Linux has yum updateinfo as default
// yum updateinfo can collenct vendor advisory information.