Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(saas): support for library-only scanning (#1300)

Browse Source
This commit is contained in:
Kota Kanbe
2021-09-10 15:38:35 +09:00
committed by GitHub
parent 9ed5f2cac5
commit 4a72295de7
9 changed files with 3678 additions and 3316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
contrib/trivy/parser/parser.go
View File

@@ -6,8 +6,11 @@ import (
"time"
"github.com/aquasecurity/fanal/analyzer/os"
ftypes "github.com/aquasecurity/fanal/types"
"github.com/aquasecurity/trivy/pkg/report"
"github.com/aquasecurity/trivy/pkg/types"
"github.com/future-architect/vuls/constant"
"github.com/future-architect/vuls/models"
)
@@ -22,9 +25,7 @@ func Parse(vulnJSON []byte, scanResult *models.ScanResult) (result *models.ScanR
vulnInfos := models.VulnInfos{}
uniqueLibraryScannerPaths := map[string]models.LibraryScanner{}
for _, trivyResult := range trivyResults {
if IsTrivySupportedOS(trivyResult.Type) {
overrideServerData(scanResult, &trivyResult)
}
setScanResultMeta(scanResult, &trivyResult)
for _, vuln := range trivyResult.Vulnerabilities {
if _, ok := vulnInfos[vuln.VulnerabilityID]; !ok {
vulnInfos[vuln.VulnerabilityID] = models.VulnInfo{
@@ -81,7 +82,7 @@ func Parse(vulnJSON []byte, scanResult *models.ScanResult) (result *models.ScanR
}},
}
// do only if image type is Vuln
if IsTrivySupportedOS(trivyResult.Type) {
if isTrivySupportedOS(trivyResult.Type) {
pkgs[vuln.PkgName] = models.Package{
Name: vuln.PkgName,
Version: vuln.InstalledVersion,
@@ -93,7 +94,6 @@ func Parse(vulnJSON []byte, scanResult *models.ScanResult) (result *models.ScanR
FixedIn: vuln.FixedVersion,
})
} else {
// LibraryScanの結果
vulnInfo.LibraryFixedIns = append(vulnInfo.LibraryFixedIns, models.LibraryFixedIn{
Key: trivyResult.Type,
Name: vuln.PkgName,
@@ -101,6 +101,7 @@ func Parse(vulnJSON []byte, scanResult *models.ScanResult) (result *models.ScanR
FixedIn: vuln.FixedVersion,
})
libScanner := uniqueLibraryScannerPaths[trivyResult.Target]
libScanner.Type = trivyResult.Type
libScanner.Libs = append(libScanner.Libs, types.Library{
Name: vuln.PkgName,
Version: vuln.InstalledVersion,
@@ -128,6 +129,7 @@ func Parse(vulnJSON []byte, scanResult *models.ScanResult) (result *models.ScanR
})
libscanner := models.LibraryScanner{
Type: v.Type,
Path: path,
Libs: libraries,
}
@@ -142,39 +144,70 @@ func Parse(vulnJSON []byte, scanResult *models.ScanResult) (result *models.ScanR
return scanResult, nil
}
// IsTrivySupportedOS :
func IsTrivySupportedOS(family string) bool {
supportedFamilies := []string{
os.RedHat,
os.Debian,
os.Ubuntu,
os.CentOS,
os.Fedora,
os.Amazon,
os.Oracle,
os.Windows,
os.OpenSUSE,
os.OpenSUSELeap,
os.OpenSUSETumbleweed,
os.SLES,
os.Photon,
os.Alpine,
}
for _, supportedFamily := range supportedFamilies {
if family == supportedFamily {
return true
}
}
return false
}
const trivyTarget = "trivy-target"
func overrideServerData(scanResult *models.ScanResult, trivyResult *report.Result) {
scanResult.Family = trivyResult.Type
scanResult.ServerName = trivyResult.Target
scanResult.Optional = map[string]interface{}{
"trivy-target": trivyResult.Target,
func setScanResultMeta(scanResult *models.ScanResult, trivyResult *report.Result) {
if isTrivySupportedOS(trivyResult.Type) {
scanResult.Family = trivyResult.Type
scanResult.ServerName = trivyResult.Target
scanResult.Optional = map[string]interface{}{
trivyTarget: trivyResult.Target,
}
} else if isTrivySupportedLib(trivyResult.Type) {
if scanResult.Family == "" {
scanResult.Family = constant.ServerTypePseudo
}
if scanResult.ServerName == "" {
scanResult.ServerName = "library scan by trivy"
}
if _, ok := scanResult.Optional[trivyTarget]; !ok {
scanResult.Optional = map[string]interface{}{
trivyTarget: trivyResult.Target,
}
}
}
scanResult.ScannedAt = time.Now()
scanResult.ScannedBy = "trivy"
scanResult.ScannedVia = "trivy"
}
// isTrivySupportedOS :
func isTrivySupportedOS(family string) bool {
supportedFamilies := map[string]interface{}{
os.RedHat: struct{}{},
os.Debian: struct{}{},
os.Ubuntu: struct{}{},
os.CentOS: struct{}{},
os.Fedora: struct{}{},
os.Amazon: struct{}{},
os.Oracle: struct{}{},
os.Windows: struct{}{},
os.OpenSUSE: struct{}{},
os.OpenSUSELeap: struct{}{},
os.OpenSUSETumbleweed: struct{}{},
os.SLES: struct{}{},
os.Photon: struct{}{},
os.Alpine: struct{}{},
}
_, ok := supportedFamilies[family]
return ok
}
func isTrivySupportedLib(typestr string) bool {
supportedLibs := map[string]interface{}{
ftypes.Bundler: struct{}{},
ftypes.Cargo: struct{}{},
ftypes.Composer: struct{}{},
ftypes.Npm: struct{}{},
ftypes.NuGet: struct{}{},
ftypes.Pip: struct{}{},
ftypes.Pipenv: struct{}{},
ftypes.Poetry: struct{}{},
ftypes.Yarn: struct{}{},
ftypes.Jar: struct{}{},
ftypes.GoBinary: struct{}{},
ftypes.GoMod: struct{}{},
}
_, ok := supportedLibs[typestr]
return ok
}

6479
contrib/trivy/parser/parser_test.go
View File

File diff suppressed because it is too large Load Diff

2
detector/detector.go
View File

@@ -202,7 +202,7 @@ func DetectPkgCves(r *models.ScanResult, ovalCnf config.GovalDictConf, gostCnf c
} else if r.Family == constant.ServerTypePseudo {
logging.Log.Infof("pseudo type. Skip OVAL and gost detection")
} else {
return xerrors.Errorf("Failed to fill CVEs. r.Release is empty")
logging.Log.Infof("r.Release is empty. detect as pseudo type. Skip OVAL and gost detection")
}
for i, v := range r.ScannedCves {

36
go.mod
View File

@@ -7,11 +7,11 @@ require (
github.com/BurntSushi/toml v0.3.1
github.com/Ullaakut/nmap/v2 v2.1.2-0.20210406060955-59a52fe80a4f
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/aquasecurity/fanal v0.0.0-20210719144537-c73c1e9f21bf
github.com/aquasecurity/trivy v0.19.2
github.com/aquasecurity/trivy-db v0.0.0-20210531102723-aaab62dec6ee
github.com/aquasecurity/fanal v0.0.0-20210815095355-42429a80d0e3
github.com/aquasecurity/trivy v0.19.3-0.20210909113250-19c0b70d2613
github.com/aquasecurity/trivy-db v0.0.0-20210809142931-da8e09204404
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
github.com/aws/aws-sdk-go v1.37.0
github.com/aws/aws-sdk-go v1.40.22
github.com/boltdb/bolt v1.3.1
github.com/briandowns/spinner v1.16.0 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible
@@ -50,7 +50,7 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/parnurzeal/gorequest v0.2.16
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
github.com/sirupsen/logrus v1.8.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/afero v1.6.0
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/cobra v1.2.1
@@ -58,7 +58,7 @@ require (
github.com/vulsio/go-exploitdb v0.2.0
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
golang.org/x/net v0.0.0-20210902165921-8d991716f632 // indirect
golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/text v0.3.7 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
@@ -68,38 +68,41 @@ require (
)
require (
github.com/Azure/go-autorest/autorest v0.10.2 // indirect
github.com/Azure/go-autorest/autorest/adal v0.8.3 // indirect
github.com/Azure/go-autorest/autorest/date v0.2.0 // indirect
github.com/Azure/go-autorest/logger v0.1.0 // indirect
github.com/Azure/go-autorest/tracing v0.5.0 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.1 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.5 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.0 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/sprig v2.22.0+incompatible // indirect
github.com/aquasecurity/go-dep-parser v0.0.0-20210520015931-0dd56983cc62 // indirect
github.com/aquasecurity/go-dep-parser v0.0.0-20210815080135-5be65146849a // indirect
github.com/aquasecurity/go-gem-version v0.0.0-20201115065557-8eed6fe000ce // indirect
github.com/aquasecurity/go-npm-version v0.0.0-20201110091526-0b796d180798 // indirect
github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 // indirect
github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 // indirect
github.com/caarlos0/env/v6 v6.0.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
github.com/go-redis/redis v6.15.9+incompatible // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-github/v33 v33.0.0 // indirect
github.com/google/go-querystring v1.0.0 // indirect
github.com/google/uuid v1.2.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/google/wire v0.4.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/grokify/html-strip-tags-go v0.0.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/htcat/htcat v1.0.2 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/imdario/mergo v0.3.9 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/log15 v0.0.0-20201112154412-8562bdadbbac // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
@@ -113,7 +116,6 @@ require (
github.com/jinzhu/now v1.1.2 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/labstack/gommon v0.3.0 // indirect
github.com/magefile/mage v1.11.0 // indirect
github.com/magiconair/properties v1.8.5 // indirect
github.com/masahiro331/go-mvn-version v0.0.0-20210429150710-d3157d602a08 // indirect
github.com/mattn/go-colorable v0.1.8 // indirect
@@ -135,7 +137,7 @@ require (
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.1 // indirect
github.com/ymomoi/goval-parser v0.0.0-20170813122243-0a0be1dd9d08 // indirect
go.etcd.io/bbolt v1.3.5 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.17.0 // indirect

328
go.sum
View File

File diff suppressed because it is too large Load Diff

1
models/cvecontents.go
View File

@@ -229,6 +229,7 @@ func (v CveContents) UniqCweIDs(myFamily string) (values []CveContentStr) {
return values
}
// Sort elements for integration-testing
func (v CveContents) Sort() {
for contType, contents := range v {
// CVSS3 desc, CVSS2 desc, SourceLink asc

2
models/vulninfos.go
View File

@@ -893,7 +893,7 @@ const (
// ChangelogExactMatchStr :
ChangelogExactMatchStr = "ChangelogExactMatch"
// ChangelogRoughMatch :
// ChangelogRoughMatchStr :
ChangelogRoughMatchStr = "ChangelogRoughMatch"
// GitHubMatchStr :

24
scanner/base.go
View File

@@ -26,14 +26,22 @@ import (
"golang.org/x/xerrors"
// Import library scanner
_ "github.com/aquasecurity/fanal/analyzer/library/bundler"
_ "github.com/aquasecurity/fanal/analyzer/library/cargo"
_ "github.com/aquasecurity/fanal/analyzer/library/composer"
_ "github.com/aquasecurity/fanal/analyzer/library/gomod"
_ "github.com/aquasecurity/fanal/analyzer/library/npm"
_ "github.com/aquasecurity/fanal/analyzer/library/pipenv"
_ "github.com/aquasecurity/fanal/analyzer/library/poetry"
_ "github.com/aquasecurity/fanal/analyzer/library/yarn"
_ "github.com/aquasecurity/fanal/analyzer/language/dotnet/nuget"
_ "github.com/aquasecurity/fanal/analyzer/language/golang/binary"
_ "github.com/aquasecurity/fanal/analyzer/language/golang/mod"
_ "github.com/aquasecurity/fanal/analyzer/language/java/jar"
_ "github.com/aquasecurity/fanal/analyzer/language/nodejs/npm"
_ "github.com/aquasecurity/fanal/analyzer/language/nodejs/yarn"
_ "github.com/aquasecurity/fanal/analyzer/language/php/composer"
_ "github.com/aquasecurity/fanal/analyzer/language/python/pip"
_ "github.com/aquasecurity/fanal/analyzer/language/python/pipenv"
_ "github.com/aquasecurity/fanal/analyzer/language/python/poetry"
_ "github.com/aquasecurity/fanal/analyzer/language/ruby/bundler"
_ "github.com/aquasecurity/fanal/analyzer/language/rust/cargo"
// _ "github.com/aquasecurity/fanal/analyzer/language/ruby/gemspec"
// _ "github.com/aquasecurity/fanal/analyzer/language/nodejs/pkg"
// _ "github.com/aquasecurity/fanal/analyzer/language/python/packaging"
nmap "github.com/Ullaakut/nmap/v2"
)

19
scanner/base_test.go
View File

@@ -4,13 +4,18 @@ import (
"reflect"
"testing"
_ "github.com/aquasecurity/fanal/analyzer/library/bundler"
_ "github.com/aquasecurity/fanal/analyzer/library/cargo"
_ "github.com/aquasecurity/fanal/analyzer/library/composer"
_ "github.com/aquasecurity/fanal/analyzer/library/npm"
_ "github.com/aquasecurity/fanal/analyzer/library/pipenv"
_ "github.com/aquasecurity/fanal/analyzer/library/poetry"
_ "github.com/aquasecurity/fanal/analyzer/library/yarn"
_ "github.com/aquasecurity/fanal/analyzer/language/dotnet/nuget"
_ "github.com/aquasecurity/fanal/analyzer/language/golang/binary"
_ "github.com/aquasecurity/fanal/analyzer/language/golang/mod"
_ "github.com/aquasecurity/fanal/analyzer/language/java/jar"
_ "github.com/aquasecurity/fanal/analyzer/language/nodejs/npm"
_ "github.com/aquasecurity/fanal/analyzer/language/nodejs/yarn"
_ "github.com/aquasecurity/fanal/analyzer/language/php/composer"
_ "github.com/aquasecurity/fanal/analyzer/language/python/pip"
_ "github.com/aquasecurity/fanal/analyzer/language/python/pipenv"
_ "github.com/aquasecurity/fanal/analyzer/language/python/poetry"
_ "github.com/aquasecurity/fanal/analyzer/language/ruby/bundler"
_ "github.com/aquasecurity/fanal/analyzer/language/rust/cargo"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
)