add Library Scan (with image scan) (#829)

* add static container image scan

* server has many staticContainers

* use go module

* for staticContainer

* fix typo

* fix setErrs error

* change name : StaticContainer -> Image

* add scan -images-only flag

* fix makefile

* fix makefile for go module

* use rpmcmd instead of rpm

* add scrutinizer.yml

* change scrutinizer.yml

* fix scrutinizer.yml

* fix scrutinizer.yml

* fix scrutinizer.yml

* fix scrutinizer.yml

* delete scrutinizer

* add report test

* add sourcePackages and Arch

* fix for sider

* fix staticContainer -> image

* init scan library

* add library scan for servers

* fix tui bug

* fix lint error

* divide WpPackageFixStats and LibraryPackageFixedIns

* fix error

* Delete libManager_test.go

* stop use alpine os if err occurred in container

* merge upstream/master

* Delete libManager.go

* update goval-dictionary

* fix go.mod

* update Readme

* add feature : auto detect lockfiles

models/cvecontents.go

@@ -19,6 +19,8 @@ package models
 
 import (
 	"time"
+
+	"github.com/knqyf263/trivy/pkg/vulnsrc/vulnerability"
 )
 
 // CveContents has CveContent
@@ -226,7 +228,7 @@ func NewCveContentType(name string) CveContentType {
 		return Oracle
 	case "ubuntu":
 		return Ubuntu
-	case "debian":
+	case "debian", vulnerability.DebianOVAL:
 		return Debian
 	case "redhat_api":
 		return RedHatAPI
@@ -238,6 +240,16 @@ func NewCveContentType(name string) CveContentType {
 		return WPVulnDB
 	case "amazon":
 		return Amazon
+	case vulnerability.NodejsSecurityWg:
+		return NodeSec
+	case vulnerability.PythonSafetyDB:
+		return PythonSec
+	case vulnerability.RustSec:
+		return RustSec
+	case vulnerability.PhpSecurityAdvisories:
+		return PhpSec
+	case vulnerability.RubySec:
+		return RubySec
 	default:
 		return Unknown
 	}
@@ -283,6 +295,21 @@ const (
 	// WPVulnDB is WordPress
 	WPVulnDB CveContentType = "wpvulndb"
 
+	// NodeSec : for JS
+	NodeSec CveContentType = "node"
+
+	// PythonSec : for PHP
+	PythonSec CveContentType = "python"
+
+	// PhpSec : for PHP
+	PhpSec CveContentType = "php"
+
+	// RubySec : for Ruby
+	RubySec CveContentType = "ruby"
+
+	// RustSec : for Rust
+	RustSec CveContentType = "rust"
+
 	// Unknown is Unknown
 	Unknown CveContentType = "unknown"
 )
@@ -303,6 +330,11 @@ var AllCveContetTypes = CveContentTypes{
 	SUSE,
 	DebianSecurityTracker,
 	WPVulnDB,
+	NodeSec,
+	PythonSec,
+	PhpSec,
+	RubySec,
+	RustSec,
 }
 
 // Except returns CveContentTypes except for given args

models/library.go

@@ -0,0 +1,141 @@
+package models
+
+import (
+	"path/filepath"
+
+	"github.com/future-architect/vuls/util"
+	"github.com/knqyf263/trivy/pkg/scanner/library"
+	"github.com/knqyf263/trivy/pkg/vulnsrc/vulnerability"
+	"golang.org/x/xerrors"
+
+	"github.com/knqyf263/go-dep-parser/pkg/types"
+	"github.com/knqyf263/go-version"
+)
+
+// LibraryScanner has libraries information
+type LibraryScanner struct {
+	Path string
+	Libs []types.Library
+}
+
+// Scan : scan target library
+func (s LibraryScanner) Scan() ([]VulnInfo, error) {
+	scanner := library.NewScanner(filepath.Base(string(s.Path)))
+	if scanner == nil {
+		return nil, xerrors.New("unknown file type")
+	}
+
+	util.Log.Info("Updating library db...")
+	err := scanner.UpdateDB()
+	if err != nil {
+		return nil, xerrors.Errorf("failed to update %s advisories: %w", scanner.Type(), err)
+	}
+
+	var vulnerabilities []VulnInfo
+	for _, pkg := range s.Libs {
+		v, err := version.NewVersion(pkg.Version)
+		if err != nil {
+			util.Log.Debugf("new version cant detected %s@%s", pkg.Name, pkg.Version)
+			continue
+		}
+
+		tvulns, err := scanner.Detect(pkg.Name, v)
+		if err != nil {
+			return nil, xerrors.Errorf("failed to detect %s vulnerabilities: %w", scanner.Type(), err)
+		}
+
+		vulns := s.convertFanalToVuln(tvulns)
+		vulnerabilities = append(vulnerabilities, vulns...)
+	}
+
+	return vulnerabilities, nil
+}
+
+func (s LibraryScanner) convertFanalToVuln(tvulns []vulnerability.DetectedVulnerability) (vulns []VulnInfo) {
+	for _, tvuln := range tvulns {
+		vinfo, _ := s.getVulnDetail(tvuln)
+		vulns = append(vulns, vinfo)
+	}
+	return vulns
+}
+
+func (s LibraryScanner) getVulnDetail(tvuln vulnerability.DetectedVulnerability) (vinfo VulnInfo, err error) {
+	details, err := vulnerability.Get(tvuln.VulnerabilityID)
+	if err != nil {
+		return vinfo, err
+	} else if len(details) == 0 {
+		return vinfo, xerrors.Errorf("Unknown vulnID : %s", tvuln.VulnerabilityID)
+	}
+	vinfo.CveID = tvuln.VulnerabilityID
+	vinfo.CveContents = getCveContents(details)
+	if tvuln.FixedVersion != "" {
+
+		vinfo.LibraryFixedIns = []LibraryFixedIn{
+			{
+				Key:     s.GetLibraryKey(),
+				Name:    tvuln.PkgName,
+				FixedIn: tvuln.FixedVersion,
+			},
+		}
+	}
+	return vinfo, nil
+}
+
+func getCveContents(details map[string]vulnerability.Vulnerability) (contents map[CveContentType]CveContent) {
+	contents = map[CveContentType]CveContent{}
+	for source, detail := range details {
+		refs := []Reference{}
+		for _, refURL := range detail.References {
+			refs = append(refs, Reference{Source: refURL, Link: refURL})
+		}
+
+		content := CveContent{
+			Type:          NewCveContentType(source),
+			CveID:         detail.ID,
+			Title:         detail.Title,
+			Summary:       detail.Description,
+			Cvss3Score:    detail.CvssScoreV3,
+			Cvss3Severity: string(detail.SeverityV3),
+			Cvss2Score:    detail.CvssScore,
+			Cvss2Severity: string(detail.Severity),
+			References:    refs,
+
+			//SourceLink    string            `json:"sourceLink"`
+			//Cvss2Vector   string            `json:"cvss2Vector"`
+			//Cvss3Vector   string            `json:"cvss3Vector"`
+			//Cvss3Severity string            `json:"cvss3Severity"`
+			//Cpes          []Cpe             `json:"cpes,omitempty"`
+			//CweIDs        []string          `json:"cweIDs,omitempty"`
+			//Published     time.Time         `json:"published"`
+			//LastModified  time.Time         `json:"lastModified"`
+			//Mitigation    string            `json:"mitigation"` // RedHat API
+			//Optional      map[string]string `json:"optional,omitempty"`
+		}
+		contents[NewCveContentType(source)] = content
+	}
+	return contents
+}
+
+// LibraryMap is filename and library type
+var LibraryMap = map[string]string{
+	"package-lock.json": "node",
+	"yarn.lock":         "node",
+	"Gemfile.lock":      "ruby",
+	"Cargo.lock":        "rust",
+	"composer.json":     "php",
+	"Pipfile.lock":      "python",
+	"poetry.lock":       "python",
+}
+
+// GetLibraryKey returns target library key
+func (s LibraryScanner) GetLibraryKey() string {
+	fileName := filepath.Base(s.Path)
+	return LibraryMap[fileName]
+}
+
+// LibraryFixedIn has library fixed information
+type LibraryFixedIn struct {
+	Key     string `json:"key,omitempty"`
+	Name    string `json:"name,omitempty"`
+	FixedIn string `json:"fixedIn,omitempty"`
+}

models/library_test.go

@@ -0,0 +1,52 @@
+package models
+
+import (
+	"testing"
+
+	godeptypes "github.com/knqyf263/go-dep-parser/pkg/types"
+	"github.com/knqyf263/trivy/pkg/db"
+	"github.com/knqyf263/trivy/pkg/log"
+)
+
+func TestScan(t *testing.T) {
+	var tests = []struct {
+		path string
+		pkgs []godeptypes.Library
+	}{
+		{
+			path: "app/package-lock.json",
+			pkgs: []godeptypes.Library{
+				{
+					Name:    "jquery",
+					Version: "2.2.4",
+				},
+				{
+					Name:    "@babel/traverse",
+					Version: "7.4.4",
+				},
+			},
+		},
+	}
+
+	if err := log.InitLogger(false); err != nil {
+		t.Errorf("trivy logger failed")
+	}
+
+	if err := db.Init(); err != nil {
+		t.Errorf("trivy db.Init failed")
+	}
+	for _, v := range tests {
+		lib := LibraryScanner{
+			Path: v.path,
+			Libs: v.pkgs,
+		}
+		actual, err := lib.Scan()
+		if err != nil {
+			t.Errorf("error occurred")
+		}
+		if len(actual) == 0 {
+			t.Errorf("no vuln found : actual: %v\n", actual)
+		}
+	}
+	db.Close()
+}

models/scanresults.go

@@ -43,6 +43,7 @@ type ScanResult struct {
 	Family           string    `json:"family"`
 	Release          string    `json:"release"`
 	Container        Container `json:"container"`
+	Image            Image     `json:"image"`
 	Platform         Platform  `json:"platform"`
 	IPv4Addrs        []string  `json:"ipv4Addrs,omitempty"` // only global unicast address (https://golang.org/pkg/net/#IP.IsGlobalUnicast)
 	IPv6Addrs        []string  `json:"ipv6Addrs,omitempty"` // only global unicast address (https://golang.org/pkg/net/#IP.IsGlobalUnicast)
@@ -65,6 +66,7 @@ type ScanResult struct {
 	Packages          Packages               `json:"packages"`
 	SrcPackages       SrcPackages            `json:",omitempty"`
 	WordPressPackages *WordPressPackages     `json:",omitempty"`
+	LibraryScanners   []LibraryScanner       `json:"libScanners"`
 	CweDict           CweDict                `json:"cweDict,omitempty"`
 	Optional          map[string]interface{} `json:",omitempty"`
 	Config            struct {
@@ -439,6 +441,11 @@ func (r ScanResult) IsContainer() bool {
 	return 0 < len(r.Container.ContainerID)
 }
 
+// IsImage returns whether this ServerInfo is about container
+func (r ScanResult) IsImage() bool {
+	return 0 < len(r.Image.Name)
+}
+
 // IsDeepScanMode checks if the scan mode is deep scan mode.
 func (r ScanResult) IsDeepScanMode() bool {
 	for _, s := range r.Config.Scan.Servers {
@@ -460,6 +467,12 @@ type Container struct {
 	UUID        string `json:"uuid"`
 }
 
+// Image has Container information
+type Image struct {
+	Name string `json:"name"`
+	Tag  string `json:"tag"`
+}
+
 // Platform has platform information
 type Platform struct {
 	Name       string `json:"name"` // aws or azure or gcp or other...

models/vulninfos.go

@@ -172,6 +172,7 @@ type VulnInfo struct {
 	CpeURIs              []string             `json:"cpeURIs,omitempty"` // CpeURIs related to this CVE defined in config.toml
 	GitHubSecurityAlerts GitHubSecurityAlerts `json:"gitHubSecurityAlerts,omitempty"`
 	WpPackageFixStats    WpPackageFixStats    `json:"wpPackageFixStats,omitempty"`
+	LibraryFixedIns      LibraryFixedIns      `json:"libraryFixedIns,omitempty"`
 
 	VulnType string `json:"vulnType,omitempty"`
 }
@@ -207,6 +208,9 @@ type GitHubSecurityAlert struct {
 	DismissReason string    `json:"dismissReason"`
 }
 
+// LibraryFixedIns is a list of Library's FixedIn
+type LibraryFixedIns []LibraryFixedIn
+
 // WpPackageFixStats is a list of WpPackageFixStatus
 type WpPackageFixStats []WpPackageFixStatus