Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add Library Scan (with image scan) (#829)

Browse Source 
* add static container image scan

* server has many staticContainers

* use go module

* for staticContainer

* fix typo

* fix setErrs error

* change name : StaticContainer -> Image

* add scan -images-only flag

* fix makefile

* fix makefile for go module

* use rpmcmd instead of rpm

* add scrutinizer.yml

* change scrutinizer.yml

* fix scrutinizer.yml

* fix scrutinizer.yml

* fix scrutinizer.yml

* fix scrutinizer.yml

* delete scrutinizer

* add report test

* add sourcePackages and Arch

* fix for sider

* fix staticContainer -> image

* init scan library

* add library scan for servers

* fix tui bug

* fix lint error

* divide WpPackageFixStats and LibraryPackageFixedIns

* fix error

* Delete libManager_test.go

* stop use alpine os if err occurred in container

* merge upstream/master

* Delete libManager.go

* update goval-dictionary

* fix go.mod

* update Readme

* add feature : auto detect lockfiles
This commit is contained in:
Tomoya Amachi
2019-06-12 18:50:07 +09:00
committed by Kota Kanbe
parent 10942f7c08
commit abcea1a14d
22 changed files with 1531 additions and 1161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
scan/base.go
View File

@@ -26,10 +26,23 @@ import (
"strings"
"time"
"github.com/knqyf263/fanal/analyzer"
"github.com/knqyf263/fanal/extractor"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/sirupsen/logrus"
"golang.org/x/xerrors"
// Import library scanner
_ "github.com/knqyf263/fanal/analyzer/library/bundler"
_ "github.com/knqyf263/fanal/analyzer/library/cargo"
_ "github.com/knqyf263/fanal/analyzer/library/composer"
_ "github.com/knqyf263/fanal/analyzer/library/npm"
_ "github.com/knqyf263/fanal/analyzer/library/pipenv"
_ "github.com/knqyf263/fanal/analyzer/library/poetry"
_ "github.com/knqyf263/fanal/analyzer/library/yarn"
)
type base struct {
@@ -37,10 +50,10 @@ type base struct {
Distro config.Distro
Platform models.Platform
osPackages
WordPress *models.WordPressPackages
log *logrus.Entry
errs []error
LibraryScanners []models.LibraryScanner
WordPress *models.WordPressPackages
log *logrus.Entry
errs []error
}
func (l *base) exec(cmd string, sudo bool) execResult {
@@ -385,6 +398,11 @@ func (l *base) convertToModel() models.ScanResult {
Type: ctype,
}
image := models.Image{
Name: l.ServerInfo.Image.Name,
Tag: l.ServerInfo.Image.Tag,
}
errs := []string{}
for _, e := range l.errs {
errs = append(errs, fmt.Sprintf("%s", e))
@@ -405,6 +423,7 @@ func (l *base) convertToModel() models.ScanResult {
Family: l.Distro.Family,
Release: l.Distro.Release,
Container: container,
Image: image,
Platform: l.Platform,
IPv4Addrs: l.ServerInfo.IPv4Addrs,
IPv6Addrs: l.ServerInfo.IPv6Addrs,
@@ -414,6 +433,7 @@ func (l *base) convertToModel() models.ScanResult {
Packages: l.Packages,
SrcPackages: l.SrcPackages,
WordPressPackages: l.WordPress,
LibraryScanners: l.LibraryScanners,
Optional: l.ServerInfo.Optional,
Errors: errs,
}
@@ -486,6 +506,65 @@ func (l *base) parseSystemctlStatus(stdout string) string {
return ss[1]
}
func (l *base) scanLibraries() (err error) {
// image already detected libraries
if len(l.LibraryScanners) != 0 {
return nil
}
// library scan for servers need lockfiles
if len(l.ServerInfo.Lockfiles) == 0 && !l.ServerInfo.FindLock {
return nil
}
libFilemap := extractor.FileMap{}
detectFiles := l.ServerInfo.Lockfiles
// auto detect lockfile
if l.ServerInfo.FindLock {
findopt := ""
for filename := range models.LibraryMap {
findopt += fmt.Sprintf("-name %q -o ", "*"+filename)
}
// delete last "-o "
// find / -name "*package-lock.json" -o -name "*yarn.lock" ... 2>&1 | grep -v "Permission denied"
cmd := fmt.Sprintf(`find / ` + findopt[:len(findopt)-3] + ` 2>&1 | grep -v "Permission denied"`)
r := exec(l.ServerInfo, cmd, noSudo)
if !r.isSuccess() {
return xerrors.Errorf("Failed to find lock files")
}
detectFiles = append(detectFiles, strings.Split(r.Stdout, "\n")...)
}
for _, path := range detectFiles {
if path == "" {
continue
}
// skip already exist
if _, ok := libFilemap[path]; ok {
continue
}
cmd := fmt.Sprintf("cat %s", path)
r := exec(l.ServerInfo, cmd, noSudo)
if !r.isSuccess() {
return xerrors.Errorf("Failed to get target file: %s, filepath: %s", r, path)
}
libFilemap[path] = []byte(r.Stdout)
}
results, err := analyzer.GetLibraries(libFilemap)
if err != nil {
return xerrors.Errorf("Failed to get libs: %w", err)
}
l.LibraryScanners, err = convertLibWithScanner(results)
if err != nil {
return xerrors.Errorf("Failed to scan libraries: %w", err)
}
return nil
}
func (l *base) scanWordPress() (err error) {
wpOpts := []string{l.ServerInfo.WordPress.OSUser,
l.ServerInfo.WordPress.DocRoot,

222
scan/container.go Normal file
View File

@@ -0,0 +1,222 @@
/* Vuls - Vulnerability Scanner
Copyright (C) 2016 Future Corporation , Japan.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package scan
import (
"context"
"fmt"
"time"
"github.com/knqyf263/fanal/analyzer"
"golang.org/x/xerrors"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
fanalos "github.com/knqyf263/fanal/analyzer/os"
godeptypes "github.com/knqyf263/go-dep-parser/pkg/types"
// Register library analyzers
_ "github.com/knqyf263/fanal/analyzer/library/bundler"
_ "github.com/knqyf263/fanal/analyzer/library/cargo"
_ "github.com/knqyf263/fanal/analyzer/library/composer"
_ "github.com/knqyf263/fanal/analyzer/library/npm"
_ "github.com/knqyf263/fanal/analyzer/library/pipenv"
_ "github.com/knqyf263/fanal/analyzer/library/poetry"
_ "github.com/knqyf263/fanal/analyzer/library/yarn"
// Register os analyzers
_ "github.com/knqyf263/fanal/analyzer/os/alpine"
_ "github.com/knqyf263/fanal/analyzer/os/amazonlinux"
_ "github.com/knqyf263/fanal/analyzer/os/debianbase"
_ "github.com/knqyf263/fanal/analyzer/os/opensuse"
_ "github.com/knqyf263/fanal/analyzer/os/redhatbase"
// Register package analyzers
_ "github.com/knqyf263/fanal/analyzer/pkg/apk"
_ "github.com/knqyf263/fanal/analyzer/pkg/dpkg"
_ "github.com/knqyf263/fanal/analyzer/pkg/rpmcmd"
)
// inherit OsTypeInterface
type image struct {
base
}
// newDummyOS is constructor
func newDummyOS(c config.ServerInfo) *image {
d := &image{
base: base{
osPackages: osPackages{
Packages: models.Packages{},
VulnInfos: models.VulnInfos{},
},
},
}
d.log = util.NewCustomLogger(c)
d.setServerInfo(c)
return d
}
func detectContainerImage(c config.ServerInfo) (itsMe bool, containerImage osTypeInterface, err error) {
if err = config.IsValidImage(c.Image); err != nil {
return false, nil, nil
}
os, pkgs, libs, err := scanImage(c)
if err != nil {
// use Alpine for setErrs
return false, newDummyOS(c), err
}
switch os.Family {
case fanalos.OpenSUSELeap, fanalos.OpenSUSETumbleweed, fanalos.OpenSUSE:
return false, newDummyOS(c), xerrors.Errorf("Unsupported OS : %s", os.Family)
}
libScanners, err := convertLibWithScanner(libs)
if err != nil {
return false, newDummyOS(c), err
}
p := newContainerImage(c, pkgs, libScanners)
p.setDistro(os.Family, os.Name)
return true, p, nil
}
func convertLibWithScanner(libs map[analyzer.FilePath][]godeptypes.Library) ([]models.LibraryScanner, error) {
scanners := []models.LibraryScanner{}
for path, pkgs := range libs {
scanners = append(scanners, models.LibraryScanner{Path: string(path), Libs: pkgs})
}
return scanners, nil
}
// scanImage returns os, packages on image layers
func scanImage(c config.ServerInfo) (os *analyzer.OS, pkgs []analyzer.Package, libs map[analyzer.FilePath][]godeptypes.Library, err error) {
ctx := context.Background()
domain := c.Image.Name + ":" + c.Image.Tag
util.Log.Info("Start fetch container... ", domain)
// Configure dockerOption
dockerOption := c.Image.DockerOption
if dockerOption.Timeout == 0 {
dockerOption.Timeout = 60 * time.Second
}
files, err := analyzer.Analyze(ctx, domain, dockerOption)
if err != nil {
return nil, nil, nil, xerrors.Errorf("Failed scan files %q, %w", domain, err)
}
containerOs, err := analyzer.GetOS(files)
if err != nil {
return nil, nil, nil, xerrors.Errorf("Failed scan os %q, %w", domain, err)
}
pkgs, err = analyzer.GetPackages(files)
if err != nil {
return nil, nil, nil, xerrors.Errorf("Failed scan pkgs %q, %w", domain, err)
}
libs, err = analyzer.GetLibraries(files)
if err != nil {
return nil, nil, nil, xerrors.Errorf("Failed scan libs %q, %w", domain, err)
}
return &containerOs, pkgs, libs, nil
}
func convertFanalToVulsPkg(pkgs []analyzer.Package) (map[string]models.Package, map[string]models.SrcPackage) {
modelPkgs := map[string]models.Package{}
modelSrcPkgs := map[string]models.SrcPackage{}
for _, pkg := range pkgs {
version := pkg.Version
if pkg.Epoch != 0 {
version = fmt.Sprintf("%d:%s", pkg.Epoch, pkg.Version)
}
modelPkgs[pkg.Name] = models.Package{
Name: pkg.Name,
Release: pkg.Release,
Version: version,
Arch: pkg.Arch,
}
// add SrcPacks
if pkg.Name != pkg.SrcName {
if pack, ok := modelSrcPkgs[pkg.SrcName]; ok {
pack.AddBinaryName(pkg.Name)
modelSrcPkgs[pkg.SrcName] = pack
} else {
modelSrcPkgs[pkg.SrcName] = models.SrcPackage{
Name: pkg.SrcName,
Version: pkg.SrcVersion,
BinaryNames: []string{pkg.Name},
}
}
}
}
return modelPkgs, modelSrcPkgs
}
func newContainerImage(c config.ServerInfo, pkgs []analyzer.Package, libs []models.LibraryScanner) *image {
modelPkgs, modelSrcPkgs := convertFanalToVulsPkg(pkgs)
d := &image{
base: base{
osPackages: osPackages{
Packages: modelPkgs,
SrcPackages: modelSrcPkgs,
VulnInfos: models.VulnInfos{},
},
LibraryScanners: libs,
},
}
d.log = util.NewCustomLogger(c)
d.setServerInfo(c)
return d
}
func (o *image) checkScanMode() error {
return nil
}
func (o *image) checkIfSudoNoPasswd() error {
return nil
}
func (o *image) checkDeps() error {
return nil
}
func (o *image) preCure() error {
return nil
}
func (o *image) postScan() error {
return nil
}
func (o *image) scanPackages() error {
return nil
}
func (o *image) parseInstalledPackages(string) (models.Packages, models.SrcPackages, error) {
return nil, nil, nil
}
func (o *image) detectPlatform() {
o.setPlatform(models.Platform{Name: "image"})
}

147
scan/serverapi.go
View File

@@ -63,6 +63,7 @@ type osTypeInterface interface {
preCure() error
postScan() error
scanWordPress() error
scanLibraries() error
scanPackages() error
convertToModel() models.ScanResult
@@ -124,6 +125,18 @@ func detectOS(c config.ServerInfo) (osType osTypeInterface) {
return
}
itsMe, osType, fatalErr = detectContainerImage(c)
if fatalErr != nil {
osType.setErrs(
[]error{xerrors.Errorf("Failed to detect OS: %w", fatalErr)},
)
return
}
if itsMe {
util.Log.Debugf("Container")
return
}
itsMe, osType, fatalErr = detectDebianWithRetry(c)
if fatalErr != nil {
osType.setErrs([]error{
@@ -182,20 +195,56 @@ func PrintSSHableServerNames() bool {
return true
}
// InitServers detect the kind of OS distribution of target servers
func InitServers(timeoutSec int) error {
servers, errServers = detectServerOSes(timeoutSec)
if len(servers) == 0 {
return xerrors.New("No scannable servers")
func needScans() (needBaseServer, scanContainer, scanImage bool) {
scanContainer = true
scanImage = true
if !config.Conf.ContainersOnly && !config.Conf.ImagesOnly {
needBaseServer = true
}
actives, inactives := detectContainerOSes(timeoutSec)
if config.Conf.ContainersOnly {
servers = actives
errServers = inactives
} else {
if config.Conf.ImagesOnly && !config.Conf.ContainersOnly {
scanContainer = false
}
if config.Conf.ContainersOnly && !config.Conf.ImagesOnly {
scanImage = false
}
return needBaseServer, scanContainer, scanImage
}
// InitServers detect the kind of OS distribution of target servers
func InitServers(timeoutSec int) error {
needBaseServers, scanContainer, scanImage := needScans()
// use global servers, errServers when scan containers and images
servers, errServers = detectServerOSes(timeoutSec)
if len(servers) == 0 {
return xerrors.New("No scannable base servers")
}
// scan additional servers
var actives, inactives []osTypeInterface
if scanImage {
oks, errs := detectImageOSes(timeoutSec)
actives = append(actives, oks...)
inactives = append(inactives, errs...)
}
if scanContainer {
oks, errs := detectContainerOSes(timeoutSec)
actives = append(actives, oks...)
inactives = append(inactives, errs...)
}
if needBaseServers {
servers = append(servers, actives...)
errServers = append(errServers, inactives...)
} else {
servers = actives
errServers = inactives
}
if len(servers) == 0 {
return xerrors.New("No scannable servers")
}
return nil
}
@@ -401,6 +450,81 @@ func detectContainerOSesOnServer(containerHost osTypeInterface) (oses []osTypeIn
return oses
}
func detectImageOSes(timeoutSec int) (actives, inactives []osTypeInterface) {
util.Log.Info("Detecting OS of static containers... ")
osTypesChan := make(chan []osTypeInterface, len(servers))
defer close(osTypesChan)
for _, s := range servers {
go func(s osTypeInterface) {
defer func() {
if p := recover(); p != nil {
util.Log.Debugf("Panic: %s on %s",
p, s.getServerInfo().GetServerName())
}
}()
osTypesChan <- detectImageOSesOnServer(s)
}(s)
}
timeout := time.After(time.Duration(timeoutSec) * time.Second)
for i := 0; i < len(servers); i++ {
select {
case res := <-osTypesChan:
for _, osi := range res {
sinfo := osi.getServerInfo()
if 0 < len(osi.getErrs()) {
inactives = append(inactives, osi)
util.Log.Errorf("Failed: %s err: %+v", sinfo.ServerName, osi.getErrs())
continue
}
actives = append(actives, osi)
util.Log.Infof("Detected: %s@%s: %s",
sinfo.Image.Name, sinfo.ServerName, osi.getDistro())
}
case <-timeout:
msg := "Timed out while detecting static containers"
util.Log.Error(msg)
for servername, sInfo := range config.Conf.Servers {
found := false
for _, o := range append(actives, inactives...) {
if servername == o.getServerInfo().ServerName {
found = true
break
}
}
if !found {
u := &unknown{}
u.setServerInfo(sInfo)
u.setErrs([]error{
xerrors.New("Timed out"),
})
inactives = append(inactives)
util.Log.Errorf("Timed out: %s", servername)
}
}
}
}
return
}
func detectImageOSesOnServer(containerHost osTypeInterface) (oses []osTypeInterface) {
containerHostInfo := containerHost.getServerInfo()
if len(containerHostInfo.Images) == 0 {
return
}
for idx, containerConf := range containerHostInfo.Images {
copied := containerHostInfo
// change servername for original
copied.ServerName = fmt.Sprintf("%s:%s@%s", idx, containerConf.Tag, containerHostInfo.ServerName)
copied.Image = containerConf
copied.Type = ""
os := detectOS(copied)
oses = append(oses, os)
}
return oses
}
// CheckScanModes checks scan mode
func CheckScanModes() error {
for _, s := range servers {
@@ -600,6 +724,9 @@ func scanVulns(jsonDir string, scannedAt time.Time, timeoutSec int) error {
if err = o.scanWordPress(); err != nil {
return xerrors.Errorf("Failed to scan WordPress: %w", err)
}
if err = o.scanLibraries(); err != nil {
return xerrors.Errorf("Failed to scan Library: %w", err)
}
return o.postScan()
}, timeoutSec)