Improve implementation around config (#1122)

* refactor config

* fix saas config

* feat(config): scanmodule for each server in config.toml

* feat(config): enable to specify containersOnly in config.toml

* add new keys of config.toml to discover.go

* fix summary output, logging

scan/alpine.go

@@ -73,7 +73,6 @@ func (o *alpine) apkUpdate() error {
 }
 
 func (o *alpine) preCure() error {
-	o.log.Infof("Scanning in %s", o.getServerInfo().Mode)
 	if err := o.detectIPAddr(); err != nil {
 		o.log.Warnf("Failed to detect IP addresses: %s", err)
 		o.warns = append(o.warns, err)
@@ -92,6 +91,7 @@ func (o *alpine) detectIPAddr() (err error) {
 }
 
 func (o *alpine) scanPackages() error {
+	o.log.Infof("Scanning OS pkg in %s", o.getServerInfo().Mode)
 	if err := o.apkUpdate(); err != nil {
 		return err
 	}

scan/base.go

@@ -526,7 +526,6 @@ func (l *base) parseSystemctlStatus(stdout string) string {
 }
 
 func (l *base) scanLibraries() (err error) {
-	// image already detected libraries
 	if len(l.LibraryScanners) != 0 {
 		return nil
 	}
@@ -536,6 +535,8 @@ func (l *base) scanLibraries() (err error) {
 		return nil
 	}
 
+	l.log.Info("Scanning Lockfile...")
+
 	libFilemap := map[string][]byte{}
 	detectFiles := l.ServerInfo.Lockfiles
 
@@ -621,34 +622,14 @@ func (d *DummyFileInfo) IsDir() bool { return false }
 func (d *DummyFileInfo) Sys() interface{} { return nil }
 
 func (l *base) scanWordPress() (err error) {
-	wpOpts := []string{l.ServerInfo.WordPress.OSUser,
-		l.ServerInfo.WordPress.DocRoot,
-		l.ServerInfo.WordPress.CmdPath,
-		l.ServerInfo.WordPress.WPVulnDBToken,
-	}
-	var isScanWp, hasEmptyOpt bool
-	for _, opt := range wpOpts {
-		if opt != "" {
-			isScanWp = true
-			break
-		} else {
-			hasEmptyOpt = true
-		}
-	}
-	if !isScanWp {
+	if l.ServerInfo.WordPress.IsZero() {
 		return nil
 	}
-
-	if hasEmptyOpt {
-		return xerrors.Errorf("%s has empty WordPress opts: %s",
-			l.getServerInfo().GetServerName(), wpOpts)
-	}
-
+	l.log.Info("Scanning WordPress...")
 	cmd := fmt.Sprintf("sudo -u %s -i -- %s cli version --allow-root",
 		l.ServerInfo.WordPress.OSUser,
 		l.ServerInfo.WordPress.CmdPath)
 	if r := exec(l.ServerInfo, cmd, noSudo); !r.isSuccess() {
-		l.ServerInfo.WordPress.WPVulnDBToken = "secret"
 		return xerrors.Errorf("Failed to exec `%s`. Check the OS user, command path of wp-cli, DocRoot and permission: %#v", cmd, l.ServerInfo.WordPress)
 	}
 
@@ -743,6 +724,7 @@ func (l *base) detectWpPlugins() ([]models.WpPackage, error) {
 }
 
 func (l *base) scanPorts() (err error) {
+	l.log.Info("Scanning listen port...")
 	dest := l.detectScanDest()
 	open, err := l.execPortsScan(dest)
 	if err != nil {
@@ -838,7 +820,7 @@ func (l *base) findPortTestSuccessOn(listenIPPorts []string, searchListenPort mo
 	for _, ipPort := range listenIPPorts {
 		ipPort, err := models.NewPortStat(ipPort)
 		if err != nil {
-			util.Log.Warnf("Failed to find: %+v", err)
+			l.log.Warnf("Failed to find: %+v", err)
 			continue
 		}
 		if searchListenPort.BindAddress == "*" {

scan/debian.go

@@ -241,7 +241,6 @@ func (o *debian) checkDeps() error {
 }
 
 func (o *debian) preCure() error {
-	o.log.Infof("Scanning in %s", o.getServerInfo().Mode)
 	if err := o.detectIPAddr(); err != nil {
 		o.log.Warnf("Failed to detect IP addresses: %s", err)
 		o.warns = append(o.warns, err)
@@ -277,6 +276,7 @@ func (o *debian) detectIPAddr() (err error) {
 }
 
 func (o *debian) scanPackages() error {
+	o.log.Infof("Scanning OS pkg in %s", o.getServerInfo().Mode)
 	// collect the running kernel information
 	release, version, err := o.runningKernel()
 	if err != nil {

scan/freebsd.go

@@ -69,7 +69,6 @@ func (o *bsd) checkDeps() error {
 }
 
 func (o *bsd) preCure() error {
-	o.log.Infof("Scanning in %s", o.getServerInfo().Mode)
 	if err := o.detectIPAddr(); err != nil {
 		o.log.Debugf("Failed to detect IP addresses: %s", err)
 	}
@@ -115,6 +114,7 @@ func (l *base) parseIfconfig(stdout string) (ipv4Addrs []string, ipv6Addrs []str
 }
 
 func (o *bsd) scanPackages() error {
+	o.log.Infof("Scanning OS pkg in %s", o.getServerInfo().Mode)
 	// collect the running kernel information
 	release, version, err := o.runningKernel()
 	if err != nil {

scan/redhatbase.go

@@ -193,12 +193,12 @@ func (o *redhatBase) postScan() error {
 }
 
 func (o *redhatBase) detectIPAddr() (err error) {
-	o.log.Infof("Scanning in %s", o.getServerInfo().Mode)
 	o.ServerInfo.IPv4Addrs, o.ServerInfo.IPv6Addrs, err = o.ip()
 	return err
 }
 
 func (o *redhatBase) scanPackages() error {
+	o.log.Infof("Scanning OS pkg in %s", o.getServerInfo().Mode)
 	installed, err := o.scanInstalledPackages()
 	if err != nil {
 		return xerrors.Errorf("Failed to scan installed packages: %w", err)

scan/serverapi.go

@@ -173,25 +173,20 @@ func PrintSSHableServerNames() bool {
 
 // InitServers detect the kind of OS distribution of target servers
 func InitServers(timeoutSec int) error {
-	// use global servers, errServers when scan containers
-	servers, errServers = detectServerOSes(timeoutSec)
-	if len(servers) == 0 {
-		return xerrors.New("No scannable base servers")
+	hosts, errHosts := detectServerOSes(timeoutSec)
+	if len(hosts) == 0 {
+		return xerrors.New("No scannable host OS")
 	}
+	containers, errContainers := detectContainerOSes(hosts, timeoutSec)
 
-	// scan additional servers
-	var actives, inactives []osTypeInterface
-	oks, errs := detectContainerOSes(timeoutSec)
-	actives = append(actives, oks...)
-	inactives = append(inactives, errs...)
-
-	if config.Conf.ContainersOnly {
-		servers = actives
-		errServers = inactives
-	} else {
-		servers = append(servers, actives...)
-		errServers = append(errServers, inactives...)
+	// set to pkg global variable
+	for _, host := range hosts {
+		if !host.getServerInfo().ContainersOnly {
+			servers = append(servers, host)
+		}
 	}
+	servers = append(servers, containers...)
+	errServers = append(errHosts, errContainers...)
 
 	if len(servers) == 0 {
 		return xerrors.New("No scannable servers")
@@ -260,11 +255,11 @@ func detectServerOSes(timeoutSec int) (servers, errServers []osTypeInterface) {
 	return
 }
 
-func detectContainerOSes(timeoutSec int) (actives, inactives []osTypeInterface) {
+func detectContainerOSes(hosts []osTypeInterface, timeoutSec int) (actives, inactives []osTypeInterface) {
 	util.Log.Info("Detecting OS of containers... ")
-	osTypesChan := make(chan []osTypeInterface, len(servers))
+	osTypesChan := make(chan []osTypeInterface, len(hosts))
 	defer close(osTypesChan)
-	for _, s := range servers {
+	for _, s := range hosts {
 		go func(s osTypeInterface) {
 			defer func() {
 				if p := recover(); p != nil {
@@ -277,7 +272,7 @@ func detectContainerOSes(timeoutSec int) (actives, inactives []osTypeInterface)
 	}
 
 	timeout := time.After(time.Duration(timeoutSec) * time.Second)
-	for i := 0; i < len(servers); i++ {
+	for i := 0; i < len(hosts); i++ {
 		select {
 		case res := <-osTypesChan:
 			for _, osi := range res {
@@ -495,7 +490,6 @@ func Scan(timeoutSec int) error {
 		}
 	}()
 
-	util.Log.Info("Scanning OS packages...")
 	scannedAt := time.Now()
 	dir, err := EnsureResultDir(scannedAt)
 	if err != nil {
@@ -631,7 +625,7 @@ func setupChangelogCache() error {
 // GetScanResults returns ScanResults from
 func GetScanResults(scannedAt time.Time, timeoutSec int) (results models.ScanResults, err error) {
 	parallelExec(func(o osTypeInterface) (err error) {
-		if !(config.Conf.LibsOnly || config.Conf.WordPressOnly) {
+		if o.getServerInfo().Module.IsScanOSPkg() {
 			if err = o.preCure(); err != nil {
 				return err
 			}
@@ -642,14 +636,20 @@ func GetScanResults(scannedAt time.Time, timeoutSec int) (results models.ScanRes
 				return err
 			}
 		}
-		if err = o.scanWordPress(); err != nil {
-			return xerrors.Errorf("Failed to scan WordPress: %w", err)
+		if o.getServerInfo().Module.IsScanPort() {
+			if err = o.scanPorts(); err != nil {
+				return xerrors.Errorf("Failed to scan Ports: %w", err)
+			}
 		}
-		if err = o.scanLibraries(); err != nil {
-			return xerrors.Errorf("Failed to scan Library: %w", err)
+		if o.getServerInfo().Module.IsScanWordPress() {
+			if err = o.scanWordPress(); err != nil {
+				return xerrors.Errorf("Failed to scan WordPress: %w", err)
+			}
 		}
-		if err = o.scanPorts(); err != nil {
-			return xerrors.Errorf("Failed to scan Ports: %w", err)
+		if o.getServerInfo().Module.IsScanLockFile() {
+			if err = o.scanLibraries(); err != nil {
+				return xerrors.Errorf("Failed to scan Library: %w", err)
+			}
 		}
 		return nil
 	}, timeoutSec)
@@ -662,6 +662,7 @@ func GetScanResults(scannedAt time.Time, timeoutSec int) (results models.ScanRes
 
 	for _, s := range append(servers, errServers...) {
 		r := s.convertToModel()
+		checkEOL(&r)
 		r.ScannedAt = scannedAt
 		r.ScannedVersion = config.Version
 		r.ScannedRevision = config.Revision
@@ -669,7 +670,6 @@ func GetScanResults(scannedAt time.Time, timeoutSec int) (results models.ScanRes
 		r.ScannedIPv4Addrs = ipv4s
 		r.ScannedIPv6Addrs = ipv6s
 		r.Config.Scan = config.Conf
-		checkEOL(&r)
 		results = append(results, r)
 
 		if 0 < len(r.Warnings) {

scan/suse.go

@@ -115,6 +115,7 @@ func (o *suse) checkIfSudoNoPasswd() error {
 }
 
 func (o *suse) scanPackages() error {
+	o.log.Infof("Scanning OS pkg in %s", o.getServerInfo().Mode)
 	installed, err := o.scanInstalledPackages()
 	if err != nil {
 		o.log.Errorf("Failed to scan installed packages: %s", err)