v0.5.0 (no backwards compatibility) (#478)

* Change config.toml, Auto-generate UUIDs, change structure of optional field * Detect processes affected by update using yum-ps (#482) Detect processes affected by update using yum-ps * Detect processes needs restart using checkrestart on Debian and Ubuntu. * pass cpename by args when calling FillCveInfo (#513) * fix new db (#502) * Include Version,Revision in JSON * Include hostname in JSON * Update goval-dictionary's commit hash in Gopkg.lock * Remove README.ja.md * update packages (#596) * fix: change ControlPath to .vuls of SSH option (#618) * feat: checkrestart for Ubuntu and Debian (#622) * feat: checkrestart for Ubuntu and Debian * fix: dependencies check logic of configtest * feat: need-restarting on RedHat * refactor: Process.ProcName to Process.Name * feat: detect a systemd service name of need-restarting-process * feat: detect a systemd service name of need-restarting-process on Ubuntu * feat: fill a service name of need-restarting-process, init-system * Support NVD JSON and CVSS3 of JVN (#605) * fix: compile errors * fix: Show CVSS3 on TUI * fix: test cases * fix: Avoid null in JSON * Fix maxCvssScore (#621) * Fix maxCvssScore * Update vulninfos.go * fix(init): remove unnecessary log initialization * refactor(nvd): use only json feed if exists json data. if not, use xml feed * fix(scan): make Confidence slice * feat(CWE): Display CWE name to TUI * feat(cwe): import CWE defs in Japanese * feat(cwe): add OWASP Top 10 ranking to CWE if applicable * feat(scan): add -fast-root mode, implement scan/amazon.go * refactor(const): change const name JVN to Jvn * feat(scan): add -fast-root mode, implement scan/centos.go * refactor(dep): update deps * fix(amazon): deps check * feat(scan): add -fast-root mode, implement scan/rhel.go * feat(scan): add -fast-root mode, implement scan/oracle.go * fix complile err * feat(scan): add -fast-root mode, implement scan/debian.go * fix testcase * fix(amazon): scan using yum * fix(configtest): change error message, status when no scannnable servers * Fix(scan): detect init process logic * fix(tui): display cvss as table format * fix(scan): parse a output of reboot-notifier on CentOS6.9 * fix(tui): don't display score, vector when score is zero * fix(scan): add -offline mode to suse scanner * fix(scan): fix help message * feat(scan): enable to define scan mode for each servers in config.toml #510 * refactor(config): chagne cpeNames to cpeURIs * refactor(config): change dependencyCheckXMLPath to owaspDCXMLPath * fix(config): containers -> containersIncluded, Excluded, containerType * feature(report): enable to define cpeURIs for each contaner * feature(report): enable to specify owasp dc xml path for each container * fix(discover): fix a template displayed at the end of discover * feature(report): add ignorePkgsRegexp #665 * feature(report): enable to define ignoreCves for each container #666 * fix(report): Displayed nothing in TUI detail area when CweID is nil * Gopkg.toml diet * feat(server): support server mode (#678) * feat(server): support server mode * Lock go version * Use the latest kernel release among the installed release when the running kernel release is unknown * Add TestViaHTTP * Set logger to go-cve-dictionary client * Add -to-localfile * Add -to-http option to report * Load -to-http conf from config.toml * Support gost (#676) * feat(gost): Support RedHat API * feat(gost): Support Debian Security Tracker * feat(db): display error msg when SQLite3 is locked at the beginning of reporting. * feat(gost): TUI * Only use RedHat information of installed packages * feat(tui): show mitigation on TUI * feat(gost): support redis backend * fix test case * fix nil pointer when db is nil * fix(gost): detect vulns of src packages for Debian * feat(gost): implement redis backend for gost redhat api * feat(report): display fixState of unfixed pkgs * fix(report): display distincted cweIDs * feat(slack): display gost info * feat(slack): display mitigation * feat(report): display available patch state as fixed/total * fix(tui): display - if source of reference is empty * update deps * fix(report): key in ScanResult JSON be lowerCamelcase. * some keys to lower camel * fix(configtest): dep check logic of yum-plugin-ps * fix(tui): format * feat(report): add -format-list option * fix(report): -format-full-text * fix(report): report -format-full-text * fix(report): display v3 score detected by gost * fix(scan): scan in fast mode if not defined in config.toml * fix(gost): fetch RedHat data for fixed CVEs * feat(report): show number of cves detected in each database * fix(report): show new version as `Unknown` in offline and fast scan mode * fix(report): fix num of upadtable and fixed * fix(report): set `Not fixed yet` if packageStatus is empty * refact(gost): make convertToModel public * fix(test): fix test case * update deps * fix(report): include gost score in MaxCvssScore * [WIP] feat(config): enable to set options in config.toml instead of cmd opt (#690) * feat(config): enable to set options in config.toml instead of cmd opt * fix(config): change Conf.Report.Slack to Conf.Slack * fix(discover): change tempalte * fix(report): fix config.toml auto-generate with -uuid * Add endpoint for health check and change endpoint * refact(cmd): refactor flag set * fix(report): enable to specify opts with cmd arg and env value * fix(scan): enable to parse the release version of amazon linux 2 * add(report) add -to-saas option (#695) * add(report) add -to-saas option * ignore other writer if -to-saas * fix(saas) fix bug * fix(scan): need-restarting needs internet connection * fix(scan,configtest): check scan mode * refactor(scan): change func name * fix(suse): support offline mode, bug fix on AWS, zypper --no-color * fix(tui): fix nil pointer when no vulns in tui * feat(report): enable to define CPE FS format in config.toml * fix(vet): fix warnings of go vet * fix(travis): go version to 1.11 * update deps
2018-08-27 13:51:09 +09:00
parent d785fc2a54
commit 44fa2c5800
82 changed files with 14019 additions and 2485 deletions
--- a/scan/serverapi.go
+++ b/scan/serverapi.go
@@ -18,7 +18,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package scan

 import (
+	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
 	"time"
@@ -30,6 +32,13 @@ import (
 	"github.com/future-architect/vuls/util"
 )

+var (
+	errOSFamilyHeader      = errors.New("X-Vuls-OS-Family header is required")
+	errOSReleaseHeader     = errors.New("X-Vuls-OS-Release header is required")
+	errKernelVersionHeader = errors.New("X-Vuls-Kernel-Version header is required")
+	errServerNameHeader    = errors.New("X-Vuls-Server-Name header is required")
+)
+
 var servers, errServers []osTypeInterface

 // Base Interface of redhat, debian, freebsd
@@ -41,7 +50,8 @@ type osTypeInterface interface {
 	detectPlatform()
 	getPlatform() models.Platform

-	checkDependencies() error
+	checkScanMode() error
+	checkDeps() error
 	checkIfSudoNoPasswd() error

 	preCure() error
@@ -49,6 +59,8 @@ type osTypeInterface interface {
 	scanPackages() error
 	convertToModel() models.ScanResult

+	parseInstalledPackages(string) (models.Packages, models.SrcPackages, error)
+
 	runningContainers() ([]config.Container, error)
 	exitedContainers() ([]config.Container, error)
 	allContainers() ([]config.Container, error)
@@ -119,7 +131,11 @@ func detectOS(c config.ServerInfo) (osType osTypeInterface) {
 }

 // PrintSSHableServerNames print SSH-able servernames
-func PrintSSHableServerNames() {
+func PrintSSHableServerNames() bool {
+	if len(servers) == 0 {
+		util.Log.Error("No scannable servers")
+		return false
+	}
 	util.Log.Info("Scannable servers are below...")
 	for _, s := range servers {
 		if s.getServerInfo().IsContainer() {
@@ -132,6 +148,7 @@ func PrintSSHableServerNames() {
 		}
 	}
 	fmt.Printf("\n")
+	return true
 }

 // InitServers detect the kind of OS distribution of target servers
@@ -273,7 +290,7 @@ func detectContainerOSes(timeoutSec int) (actives, inactives []osTypeInterface)

 func detectContainerOSesOnServer(containerHost osTypeInterface) (oses []osTypeInterface) {
 	containerHostInfo := containerHost.getServerInfo()
-	if len(containerHostInfo.Containers.Includes) == 0 {
+	if len(containerHostInfo.ContainersIncluded) == 0 {
 		return
 	}

@@ -285,11 +302,10 @@ func detectContainerOSesOnServer(containerHost osTypeInterface) (oses []osTypeIn
 		return append(oses, containerHost)
 	}

-	if containerHostInfo.Containers.Includes[0] == "${running}" {
+	if containerHostInfo.ContainersIncluded[0] == "${running}" {
 		for _, containerInfo := range running {
-
 			found := false
-			for _, ex := range containerHost.getServerInfo().Containers.Excludes {
+			for _, ex := range containerHost.getServerInfo().ContainersExcluded {
 				if containerInfo.Name == ex || containerInfo.ContainerID == ex {
 					found = true
 				}
@@ -319,7 +335,7 @@ func detectContainerOSesOnServer(containerHost osTypeInterface) (oses []osTypeIn
 	}

 	var exited, unknown []string
-	for _, container := range containerHostInfo.Containers.Includes {
+	for _, container := range containerHostInfo.ContainersIncluded {
 		found := false
 		for _, c := range running {
 			if c.ContainerID == container || c.Name == container {
@@ -355,10 +371,21 @@ func detectContainerOSesOnServer(containerHost osTypeInterface) (oses []osTypeIn
 	return oses
 }

+// CheckScanModes checks scan mode
+func CheckScanModes() error {
+	for _, s := range servers {
+		if err := s.checkScanMode(); err != nil {
+			return fmt.Errorf("servers.%s.scanMode err: %s",
+				s.getServerInfo().GetServerName(), err)
+		}
+	}
+	return nil
+}
+
 // CheckDependencies checks dependencies are installed on target servers.
 func CheckDependencies(timeoutSec int) {
 	parallelExec(func(o osTypeInterface) error {
-		return o.checkDependencies()
+		return o.checkDeps()
 	}, timeoutSec)
 	return
 }
@@ -420,15 +447,92 @@ func Scan(timeoutSec int) error {

 	util.Log.Info("Scanning vulnerable OS packages...")
 	scannedAt := time.Now()
-	dir, err := ensureResultDir(scannedAt)
+	dir, err := EnsureResultDir(scannedAt)
 	if err != nil {
 		return err
 	}
-	if err := scanVulns(dir, scannedAt, timeoutSec); err != nil {
-		return err
+	return scanVulns(dir, scannedAt, timeoutSec)
+}
+
+// ViaHTTP scans servers by HTTP header and body
+func ViaHTTP(header http.Header, body string) (models.ScanResult, error) {
+	family := header.Get("X-Vuls-OS-Family")
+	if family == "" {
+		return models.ScanResult{}, errOSFamilyHeader
 	}

-	return nil
+	release := header.Get("X-Vuls-OS-Release")
+	if release == "" {
+		return models.ScanResult{}, errOSReleaseHeader
+	}
+
+	kernelRelease := header.Get("X-Vuls-Kernel-Release")
+	if kernelRelease == "" {
+		util.Log.Warn("If X-Vuls-Kernel-Release is not specified, there is a possibility of false detection")
+	}
+
+	kernelVersion := header.Get("X-Vuls-Kernel-Version")
+	if family == config.Debian && kernelVersion == "" {
+		return models.ScanResult{}, errKernelVersionHeader
+	}
+
+	serverName := header.Get("X-Vuls-Server-Name")
+	if config.Conf.ToLocalFile && serverName == "" {
+		return models.ScanResult{}, errServerNameHeader
+	}
+
+	distro := config.Distro{
+		Family:  family,
+		Release: release,
+	}
+
+	kernel := models.Kernel{
+		Release: kernelRelease,
+		Version: kernelVersion,
+	}
+	base := base{
+		Distro: distro,
+		osPackages: osPackages{
+			Kernel: kernel,
+		},
+		log: util.Log,
+	}
+
+	var osType osTypeInterface
+	switch family {
+	case config.Debian, config.Ubuntu:
+		osType = &debian{base: base}
+	case config.RedHat:
+		osType = &rhel{
+			redhatBase: redhatBase{base: base},
+		}
+	case config.CentOS:
+		osType = &centos{
+			redhatBase: redhatBase{base: base},
+		}
+	default:
+		return models.ScanResult{}, fmt.Errorf("Server mode for %s is not implemented yet", family)
+	}
+
+	installedPackages, srcPackages, err := osType.parseInstalledPackages(body)
+	if err != nil {
+		return models.ScanResult{}, err
+	}
+
+	result := models.ScanResult{
+		ServerName: serverName,
+		Family:     family,
+		Release:    release,
+		RunningKernel: models.Kernel{
+			Release: kernelRelease,
+			Version: kernelVersion,
+		},
+		Packages:    installedPackages,
+		SrcPackages: srcPackages,
+		ScannedCves: models.VulnInfos{},
+	}
+
+	return result, nil
 }

 func setupChangelogCache() error {
@@ -440,7 +544,7 @@ func setupChangelogCache() error {
 			break
 		case config.Ubuntu, config.Debian:
 			//TODO changelopg cache for RedHat, Oracle, Amazon, CentOS is not implemented yet.
-			if config.Conf.Deep {
+			if s.getServerInfo().Mode.IsDeep() {
 				needToSetupCache = true
 			}
 			break
@@ -466,9 +570,13 @@ func scanVulns(jsonDir string, scannedAt time.Time, timeoutSec int) error {
 		return o.postScan()
 	}, timeoutSec)

+	hostname, _ := os.Hostname()
 	for _, s := range append(servers, errServers...) {
 		r := s.convertToModel()
 		r.ScannedAt = scannedAt
+		r.ScannedVersion = config.Version
+		r.ScannedRevision = config.Revision
+		r.ScannedBy = hostname
 		r.Config.Scan = config.Conf
 		results = append(results, r)
 	}
@@ -487,7 +595,8 @@ func scanVulns(jsonDir string, scannedAt time.Time, timeoutSec int) error {
 	return nil
 }

-func ensureResultDir(scannedAt time.Time) (currentDir string, err error) {
+// EnsureResultDir ensures the directory for scan results
+func EnsureResultDir(scannedAt time.Time) (currentDir string, err error) {
 	jsonDirName := scannedAt.Format(time.RFC3339)

 	resultsDir := config.Conf.ResultsDir