Merge pull request #77 from future-architect/json_writer_sort_order

Add JSONWriter, Fix CVE sort order of report

.gitignore

@@ -2,6 +2,7 @@
 coverage.out
 issues/
 *.txt
+*.json
 vendor/
 log/
 .gitmodules

README.md

@@ -480,8 +480,9 @@ scan:
                 [-cve-dictionary-url=http://127.0.0.1:1323]
                 [-cvss-over=7]
                 [-ignore-unscored-cves]
-                [-report-slack]
+                [-report-json]
                 [-report-mail]
+                [-report-slack]
                 [-http-proxy=http://192.168.0.1:8080]
                 [-ask-sudo-password]
                 [-ask-key-password]
@@ -509,10 +510,12 @@ scan:
         Don't report the unscored CVEs
   -lang string
         [en|ja] (default "en")
+  -report-json
+        Write report to JSON files ($PWD/results/current)
   -report-mail
-        Email report
+        Send report via Email
   -report-slack
-        Slack report
+        Send report via Slack
   -use-unattended-upgrades
         [Deprecated] For Ubuntu. Scan by unattended-upgrades or not (use apt-get upgrade --dry-run by default)
   -use-yum-plugin-security
@@ -520,20 +523,24 @@ scan:
 
 ```
 
-## ask-key-password option 
+## -ask-key-password option 
 
 | SSH key password |  -ask-key-password | |
 |:-----------------|:-------------------|:----|
 | empty password   |                 -  | |
 | with password    |           required | or use ssh-agent |
 
-## ask-sudo-password option
+## -ask-sudo-password option
 
 | sudo password on target servers | -ask-sudo-password | |
 |:-----------------|:-------|:------|
 | NOPASSWORD       | - | defined as NOPASSWORD in /etc/sudoers on target servers |
 | with password    | required | . |
 
+## -report-json option
+
+At the end of the scan, scan results will be available in JSON format in the $PWD/result/current/ directory.  
+all.json includes the scan results of all servres and servername.json includes the scan result of the server.
 
 ## example
 
@@ -563,6 +570,8 @@ With this sample command, it will ..
 - Scan only 2 servers (server1, server2)
 - Print scan result to terminal
 
+
+
 ----
 
 # Usage: Scan vulnerability of non-OS package

commands/scan.go

@@ -19,6 +19,7 @@ package commands
 
 import (
 	"flag"
+	"fmt"
 	"os"
 	"path/filepath"
 
@@ -52,6 +53,7 @@ type ScanCmd struct {
 	// reporting
 	reportSlack bool
 	reportMail  bool
+	reportJSON  bool
 
 	askSudoPassword bool
 	askKeyPassword  bool
@@ -76,8 +78,9 @@ func (*ScanCmd) Usage() string {
 		[-cve-dictionary-url=http://127.0.0.1:1323]
 		[-cvss-over=7]
 		[-ignore-unscored-cves]
-		[-report-slack]
+		[-report-json]
 		[-report-mail]
+		[-report-slack]
 		[-http-proxy=http://192.168.0.1:8080]
 		[-ask-sudo-password]
 		[-ask-key-password]
@@ -126,8 +129,13 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) {
 		"http://proxy-url:port (default: empty)",
 	)
 
-	f.BoolVar(&p.reportSlack, "report-slack", false, "Slack report")
-	f.BoolVar(&p.reportMail, "report-mail", false, "Email report")
+	f.BoolVar(&p.reportSlack, "report-slack", false, "Send report via Slack")
+	f.BoolVar(&p.reportMail, "report-mail", false, "Send report via Email")
+	f.BoolVar(&p.reportJSON,
+		"report-json",
+		false,
+		fmt.Sprintf("Write report to JSON files (%s/results/current)", wd),
+	)
 
 	f.BoolVar(
 		&p.askKeyPassword,
@@ -222,6 +230,9 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 	if p.reportMail {
 		reports = append(reports, report.MailWriter{})
 	}
+	if p.reportJSON {
+		reports = append(reports, report.JSONWriter{})
+	}
 
 	c.Conf.DBPath = p.dbpath
 	c.Conf.CveDictionaryURL = p.cveDictionaryURL
@@ -263,15 +274,6 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 		return subcommands.ExitFailure
 	}
 
-	Log.Info("Reporting...")
-	filtered := scanResults.FilterByCvssOver()
-	for _, w := range reports {
-		if err := w.Write(filtered); err != nil {
-			Log.Fatalf("Failed to output report, err: %s", err)
-			return subcommands.ExitFailure
-		}
-	}
-
 	Log.Info("Insert to DB...")
 	if err := db.OpenDB(); err != nil {
 		Log.Errorf("Failed to open DB. datafile: %s, err: %s", c.Conf.DBPath, err)
@@ -287,5 +289,14 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 		return subcommands.ExitFailure
 	}
 
+	Log.Info("Reporting...")
+	filtered := scanResults.FilterByCvssOver()
+	for _, w := range reports {
+		if err := w.Write(filtered); err != nil {
+			Log.Fatalf("Failed to report, err: %s", err)
+			return subcommands.ExitFailure
+		}
+	}
+
 	return subcommands.ExitSuccess
 }

models/models.go

@@ -72,8 +72,8 @@ func (s ScanResults) FilterByCvssOver() (filtered ScanResults) {
 
 // ScanResult has the result of scanned CVE information.
 type ScanResult struct {
-	gorm.Model
-	ScanHistoryID uint
+	gorm.Model    `json:"-"`
+	ScanHistoryID uint `json:"-"`
 
 	ServerName string // TOML Section key
 	//  Hostname    string
@@ -161,8 +161,8 @@ func (r ScanResult) CveSummary() string {
 
 // NWLink has network link information.
 type NWLink struct {
-	gorm.Model
-	ScanResultID uint
+	gorm.Model   `json:"-"`
+	ScanResultID uint `json:"-"`
 
 	IPAddress string
 	Netmask   string
@@ -183,13 +183,16 @@ func (c CveInfos) Swap(i, j int) {
 
 func (c CveInfos) Less(i, j int) bool {
 	lang := config.Conf.Lang
+	if c[i].CveDetail.CvssScore(lang) == c[j].CveDetail.CvssScore(lang) {
+		return c[i].CveDetail.CveID < c[j].CveDetail.CveID
+	}
 	return c[i].CveDetail.CvssScore(lang) > c[j].CveDetail.CvssScore(lang)
 }
 
 // CveInfo has Cve Information.
 type CveInfo struct {
-	gorm.Model
-	ScanResultID uint
+	gorm.Model   `json:"-"`
+	ScanResultID uint `json:"-"`
 
 	CveDetail        cve.CveDetail
 	Packages         []PackageInfo
@@ -199,8 +202,8 @@ type CveInfo struct {
 
 // CpeName has CPE name
 type CpeName struct {
-	gorm.Model
-	CveInfoID uint
+	gorm.Model `json:"-"`
+	CveInfoID  uint `json:"-"`
 
 	Name string
 }
@@ -265,8 +268,8 @@ func (ps PackageInfoList) FindByName(name string) (result PackageInfo, found boo
 
 // PackageInfo has installed packages.
 type PackageInfo struct {
-	gorm.Model
-	CveInfoID uint
+	gorm.Model `json:"-"`
+	CveInfoID  uint `json:"-"`
 
 	Name    string
 	Version string
@@ -302,8 +305,8 @@ func (p PackageInfo) ToStringNewVersion() string {
 
 // DistroAdvisory has Amazon Linux AMI Security Advisory information.
 type DistroAdvisory struct {
-	gorm.Model
-	CveInfoID uint
+	gorm.Model `json:"-"`
+	CveInfoID  uint `json:"-"`
 
 	AdvisoryID string
 	Severity   string
@@ -313,8 +316,8 @@ type DistroAdvisory struct {
 
 // Container has Container information
 type Container struct {
-	gorm.Model
-	ScanResultID uint
+	gorm.Model   `json:"-"`
+	ScanResultID uint `json:"-"`
 
 	ContainerID string
 	Name        string

report/json.go

@@ -20,18 +20,43 @@ package report
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"path/filepath"
 
 	"github.com/future-architect/vuls/models"
 )
 
-// JSONWriter writes report as JSON format
+// JSONWriter writes results to file.
 type JSONWriter struct{}
 
 func (w JSONWriter) Write(scanResults []models.ScanResult) (err error) {
-	var j []byte
-	if j, err = json.MarshalIndent(scanResults, "", "  "); err != nil {
-		return
+
+	path, err := ensureResultDir()
+
+	var jsonBytes []byte
+	if jsonBytes, err = json.MarshalIndent(scanResults, "", "  "); err != nil {
+		return fmt.Errorf("Failed to Marshal to JSON: %s", err)
+	}
+	all := filepath.Join(path, "all.json")
+	if err := ioutil.WriteFile(all, jsonBytes, 0644); err != nil {
+		return fmt.Errorf("Failed to write JSON. path: %s, err: %s", all, err)
+	}
+
+	for _, r := range scanResults {
+		jsonPath := ""
+		if r.Container.ContainerID == "" {
+			jsonPath = filepath.Join(path, fmt.Sprintf("%s.json", r.ServerName))
+		} else {
+			jsonPath = filepath.Join(path,
+				fmt.Sprintf("%s_%s.json", r.ServerName, r.Container.Name))
+
+		}
+		if jsonBytes, err = json.MarshalIndent(r, "", "  "); err != nil {
+			return fmt.Errorf("Failed to Marshal to JSON: %s", err)
+		}
+		if err := ioutil.WriteFile(jsonPath, jsonBytes, 0644); err != nil {
+			return fmt.Errorf("Failed to write JSON. path: %s, err: %s", all, err)
+		}
 	}
-	fmt.Println(string(j))
 	return nil
 }

report/slack.go

@@ -113,9 +113,8 @@ func toSlackAttachments(scanResult models.ScanResult) (attaches []*attachment) {
 	if !config.Conf.IgnoreUnscoredCves {
 		cves = append(cves, scanResult.UnknownCves...)
 	}
-	scanResult.KnownCves = cves
 
-	for _, cveInfo := range scanResult.KnownCves {
+	for _, cveInfo := range cves {
 		cveID := cveInfo.CveDetail.CveID
 
 		curentPackages := []string{}
@@ -176,8 +175,7 @@ func attachmentText(cveInfo models.CveInfo, osFamily string) string {
 
 	switch {
 	case config.Conf.Lang == "ja" &&
-		cveInfo.CveDetail.Jvn.ID != 0 &&
-		0 < cveInfo.CveDetail.CvssScore("ja"):
+		0 < cveInfo.CveDetail.Jvn.CvssScore():
 
 		jvn := cveInfo.CveDetail.Jvn
 		return fmt.Sprintf("*%4.1f (%s)* <%s|%s>\n%s\n%s",

report/util.go

@@ -20,13 +20,44 @@ package report
 import (
 	"bytes"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/models"
 	"github.com/gosuri/uitable"
 )
 
+func ensureResultDir() (path string, err error) {
+	if resultDirPath != "" {
+		return resultDirPath, nil
+	}
+
+	const timeLayout = "20060102_1504"
+	timedir := time.Now().Format(timeLayout)
+	wd, _ := os.Getwd()
+	dir := filepath.Join(wd, "results", timedir)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return "", fmt.Errorf("Failed to create dir: %s", err)
+	}
+
+	symlinkPath := filepath.Join(wd, "results", "current")
+	if _, err := os.Stat(symlinkPath); err == nil {
+		if err := os.Remove(symlinkPath); err != nil {
+			return "", fmt.Errorf(
+				"Failed to remove symlink. path: %s, err: %s", symlinkPath, err)
+		}
+	}
+
+	if err := os.Symlink(dir, symlinkPath); err != nil {
+		return "", fmt.Errorf(
+			"Failed to create symlink: path: %s, err: %s", symlinkPath, err)
+	}
+	return dir, nil
+}
+
 func toPlainText(scanResult models.ScanResult) (string, error) {
 	serverInfo := scanResult.ServerInfo()
 
@@ -83,8 +114,7 @@ func ToPlainTextSummary(r models.ScanResult) string {
 
 		switch {
 		case config.Conf.Lang == "ja" &&
-			d.CveDetail.Jvn.ID != 0 &&
-			0 < d.CveDetail.CvssScore("ja"):
+			0 < d.CveDetail.Jvn.CvssScore():
 
 			summary := d.CveDetail.Jvn.Title
 			scols = []string{
@@ -121,12 +151,11 @@ func ToPlainTextSummary(r models.ScanResult) string {
 	return fmt.Sprintf("%s", stable)
 }
 
-//TODO Distro Advisory
 func toPlainTextDetails(data models.ScanResult, osFamily string) (scoredReport, unscoredReport []string) {
 	for _, cve := range data.KnownCves {
 		switch config.Conf.Lang {
 		case "en":
-			if cve.CveDetail.Nvd.ID != 0 {
+			if 0 < cve.CveDetail.Nvd.CvssScore() {
 				scoredReport = append(
 					scoredReport, toPlainTextDetailsLangEn(cve, osFamily))
 			} else {
@@ -134,10 +163,10 @@ func toPlainTextDetails(data models.ScanResult, osFamily string) (scoredReport,
 					scoredReport, toPlainTextUnknownCve(cve, osFamily))
 			}
 		case "ja":
-			if cve.CveDetail.Jvn.ID != 0 {
+			if 0 < cve.CveDetail.Jvn.CvssScore() {
 				scoredReport = append(
 					scoredReport, toPlainTextDetailsLangJa(cve, osFamily))
-			} else if cve.CveDetail.Nvd.ID != 0 {
+			} else if 0 < cve.CveDetail.Nvd.CvssScore() {
 				scoredReport = append(
 					scoredReport, toPlainTextDetailsLangEn(cve, osFamily))
 			} else {

report/writer.go

@@ -37,3 +37,5 @@ const (
 type ResultWriter interface {
 	Write([]models.ScanResult) error
 }
+
+var resultDirPath string

scan/debian.go

@@ -20,7 +20,6 @@ package scan
 import (
 	"fmt"
 	"regexp"
-	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -526,7 +525,6 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac
 			//  CvssScore: cinfo.CvssScore(conf.Lang),
 		})
 	}
-	sort.Sort(CvePacksList(cvePacksList))
 	return
 }
 

scan/linux.go

@@ -132,10 +132,10 @@ func (l *linux) parseDockerPs(stdout string) (containers []config.Container, err
 }
 
 func (l *linux) convertToModel() (models.ScanResult, error) {
-	var cves, unknownScoreCves []models.CveInfo
+	var scoredCves, unscoredCves models.CveInfos
 	for _, p := range l.UnsecurePackages {
-		if p.CveDetail.CvssScore(config.Conf.Lang) < 0 {
-			unknownScoreCves = append(unknownScoreCves, models.CveInfo{
+		if p.CveDetail.CvssScore(config.Conf.Lang) <= 0 {
+			unscoredCves = append(unscoredCves, models.CveInfo{
 				CveDetail:        p.CveDetail,
 				Packages:         p.Packs,
 				DistroAdvisories: p.DistroAdvisories, // only Amazon Linux
@@ -155,7 +155,7 @@ func (l *linux) convertToModel() (models.ScanResult, error) {
 			DistroAdvisories: p.DistroAdvisories, // only Amazon Linux
 			CpeNames:         cpenames,
 		}
-		cves = append(cves, cve)
+		scoredCves = append(scoredCves, cve)
 	}
 
 	container := models.Container{
@@ -163,13 +163,16 @@ func (l *linux) convertToModel() (models.ScanResult, error) {
 		Name:        l.ServerInfo.Container.Name,
 	}
 
+	sort.Sort(scoredCves)
+	sort.Sort(unscoredCves)
+
 	return models.ScanResult{
 		ServerName:  l.ServerInfo.ServerName,
 		Family:      l.Family,
 		Release:     l.Release,
 		Container:   container,
-		KnownCves:   cves,
-		UnknownCves: unknownScoreCves,
+		KnownCves:   scoredCves,
+		UnknownCves: unscoredCves,
 	}, nil
 }
 
@@ -208,7 +211,6 @@ func (l *linux) scanVulnByCpeName() error {
 		unsecurePacks = append(unsecurePacks, set[key])
 	}
 	unsecurePacks = append(unsecurePacks, l.UnsecurePackages...)
-	sort.Sort(CvePacksList(unsecurePacks))
 	l.setUnsecurePackages(unsecurePacks)
 	return nil
 }

scan/redhat.go

@@ -395,7 +395,6 @@ func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (CvePacksList, error)
 			//  CvssScore: cinfo.CvssScore(conf.Lang),
 		})
 	}
-	sort.Sort(CvePacksList(cvePacksList))
 	return cvePacksList, nil
 }
 

scan/serverapi.go

@@ -98,7 +98,8 @@ func (s CvePacksList) Swap(i, j int) {
 
 // Less implement Sort Interface
 func (s CvePacksList) Less(i, j int) bool {
-	return s[i].CveDetail.CvssScore("en") > s[j].CveDetail.CvssScore("en")
+	return s[i].CveDetail.CvssScore(config.Conf.Lang) >
+		s[j].CveDetail.CvssScore(config.Conf.Lang)
 }
 
 func detectOS(c config.ServerInfo) (osType osTypeInterface) {