Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add report subcommand, change scan options. Bump up ver #239

Browse Source
This commit is contained in:
Kota Kanbe
2016-11-17 14:24:31 +09:00
parent cb29289167
commit 155cadf901
43 changed files with 2761 additions and 1979 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
commands/configtest.go
View File

@@ -146,7 +146,7 @@ func (p *ConfigtestCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfa
Log := util.NewCustomLogger(c.ServerInfo{})
Log.Info("Validating Config...")
if !c.Conf.Validate() {
if !c.Conf.ValidateOnConfigtest() {
return subcommands.ExitUsageError
}

2
commands/discover.go
View File

@@ -98,7 +98,7 @@ iconEmoji = ":ghost:"
authUser = "username"
notifyUsers = ["@username"]
[mail]
[email]
smtpAddr = "smtp.gmail.com"
smtpPort = "587"
user = "username"

9
commands/history.go
View File

@@ -27,7 +27,6 @@ import (
"strings"
c "github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/report"
"github.com/google/subcommands"
)
@@ -70,11 +69,11 @@ func (p *HistoryCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{
c.Conf.ResultsDir = p.resultsDir
var err error
var jsonDirs report.JSONDirs
if jsonDirs, err = report.GetValidJSONDirs(); err != nil {
var dirs jsonDirs
if dirs, err = lsValidJSONDirs(); err != nil {
return subcommands.ExitFailure
}
for _, d := range jsonDirs {
for _, d := range dirs {
var files []os.FileInfo
if files, err = ioutil.ReadDir(d); err != nil {
return subcommands.ExitFailure
@@ -89,7 +88,7 @@ func (p *HistoryCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{
}
splitPath := strings.Split(d, string(os.PathSeparator))
timeStr := splitPath[len(splitPath)-1]
fmt.Printf("%s scanned %d servers: %s\n",
fmt.Printf("%s %d servers: %s\n",
timeStr,
len(hosts),
strings.Join(hosts, ", "),

5
commands/prepare.go
View File

@@ -50,8 +50,9 @@ func (*PrepareCmd) Synopsis() string {
return `Install required packages to scan.
CentOS: yum-plugin-security, yum-plugin-changelog
Amazon: None
RHEL: TODO
RHEL: None
Ubuntu: None
Debian: aptitude
`
}
@@ -155,7 +156,7 @@ func (p *PrepareCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{
c.Conf.AssumeYes = p.assumeYes
logrus.Info("Validating Config...")
if !c.Conf.Validate() {
if !c.Conf.ValidateOnPrepare() {
return subcommands.ExitUsageError
}
// Set up custom logger

386
commands/report.go Normal file
View File

@@ -0,0 +1,386 @@
/* Vuls - Vulnerability Scanner
Copyright (C) 2016 Future Architect, Inc. Japan.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package commands
import (
"context"
"flag"
"fmt"
"os"
"path/filepath"
c "github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/cveapi"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/report"
"github.com/future-architect/vuls/util"
"github.com/google/subcommands"
"github.com/kotakanbe/go-cve-dictionary/log"
)
// ReportCmd is subcommand for reporting
type ReportCmd struct {
lang string
debug bool
debugSQL bool
configPath string
resultsDir string
refreshCve bool
cvssScoreOver float64
ignoreUnscoredCves bool
httpProxy string
cvedbtype string
cvedbpath string
cveDictionaryURL string
toSlack bool
toEMail bool
toLocalFile bool
toS3 bool
toAzureBlob bool
formatJSON bool
formatXML bool
formatOneLineText bool
formatShortText bool
formatFullText bool
gzip bool
awsProfile string
awsS3Bucket string
awsRegion string
azureAccount string
azureKey string
azureContainer string
}
// Name return subcommand name
func (*ReportCmd) Name() string { return "report" }
// Synopsis return synopsis
func (*ReportCmd) Synopsis() string { return "Reporting" }
// Usage return usage
func (*ReportCmd) Usage() string {
return `report:
report
[-lang=en|ja]
[-config=/path/to/config.toml]
[-results-dir=/path/to/results]
[-refresh-cve]
[-cvedb-type=sqlite3|mysql]
[-cvedb-path=/path/to/cve.sqlite3]
[-cvedb-url=http://127.0.0.1:1323 or mysql connection string]
[-cvss-over=7]
[-ignore-unscored-cves]
[-to-email]
[-to-slack]
[-to-localfile]
[-to-s3]
[-to-azure-blob]
[-format-json]
[-format-xml]
[-format-one-line-text]
[-format-short-text]
[-format-full-text]
[-gzip]
[-aws-profile=default]
[-aws-region=us-west-2]
[-aws-s3-bucket=bucket_name]
[-azure-account=accout]
[-azure-key=key]
[-azure-container=container]
[-http-proxy=http://192.168.0.1:8080]
[-debug]
[-debug-sql]
[SERVER]...
`
}
// SetFlags set flag
func (p *ReportCmd) SetFlags(f *flag.FlagSet) {
f.StringVar(&p.lang, "lang", "en", "[en|ja]")
f.BoolVar(&p.debug, "debug", false, "debug mode")
f.BoolVar(&p.debugSQL, "debug-sql", false, "SQL debug mode")
wd, _ := os.Getwd()
defaultConfPath := filepath.Join(wd, "config.toml")
f.StringVar(&p.configPath, "config", defaultConfPath, "/path/to/toml")
defaultResultsDir := filepath.Join(wd, "results")
f.StringVar(&p.resultsDir, "results-dir", defaultResultsDir, "/path/to/results")
f.BoolVar(
&p.refreshCve,
"refresh-cve",
false,
"Refresh CVE information in JSON file under results dir")
f.StringVar(
&p.cvedbtype,
"cvedb-type",
"sqlite3",
"DB type for fetching CVE dictionary (sqlite3 or mysql)")
defaultCveDBPath := filepath.Join(wd, "cve.sqlite3")
f.StringVar(
&p.cvedbpath,
"cvedb-path",
defaultCveDBPath,
"/path/to/sqlite3 (For get cve detail from cve.sqlite3)")
f.StringVar(
&p.cveDictionaryURL,
"cvedb-url",
"",
"http://cve-dictionary.com:8080 or mysql connection string")
f.Float64Var(
&p.cvssScoreOver,
"cvss-over",
0,
"-cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all))")
f.BoolVar(
&p.ignoreUnscoredCves,
"ignore-unscored-cves",
false,
"Don't report the unscored CVEs")
f.StringVar(
&p.httpProxy,
"http-proxy",
"",
"http://proxy-url:port (default: empty)")
f.BoolVar(&p.formatJSON,
"format-json",
false,
fmt.Sprintf("JSON format"))
f.BoolVar(&p.formatXML,
"format-xml",
false,
fmt.Sprintf("XML format"))
f.BoolVar(&p.formatOneLineText,
"format-one-line-text",
false,
fmt.Sprintf("One line summary in plain text"))
f.BoolVar(&p.formatShortText,
"format-short-text",
false,
fmt.Sprintf("Summary in plain text"))
f.BoolVar(&p.formatFullText,
"format-full-text",
false,
fmt.Sprintf("Detail report in plain text"))
f.BoolVar(&p.gzip, "gzip", false, "gzip compression")
f.BoolVar(&p.toSlack, "to-slack", false, "Send report via Slack")
f.BoolVar(&p.toEMail, "to-email", false, "Send report via Email")
f.BoolVar(&p.toLocalFile,
"to-localfile",
false,
fmt.Sprintf("Write report to localfile"))
f.BoolVar(&p.toS3,
"to-s3",
false,
"Write report to S3 (bucket/yyyyMMdd_HHmm/servername.json/xml/txt)")
f.StringVar(&p.awsProfile, "aws-profile", "default", "AWS profile to use")
f.StringVar(&p.awsRegion, "aws-region", "us-east-1", "AWS region to use")
f.StringVar(&p.awsS3Bucket, "aws-s3-bucket", "", "S3 bucket name")
f.BoolVar(&p.toAzureBlob,
"to-azure-blob",
false,
"Write report to Azure Storage blob (container/yyyyMMdd_HHmm/servername.json/xml/txt)")
f.StringVar(&p.azureAccount,
"azure-account",
"",
"Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified")
f.StringVar(&p.azureKey,
"azure-key",
"",
"Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified")
f.StringVar(&p.azureContainer, "azure-container", "", "Azure storage container name")
}
// Execute execute
func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
c.Conf.Debug = p.debug
c.Conf.DebugSQL = p.debugSQL
Log := util.NewCustomLogger(c.ServerInfo{})
if err := c.Load(p.configPath, ""); err != nil {
Log.Errorf("Error loading %s, %s", p.configPath, err)
return subcommands.ExitUsageError
}
c.Conf.Lang = p.lang
c.Conf.ResultsDir = p.resultsDir
c.Conf.CveDBType = p.cvedbtype
c.Conf.CveDBPath = p.cvedbpath
c.Conf.CveDictionaryURL = p.cveDictionaryURL
c.Conf.CvssScoreOver = p.cvssScoreOver
c.Conf.IgnoreUnscoredCves = p.ignoreUnscoredCves
c.Conf.HTTPProxy = p.httpProxy
jsonDir, err := jsonDir(f.Args())
if err != nil {
log.Errorf("Failed to read from JSON: %s", err)
return subcommands.ExitFailure
}
c.Conf.FormatXML = p.formatXML
c.Conf.FormatJSON = p.formatJSON
c.Conf.FormatOneLineText = p.formatOneLineText
c.Conf.FormatShortText = p.formatShortText
c.Conf.FormatFullText = p.formatFullText
c.Conf.GZIP = p.gzip
// report
reports := []report.ResultWriter{
report.StdoutWriter{},
}
if p.toSlack {
reports = append(reports, report.SlackWriter{})
}
if p.toEMail {
reports = append(reports, report.EMailWriter{})
}
if p.toLocalFile {
reports = append(reports, report.LocalFileWriter{
CurrentDir: jsonDir,
})
}
if p.toS3 {
c.Conf.AwsRegion = p.awsRegion
c.Conf.AwsProfile = p.awsProfile
c.Conf.S3Bucket = p.awsS3Bucket
if err := report.CheckIfBucketExists(); err != nil {
Log.Errorf("Check if there is a bucket beforehand: %s, err: %s", c.Conf.S3Bucket, err)
return subcommands.ExitUsageError
}
reports = append(reports, report.S3Writer{})
}
if p.toAzureBlob {
c.Conf.AzureAccount = p.azureAccount
if len(c.Conf.AzureAccount) == 0 {
c.Conf.AzureAccount = os.Getenv("AZURE_STORAGE_ACCOUNT")
}
c.Conf.AzureKey = p.azureKey
if len(c.Conf.AzureKey) == 0 {
c.Conf.AzureKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
}
c.Conf.AzureContainer = p.azureContainer
if len(c.Conf.AzureContainer) == 0 {
Log.Error("Azure storage container name is requied with --azure-container option")
return subcommands.ExitUsageError
}
if err := report.CheckIfAzureContainerExists(); err != nil {
Log.Errorf("Check if there is a container beforehand: %s, err: %s", c.Conf.AzureContainer, err)
return subcommands.ExitUsageError
}
reports = append(reports, report.AzureBlobWriter{})
}
if !(p.formatJSON || p.formatOneLineText ||
p.formatShortText || p.formatFullText || p.formatXML) {
c.Conf.FormatShortText = true
}
Log.Info("Validating Config...")
if !c.Conf.ValidateOnReport() {
return subcommands.ExitUsageError
}
if ok, err := cveapi.CveClient.CheckHealth(); !ok {
Log.Errorf("CVE HTTP server is not running. err: %s", err)
Log.Errorf("Run go-cve-dictionary as server mode before reporting or run with --cvedb-path option")
return subcommands.ExitFailure
}
if c.Conf.CveDictionaryURL != "" {
Log.Infof("cve-dictionary: %s", c.Conf.CveDictionaryURL)
} else {
if c.Conf.CveDBType == "sqlite3" {
Log.Infof("cve-dictionary: %s", c.Conf.CveDBPath)
}
}
history, err := loadOneScanHistory(jsonDir)
var results []models.ScanResult
for _, r := range history.ScanResults {
if p.refreshCve || needToRefreshCve(r) {
Log.Debugf("need to refresh")
if c.Conf.CveDBType == "sqlite3" {
if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) {
log.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s",
c.Conf.CveDBPath)
return subcommands.ExitFailure
}
}
filled, err := fillCveInfoFromCveDB(r)
if err != nil {
Log.Errorf("Failed to fill CVE information: %s", err)
return subcommands.ExitFailure
}
filled.Lang = c.Conf.Lang
if err := overwriteJSONFile(jsonDir, filled); err != nil {
Log.Errorf("Failed to write JSON: %s", err)
return subcommands.ExitFailure
}
results = append(results, filled)
} else {
Log.Debugf("no need to refresh")
results = append(results, r)
}
}
var res models.ScanResults
for _, r := range results {
res = append(res, r.FilterByCvssOver())
}
for _, w := range reports {
if err := w.Write(res...); err != nil {
Log.Errorf("Failed to report: %s", err)
return subcommands.ExitFailure
}
}
return subcommands.ExitSuccess
}

249
commands/scan.go
View File

@@ -25,12 +25,9 @@ import (
"os"
"path/filepath"
"strings"
"time"
"github.com/Sirupsen/logrus"
c "github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/cveapi"
"github.com/future-architect/vuls/report"
"github.com/future-architect/vuls/scan"
"github.com/future-architect/vuls/util"
"github.com/google/subcommands"
@@ -39,46 +36,15 @@ import (
// ScanCmd is Subcommand of host discovery mode
type ScanCmd struct {
lang string
debug bool
debugSQL bool
configPath string
resultsDir string
cvedbtype string
cvedbpath string
cveDictionaryURL string
cacheDBPath string
cvssScoreOver float64
ignoreUnscoredCves bool
httpProxy string
askSudoPassword bool
askKeyPassword bool
debug bool
configPath string
resultsDir string
cacheDBPath string
httpProxy string
askKeyPassword bool
containersOnly bool
skipBroken bool
// reporting
reportSlack bool
reportMail bool
reportJSON bool
reportText bool
reportS3 bool
reportAzureBlob bool
reportXML bool
awsProfile string
awsS3Bucket string
awsRegion string
azureAccount string
azureKey string
azureContainer string
sshExternal bool
sshExternal bool
}
// Name return subcommand name
@@ -91,35 +57,15 @@ func (*ScanCmd) Synopsis() string { return "Scan vulnerabilities" }
func (*ScanCmd) Usage() string {
return `scan:
scan
[-lang=en|ja]
[-config=/path/to/config.toml]
[-results-dir=/path/to/results]
[-cve-dictionary-dbtype=sqlite3|mysql]
[-cve-dictionary-dbpath=/path/to/cve.sqlite3 or mysql connection string]
[-cve-dictionary-url=http://127.0.0.1:1323]
[-cache-dbpath=/path/to/cache.db]
[-cvss-over=7]
[-ignore-unscored-cves]
[-cachedb-path=/path/to/cache.db]
[-ssh-external]
[-containers-only]
[-skip-broken]
[-report-azure-blob]
[-report-json]
[-report-mail]
[-report-s3]
[-report-slack]
[-report-text]
[-report-xml]
[-http-proxy=http://192.168.0.1:8080]
[-ask-key-password]
[-debug]
[-debug-sql]
[-aws-profile=default]
[-aws-region=us-west-2]
[-aws-s3-bucket=bucket_name]
[-azure-account=accout]
[-azure-key=key]
[-azure-container=container]
[SERVER]...
`
@@ -127,9 +73,7 @@ func (*ScanCmd) Usage() string {
// SetFlags set flag
func (p *ScanCmd) SetFlags(f *flag.FlagSet) {
f.StringVar(&p.lang, "lang", "en", "[en|ja]")
f.BoolVar(&p.debug, "debug", false, "debug mode")
f.BoolVar(&p.debugSQL, "debug-sql", false, "SQL debug mode")
wd, _ := os.Getwd()
@@ -139,44 +83,13 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) {
defaultResultsDir := filepath.Join(wd, "results")
f.StringVar(&p.resultsDir, "results-dir", defaultResultsDir, "/path/to/results")
f.StringVar(
&p.cvedbtype,
"cve-dictionary-dbtype",
"sqlite3",
"DB type for fetching CVE dictionary (sqlite3 or mysql)")
f.StringVar(
&p.cvedbpath,
"cve-dictionary-dbpath",
"",
"/path/to/sqlite3 (For get cve detail from cve.sqlite3)")
defaultURL := "http://127.0.0.1:1323"
f.StringVar(
&p.cveDictionaryURL,
"cve-dictionary-url",
defaultURL,
"http://CVE.Dictionary")
defaultCacheDBPath := filepath.Join(wd, "cache.db")
f.StringVar(
&p.cacheDBPath,
"cache-dbpath",
"cachedb-path",
defaultCacheDBPath,
"/path/to/cache.db (local cache of changelog for Ubuntu/Debian)")
f.Float64Var(
&p.cvssScoreOver,
"cvss-over",
0,
"-cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all))")
f.BoolVar(
&p.ignoreUnscoredCves,
"ignore-unscored-cves",
false,
"Don't report the unscored CVEs")
f.BoolVar(
&p.sshExternal,
"ssh-external",
@@ -202,55 +115,12 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) {
"http://proxy-url:port (default: empty)",
)
f.BoolVar(&p.reportSlack, "report-slack", false, "Send report via Slack")
f.BoolVar(&p.reportMail, "report-mail", false, "Send report via Email")
f.BoolVar(&p.reportJSON,
"report-json",
false,
fmt.Sprintf("Write report to JSON files (%s/results/current)", wd),
)
f.BoolVar(&p.reportText,
"report-text",
false,
fmt.Sprintf("Write report to text files (%s/results/current)", wd),
)
f.BoolVar(&p.reportXML,
"report-xml",
false,
fmt.Sprintf("Write report to XML files (%s/results/current)", wd),
)
f.BoolVar(&p.reportS3,
"report-s3",
false,
"Write report to S3 (bucket/yyyyMMdd_HHmm/servername.json)",
)
f.StringVar(&p.awsProfile, "aws-profile", "default", "AWS profile to use")
f.StringVar(&p.awsRegion, "aws-region", "us-east-1", "AWS region to use")
f.StringVar(&p.awsS3Bucket, "aws-s3-bucket", "", "S3 bucket name")
f.BoolVar(&p.reportAzureBlob,
"report-azure-blob",
false,
"Write report to Azure Storage blob (container/yyyyMMdd_HHmm/servername.json)",
)
f.StringVar(&p.azureAccount, "azure-account", "", "Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified")
f.StringVar(&p.azureKey, "azure-key", "", "Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified")
f.StringVar(&p.azureContainer, "azure-container", "", "Azure storage container name")
f.BoolVar(
&p.askKeyPassword,
"ask-key-password",
false,
"Ask ssh privatekey password before scanning",
)
f.BoolVar(
&p.askSudoPassword,
"ask-sudo-password",
false,
"[Deprecated] THIS OPTION WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on target servers and use SSH key-based authentication",
)
}
// Execute execute
@@ -264,10 +134,6 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
return subcommands.ExitFailure
}
}
if p.askSudoPassword {
logrus.Errorf("[Deprecated] -ask-sudo-password WAS REMOVED FOR SECURITY REASONS. Define NOPASSWD in /etc/sudoers on target servers and use SSH key-based authentication")
return subcommands.ExitFailure
}
c.Conf.Debug = p.debug
err = c.Load(p.configPath, keyPass)
@@ -278,13 +144,6 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
logrus.Info("Start scanning")
logrus.Infof("config: %s", p.configPath)
if p.cvedbpath != "" {
if p.cvedbtype == "sqlite3" {
logrus.Infof("cve-dictionary: %s", p.cvedbpath)
}
} else {
logrus.Infof("cve-dictionary: %s", p.cveDictionaryURL)
}
var servernames []string
if 0 < len(f.Args()) {
@@ -324,91 +183,21 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
}
logrus.Debugf("%s", pp.Sprintf("%v", target))
c.Conf.Lang = p.lang
c.Conf.DebugSQL = p.debugSQL
// logger
Log := util.NewCustomLogger(c.ServerInfo{})
scannedAt := time.Now()
// report
reports := []report.ResultWriter{
report.StdoutWriter{},
report.LogrusWriter{},
}
if p.reportSlack {
reports = append(reports, report.SlackWriter{})
}
if p.reportMail {
reports = append(reports, report.MailWriter{})
}
if p.reportJSON {
reports = append(reports, report.JSONWriter{ScannedAt: scannedAt})
}
if p.reportText {
reports = append(reports, report.TextFileWriter{ScannedAt: scannedAt})
}
if p.reportXML {
reports = append(reports, report.XMLWriter{ScannedAt: scannedAt})
}
if p.reportS3 {
c.Conf.AwsRegion = p.awsRegion
c.Conf.AwsProfile = p.awsProfile
c.Conf.S3Bucket = p.awsS3Bucket
if err := report.CheckIfBucketExists(); err != nil {
Log.Errorf("Failed to access to the S3 bucket. err: %s", err)
Log.Error("Ensure the bucket or check AWS config before scanning")
return subcommands.ExitUsageError
}
reports = append(reports, report.S3Writer{})
}
if p.reportAzureBlob {
c.Conf.AzureAccount = p.azureAccount
if len(c.Conf.AzureAccount) == 0 {
c.Conf.AzureAccount = os.Getenv("AZURE_STORAGE_ACCOUNT")
}
c.Conf.AzureKey = p.azureKey
if len(c.Conf.AzureKey) == 0 {
c.Conf.AzureKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
}
c.Conf.AzureContainer = p.azureContainer
if len(c.Conf.AzureContainer) == 0 {
Log.Error("Azure storage container name is requied with --azure-container option")
return subcommands.ExitUsageError
}
if err := report.CheckIfAzureContainerExists(); err != nil {
Log.Errorf("Failed to access to the Azure Blob container. err: %s", err)
Log.Error("Ensure the container or check Azure config before scanning")
return subcommands.ExitUsageError
}
reports = append(reports, report.AzureBlobWriter{})
}
c.Conf.ResultsDir = p.resultsDir
c.Conf.CveDBType = p.cvedbtype
c.Conf.CveDBPath = p.cvedbpath
c.Conf.CveDictionaryURL = p.cveDictionaryURL
c.Conf.CacheDBPath = p.cacheDBPath
c.Conf.CvssScoreOver = p.cvssScoreOver
c.Conf.IgnoreUnscoredCves = p.ignoreUnscoredCves
c.Conf.SSHExternal = p.sshExternal
c.Conf.HTTPProxy = p.httpProxy
c.Conf.ContainersOnly = p.containersOnly
c.Conf.SkipBroken = p.skipBroken
Log.Info("Validating Config...")
if !c.Conf.Validate() {
if !c.Conf.ValidateOnScan() {
return subcommands.ExitUsageError
}
if ok, err := cveapi.CveClient.CheckHealth(); !ok {
Log.Errorf("CVE HTTP server is not running. err: %s", err)
Log.Errorf("Run go-cve-dictionary as server mode or specify -cve-dictionary-dbpath option")
return subcommands.ExitFailure
}
Log.Info("Detecting Server/Contianer OS... ")
if err := scan.InitServers(Log); err != nil {
Log.Errorf("Failed to init servers: %s", err)
@@ -431,21 +220,9 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
}
return subcommands.ExitFailure
}
scanResults, err := scan.GetScanResults()
if err != nil {
Log.Fatal(err)
return subcommands.ExitFailure
}
Log.Info("Reporting...")
filtered := scanResults.FilterByCvssOver()
for _, w := range reports {
if err := w.Write(filtered); err != nil {
Log.Fatalf("Failed to report, err: %s", err)
return subcommands.ExitFailure
}
}
fmt.Printf("\n\n\n")
fmt.Println("To view the detail, vuls tui is useful.")
fmt.Println("To send a report, run vuls report -h.")
return subcommands.ExitSuccess
}

109
commands/tui.go
View File

@@ -20,13 +20,12 @@ package commands
import (
"context"
"flag"
"io/ioutil"
"os"
"path/filepath"
"strings"
log "github.com/Sirupsen/logrus"
c "github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/report"
"github.com/google/subcommands"
)
@@ -36,6 +35,11 @@ type TuiCmd struct {
lang string
debugSQL bool
resultsDir string
refreshCve bool
cvedbtype string
cvedbpath string
cveDictionaryURL string
}
// Name return subcommand name
@@ -47,7 +51,13 @@ func (*TuiCmd) Synopsis() string { return "Run Tui view to anayze vulnerabilites
// Usage return usage
func (*TuiCmd) Usage() string {
return `tui:
tui [-results-dir=/path/to/results]
tui
[-cvedb-type=sqlite3|mysql]
[-cvedb-path=/path/to/cve.sqlite3]
[-cvedb-url=http://127.0.0.1:1323 or mysql connection string]
[-results-dir=/path/to/results]
[-refresh-cve]
[-debug-sql]
`
}
@@ -58,9 +68,33 @@ func (p *TuiCmd) SetFlags(f *flag.FlagSet) {
f.BoolVar(&p.debugSQL, "debug-sql", false, "debug SQL")
wd, _ := os.Getwd()
defaultResultsDir := filepath.Join(wd, "results")
f.StringVar(&p.resultsDir, "results-dir", defaultResultsDir, "/path/to/results")
f.BoolVar(
&p.refreshCve,
"refresh-cve",
false,
"Refresh CVE information in JSON file under results dir")
f.StringVar(
&p.cvedbtype,
"cvedb-type",
"sqlite3",
"DB type for fetching CVE dictionary (sqlite3 or mysql)")
defaultCveDBPath := filepath.Join(wd, "cve.sqlite3")
f.StringVar(
&p.cvedbpath,
"cvedb-path",
defaultCveDBPath,
"/path/to/sqlite3 (For get cve detail from cve.sqlite3)")
f.StringVar(
&p.cveDictionaryURL,
"cvedb-url",
"",
"http://cve-dictionary.com:8080 or mysql connection string")
}
// Execute execute
@@ -68,38 +102,53 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s
c.Conf.Lang = "en"
c.Conf.DebugSQL = p.debugSQL
c.Conf.ResultsDir = p.resultsDir
c.Conf.CveDBType = p.cvedbtype
c.Conf.CveDBPath = p.cvedbpath
c.Conf.CveDictionaryURL = p.cveDictionaryURL
var jsonDirName string
var err error
if 0 < len(f.Args()) {
var jsonDirs report.JSONDirs
if jsonDirs, err = report.GetValidJSONDirs(); err != nil {
return subcommands.ExitFailure
}
for _, d := range jsonDirs {
splitPath := strings.Split(d, string(os.PathSeparator))
if splitPath[len(splitPath)-1] == f.Args()[0] {
jsonDirName = f.Args()[0]
break
log.Info("Validating Config...")
if !c.Conf.ValidateOnTui() {
return subcommands.ExitUsageError
}
jsonDir, err := jsonDir(f.Args())
if err != nil {
log.Errorf("Failed to read json dir under results: %s", err)
return subcommands.ExitFailure
}
history, err := loadOneScanHistory(jsonDir)
if err != nil {
log.Errorf("Failed to read from JSON: %s", err)
return subcommands.ExitFailure
}
var results []models.ScanResult
for _, r := range history.ScanResults {
if p.refreshCve || needToRefreshCve(r) {
if c.Conf.CveDBType == "sqlite3" {
if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) {
log.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s",
c.Conf.CveDBPath)
return subcommands.ExitFailure
}
}
}
if len(jsonDirName) == 0 {
log.Errorf("First Argument have to be JSON directory name : %s", err)
return subcommands.ExitFailure
}
} else {
stat, _ := os.Stdin.Stat()
if (stat.Mode() & os.ModeCharDevice) == 0 {
bytes, err := ioutil.ReadAll(os.Stdin)
filled, err := fillCveInfoFromCveDB(r)
if err != nil {
log.Errorf("Failed to read stdin: %s", err)
log.Errorf("Failed to fill CVE information: %s", err)
return subcommands.ExitFailure
}
fields := strings.Fields(string(bytes))
if 0 < len(fields) {
jsonDirName = fields[0]
if err := overwriteJSONFile(jsonDir, filled); err != nil {
log.Errorf("Failed to write JSON: %s", err)
return subcommands.ExitFailure
}
results = append(results, filled)
} else {
results = append(results, r)
}
}
return report.RunTui(jsonDirName)
history.ScanResults = results
return report.RunTui(history)
}

225
commands/util.go Normal file
View File

@@ -0,0 +1,225 @@
/* Vuls - Vulnerability Scanner
Copyright (C) 2016 Future Architect, Inc. Japan.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package commands
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
c "github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/cveapi"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/report"
"github.com/future-architect/vuls/util"
)
// jsonDirPattern is file name pattern of JSON directory
// 2016-11-16T10:43:28+09:00
// 2016-11-16T10:43:28Z
var jsonDirPattern = regexp.MustCompile(
`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})$`)
// JSONDirs is array of json files path.
type jsonDirs []string
// sort as recent directories are at the head
func (d jsonDirs) Len() int {
return len(d)
}
func (d jsonDirs) Swap(i, j int) {
d[i], d[j] = d[j], d[i]
}
func (d jsonDirs) Less(i, j int) bool {
return d[j] < d[i]
}
// getValidJSONDirs return valid json directory as array
// Returned array is sorted so that recent directories are at the head
func lsValidJSONDirs() (dirs jsonDirs, err error) {
var dirInfo []os.FileInfo
if dirInfo, err = ioutil.ReadDir(c.Conf.ResultsDir); err != nil {
err = fmt.Errorf("Failed to read %s: %s", c.Conf.ResultsDir, err)
return
}
for _, d := range dirInfo {
if d.IsDir() && jsonDirPattern.MatchString(d.Name()) {
jsonDir := filepath.Join(c.Conf.ResultsDir, d.Name())
dirs = append(dirs, jsonDir)
}
}
sort.Sort(dirs)
return
}
// jsonDir returns
// If there is an arg, check if it is a valid format and return the corresponding path under results.
// If passed via PIPE (such as history subcommand), return that path.
// Otherwise, returns the path of the latest directory
func jsonDir(args []string) (string, error) {
var err error
var dirs jsonDirs
if 0 < len(args) {
if dirs, err = lsValidJSONDirs(); err != nil {
return "", err
}
path := filepath.Join(c.Conf.ResultsDir, args[0])
for _, d := range dirs {
ss := strings.Split(d, string(os.PathSeparator))
timedir := ss[len(ss)-1]
if timedir == args[0] {
return path, nil
}
}
return "", fmt.Errorf("Invalid path: %s", path)
}
// PIPE
stat, _ := os.Stdin.Stat()
if (stat.Mode() & os.ModeCharDevice) == 0 {
bytes, err := ioutil.ReadAll(os.Stdin)
if err != nil {
return "", fmt.Errorf("Failed to read stdin: %s", err)
}
fields := strings.Fields(string(bytes))
if 0 < len(fields) {
return filepath.Join(c.Conf.ResultsDir, fields[0]), nil
}
return "", fmt.Errorf("Stdin is invalid: %s", string(bytes))
}
// returns latest dir when no args or no PIPE
if dirs, err = lsValidJSONDirs(); err != nil {
return "", err
}
if len(dirs) == 0 {
return "", fmt.Errorf("No results under %s",
c.Conf.ResultsDir)
}
return dirs[0], nil
}
// loadOneScanHistory read JSON data
func loadOneScanHistory(jsonDir string) (scanHistory models.ScanHistory, err error) {
var results []models.ScanResult
var files []os.FileInfo
if files, err = ioutil.ReadDir(jsonDir); err != nil {
err = fmt.Errorf("Failed to read %s: %s", jsonDir, err)
return
}
for _, f := range files {
if filepath.Ext(f.Name()) != ".json" {
continue
}
var r models.ScanResult
var data []byte
path := filepath.Join(jsonDir, f.Name())
if data, err = ioutil.ReadFile(path); err != nil {
err = fmt.Errorf("Failed to read %s: %s", path, err)
return
}
if json.Unmarshal(data, &r) != nil {
err = fmt.Errorf("Failed to parse %s: %s", path, err)
return
}
results = append(results, r)
}
if len(results) == 0 {
err = fmt.Errorf("There is no json file under %s", jsonDir)
return
}
scanHistory = models.ScanHistory{
ScanResults: results,
}
return
}
func fillCveInfoFromCveDB(r models.ScanResult) (filled models.ScanResult, err error) {
sInfo := c.Conf.Servers[r.ServerName]
vs, err := scanVulnByCpeNames(sInfo.CpeNames, r.ScannedCves)
if err != nil {
return
}
r.ScannedCves = vs
filled, err = r.FillCveDetail()
if err != nil {
return
}
return
}
func overwriteJSONFile(dir string, r models.ScanResult) error {
before := c.Conf.FormatJSON
c.Conf.FormatJSON = true
w := report.LocalFileWriter{CurrentDir: dir}
if err := w.Write(r); err != nil {
return fmt.Errorf("Failed to write summary report: %s", err)
}
c.Conf.FormatJSON = before
return nil
}
func scanVulnByCpeNames(cpeNames []string, scannedVulns []models.VulnInfo) ([]models.VulnInfo,
error) {
// To remove duplicate
set := map[string]models.VulnInfo{}
for _, v := range scannedVulns {
set[v.CveID] = v
}
for _, name := range cpeNames {
details, err := cveapi.CveClient.FetchCveDetailsByCpeName(name)
if err != nil {
return nil, err
}
for _, detail := range details {
if val, ok := set[detail.CveID]; ok {
names := val.CpeNames
names = util.AppendIfMissing(names, name)
val.CpeNames = names
set[detail.CveID] = val
} else {
set[detail.CveID] = models.VulnInfo{
CveID: detail.CveID,
CpeNames: []string{name},
}
}
}
}
vinfos := []models.VulnInfo{}
for key := range set {
vinfos = append(vinfos, set[key])
}
return vinfos, nil
}
func needToRefreshCve(r models.ScanResult) bool {
return r.Lang != c.Conf.Lang || len(r.KnownCves) == 0 &&
len(r.UnknownCves) == 0 &&
len(r.IgnoredCves) == 0
}