Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

High speed scan on Ubuntu/Debian

Browse Source
This commit is contained in:
Kota Kanbe
2016-09-08 19:18:49 +09:00
parent 2afe2d2640
commit dd1d3a05fa
25 changed files with 1035 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
scan/debian.go
View File

@@ -24,6 +24,7 @@ import (
"strings"
"time"
"github.com/future-architect/vuls/cache"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/cveapi"
"github.com/future-architect/vuls/models"
@@ -39,6 +40,7 @@ type debian struct {
func newDebian(c config.ServerInfo) *debian {
d := &debian{}
d.log = util.NewCustomLogger(c)
d.setServerInfo(c)
return d
}
@@ -46,7 +48,6 @@ func newDebian(c config.ServerInfo) *debian {
// https://github.com/serverspec/specinfra/blob/master/lib/specinfra/helper/detect_os/debian.rb
func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err error) {
deb = newDebian(c)
deb.setServerInfo(c)
if r := sshExec(c, "ls /etc/debian_version", noSudo); !r.isSuccess() {
if r.Error != nil {
@@ -69,12 +70,12 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err
result := re.FindStringSubmatch(trim(r.Stdout))
if len(result) == 0 {
deb.setDistributionInfo("debian/ubuntu", "unknown")
deb.setDistro("debian/ubuntu", "unknown")
Log.Warnf(
"Unknown Debian/Ubuntu version. lsb_release -ir: %s", r)
} else {
distro := strings.ToLower(trim(result[1]))
deb.setDistributionInfo(distro, trim(result[2]))
deb.setDistro(distro, trim(result[2]))
}
return true, deb, nil
}
@@ -90,10 +91,10 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err
if len(result) == 0 {
Log.Warnf(
"Unknown Debian/Ubuntu. cat /etc/lsb-release: %s", r)
deb.setDistributionInfo("debian/ubuntu", "unknown")
deb.setDistro("debian/ubuntu", "unknown")
} else {
distro := strings.ToLower(trim(result[1]))
deb.setDistributionInfo(distro, trim(result[2]))
deb.setDistro(distro, trim(result[2]))
}
return true, deb, nil
}
@@ -101,7 +102,7 @@ func detectDebian(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err err
// Debian
cmd := "cat /etc/debian_version"
if r := sshExec(c, cmd, noSudo); r.isSuccess() {
deb.setDistributionInfo("debian", trim(r.Stdout))
deb.setDistro("debian", trim(r.Stdout))
return true, deb, nil
}
@@ -133,7 +134,7 @@ func (o *debian) install() error {
return fmt.Errorf(msg)
}
if o.Family == "debian" {
if o.Distro.Family == "debian" {
// install aptitude
cmd = util.PrependProxyEnv("apt-get install --force-yes -y aptitude")
if r := o.ssh(cmd, sudo); !r.isSuccess() {
@@ -208,7 +209,7 @@ func (o *debian) parseScannedPackagesLine(line string) (name, version string, er
}
func (o *debian) checkRequiredPackagesInstalled() error {
if o.Family == "debian" {
if o.Distro.Family == "debian" {
if r := o.ssh("test -f /usr/bin/aptitude", noSudo); !r.isSuccess() {
msg := fmt.Sprintf("aptitude is not installed: %s", r)
o.log.Errorf(msg)
@@ -218,9 +219,8 @@ func (o *debian) checkRequiredPackagesInstalled() error {
return nil
}
//TODO return whether already expired.
func (o *debian) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInfo, error) {
// cmd := prependProxyEnv(conf.HTTPProxy, "apt-get update | cat; echo 1")
o.log.Infof("apt-get update...")
cmd := util.PrependProxyEnv("apt-get update")
if r := o.ssh(cmd, sudo); !r.isSuccess() {
return nil, fmt.Errorf("Failed to SSH: %s", r)
@@ -241,12 +241,21 @@ func (o *debian) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInf
}
}
}
unsecurePacks, err = o.fillCandidateVersion(unsecurePacks)
if err != nil {
return nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err)
}
current := cache.Meta{
Name: o.getServerInfo().ServerName,
Distro: o.getServerInfo().Distro,
Packs: unsecurePacks,
}
o.log.Debugf("Ensure changelog cache: %s", current.Name)
if err := o.ensureChangelogCache(current); err != nil {
return nil, err
}
// Collect CVE information of upgradable packages
cvePacksInfos, err := o.scanPackageCveInfos(unsecurePacks)
if err != nil {
@@ -256,63 +265,61 @@ func (o *debian) scanUnsecurePackages(packs []models.PackageInfo) ([]CvePacksInf
return cvePacksInfos, nil
}
func (o *debian) fillCandidateVersion(packs []models.PackageInfo) ([]models.PackageInfo, error) {
reqChan := make(chan models.PackageInfo, len(packs))
resChan := make(chan models.PackageInfo, len(packs))
errChan := make(chan error, len(packs))
defer close(resChan)
defer close(errChan)
defer close(reqChan)
go func() {
for _, pack := range packs {
reqChan <- pack
func (o *debian) ensureChangelogCache(current cache.Meta) error {
// Search from cache
old, found, err := cache.DB.GetMeta(current.Name)
if err != nil {
return fmt.Errorf("Failed to get meta. err: %s", err)
}
if !found {
o.log.Debugf("Not found in meta: %s", current.Name)
err = cache.DB.EnsureBuckets(current)
if err != nil {
return fmt.Errorf("Failed to ensure buckets. err: %s", err)
}
}()
timeout := time.After(5 * 60 * time.Second)
concurrency := 5
tasks := util.GenWorkers(concurrency)
for range packs {
tasks <- func() {
select {
case pack := <-reqChan:
func(p models.PackageInfo) {
cmd := fmt.Sprintf("LANG=en_US.UTF-8 apt-cache policy %s", p.Name)
r := o.ssh(cmd, sudo)
if !r.isSuccess() {
errChan <- fmt.Errorf("Failed to SSH: %s.", r)
return
}
ver, err := o.parseAptCachePolicy(r.Stdout, p.Name)
if err != nil {
errChan <- fmt.Errorf("Failed to parse %s", err)
}
p.NewVersion = ver.Candidate
resChan <- p
}(pack)
} else {
if current.Distro.Family != old.Distro.Family ||
current.Distro.Release != old.Distro.Release {
o.log.Debugf("Need to refesh meta: %s", current.Name)
err = cache.DB.EnsureBuckets(current)
if err != nil {
return fmt.Errorf("Failed to ensure buckets. err: %s", err)
}
} else {
o.log.Debugf("Reuse meta: %s", current.Name)
}
}
errs := []error{}
result := []models.PackageInfo{}
for i := 0; i < len(packs); i++ {
select {
case pack := <-resChan:
result = append(result, pack)
o.log.Infof("(%d/%d) Upgradable: %s-%s -> %s",
i+1, len(packs), pack.Name, pack.Version, pack.NewVersion)
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
return nil, fmt.Errorf("Timeout fillCandidateVersion")
if config.Conf.Debug {
cache.DB.PrettyPrint(current)
}
return nil
}
func (o *debian) fillCandidateVersion(before models.PackageInfoList) (filled []models.PackageInfo, err error) {
names := []string{}
for _, p := range before {
names = append(names, p.Name)
}
cmd := fmt.Sprintf("LANG=en_US.UTF-8 apt-cache policy %s", strings.Join(names, " "))
r := o.ssh(cmd, sudo)
if !r.isSuccess() {
return nil, fmt.Errorf("Failed to SSH: %s.", r)
}
packChangelog := o.splitAptCachePolicy(r.Stdout)
for k, v := range packChangelog {
ver, err := o.parseAptCachePolicy(v, k)
if err != nil {
return nil, fmt.Errorf("Failed to parse %s", err)
}
p, found := before.FindByName(k)
if !found {
return nil, fmt.Errorf("Not found: %s", k)
}
p.NewVersion = ver.Candidate
filled = append(filled, p)
}
if 0 < len(errs) {
return nil, fmt.Errorf("%v", errs)
}
return result, nil
return
}
func (o *debian) GetUpgradablePackNames() (packNames []string, err error) {
@@ -369,9 +376,11 @@ func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, er
}
func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePacksList CvePacksList, err error) {
// { CVE ID: [packageInfo] }
cvePackages := make(map[string][]models.PackageInfo)
meta := cache.Meta{
Name: o.getServerInfo().ServerName,
Distro: o.getServerInfo().Distro,
Packs: unsecurePacks,
}
type strarray []string
resChan := make(chan struct {
@@ -398,6 +407,18 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac
select {
case pack := <-reqChan:
func(p models.PackageInfo) {
changelog := o.getChangelogCache(meta, p)
if 0 < len(changelog) {
cveIDs := o.getCveIDFromChangelog(changelog, p.Name, p.Version)
resChan <- struct {
models.PackageInfo
strarray
}{p, cveIDs}
return
}
// if the changelog is not in cache or failed to get from local cache,
// get the changelog of the package via internet.
if cveIDs, err := o.scanPackageCveIDs(p); err != nil {
errChan <- err
} else {
@@ -411,6 +432,8 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac
}
}
// { CVE ID: [packageInfo] }
cvePackages := make(map[string][]models.PackageInfo)
errs := []error{}
for i := 0; i < len(unsecurePacks); i++ {
select {
@@ -429,7 +452,6 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac
return nil, fmt.Errorf("Timeout scanPackageCveIDs")
}
}
if 0 < len(errs) {
return nil, fmt.Errorf("%v", errs)
}
@@ -438,7 +460,6 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac
for k := range cvePackages {
cveIDs = append(cveIDs, k)
}
o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs)
o.log.Info("Fetching CVE details...")
@@ -459,9 +480,32 @@ func (o *debian) scanPackageCveInfos(unsecurePacks []models.PackageInfo) (cvePac
return
}
func (o *debian) getChangelogCache(meta cache.Meta, pack models.PackageInfo) string {
cachedPack, found := meta.FindPack(pack.Name)
if !found {
return ""
}
if cachedPack.NewVersion != pack.NewVersion {
return ""
}
changelog, err := cache.DB.GetChangelog(meta.Name, pack.Name)
if err != nil {
o.log.Warnf("Failed to get chnagelog. bucket: %s, key:%s, err: %s",
meta.Name, pack.Name, err)
return ""
}
if len(changelog) == 0 {
return ""
}
o.log.Debugf("Cache hit: %s, len: %d, %s...",
meta.Name, len(changelog), util.Truncate(changelog, 30))
return changelog
}
func (o *debian) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) {
cmd := ""
switch o.Family {
switch o.Distro.Family {
case "ubuntu":
cmd = fmt.Sprintf(`apt-get changelog %s | grep '\(urgency\|CVE\)'`, pack.Name)
case "debian":
@@ -476,36 +520,38 @@ func (o *debian) scanPackageCveIDs(pack models.PackageInfo) ([]string, error) {
return nil, nil
}
err := cache.DB.PutChangelog(o.getServerInfo().ServerName, pack.Name, r.Stdout)
if err != nil {
return nil, fmt.Errorf("Failed to put changelog into cache")
}
// No error will be returned. Only logging.
return o.getCveIDParsingChangelog(r.Stdout, pack.Name, pack.Version)
return o.getCveIDFromChangelog(r.Stdout, pack.Name, pack.Version), nil
}
func (o *debian) getCveIDParsingChangelog(changelog string,
packName string, versionOrLater string) (cveIDs []string, err error) {
func (o *debian) getCveIDFromChangelog(changelog string,
packName string, versionOrLater string) []string {
cveIDs, err = o.parseChangelog(changelog, packName, versionOrLater)
if err == nil {
return
if cveIDs, err := o.parseChangelog(changelog, packName, versionOrLater); err == nil {
return cveIDs
}
ver := strings.Split(versionOrLater, "ubuntu")[0]
cveIDs, err = o.parseChangelog(changelog, packName, ver)
if err == nil {
return
if cveIDs, err := o.parseChangelog(changelog, packName, ver); err == nil {
return cveIDs
}
splittedByColon := strings.Split(versionOrLater, ":")
if 1 < len(splittedByColon) {
ver = splittedByColon[1]
}
cveIDs, err = o.parseChangelog(changelog, packName, ver)
cveIDs, err := o.parseChangelog(changelog, packName, ver)
if err == nil {
return
return cveIDs
}
// Only logging the error.
o.log.Error(err)
return []string{}, nil
return []string{}
}
// Collect CVE-IDs included in the changelog.
@@ -538,6 +584,29 @@ func (o *debian) parseChangelog(changelog string,
return
}
func (o *debian) splitAptCachePolicy(stdout string) map[string]string {
// re := regexp.MustCompile(`(?m:^[^ \t]+:$)`)
re := regexp.MustCompile(`(?m:^[^ \t]+:\r\n)`)
ii := re.FindAllStringIndex(stdout, -1)
ri := []int{}
for i := len(ii) - 1; 0 <= i; i-- {
ri = append(ri, ii[i][0])
}
splitted := []string{}
lasti := len(stdout)
for _, i := range ri {
splitted = append(splitted, stdout[i:lasti])
lasti = i
}
packChangelog := map[string]string{}
for _, r := range splitted {
packName := r[:strings.Index(r, ":")]
packChangelog[packName] = r
}
return packChangelog
}
type packCandidateVer struct {
Name string
Installed string