Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change ScanResult.Packages structure to Map

Browse Source
This commit is contained in:
Kota Kanbe
2017-05-08 22:15:12 +09:00
committed by kota kanbe
parent f36671784e
commit 210e3dc990
15 changed files with 209 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
scan/debian.go
View File

@@ -181,7 +181,10 @@ func (o *debian) scanPackages() error {
return nil
}
func (o *debian) scanInstalledPackages() (installed models.Packages, upgradable models.Packages, err error) {
func (o *debian) scanInstalledPackages() (models.Packages, models.Packages, error) {
installed := models.Packages{}
upgradable := models.Packages{}
r := o.exec("dpkg-query -W", noSudo)
if !r.isSuccess() {
return nil, nil, fmt.Errorf("Failed to SSH: %s", r)
@@ -198,10 +201,10 @@ func (o *debian) scanInstalledPackages() (installed models.Packages, upgradable
return nil, nil, fmt.Errorf(
"Debian: Failed to parse package line: %s", line)
}
installed = append(installed, models.Package{
installed[name] = models.Package{
Name: name,
Version: version,
})
}
}
}
@@ -212,20 +215,20 @@ func (o *debian) scanInstalledPackages() (installed models.Packages, upgradable
for _, name := range upgradableNames {
for _, pack := range installed {
if pack.Name == name {
upgradable = append(upgradable, pack)
upgradable[name] = pack
break
}
}
}
// Fill the candidate versions of upgradable packages
upgradable, err = o.fillCandidateVersion(upgradable)
err = o.fillCandidateVersion(upgradable)
if err != nil {
return nil, nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err)
}
installed.MergeNewVersion(upgradable)
return
return installed, upgradable, nil
}
var packageLinePattern = regexp.MustCompile(`^([^\t']+)\t(.+)$`)
@@ -254,7 +257,7 @@ func (o *debian) aptGetUpdate() error {
return nil
}
func (o *debian) scanUnsecurePackages(upgradable []models.Package) ([]models.VulnInfo, error) {
func (o *debian) scanUnsecurePackages(upgradable models.Packages) ([]models.VulnInfo, error) {
o.aptGetUpdate()
@@ -315,28 +318,28 @@ func (o *debian) ensureChangelogCache(current cache.Meta) (*cache.Meta, error) {
return &cached, nil
}
func (o *debian) fillCandidateVersion(before models.Packages) (filled []models.Package, err error) {
func (o *debian) fillCandidateVersion(packages models.Packages) (err error) {
names := []string{}
for _, p := range before {
names = append(names, p.Name)
for name := range packages {
names = append(names, name)
}
cmd := fmt.Sprintf("LANGUAGE=en_US.UTF-8 apt-cache policy %s", strings.Join(names, " "))
r := o.exec(cmd, noSudo)
if !r.isSuccess() {
return nil, fmt.Errorf("Failed to SSH: %s", r)
return fmt.Errorf("Failed to SSH: %s", r)
}
packChangelog := o.splitAptCachePolicy(r.Stdout)
for k, v := range packChangelog {
ver, err := o.parseAptCachePolicy(v, k)
if err != nil {
return nil, fmt.Errorf("Failed to parse %s", err)
return fmt.Errorf("Failed to parse %s", err)
}
p, found := before.FindByName(k)
if !found {
return nil, fmt.Errorf("Not found: %s", k)
pack, ok := packages[k]
if !ok {
return fmt.Errorf("Not found: %s", k)
}
p.NewVersion = ver.Candidate
filled = append(filled, p)
pack.NewVersion = ver.Candidate
packages[k] = pack
}
return
}
@@ -394,7 +397,7 @@ func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, er
return
}
func (o *debian) scanVulnInfos(upgradablePacks []models.Package, meta *cache.Meta) (models.VulnInfos, error) {
func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta) (models.VulnInfos, error) {
resChan := make(chan struct {
models.Package
DetectedCveIDs
@@ -412,7 +415,7 @@ func (o *debian) scanVulnInfos(upgradablePacks []models.Package, meta *cache.Met
}()
timeout := time.After(30 * 60 * time.Second)
concurrency := 10
concurrency := 1
tasks := util.GenWorkers(concurrency)
for range upgradablePacks {
tasks <- func() {
@@ -446,18 +449,23 @@ func (o *debian) scanVulnInfos(upgradablePacks []models.Package, meta *cache.Met
}
// { DetectedCveID{} : [package] }
cvePackages := make(map[DetectedCveID][]models.Package)
cvePackages := make(map[DetectedCveID]models.Packages)
errs := []error{}
for i := 0; i < len(upgradablePacks); i++ {
select {
case pair := <-resChan:
pack := pair.Package
cveIDs := pair.DetectedCveIDs
for _, cveID := range cveIDs {
cvePackages[cveID] = appendPackIfMissing(cvePackages[cveID], pack)
cves := pair.DetectedCveIDs
for _, cve := range cves {
packs, ok := cvePackages[cve]
if ok {
packs[cve.CveID] = pair.Package
} else {
packs = models.Packages{}
}
cvePackages[cve] = packs
}
o.log.Infof("(%d/%d) Scanned %s-%s : %s",
i+1, len(upgradablePacks), pair.Name, pair.Package.Version, cveIDs)
i+1, len(upgradablePacks), pair.Name, pair.Package.Version, cves)
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
@@ -492,7 +500,7 @@ func (o *debian) scanVulnInfos(upgradablePacks []models.Package, meta *cache.Met
}
func (o *debian) getChangelogCache(meta *cache.Meta, pack models.Package) string {
cachedPack, found := meta.FindPack(pack.Name)
cachedPack, found := meta.Packs[pack.Name]
if !found {
o.log.Debugf("Not found: %s", pack.Name)
return ""
@@ -602,14 +610,12 @@ func (o *debian) getCveIDsFromChangelog(
// Only logging the error.
o.log.Error(err)
for i, p := range o.Packages {
if p.Name == name {
o.Packages[i].Changelog = models.Changelog{
Contents: "",
Method: models.FailedToFindVersionInChangelog,
}
}
pack := o.Packages[name]
pack.Changelog = models.Changelog{
Contents: "",
Method: models.FailedToFindVersionInChangelog,
}
o.Packages[name] = pack
// If the version is not in changelog, return entire changelog to put into cache
return []DetectedCveID{}, models.Changelog{
@@ -666,11 +672,9 @@ func (o *debian) parseChangelog(changelog, name, ver string, confidence models.C
Method: string(confidence.DetectionMethod),
}
for i, p := range o.Packages {
if p.Name == name {
o.Packages[i].Changelog = clog
}
}
pack := o.Packages[name]
pack.Changelog = clog
o.Packages[name] = pack
cves := []DetectedCveID{}
for _, id := range cveIDs {
@@ -729,14 +733,3 @@ func (o *debian) parseAptCachePolicy(stdout, name string) (packCandidateVer, err
}
return ver, fmt.Errorf("Unknown Format: %s", stdout)
}
func appendPackIfMissing(slice []models.Package, s models.Package) []models.Package {
for _, ele := range slice {
if ele.Name == s.Name &&
ele.Version == s.Version &&
ele.Release == s.Release {
return slice
}
}
return append(slice, s)
}

4
scan/debian_test.go
View File

@@ -624,7 +624,9 @@ func TestGetChangelogCache(t *testing.T) {
Family: "ubuntu",
Release: "16.04",
},
Packs: []models.Package{pack},
Packs: models.Packages{
"apt": pack,
},
}
const path = "/tmp/vuls-test-cache-11111111.db"

23
scan/freebsd.go
View File

@@ -71,7 +71,7 @@ func (o *bsd) checkDependencies() error {
func (o *bsd) scanPackages() error {
var err error
var packs []models.Package
var packs models.Packages
if packs, err = o.scanInstalledPackages(); err != nil {
o.log.Errorf("Failed to scan installed packages")
return err
@@ -87,7 +87,7 @@ func (o *bsd) scanPackages() error {
return nil
}
func (o *bsd) scanInstalledPackages() ([]models.Package, error) {
func (o *bsd) scanInstalledPackages() (models.Packages, error) {
cmd := util.PrependProxyEnv("pkg version -v")
r := o.exec(cmd, noSudo)
if !r.isSuccess() {
@@ -121,7 +121,7 @@ func (o *bsd) scanUnsecurePackages() (vulnInfos []models.VulnInfo, err error) {
if len(cveIDs) == 0 {
continue
}
pack, found := o.Packages.FindByName(name)
pack, found := o.Packages[name]
if !found {
return nil, fmt.Errorf("Vulnerable package: %s is not found", name)
}
@@ -143,9 +143,9 @@ func (o *bsd) scanUnsecurePackages() (vulnInfos []models.VulnInfo, err error) {
}
for k := range cveIDAdtMap {
packs := []models.Package{}
packs := models.Packages{}
for _, r := range cveIDAdtMap[k] {
packs = append(packs, r.pack)
packs[r.pack.Name] = r.pack
}
disAdvs := []models.DistroAdvisory{}
@@ -165,7 +165,8 @@ func (o *bsd) scanUnsecurePackages() (vulnInfos []models.VulnInfo, err error) {
return
}
func (o *bsd) parsePkgVersion(stdout string) (packs []models.Package) {
func (o *bsd) parsePkgVersion(stdout string) models.Packages {
packs := models.Packages{}
lines := strings.Split(stdout, "\n")
for _, l := range lines {
fields := strings.Fields(l)
@@ -180,20 +181,20 @@ func (o *bsd) parsePkgVersion(stdout string) (packs []models.Package) {
switch fields[1] {
case "?", "=":
packs = append(packs, models.Package{
packs[name] = models.Package{
Name: name,
Version: ver,
})
}
case "<":
candidate := strings.TrimSuffix(fields[6], ")")
packs = append(packs, models.Package{
packs[name] = models.Package{
Name: name,
Version: ver,
NewVersion: candidate,
})
}
}
}
return
return packs
}
type vulnIDCveIDs struct {

12
scan/freebsd_test.go
View File

@@ -12,7 +12,7 @@ import (
func TestParsePkgVersion(t *testing.T) {
var tests = []struct {
in string
expected []models.Package
expected models.Packages
}{
{
`Updating FreeBSD repository catalogue...
@@ -23,22 +23,22 @@ gettext-0.18.3.1 < needs updating (remote has 0.19.7)
tcl84-8.4.20_2,1 = up-to-date with remote
teTeX-base-3.0_25 ? orphaned: print/teTeX-base`,
[]models.Package{
{
models.Packages{
"bash": {
Name: "bash",
Version: "4.2.45",
NewVersion: "4.3.42_1",
},
{
"gettext": {
Name: "gettext",
Version: "0.18.3.1",
NewVersion: "0.19.7",
},
{
"tcl84": {
Name: "tcl84",
Version: "8.4.20_2,1",
},
{
"teTeX-base": {
Name: "teTeX-base",
Version: "3.0_25",
},

45
scan/redhat.go
View File

@@ -231,7 +231,7 @@ func (o *redhat) scanPackages() error {
o.log.Errorf("Failed to scan installed packages")
return err
}
o.setPackages(packs)
o.setPackages(models.NewPackages(packs...))
var vinfos []models.VulnInfo
if vinfos, err = o.scanVulnInfos(); err != nil {
@@ -242,7 +242,7 @@ func (o *redhat) scanPackages() error {
return nil
}
func (o *redhat) scanInstalledPackages() (installedPackages models.Packages, err error) {
func (o *redhat) scanInstalledPackages() (installed []models.Package, err error) {
cmd := "rpm -qa --queryformat '%{NAME}\t%{EPOCHNUM}\t%{VERSION}\t%{RELEASE}\n'"
r := o.exec(cmd, noSudo)
if r.isSuccess() {
@@ -255,13 +255,13 @@ func (o *redhat) scanInstalledPackages() (installedPackages models.Packages, err
if pack, err = o.parseScannedPackagesLine(line); err != nil {
return
}
installedPackages = append(installedPackages, pack)
installed = append(installed, pack)
}
}
return
}
return installedPackages, fmt.Errorf(
return nil, fmt.Errorf(
"Scan packages failed. status: %d, stdout: %s, stderr: %s",
r.ExitStatus, r.Stdout, r.Stderr)
}
@@ -341,22 +341,23 @@ func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (models.VulnInfos, er
}
for name, clog := range rpm2changelog {
for i, p := range o.Packages {
n := fmt.Sprintf("%s-%s-%s",
p.Name, p.NewVersion, p.NewRelease)
for _, p := range o.Packages {
n := fmt.Sprintf("%s-%s-%s", p.Name, p.NewVersion, p.NewRelease)
if name == n {
o.Packages[i].Changelog = models.Changelog{
p.Changelog = models.Changelog{
Contents: *clog,
Method: models.ChangelogExactMatchStr,
}
o.Packages[p.Name] = p
break
}
}
}
var results []PackageCveIDs
for i, pack := range packages {
changelog := o.getChangelogCVELines(rpm2changelog, pack)
i := 0
for name := range packages {
changelog := o.getChangelogCVELines(rpm2changelog, packages[name])
// Collect unique set of CVE-ID in each changelog
uniqueCveIDMap := make(map[string]bool)
@@ -374,7 +375,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (models.VulnInfos, er
cveIDs = append(cveIDs, k)
}
p := PackageCveIDs{
Package: pack,
Package: packages[name],
CveIDs: cveIDs,
}
results = append(results, p)
@@ -388,6 +389,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (models.VulnInfos, er
p.Package.NewVersion,
p.Package.NewRelease,
p.CveIDs)
i++
}
// transform datastructure
@@ -415,7 +417,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (models.VulnInfos, er
// Amazon, RHEL do not use this method, so VendorAdvisory do not set.
vinfos = append(vinfos, models.VulnInfo{
CveID: k,
Packages: v,
Packages: models.NewPackages(v...),
Confidence: models.ChangelogExactMatch,
})
}
@@ -423,7 +425,8 @@ func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (models.VulnInfos, er
}
// parseYumCheckUpdateLines parse yum check-update to get package name, candidate version
func (o *redhat) parseYumCheckUpdateLines(stdout string) (results models.Packages, err error) {
func (o *redhat) parseYumCheckUpdateLines(stdout string) (models.Packages, error) {
results := models.Packages{}
needToParse := false
lines := strings.Split(stdout, "\n")
for _, line := range lines {
@@ -443,20 +446,20 @@ func (o *redhat) parseYumCheckUpdateLines(stdout string) (results models.Package
return results, err
}
installed, found := o.Packages.FindByName(candidate.Name)
installed, found := o.Packages[candidate.Name]
if !found {
o.log.Warnf("Not found the package in rpm -qa. candidate: %s-%s-%s",
candidate.Name, candidate.Version, candidate.Release)
results = append(results, candidate)
results[candidate.Name] = candidate
continue
}
installed.NewVersion = candidate.NewVersion
installed.NewRelease = candidate.NewRelease
installed.Repository = candidate.Repository
results = append(results, installed)
results[installed.Name] = installed
}
}
return
return results, nil
}
func (o *redhat) parseYumCheckUpdateLine(line string) (models.Package, error) {
@@ -686,16 +689,16 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos,
// set candidate version info
o.Packages.MergeNewVersion(updatable)
dict := map[string][]models.Package{}
dict := make(map[string]models.Packages)
for _, advIDPackNames := range advIDPackNamesList {
packages := models.Packages{}
for _, packName := range advIDPackNames.PackNames {
pack, found := updatable.FindByName(packName)
pack, found := updatable[packName]
if !found {
return nil, fmt.Errorf(
"Package not found. pack: %#v", packName)
}
packages = append(packages, pack)
packages[pack.Name] = pack
continue
}
dict[advIDPackNames.AdvisoryID] = packages
@@ -729,7 +732,7 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos,
vinfos[i].DistroAdvisories = advAppended
packs := dict[advIDCveIDs.DistroAdvisory.AdvisoryID]
vinfos[i].Packages = append(vinfos[i].Packages, packs...)
vinfos[i].Packages = vinfos[i].Packages.Merge(packs)
found = true
break
}

88
scan/redhat_test.go
View File

@@ -440,11 +440,13 @@ Description : kernel-uek
for _, tt := range tests {
actual, _ := r.parseYumUpdateinfo(tt.in)
for i, advisoryCveIDs := range actual {
if !reflect.DeepEqual(tt.out[i], advisoryCveIDs) {
e := pp.Sprintf("%v", tt.out[i])
a := pp.Sprintf("%v", advisoryCveIDs)
if tt.out[i].DistroAdvisory != advisoryCveIDs.DistroAdvisory {
t.Errorf("[%d] Alas is not same. \nexpected: %s\nactual: %s",
i, e, a)
i, tt.out[i].DistroAdvisory, advisoryCveIDs.DistroAdvisory)
}
if !reflect.DeepEqual(tt.out[i].CveIDs, advisoryCveIDs.CveIDs) {
t.Errorf("[%d] Alas is not same. \nexpected: %s\nactual: %s",
i, tt.out[i].CveIDs, advisoryCveIDs.CveIDs)
}
}
}
@@ -562,14 +564,14 @@ Description : The Berkeley Internet Name Domain (BIND) is an implementation of
}
for _, tt := range tests {
actual, _ := r.parseYumUpdateinfo(tt.in)
for j, advisoryCveIDs := range actual {
sort.Strings(tt.out[j].CveIDs)
for i, advisoryCveIDs := range actual {
sort.Strings(tt.out[i].CveIDs)
sort.Strings(advisoryCveIDs.CveIDs)
if !reflect.DeepEqual(tt.out[j], advisoryCveIDs) {
e := pp.Sprintf("%v", tt.out[j])
if !reflect.DeepEqual(tt.out[i], advisoryCveIDs) {
e := pp.Sprintf("%v", tt.out[i])
a := pp.Sprintf("%v", advisoryCveIDs)
t.Errorf("[%d] Alas is not same. \nexpected: %s\nactual: %s",
j, e, a)
i, e, a)
}
}
}
@@ -688,46 +690,46 @@ bind-utils.x86_64 30:9.3.6-25.P1.el5_11.8 updates
pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5
`
r.Packages = []models.Package{
{
r.setPackages(models.NewPackages(
models.Package{
Name: "audit-libs",
Version: "2.3.6",
Release: "4.el6",
},
{
models.Package{
Name: "bash",
Version: "4.1.1",
Release: "33",
},
{
models.Package{
Name: "python-libs",
Version: "2.6.0",
Release: "1.1-0",
},
{
models.Package{
Name: "python-ordereddict",
Version: "1.0",
Release: "1",
},
{
models.Package{
Name: "bind-utils",
Version: "1.0",
Release: "1",
},
{
models.Package{
Name: "pytalloc",
Version: "2.0.1",
Release: "0",
},
}
))
var tests = []struct {
in string
out models.Packages
}{
{
stdout,
models.Packages{
{
models.NewPackages(
models.Package{
Name: "audit-libs",
Version: "2.3.6",
Release: "4.el6",
@@ -735,7 +737,7 @@ pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5
NewRelease: "5.el6",
Repository: "base",
},
{
models.Package{
Name: "bash",
Version: "4.1.1",
Release: "33",
@@ -743,7 +745,7 @@ pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5
NewRelease: "33.el6_7.1",
Repository: "updates",
},
{
models.Package{
Name: "python-libs",
Version: "2.6.0",
Release: "1.1-0",
@@ -751,7 +753,7 @@ pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5
NewRelease: "64.el6",
Repository: "rhui-REGION-rhel-server-releases",
},
{
models.Package{
Name: "python-ordereddict",
Version: "1.0",
Release: "1",
@@ -759,7 +761,7 @@ pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5
NewRelease: "3.el6ev",
Repository: "installed",
},
{
models.Package{
Name: "bind-utils",
Version: "1.0",
Release: "1",
@@ -767,7 +769,7 @@ pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5
NewRelease: "25.P1.el5_11.8",
Repository: "updates",
},
{
models.Package{
Name: "pytalloc",
Version: "2.0.1",
Release: "0",
@@ -775,7 +777,7 @@ pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5
NewRelease: "2.el6",
Repository: "@CentOS 6.5/6.5",
},
},
),
},
}
@@ -785,11 +787,11 @@ pytalloc.x86_64 2.0.7-2.el6 @CentOS 6.5/6.5
t.Errorf("Error has occurred, err: %s\ntt.in: %v", err, tt.in)
return
}
for i, ePack := range tt.out {
if !reflect.DeepEqual(ePack, packages[i]) {
for name, ePack := range tt.out {
if !reflect.DeepEqual(ePack, packages[name]) {
e := pp.Sprintf("%v", ePack)
a := pp.Sprintf("%v", packages[i])
t.Errorf("[%d] expected %s, actual %s", i, e, a)
a := pp.Sprintf("%v", packages[name])
t.Errorf("expected %s, actual %s", e, a)
}
}
}
@@ -805,31 +807,31 @@ bind-libs.x86_64 32:9.8.2-0.37.rc1.45.amzn1 amzn-main
java-1.7.0-openjdk.x86_64 1.7.0.95-2.6.4.0.65.amzn1 amzn-main
if-not-architecture 100-200 amzn-main
`
r.Packages = []models.Package{
{
r.Packages = models.NewPackages(
models.Package{
Name: "bind-libs",
Version: "9.8.0",
Release: "0.33.rc1.45.amzn1",
},
{
models.Package{
Name: "java-1.7.0-openjdk",
Version: "1.7.0.0",
Release: "2.6.4.0.0.amzn1",
},
{
models.Package{
Name: "if-not-architecture",
Version: "10",
Release: "20",
},
}
)
var tests = []struct {
in string
out models.Packages
}{
{
stdout,
models.Packages{
{
models.NewPackages(
models.Package{
Name: "bind-libs",
Version: "9.8.0",
Release: "0.33.rc1.45.amzn1",
@@ -837,7 +839,7 @@ if-not-architecture 100-200 amzn-main
NewRelease: "0.37.rc1.45.amzn1",
Repository: "amzn-main",
},
{
models.Package{
Name: "java-1.7.0-openjdk",
Version: "1.7.0.0",
Release: "2.6.4.0.0.amzn1",
@@ -845,7 +847,7 @@ if-not-architecture 100-200 amzn-main
NewRelease: "2.6.4.0.65.amzn1",
Repository: "amzn-main",
},
{
models.Package{
Name: "if-not-architecture",
Version: "10",
Release: "20",
@@ -853,7 +855,7 @@ if-not-architecture 100-200 amzn-main
NewRelease: "200",
Repository: "amzn-main",
},
},
),
},
}
@@ -863,11 +865,11 @@ if-not-architecture 100-200 amzn-main
t.Errorf("Error has occurred, err: %s\ntt.in: %v", err, tt.in)
return
}
for i, ePack := range tt.out {
if !reflect.DeepEqual(ePack, packages[i]) {
for name, ePack := range tt.out {
if !reflect.DeepEqual(ePack, packages[name]) {
e := pp.Sprintf("%v", ePack)
a := pp.Sprintf("%v", packages[i])
t.Errorf("[%d] expected %s, actual %s", i, e, a)
a := pp.Sprintf("%v", packages[name])
t.Errorf("[%s] expected %s, actual %s", name, e, a)
}
}
}