Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(report): azure and aws writer (#1190)

Browse Source
This commit is contained in:
Kota Kanbe
2021-03-04 07:42:38 +09:00
committed by GitHub
parent 1ec31d7be9
commit 2a8ee4b22b
4 changed files with 73 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
config/azureconf.go
View File

@@ -1,5 +1,11 @@
package config
import (
"os"
"golang.org/x/xerrors"
)
// AzureConf is azure config
type AzureConf struct {
// Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified
@@ -16,9 +22,18 @@ type AzureConf struct {
// Validate configuration
func (c *AzureConf) Validate() (errs []error) {
// TODO
if !c.Enabled {
return
}
if c.AccountName == "" {
c.AccountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
}
if c.AccountKey == "" {
c.AccountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
}
if c.ContainerName == "" {
errs = append(errs, xerrors.Errorf("Azure storage container name is required"))
}
return
}

32
reporter/azureblob.go
View File

@@ -20,6 +20,8 @@ type AzureBlobWriter struct {
FormatOneLineText bool
FormatList bool
Gzip bool
c.AzureConf
}
// Write results to Azure Blob storage
@@ -28,7 +30,7 @@ func (w AzureBlobWriter) Write(rs ...models.ScanResult) (err error) {
return nil
}
cli, err := getBlobClient()
cli, err := w.getBlobClient()
if err != nil {
return err
}
@@ -38,7 +40,7 @@ func (w AzureBlobWriter) Write(rs ...models.ScanResult) (err error) {
k := fmt.Sprintf(timestr + "/summary.txt")
text := formatOneLineSummary(rs...)
b := []byte(text)
if err := createBlockBlob(cli, k, b, w.Gzip); err != nil {
if err := w.createBlockBlob(cli, k, b, w.Gzip); err != nil {
return err
}
}
@@ -51,7 +53,7 @@ func (w AzureBlobWriter) Write(rs ...models.ScanResult) (err error) {
if b, err = json.Marshal(r); err != nil {
return xerrors.Errorf("Failed to Marshal to JSON: %w", err)
}
if err := createBlockBlob(cli, k, b, w.Gzip); err != nil {
if err := w.createBlockBlob(cli, k, b, w.Gzip); err != nil {
return err
}
}
@@ -59,7 +61,7 @@ func (w AzureBlobWriter) Write(rs ...models.ScanResult) (err error) {
if w.FormatList {
k := key + "_short.txt"
b := []byte(formatList(r))
if err := createBlockBlob(cli, k, b, w.Gzip); err != nil {
if err := w.createBlockBlob(cli, k, b, w.Gzip); err != nil {
return err
}
}
@@ -67,7 +69,7 @@ func (w AzureBlobWriter) Write(rs ...models.ScanResult) (err error) {
if w.FormatFullText {
k := key + "_full.txt"
b := []byte(formatFullPlainText(r))
if err := createBlockBlob(cli, k, b, w.Gzip); err != nil {
if err := w.createBlockBlob(cli, k, b, w.Gzip); err != nil {
return err
}
}
@@ -75,9 +77,9 @@ func (w AzureBlobWriter) Write(rs ...models.ScanResult) (err error) {
return
}
// CheckIfAzureContainerExists check the existence of Azure storage container
func CheckIfAzureContainerExists() error {
cli, err := getBlobClient()
// Validate check the existence of Azure storage container
func (w AzureBlobWriter) Validate() error {
cli, err := w.getBlobClient()
if err != nil {
return err
}
@@ -88,26 +90,26 @@ func CheckIfAzureContainerExists() error {
found := false
for _, con := range r.Containers {
if con.Name == c.Conf.Azure.ContainerName {
if con.Name == w.ContainerName {
found = true
break
}
}
if !found {
return xerrors.Errorf("Container not found. Container: %s", c.Conf.Azure.ContainerName)
return xerrors.Errorf("Container not found. Container: %s", w.ContainerName)
}
return nil
}
func getBlobClient() (storage.BlobStorageClient, error) {
api, err := storage.NewBasicClient(c.Conf.Azure.AccountName, c.Conf.Azure.AccountKey)
func (w AzureBlobWriter) getBlobClient() (storage.BlobStorageClient, error) {
api, err := storage.NewBasicClient(w.AccountName, w.AccountKey)
if err != nil {
return storage.BlobStorageClient{}, err
}
return api.GetBlobService(), nil
}
func createBlockBlob(cli storage.BlobStorageClient, k string, b []byte, gzip bool) error {
func (w AzureBlobWriter) createBlockBlob(cli storage.BlobStorageClient, k string, b []byte, gzip bool) error {
var err error
if gzip {
if b, err = gz(b); err != nil {
@@ -116,11 +118,11 @@ func createBlockBlob(cli storage.BlobStorageClient, k string, b []byte, gzip boo
k += ".gz"
}
ref := cli.GetContainerReference(c.Conf.Azure.ContainerName)
ref := cli.GetContainerReference(w.ContainerName)
blob := ref.GetBlobReference(k)
if err := blob.CreateBlockBlobFromReader(bytes.NewReader(b), nil); err != nil {
return xerrors.Errorf("Failed to upload data to %s/%s, err: %w",
c.Conf.Azure.ContainerName, k, err)
w.ContainerName, k, err)
}
return nil
}

48
reporter/s3.go
View File

@@ -26,18 +26,20 @@ type S3Writer struct {
FormatOneLineText bool
FormatList bool
Gzip bool
c.AWSConf
}
func getS3() (*s3.S3, error) {
func (w S3Writer) getS3() (*s3.S3, error) {
ses, err := session.NewSession()
if err != nil {
return nil, err
}
config := &aws.Config{
Region: aws.String(c.Conf.AWS.Region),
Region: aws.String(w.Region),
Credentials: credentials.NewChainCredentials([]credentials.Provider{
&credentials.EnvProvider{},
&credentials.SharedCredentialsProvider{Filename: "", Profile: c.Conf.AWS.Profile},
&credentials.SharedCredentialsProvider{Filename: "", Profile: w.Profile},
&ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(ses)},
}),
}
@@ -55,7 +57,7 @@ func (w S3Writer) Write(rs ...models.ScanResult) (err error) {
return nil
}
svc, err := getS3()
svc, err := w.getS3()
if err != nil {
return err
}
@@ -64,7 +66,7 @@ func (w S3Writer) Write(rs ...models.ScanResult) (err error) {
timestr := rs[0].ScannedAt.Format(time.RFC3339)
k := fmt.Sprintf(timestr + "/summary.txt")
text := formatOneLineSummary(rs...)
if err := putObject(svc, k, []byte(text), w.Gzip); err != nil {
if err := w.putObject(svc, k, []byte(text), w.Gzip); err != nil {
return err
}
}
@@ -77,7 +79,7 @@ func (w S3Writer) Write(rs ...models.ScanResult) (err error) {
if b, err = json.Marshal(r); err != nil {
return xerrors.Errorf("Failed to Marshal to JSON: %w", err)
}
if err := putObject(svc, k, b, w.Gzip); err != nil {
if err := w.putObject(svc, k, b, w.Gzip); err != nil {
return err
}
}
@@ -85,7 +87,7 @@ func (w S3Writer) Write(rs ...models.ScanResult) (err error) {
if w.FormatList {
k := key + "_short.txt"
text := formatList(r)
if err := putObject(svc, k, []byte(text), w.Gzip); err != nil {
if err := w.putObject(svc, k, []byte(text), w.Gzip); err != nil {
return err
}
}
@@ -93,7 +95,7 @@ func (w S3Writer) Write(rs ...models.ScanResult) (err error) {
if w.FormatFullText {
k := key + "_full.txt"
text := formatFullPlainText(r)
if err := putObject(svc, k, []byte(text), w.Gzip); err != nil {
if err := w.putObject(svc, k, []byte(text), w.Gzip); err != nil {
return err
}
}
@@ -101,36 +103,34 @@ func (w S3Writer) Write(rs ...models.ScanResult) (err error) {
return nil
}
// CheckIfBucketExists check the existence of S3 bucket
func CheckIfBucketExists() error {
svc, err := getS3()
// Validate check the existence of S3 bucket
func (w S3Writer) Validate() error {
svc, err := w.getS3()
if err != nil {
return err
}
result, err := svc.ListBuckets(&s3.ListBucketsInput{})
if err != nil {
return xerrors.Errorf(
"Failed to list buckets. err: %w, profile: %s, region: %s",
err, c.Conf.AWS.Profile, c.Conf.AWS.Region)
return xerrors.Errorf("Failed to list buckets. err: %w, profile: %s, region: %s",
err, w.Profile, w.Region)
}
found := false
for _, bucket := range result.Buckets {
if *bucket.Name == c.Conf.AWS.S3Bucket {
if *bucket.Name == w.S3Bucket {
found = true
break
}
}
if !found {
return xerrors.Errorf(
"Failed to find the buckets. profile: %s, region: %s, bucket: %s",
c.Conf.AWS.Profile, c.Conf.AWS.Region, c.Conf.AWS.S3Bucket)
return xerrors.Errorf("Failed to find the buckets. profile: %s, region: %s, bucket: %s",
w.Profile, w.Region, w.S3Bucket)
}
return nil
}
func putObject(svc *s3.S3, k string, b []byte, gzip bool) error {
func (w S3Writer) putObject(svc *s3.S3, k string, b []byte, gzip bool) error {
var err error
if gzip {
if b, err = gz(b); err != nil {
@@ -140,18 +140,18 @@ func putObject(svc *s3.S3, k string, b []byte, gzip bool) error {
}
putObjectInput := &s3.PutObjectInput{
Bucket: aws.String(c.Conf.AWS.S3Bucket),
Key: aws.String(path.Join(c.Conf.AWS.S3ResultsDir, k)),
Bucket: aws.String(w.S3Bucket),
Key: aws.String(path.Join(w.S3ResultsDir, k)),
Body: bytes.NewReader(b),
}
if c.Conf.AWS.S3ServerSideEncryption != "" {
putObjectInput.ServerSideEncryption = aws.String(c.Conf.AWS.S3ServerSideEncryption)
if w.S3ServerSideEncryption != "" {
putObjectInput.ServerSideEncryption = aws.String(w.S3ServerSideEncryption)
}
if _, err := svc.PutObject(putObjectInput); err != nil {
return xerrors.Errorf("Failed to upload data to %s/%s, err: %w",
c.Conf.AWS.S3Bucket, k, err)
w.S3Bucket, k, err)
}
return nil
}

43
subcmds/report.go
View File

@@ -321,46 +321,35 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
}
if p.toS3 {
if err := reporter.CheckIfBucketExists(); err != nil {
logging.Log.Errorf("Check if there is a bucket beforehand: %s, err: %+v",
c.Conf.AWS.S3Bucket, err)
return subcommands.ExitUsageError
}
reports = append(reports, reporter.S3Writer{
w := reporter.S3Writer{
FormatJSON: p.formatJSON,
FormatFullText: p.formatFullText,
FormatOneLineText: p.formatOneLineText,
FormatList: p.formatList,
Gzip: p.gzip,
})
AWSConf: c.Conf.AWS,
}
if err := w.Validate(); err != nil {
logging.Log.Errorf("Check if there is a bucket beforehand: %s, err: %+v", c.Conf.AWS.S3Bucket, err)
return subcommands.ExitUsageError
}
reports = append(reports, w)
}
if p.toAzureBlob {
// TODO refactor
if c.Conf.Azure.AccountName == "" {
c.Conf.Azure.AccountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
}
if c.Conf.Azure.AccountKey == "" {
c.Conf.Azure.AccountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
}
if c.Conf.Azure.ContainerName == "" {
logging.Log.Error("Azure storage container name is required with -azure-container option")
return subcommands.ExitUsageError
}
if err := reporter.CheckIfAzureContainerExists(); err != nil {
logging.Log.Errorf("Check if there is a container beforehand: %s, err: %+v",
c.Conf.Azure.ContainerName, err)
return subcommands.ExitUsageError
}
reports = append(reports, reporter.AzureBlobWriter{
w := reporter.AzureBlobWriter{
FormatJSON: p.formatJSON,
FormatFullText: p.formatFullText,
FormatOneLineText: p.formatOneLineText,
FormatList: p.formatList,
Gzip: p.gzip,
})
AzureConf: c.Conf.Azure,
}
if err := w.Validate(); err != nil {
logging.Log.Errorf("Check if there is a container beforehand: %s, err: %+v", c.Conf.Azure.ContainerName, err)
return subcommands.ExitUsageError
}
reports = append(reports, w)
}
for _, w := range reports {