From e754c649a79192549878c5925e7883e7a93d87ff Mon Sep 17 00:00:00 2001 From: cloudina Date: Sat, 6 Jul 2024 17:43:43 +0100 Subject: [PATCH 01/11] initial step --- buceket_scan_object_handler.go | 22 ++++++++++++++++++++++ main.go | 16 ++++++++++------ 2 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 buceket_scan_object_handler.go diff --git a/buceket_scan_object_handler.go b/buceket_scan_object_handler.go new file mode 100644 index 0000000..b92a1cb --- /dev/null +++ b/buceket_scan_object_handler.go @@ -0,0 +1,22 @@ +package main + +import ( + // standard + "encoding/json" + "errors" + "log" + "net/http" +) + +func BucketScanObjectHandler(w http.ResponseWriter, r *http.Request) { + switch cloud_provider { + case "AWS": + retrun S3ScanFileHandler(w,r) + case "GCP": + retrun GCSScanFileHandler(w,r) + case "AZURE": + retrun ABSScanFileHandler(w,r) + default: + panic(fmt.Errorf("unwknown cloud_provider: %s", s)) + } +} diff --git a/main.go b/main.go index 3c825d5..e831aa6 100644 --- a/main.go +++ b/main.go @@ -21,6 +21,7 @@ var ( clamdaddr string clean_files_bucket string quarantine_files_bucket string + cloud_provider string // channels healthcheckrequests chan *HealthCheckRequest scanstreamrequests chan *ScanStreamRequest @@ -48,11 +49,14 @@ func init() { //build address string addrport = address + ":" + port - clean_files_bucket = getEnv("CLEAN_FILES_S3_BUCKET", "") - quarantine_files_bucket = getEnv("QUARANTINE_FILES_S3_BUCKET", "") + clean_files_bucket = getEnv("CLEAN_FILES_BUCKET", "") + quarantine_files_bucket = getEnv("QUARANTINE_FILES_BUCKET", "") + cloud_provider = getEnv("CLOUD_PROVIDER", "") + + info.Println("reading CLEAN_FILES_BUCKET value as " +clean_files_bucket) + info.Println("reading QUARANTINE_FILES_BUCKET value as " +quarantine_files_bucket) + info.Println("reading CLOUD_PROVIDER value as " +cloud_provider) - info.Println("reading CLEAN_FILES_S3_BUCKET value as " +clean_files_bucket) - info.Println("reading QUARANTINE_FILES_S3_BUCKET value as " +quarantine_files_bucket) } func main() { @@ -100,8 +104,8 @@ func main() { // Prometheus metrics r.Handle("/metrics", promhttp.Handler()) - s3_sub := r.PathPrefix("/s3").Subrouter() - s3_sub.HandleFunc("/scanfile", S3ScanFileHandler).Methods("POST") + bucket_sub := r.PathPrefix("/bucket").Subrouter() + bucket_sub.HandleFunc("/scanobject", BucketScanObjectHandler).Methods("POST") ruleset_sub := r.PathPrefix("/ruleset").Subrouter() ruleset_sub.HandleFunc("", RuleSetListHandler).Methods("GET") ruleset_sub.HandleFunc("/", RuleSetListHandler).Methods("GET") From 3953a58e124fa52f3ecfaa30f567b1ff9fd2b651 Mon Sep 17 00:00:00 2001 From: cloudina Date: Sat, 6 Jul 2024 18:09:47 +0100 Subject: [PATCH 02/11] initial step --- buceket_scan_object_handler.go | 37 +++++++++++++++++++++++++--------- main.go | 8 +++++--- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/buceket_scan_object_handler.go b/buceket_scan_object_handler.go index b92a1cb..e406972 100644 --- a/buceket_scan_object_handler.go +++ b/buceket_scan_object_handler.go @@ -2,21 +2,38 @@ package main import ( // standard - "encoding/json" - "errors" - "log" + "fmt" "net/http" + "strings" ) +type CloudProvider int +const ( + CloudProviderAWS = iota + CloudProviderAzure + CloudProviderGCP +) + +var CloudProviderMap = map[string] CloudProvider{ + "AWS": CloudProviderAWS , + "AZURE": CloudProviderAzure , + "GCP": CloudProviderGCP , +} + +func ParseCloudProviderString(str string) (CloudProvider, bool) { + c, ok := CloudProviderMap[strings.ToUpper(str)] + return c, ok +} + func BucketScanObjectHandler(w http.ResponseWriter, r *http.Request) { switch cloud_provider { - case "AWS": - retrun S3ScanFileHandler(w,r) - case "GCP": - retrun GCSScanFileHandler(w,r) - case "AZURE": - retrun ABSScanFileHandler(w,r) + case CloudProviderAWS: + S3ScanFileHandler(w,r) + case CloudProviderAzure: + ABSScanFileHandler(w,r) + case CloudProviderGCP: + GCSScanFileHandler(w,r) default: - panic(fmt.Errorf("unwknown cloud_provider: %s", s)) + panic(fmt.Errorf("unwknown cloud_provider: %s", cloud_provider)) } } diff --git a/main.go b/main.go index e831aa6..087a979 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,7 @@ var ( clamdaddr string clean_files_bucket string quarantine_files_bucket string - cloud_provider string + cloud_provider CloudProvider // channels healthcheckrequests chan *HealthCheckRequest scanstreamrequests chan *ScanStreamRequest @@ -51,11 +51,13 @@ func init() { clean_files_bucket = getEnv("CLEAN_FILES_BUCKET", "") quarantine_files_bucket = getEnv("QUARANTINE_FILES_BUCKET", "") - cloud_provider = getEnv("CLOUD_PROVIDER", "") + cloud_provider_str := getEnv("CLOUD_PROVIDER", "") + + cloud_provider, _ = ParseCloudProviderString(cloud_provider_str) info.Println("reading CLEAN_FILES_BUCKET value as " +clean_files_bucket) info.Println("reading QUARANTINE_FILES_BUCKET value as " +quarantine_files_bucket) - info.Println("reading CLOUD_PROVIDER value as " +cloud_provider) + info.Println("reading CLOUD_PROVIDER value as " +cloud_provider_str) } From 41dc2d915d8a9d1073f5b8e9baf3c3d68ba4d956 Mon Sep 17 00:00:00 2001 From: cloudina Date: Sat, 6 Jul 2024 19:05:39 +0100 Subject: [PATCH 03/11] update --- abs.go | 7 ++ bucket_interface.go | 210 ++++++++++++++++++++++++++++++++++ bucket_scan_object_handler.go | 44 +++++++ gcs.go | 7 ++ 4 files changed, 268 insertions(+) create mode 100644 abs.go create mode 100644 bucket_interface.go create mode 100644 bucket_scan_object_handler.go create mode 100644 gcs.go diff --git a/abs.go b/abs.go new file mode 100644 index 0000000..0e9d585 --- /dev/null +++ b/abs.go @@ -0,0 +1,7 @@ +type GCS_Manager struct { + +} + +func (self *GCS_Manager ) readFile(bucket string, item string) ([] byte, error) { +func (self *S3_Manager ) copyFile(bucket string, item string, other string) (error){ +func (self *S3_Manager ) deleteFile(bucket string, item string) (error) { diff --git a/bucket_interface.go b/bucket_interface.go new file mode 100644 index 0000000..bc26462 --- /dev/null +++ b/bucket_interface.go @@ -0,0 +1,210 @@ +package main + +// Defining an interface +type BucketInterface interface { + ScanObject(w http.ResponseWriter, r *http.Request) +} + +func validateInputBucket(w http.ResponseWriter, bucket string) error { + if (bucket == "") { + errorResponse(w, "Invalid input bucket", http.StatusUnprocessableEntity) + return errors.New("Invalid input bucket") + } + + bucketExists, err := bucketExists(bucket) + + if(err != nil) { + errorResponse(w, err.Error(), http.StatusInternalServerError) + return err + } + if (!bucketExists) { + errorResponse(w, "Bucket: "+bucket+" does not exists", http.StatusUnprocessableEntity) + return errors.New("Bucket: "+bucket+" does not exists") + } + return nil +} + +func validateInputKey(w http.ResponseWriter, bucket string, key string) error { + if (key == "") { + errorResponse(w, "Invalid input key", http.StatusUnprocessableEntity) + return errors.New("Invalid input key") + } + + keyExists, err := keyExists(bucket,key) + if(err != nil) { + errorResponse(w, err.Error(), http.StatusInternalServerError) + return err + } + if (!keyExists) { + errorResponse(w, "Key: "+key+" does not exist in Bucket: "+bucket, http.StatusUnprocessableEntity) + return errors.New("Key: "+key+" does not exist in Bucket: "+bucket) + } + return nil +} + +func getQurantineFilesBucket(qurantineFilesBucket string) string{ + // input has more priority + if (qurantineFilesBucket != "" ) { + return qurantineFilesBucket + } + if (quarantine_files_bucket != "" ) { + return quarantine_files_bucket + } + return "" +} + +func getCleanFilesBucket(cleanFilesBucket string) string{ + // input has more priority + if (cleanFilesBucket != "" ) { + return cleanFilesBucket + } + if (clean_files_bucket != "" ) { + return clean_files_bucket + } + return "" +} + +func validateQrantineFilesBucket(w http.ResponseWriter, qurantineFilesBucket string) error { + var bucket = getQurantineFilesBucket(qurantineFilesBucket) + + if (bucket == "" ) { + errorResponse(w, "Invalid qurantine files bucket", http.StatusBadRequest) + return errors.New("Invalid qurantine files bucket") + + } else { + err := validateInputBucket(w,bucket) + if err != nil { + return err + } + } + return nil +} + +func validateCleanFilesBucket(w http.ResponseWriter, cleanFilesBucket string) error { + + var bucket = getCleanFilesBucket(cleanFilesBucket) + + if (bucket == "" ) { + errorResponse(w, "Invalid clean files bucket", http.StatusBadRequest) + return errors.New("Invalid clean files bucket") + + } else { + err := validateInputBucket(w,bucket) + if err != nil { + return err + } + } + return nil + +} + +func validateInputData(w http.ResponseWriter, data *ScanObject) error { + + err := validateInputBucket(w,data.BucketName) + if err != nil { + return err + } + + err = validateInputKey(w,data.BucketName,data.Key) + if err != nil { + return err + } + + err = validateQrantineFilesBucket(w,data.QurantineFilesBucket) + if err != nil { + return err + } + + err = validateCleanFilesBucket(w,data.CleanFilesBucket) + if err != nil { + return err + } + + return nil +} + +func ScanObject(bucketIF BucketInterface, w http.ResponseWriter, r *http.Request) () { + + data := new(ScanObject) + err := decodeJSONBody(w, r, &data) + if err != nil { + var mr *malformedRequest + if errors.As(err, &mr) { + errorResponse(w, mr.msg, mr.status) + } else { + log.Println(err.Error()) + errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + } + return + } + + err = validateInputData(w,data) + if err != nil { + elog.Println(" validateInputData failed " + err.Error()) + return + } + + resp, _ := json.Marshal(data) + info.Println(" Received ScanS3 request " + string(resp)) + + byteData, err := bucketIF.readFile(data.BucketName, data.Key) + if err != nil { + elog.Println(err) + errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + // send request for scanning + newRequest := NewScanStreamRequest(byteData) + scanstreamrequests <- newRequest + + response := <-newRequest.ResponseChan + + err = response.err + + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } else { + if response.data.Status == "INFECTED" { + elog.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Infected") + err = bucketIF.copyFile(data.BucketName, data.Key, getQurantineFilesBucket(data.QurantineFilesBucket)) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + err = bucketIF.deleteFile(data.BucketName, data.Key) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + } else if response.data.Status == "CLEAN" { + info.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Clean") + err = bucketIF.copyFile(data.BucketName, data.Key, getCleanFilesBucket(data.CleanFilesBucket)) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + err = bucketIF.deleteFile(data.BucketName, data.Key) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + } + } + + output, err := json.Marshal(response.data) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + + sendJsonResponse(w, output) + //fmt.Fprintf(w, string(output)) +} diff --git a/bucket_scan_object_handler.go b/bucket_scan_object_handler.go new file mode 100644 index 0000000..52b79c0 --- /dev/null +++ b/bucket_scan_object_handler.go @@ -0,0 +1,44 @@ +package main + +import ( + // standard + "fmt" + "net/http" + "strings" +) + +type CloudProvider int +const ( + CloudProviderAWS = iota + CloudProviderAzure + CloudProviderGCP +) + +var CloudProviderMap = map[string] CloudProvider{ + "AWS": CloudProviderAWS , + "AZURE": CloudProviderAzure , + "GCP": CloudProviderGCP , +} + +func ParseCloudProviderString(str string) (CloudProvider, bool) { + c, ok := CloudProviderMap[strings.ToUpper(str)] + return c, ok +} + +func initialiseBucketInterface(cloud_provider string, interface* BucketInterface ) (){ + switch cloud_provider { + case CloudProviderAWS: + interface := S3_Manager() + case CloudProviderAzure: + interface = ABS_Manager() + case CloudProviderGCP: + interface = GCP_Manager() + default: + panic(fmt.Errorf("unwknown cloud_provider: %s", cloud_provider)) + } +} + +func BucketScanObjectHandler(w http.ResponseWriter, r *http.Request) { + ScanObject(bucketInterface, w, r) +} + diff --git a/gcs.go b/gcs.go new file mode 100644 index 0000000..9474486 --- /dev/null +++ b/gcs.go @@ -0,0 +1,7 @@ +type ABS_Manager struct { + +} + +func (self *ABS_Manager ) readFile(bucket string, item string) ([] byte, error) +func (self *ABS_Manager ) copyFile(bucket string, item string, other string) (error) +func (self *ABS_Manager ) deleteFile(bucket string, item string) (error) From 59404b289344561b97acd9166c1e22408eba52c8 Mon Sep 17 00:00:00 2001 From: cloudina Date: Sat, 6 Jul 2024 19:06:52 +0100 Subject: [PATCH 04/11] update --- abs.go | 8 +- buceket_scan_object_handler.go | 39 ------ gcs.go | 8 +- main.go | 6 +- request.go | 2 +- s3.go | 12 +- s3scanfilehandler.go | 213 --------------------------------- 7 files changed, 21 insertions(+), 267 deletions(-) delete mode 100644 buceket_scan_object_handler.go delete mode 100644 s3scanfilehandler.go diff --git a/abs.go b/abs.go index 0e9d585..9474486 100644 --- a/abs.go +++ b/abs.go @@ -1,7 +1,7 @@ -type GCS_Manager struct { +type ABS_Manager struct { } -func (self *GCS_Manager ) readFile(bucket string, item string) ([] byte, error) { -func (self *S3_Manager ) copyFile(bucket string, item string, other string) (error){ -func (self *S3_Manager ) deleteFile(bucket string, item string) (error) { +func (self *ABS_Manager ) readFile(bucket string, item string) ([] byte, error) +func (self *ABS_Manager ) copyFile(bucket string, item string, other string) (error) +func (self *ABS_Manager ) deleteFile(bucket string, item string) (error) diff --git a/buceket_scan_object_handler.go b/buceket_scan_object_handler.go deleted file mode 100644 index e406972..0000000 --- a/buceket_scan_object_handler.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - // standard - "fmt" - "net/http" - "strings" -) - -type CloudProvider int -const ( - CloudProviderAWS = iota - CloudProviderAzure - CloudProviderGCP -) - -var CloudProviderMap = map[string] CloudProvider{ - "AWS": CloudProviderAWS , - "AZURE": CloudProviderAzure , - "GCP": CloudProviderGCP , -} - -func ParseCloudProviderString(str string) (CloudProvider, bool) { - c, ok := CloudProviderMap[strings.ToUpper(str)] - return c, ok -} - -func BucketScanObjectHandler(w http.ResponseWriter, r *http.Request) { - switch cloud_provider { - case CloudProviderAWS: - S3ScanFileHandler(w,r) - case CloudProviderAzure: - ABSScanFileHandler(w,r) - case CloudProviderGCP: - GCSScanFileHandler(w,r) - default: - panic(fmt.Errorf("unwknown cloud_provider: %s", cloud_provider)) - } -} diff --git a/gcs.go b/gcs.go index 9474486..de1fa82 100644 --- a/gcs.go +++ b/gcs.go @@ -1,7 +1,7 @@ -type ABS_Manager struct { +type GCS_Manager struct { } -func (self *ABS_Manager ) readFile(bucket string, item string) ([] byte, error) -func (self *ABS_Manager ) copyFile(bucket string, item string, other string) (error) -func (self *ABS_Manager ) deleteFile(bucket string, item string) (error) +func (self *GCS_Manager ) readFile(bucket string, item string) ([] byte, error) +func (self *GCS_Manager ) copyFile(bucket string, item string, other string) (error) +func (self *GCS_Manager ) deleteFile(bucket string, item string) (error) diff --git a/main.go b/main.go index 087a979..287f4fb 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,7 @@ var ( clamdaddr string clean_files_bucket string quarantine_files_bucket string - cloud_provider CloudProvider + bucketInterface *BucketInterface // channels healthcheckrequests chan *HealthCheckRequest scanstreamrequests chan *ScanStreamRequest @@ -53,7 +53,9 @@ func init() { quarantine_files_bucket = getEnv("QUARANTINE_FILES_BUCKET", "") cloud_provider_str := getEnv("CLOUD_PROVIDER", "") - cloud_provider, _ = ParseCloudProviderString(cloud_provider_str) + cloud_provider, _ := ParseCloudProviderString(cloud_provider_str) + + initialiseBucketInterface(cloud_provider, bucketInterface) info.Println("reading CLEAN_FILES_BUCKET value as " +clean_files_bucket) info.Println("reading QUARANTINE_FILES_BUCKET value as " +quarantine_files_bucket) diff --git a/request.go b/request.go index f0fad10..cc12ddc 100644 --- a/request.go +++ b/request.go @@ -1,6 +1,6 @@ package main -type ScanS3Object struct { +type ScanObject struct { BucketName string `json:"bucketname"` Key string `json:"key"` CleanFilesBucket string `json:"clean_files_bucket,omitempty"` diff --git a/s3.go b/s3.go index 98b2d4e..476ae6e 100644 --- a/s3.go +++ b/s3.go @@ -18,6 +18,10 @@ import ( ) + +type S3_Manager struct { +} + func getPartSize() int64 { var partSize int64 @@ -48,7 +52,7 @@ func getRegion() string { } // check if a bucket exists. -func bucketExists(bucket string) (bool, error) { +func (self *S3_Manager ) bucketExists(bucket string) (bool, error) { cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(getRegion()), ) @@ -186,7 +190,7 @@ func keyExists(bucket string, key string) (bool, error) { return true, nil } -func readFile(bucket string, item string) ([] byte, error) { +func (self *S3_Manager ) readFile(bucket string, item string) ([] byte, error) { // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), @@ -230,7 +234,7 @@ func readFile(bucket string, item string) ([] byte, error) { return buff, nil } -func copyFile(bucket string, item string, other string) (error){ +func (self *S3_Manager ) copyFile(bucket string, item string, other string) (error){ // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), @@ -262,7 +266,7 @@ func copyFile(bucket string, item string, other string) (error){ return nil } -func deleteFile(bucket string, item string) (error) { +func (self *S3_Manager ) deleteFile(bucket string, item string) (error) { // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), diff --git a/s3scanfilehandler.go b/s3scanfilehandler.go deleted file mode 100644 index 75fe888..0000000 --- a/s3scanfilehandler.go +++ /dev/null @@ -1,213 +0,0 @@ -package main - -import ( - // standard - "encoding/json" - "errors" - "log" - "net/http" - -) - -func validateInputBucket(w http.ResponseWriter, bucket string) error { - if (bucket == "") { - errorResponse(w, "Invalid input bucket", http.StatusUnprocessableEntity) - return errors.New("Invalid input bucket") - } - - bucketExists, err := bucketExists(bucket) - - if(err != nil) { - errorResponse(w, err.Error(), http.StatusInternalServerError) - return err - } - if (!bucketExists) { - errorResponse(w, "Bucket: "+bucket+" does not exists", http.StatusUnprocessableEntity) - return errors.New("Bucket: "+bucket+" does not exists") - } - return nil -} - -func validateInputKey(w http.ResponseWriter, bucket string, key string) error { - if (key == "") { - errorResponse(w, "Invalid input key", http.StatusUnprocessableEntity) - return errors.New("Invalid input key") - } - - keyExists, err := keyExists(bucket,key) - if(err != nil) { - errorResponse(w, err.Error(), http.StatusInternalServerError) - return err - } - if (!keyExists) { - errorResponse(w, "Key: "+key+" does not exist in Bucket: "+bucket, http.StatusUnprocessableEntity) - return errors.New("Key: "+key+" does not exist in Bucket: "+bucket) - } - return nil -} - -func getQurantineFilesBucket(qurantineFilesBucket string) string{ - // input has more priority - if (qurantineFilesBucket != "" ) { - return qurantineFilesBucket - } - if (quarantine_files_bucket != "" ) { - return quarantine_files_bucket - } - return "" -} - -func getCleanFilesBucket(cleanFilesBucket string) string{ - // input has more priority - if (cleanFilesBucket != "" ) { - return cleanFilesBucket - } - if (clean_files_bucket != "" ) { - return clean_files_bucket - } - return "" -} - -func validateQrantineFilesBucket(w http.ResponseWriter, qurantineFilesBucket string) error { - var bucket = getQurantineFilesBucket(qurantineFilesBucket) - - if (bucket == "" ) { - errorResponse(w, "Invalid qurantine files bucket", http.StatusBadRequest) - return errors.New("Invalid qurantine files bucket") - - } else { - err := validateInputBucket(w,bucket) - if err != nil { - return err - } - } - return nil -} - -func validateCleanFilesBucket(w http.ResponseWriter, cleanFilesBucket string) error { - - var bucket = getCleanFilesBucket(cleanFilesBucket) - - if (bucket == "" ) { - errorResponse(w, "Invalid clean files bucket", http.StatusBadRequest) - return errors.New("Invalid clean files bucket") - - } else { - err := validateInputBucket(w,bucket) - if err != nil { - return err - } - } - return nil - -} - -func validateInputData(w http.ResponseWriter, data *ScanS3Object) error { - - err := validateInputBucket(w,data.BucketName) - if err != nil { - return err - } - - err = validateInputKey(w,data.BucketName,data.Key) - if err != nil { - return err - } - - err = validateQrantineFilesBucket(w,data.QurantineFilesBucket) - if err != nil { - return err - } - - err = validateCleanFilesBucket(w,data.CleanFilesBucket) - if err != nil { - return err - } - - return nil -} - -func S3ScanFileHandler(w http.ResponseWriter, r *http.Request) { - data := new(ScanS3Object) - err := decodeJSONBody(w, r, &data) - if err != nil { - var mr *malformedRequest - if errors.As(err, &mr) { - errorResponse(w, mr.msg, mr.status) - } else { - log.Println(err.Error()) - errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - } - return - } - - err = validateInputData(w,data) - if err != nil { - elog.Println(" validateInputData failed " + err.Error()) - return - } - - resp, _ := json.Marshal(data) - info.Println(" Received ScanS3 request " + string(resp)) - - byteData, err := readFile(data.BucketName, data.Key) - if err != nil { - elog.Println(err) - errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - // send request for scanning - newRequest := NewScanStreamRequest(byteData) - scanstreamrequests <- newRequest - - response := <-newRequest.ResponseChan - - err = response.err - - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } else { - if response.data.Status == "INFECTED" { - elog.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Infected") - err = copyFile(data.BucketName, data.Key, getQurantineFilesBucket(data.QurantineFilesBucket)) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - err = deleteFile(data.BucketName, data.Key) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - } else if response.data.Status == "CLEAN" { - info.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Clean") - err = copyFile(data.BucketName, data.Key, getCleanFilesBucket(data.CleanFilesBucket)) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - err = deleteFile(data.BucketName, data.Key) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - } - } - - output, err := json.Marshal(response.data) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - - sendJsonResponse(w, output) - //fmt.Fprintf(w, string(output)) -} From 7ece5349ef11bb0d083c2ac501f78030b0597f72 Mon Sep 17 00:00:00 2001 From: cloudina Date: Sat, 6 Jul 2024 21:17:08 +0100 Subject: [PATCH 05/11] update --- abs.go | 6 +- bucket_interface.go | 210 +-------------------------------- bucket_interface_manager.go | 213 ++++++++++++++++++++++++++++++++++ bucket_manager.go | 5 + bucket_scan_object_handler.go | 18 +-- gcs.go | 7 +- main.go | 6 +- s3.go | 26 ++--- 8 files changed, 258 insertions(+), 233 deletions(-) create mode 100644 bucket_interface_manager.go create mode 100644 bucket_manager.go diff --git a/abs.go b/abs.go index 9474486..4f82559 100644 --- a/abs.go +++ b/abs.go @@ -1,7 +1,11 @@ -type ABS_Manager struct { +package main +type ABS_Manager struct { + *BucketMgr } +func (self *ABS_Manager ) bucketExists(bucket string) (bool, error) +func (self *ABS_Manager ) keyExists(bucket string, key string) (bool, error) func (self *ABS_Manager ) readFile(bucket string, item string) ([] byte, error) func (self *ABS_Manager ) copyFile(bucket string, item string, other string) (error) func (self *ABS_Manager ) deleteFile(bucket string, item string) (error) diff --git a/bucket_interface.go b/bucket_interface.go index bc26462..81b16c3 100644 --- a/bucket_interface.go +++ b/bucket_interface.go @@ -2,209 +2,9 @@ package main // Defining an interface type BucketInterface interface { - ScanObject(w http.ResponseWriter, r *http.Request) -} - -func validateInputBucket(w http.ResponseWriter, bucket string) error { - if (bucket == "") { - errorResponse(w, "Invalid input bucket", http.StatusUnprocessableEntity) - return errors.New("Invalid input bucket") - } - - bucketExists, err := bucketExists(bucket) - - if(err != nil) { - errorResponse(w, err.Error(), http.StatusInternalServerError) - return err - } - if (!bucketExists) { - errorResponse(w, "Bucket: "+bucket+" does not exists", http.StatusUnprocessableEntity) - return errors.New("Bucket: "+bucket+" does not exists") - } - return nil -} - -func validateInputKey(w http.ResponseWriter, bucket string, key string) error { - if (key == "") { - errorResponse(w, "Invalid input key", http.StatusUnprocessableEntity) - return errors.New("Invalid input key") - } - - keyExists, err := keyExists(bucket,key) - if(err != nil) { - errorResponse(w, err.Error(), http.StatusInternalServerError) - return err - } - if (!keyExists) { - errorResponse(w, "Key: "+key+" does not exist in Bucket: "+bucket, http.StatusUnprocessableEntity) - return errors.New("Key: "+key+" does not exist in Bucket: "+bucket) - } - return nil -} - -func getQurantineFilesBucket(qurantineFilesBucket string) string{ - // input has more priority - if (qurantineFilesBucket != "" ) { - return qurantineFilesBucket - } - if (quarantine_files_bucket != "" ) { - return quarantine_files_bucket - } - return "" -} - -func getCleanFilesBucket(cleanFilesBucket string) string{ - // input has more priority - if (cleanFilesBucket != "" ) { - return cleanFilesBucket - } - if (clean_files_bucket != "" ) { - return clean_files_bucket - } - return "" -} - -func validateQrantineFilesBucket(w http.ResponseWriter, qurantineFilesBucket string) error { - var bucket = getQurantineFilesBucket(qurantineFilesBucket) - - if (bucket == "" ) { - errorResponse(w, "Invalid qurantine files bucket", http.StatusBadRequest) - return errors.New("Invalid qurantine files bucket") - - } else { - err := validateInputBucket(w,bucket) - if err != nil { - return err - } - } - return nil -} - -func validateCleanFilesBucket(w http.ResponseWriter, cleanFilesBucket string) error { - - var bucket = getCleanFilesBucket(cleanFilesBucket) - - if (bucket == "" ) { - errorResponse(w, "Invalid clean files bucket", http.StatusBadRequest) - return errors.New("Invalid clean files bucket") - - } else { - err := validateInputBucket(w,bucket) - if err != nil { - return err - } - } - return nil - -} - -func validateInputData(w http.ResponseWriter, data *ScanObject) error { - - err := validateInputBucket(w,data.BucketName) - if err != nil { - return err - } - - err = validateInputKey(w,data.BucketName,data.Key) - if err != nil { - return err - } - - err = validateQrantineFilesBucket(w,data.QurantineFilesBucket) - if err != nil { - return err - } - - err = validateCleanFilesBucket(w,data.CleanFilesBucket) - if err != nil { - return err - } - - return nil -} - -func ScanObject(bucketIF BucketInterface, w http.ResponseWriter, r *http.Request) () { - - data := new(ScanObject) - err := decodeJSONBody(w, r, &data) - if err != nil { - var mr *malformedRequest - if errors.As(err, &mr) { - errorResponse(w, mr.msg, mr.status) - } else { - log.Println(err.Error()) - errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - } - return - } - - err = validateInputData(w,data) - if err != nil { - elog.Println(" validateInputData failed " + err.Error()) - return - } - - resp, _ := json.Marshal(data) - info.Println(" Received ScanS3 request " + string(resp)) - - byteData, err := bucketIF.readFile(data.BucketName, data.Key) - if err != nil { - elog.Println(err) - errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - // send request for scanning - newRequest := NewScanStreamRequest(byteData) - scanstreamrequests <- newRequest - - response := <-newRequest.ResponseChan - - err = response.err - - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } else { - if response.data.Status == "INFECTED" { - elog.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Infected") - err = bucketIF.copyFile(data.BucketName, data.Key, getQurantineFilesBucket(data.QurantineFilesBucket)) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - err = bucketIF.deleteFile(data.BucketName, data.Key) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - } else if response.data.Status == "CLEAN" { - info.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Clean") - err = bucketIF.copyFile(data.BucketName, data.Key, getCleanFilesBucket(data.CleanFilesBucket)) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - err = bucketIF.deleteFile(data.BucketName, data.Key) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - } - } - - output, err := json.Marshal(response.data) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - - sendJsonResponse(w, output) - //fmt.Fprintf(w, string(output)) + bucketExists(bucket string) (bool, error) + keyExists(bucket string, key string) (bool, error) + readFile(bucket string, item string) ([] byte, error) + copyFile(bucket string, item string, other string) (error) + deleteFile(bucket string, item string) (error) } diff --git a/bucket_interface_manager.go b/bucket_interface_manager.go new file mode 100644 index 0000000..dde3dbd --- /dev/null +++ b/bucket_interface_manager.go @@ -0,0 +1,213 @@ +package main + +import ( + // standard + "net/http" + "errors" + "encoding/json" + "log" +) + +func _getQurantineFilesBucket(qurantineFilesBucket string) string{ + // input has more priority + if (qurantineFilesBucket != "" ) { + return qurantineFilesBucket + } + if (quarantine_files_bucket != "" ) { + return quarantine_files_bucket + } + return "" +} + +func _getCleanFilesBucket(cleanFilesBucket string) string{ + // input has more priority + if (cleanFilesBucket != "" ) { + return cleanFilesBucket + } + if (clean_files_bucket != "" ) { + return clean_files_bucket + } + return "" +} + +func validateInputBucket(w http.ResponseWriter, bucket string, bucketInterface BucketInterface) error { + if (bucket == "") { + errorResponse(w, "Invalid input bucket", http.StatusUnprocessableEntity) + return errors.New("Invalid input bucket") + } + + isbucketExists, err := bucketInterface.bucketExists(bucket) + + if(err != nil) { + errorResponse(w, err.Error(), http.StatusInternalServerError) + return err + } + if (!isbucketExists) { + errorResponse(w, "Bucket: "+bucket+" does not exists", http.StatusUnprocessableEntity) + return errors.New("Bucket: "+bucket+" does not exists") + } + return nil +} + +func validateInputKey(w http.ResponseWriter, bucket string, key string, bucketInterface BucketInterface) error { + if (key == "") { + errorResponse(w, "Invalid input key", http.StatusUnprocessableEntity) + return errors.New("Invalid input key") + } + + keyExists, err := bucketInterface.keyExists(bucket,key) + if(err != nil) { + errorResponse(w, err.Error(), http.StatusInternalServerError) + return err + } + if (!keyExists) { + errorResponse(w, "Key: "+key+" does not exist in Bucket: "+bucket, http.StatusUnprocessableEntity) + return errors.New("Key: "+key+" does not exist in Bucket: "+bucket) + } + return nil +} + +func validateQrantineFilesBucket(w http.ResponseWriter, qurantineFilesBucket string, bucketInterface BucketInterface) error { + var bucket = _getQurantineFilesBucket(qurantineFilesBucket) + + if (bucket == "" ) { + errorResponse(w, "Invalid qurantine files bucket", http.StatusBadRequest) + return errors.New("Invalid qurantine files bucket") + + } else { + err := validateInputBucket(w,bucket, bucketInterface) + if err != nil { + return err + } + } + return nil +} + +func validateCleanFilesBucket(w http.ResponseWriter, cleanFilesBucket string, bucketInterface BucketInterface) error { + + var bucket = _getCleanFilesBucket(cleanFilesBucket) + + if (bucket == "" ) { + errorResponse(w, "Invalid clean files bucket", http.StatusBadRequest) + return errors.New("Invalid clean files bucket") + + } else { + err := validateInputBucket(w,bucket, bucketInterface) + if err != nil { + return err + } + } + return nil + +} + +func validateInputData(w http.ResponseWriter, data *ScanObject, bucketInterface BucketInterface) error { + + err := validateInputBucket(w,data.BucketName, bucketInterface) + if err != nil { + return err + } + + err = validateInputKey(w,data.BucketName,data.Key, bucketInterface) + if err != nil { + return err + } + + err = validateQrantineFilesBucket(w,data.QurantineFilesBucket, bucketInterface) + if err != nil { + return err + } + + err = validateCleanFilesBucket(w,data.CleanFilesBucket, bucketInterface) + if err != nil { + return err + } + + return nil +} + +func ScanBucketObject(w http.ResponseWriter, r *http.Request, bucketInterface BucketInterface) () { + + data := new(ScanObject) + err := decodeJSONBody(w, r, &data) + if err != nil { + var mr *malformedRequest + if errors.As(err, &mr) { + errorResponse(w, mr.msg, mr.status) + } else { + log.Println(err.Error()) + errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + } + return + } + + err = validateInputData(w,data, bucketInterface) + if err != nil { + elog.Println(" validateInputData failed " + err.Error()) + return + } + + resp, _ := json.Marshal(data) + info.Println(" Received ScanS3 request " + string(resp)) + + byteData, err := bucketInterface.readFile(data.BucketName, data.Key) + if err != nil { + elog.Println(err) + errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + // send request for scanning + newRequest := NewScanStreamRequest(byteData) + scanstreamrequests <- newRequest + + response := <-newRequest.ResponseChan + + err = response.err + + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } else { + if response.data.Status == "INFECTED" { + elog.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Infected") + err = bucketInterface.copyFile(data.BucketName, data.Key, _getQurantineFilesBucket(data.QurantineFilesBucket)) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + err = bucketInterface.deleteFile(data.BucketName, data.Key) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + } else if response.data.Status == "CLEAN" { + info.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Clean") + err = bucketInterface.copyFile(data.BucketName, data.Key, _getCleanFilesBucket(data.CleanFilesBucket)) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + err = bucketInterface.deleteFile(data.BucketName, data.Key) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + } + } + + output, err := json.Marshal(response.data) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + + sendJsonResponse(w, output) + //fmt.Fprintf(w, string(output)) +} diff --git a/bucket_manager.go b/bucket_manager.go new file mode 100644 index 0000000..cdb447f --- /dev/null +++ b/bucket_manager.go @@ -0,0 +1,5 @@ +package main + +type BucketMgr struct { +} + diff --git a/bucket_scan_object_handler.go b/bucket_scan_object_handler.go index 52b79c0..e89da42 100644 --- a/bucket_scan_object_handler.go +++ b/bucket_scan_object_handler.go @@ -25,20 +25,22 @@ func ParseCloudProviderString(str string) (CloudProvider, bool) { return c, ok } -func initialiseBucketInterface(cloud_provider string, interface* BucketInterface ) (){ + +func BucketScanObjectHandler(w http.ResponseWriter, r *http.Request) { + switch cloud_provider { case CloudProviderAWS: - interface := S3_Manager() + s3_Mgr := &S3_Manager{} + ScanBucketObject(w,r, s3_Mgr) case CloudProviderAzure: - interface = ABS_Manager() + abs_Mgr := &ABS_Manager{} + ScanBucketObject(w,r, abs_Mgr) case CloudProviderGCP: - interface = GCP_Manager() + gcs_Mgr := &GCS_Manager{} + ScanBucketObject(w,r, gcs_Mgr) default: panic(fmt.Errorf("unwknown cloud_provider: %s", cloud_provider)) - } -} + } -func BucketScanObjectHandler(w http.ResponseWriter, r *http.Request) { - ScanObject(bucketInterface, w, r) } diff --git a/gcs.go b/gcs.go index de1fa82..ffea1a8 100644 --- a/gcs.go +++ b/gcs.go @@ -1,7 +1,10 @@ -type GCS_Manager struct { +package main +type GCS_Manager struct { + *BucketMgr } - +func (self *GCS_Manager ) bucketExists(bucket string) (bool, error) +func (self *GCS_Manager ) keyExists(bucket string, key string) (bool, error) func (self *GCS_Manager ) readFile(bucket string, item string) ([] byte, error) func (self *GCS_Manager ) copyFile(bucket string, item string, other string) (error) func (self *GCS_Manager ) deleteFile(bucket string, item string) (error) diff --git a/main.go b/main.go index 287f4fb..087a979 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,7 @@ var ( clamdaddr string clean_files_bucket string quarantine_files_bucket string - bucketInterface *BucketInterface + cloud_provider CloudProvider // channels healthcheckrequests chan *HealthCheckRequest scanstreamrequests chan *ScanStreamRequest @@ -53,9 +53,7 @@ func init() { quarantine_files_bucket = getEnv("QUARANTINE_FILES_BUCKET", "") cloud_provider_str := getEnv("CLOUD_PROVIDER", "") - cloud_provider, _ := ParseCloudProviderString(cloud_provider_str) - - initialiseBucketInterface(cloud_provider, bucketInterface) + cloud_provider, _ = ParseCloudProviderString(cloud_provider_str) info.Println("reading CLEAN_FILES_BUCKET value as " +clean_files_bucket) info.Println("reading QUARANTINE_FILES_BUCKET value as " +quarantine_files_bucket) diff --git a/s3.go b/s3.go index 476ae6e..81e110b 100644 --- a/s3.go +++ b/s3.go @@ -18,11 +18,11 @@ import ( ) - type S3_Manager struct { + *BucketMgr } -func getPartSize() int64 { +func (self* S3_Manager) getPartSize() int64 { var partSize int64 strSizeInMb, err := os.LookupEnv("DOWNLOAD_PART_SIZE") @@ -42,7 +42,7 @@ func getPartSize() int64 { return partSize } -func getRegion() string { +func (self* S3_Manager) getRegion() string { region, err := os.LookupEnv("AWS_REGION") if !err { elog.Println(time.Now().Format(time.RFC3339) + " AWS_REGION is not present..using us-east-1") @@ -54,7 +54,7 @@ func getRegion() string { // check if a bucket exists. func (self *S3_Manager ) bucketExists(bucket string) (bool, error) { cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { elog.Println( time.Now().Format(time.RFC3339) + " bucketExists: Filed to load config for bucket "+bucket + " error : " + err.Error()) @@ -97,10 +97,10 @@ func (self *S3_Manager ) bucketExists(bucket string) (bool, error) { return true,nil } -func getHeadObject(bucket string, key string) (*s3.HeadObjectOutput, error) { +func (self* S3_Manager) getHeadObject(bucket string, key string) (*s3.HeadObjectOutput, error) { cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { elog.Println( time.Now().Format(time.RFC3339) + " getHeadObject: Filed to load config for bucket "+bucket + " error : " + err.Error()) @@ -147,10 +147,10 @@ func getHeadObject(bucket string, key string) (*s3.HeadObjectOutput, error) { } // check if a file exists. -func keyExists(bucket string, key string) (bool, error) { +func (self* S3_Manager) keyExists(bucket string, key string) (bool, error) { cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { elog.Println( time.Now().Format(time.RFC3339) + "keyExists: Filed to load config for bucket "+bucket +" key "+key+" error : " + err.Error()) @@ -194,7 +194,7 @@ func (self *S3_Manager ) readFile(bucket string, item string) ([] byte, error) { // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { elog.Println( time.Now().Format(time.RFC3339) + " readFile: Filed to load config to read file " +item+ " from bucket "+bucket + " error : " + err.Error()) @@ -206,10 +206,10 @@ func (self *S3_Manager ) readFile(bucket string, item string) ([] byte, error) { // Create a downloader with the client and custom downloader options downloader := manager.NewDownloader(s3client, func(d *manager.Downloader) { - d.PartSize = getPartSize() + d.PartSize = self.getPartSize() }) - headObject, err := getHeadObject(bucket,item) + headObject, err := self.getHeadObject(bucket,item) if err != nil { elog.Println( time.Now().Format(time.RFC3339) + " readFile: getHeadObject failed " +item+ " from bucket "+bucket + " error : " + err.Error()) return nil, errors.New("Filed to read file") @@ -238,7 +238,7 @@ func (self *S3_Manager ) copyFile(bucket string, item string, other string) (err // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { elog.Println( time.Now().Format(time.RFC3339) + " copyFile: Filed to load config to read file " +item+ " from bucket "+bucket + " error : " + err.Error()) @@ -270,7 +270,7 @@ func (self *S3_Manager ) deleteFile(bucket string, item string) (error) { // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { elog.Println( time.Now().Format(time.RFC3339) + " deleteFile: Filed to load config to read file " +item+ " from bucket "+bucket + " error : " + err.Error()) From d1b9d57dc9a0643240a96a70c5465f7ed6a1f0a6 Mon Sep 17 00:00:00 2001 From: cloudina Date: Sat, 6 Jul 2024 21:22:31 +0100 Subject: [PATCH 06/11] update --- bucket_interface_manager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bucket_interface_manager.go b/bucket_interface_manager.go index dde3dbd..e53bc7e 100644 --- a/bucket_interface_manager.go +++ b/bucket_interface_manager.go @@ -36,13 +36,13 @@ func validateInputBucket(w http.ResponseWriter, bucket string, bucketInterface B return errors.New("Invalid input bucket") } - isbucketExists, err := bucketInterface.bucketExists(bucket) + bucketExists, err := bucketInterface.bucketExists(bucket) if(err != nil) { errorResponse(w, err.Error(), http.StatusInternalServerError) return err } - if (!isbucketExists) { + if (!bucketExists) { errorResponse(w, "Bucket: "+bucket+" does not exists", http.StatusUnprocessableEntity) return errors.New("Bucket: "+bucket+" does not exists") } From 8b87d0e7d7338d370325bb5cb382dbfd4478a5ab Mon Sep 17 00:00:00 2001 From: cloudina Date: Sun, 7 Jul 2024 22:44:38 +0100 Subject: [PATCH 07/11] update --- abs.go | 142 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- gcs.go | 50 ++++++++++++++++++++ 2 files changed, 187 insertions(+), 5 deletions(-) diff --git a/abs.go b/abs.go index 4f82559..2d0e702 100644 --- a/abs.go +++ b/abs.go @@ -1,11 +1,143 @@ package main +import ( + "context" + "fmt" + "io" + "log" + "os" + "strings" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" +) + type ABS_Manager struct { *BucketMgr } -func (self *ABS_Manager ) bucketExists(bucket string) (bool, error) -func (self *ABS_Manager ) keyExists(bucket string, key string) (bool, error) -func (self *ABS_Manager ) readFile(bucket string, item string) ([] byte, error) -func (self *ABS_Manager ) copyFile(bucket string, item string, other string) (error) -func (self *ABS_Manager ) deleteFile(bucket string, item string) (error) +func (self *ABS_Manager ) handleError(err error) { + if err != nil { + log.Fatal(err.Error()) + } +} + +func (self *ABS_Manager ) getServiceClient() *azblob.Client { + // Create a new service client with token credential + accountName, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") + if !ok { + panic("AZURE_STORAGE_ACCOUNT_NAME could not be found") + } + + serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) + + credential, err := azidentity.NewDefaultAzureCredential(nil) + self.handleError(err) + + client, err := azblob.NewClient(serviceURL, credential, nil) + self.handleError(err) + return client +} + +func (self *ABS_Manager ) getContainerClient(containerName string) *azblob.Client { + accountName, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") + if !ok { + panic("AZURE_STORAGE_ACCOUNT_NAME could not be found") + } + containerName := "testcontainer" + containerURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName) + + cred, err := azidentity.NewDefaultAzureCredential(nil) + self.handleError(err) + + containerClient, err := container.NewClient(containerURL, cred, nil) + self.handleError(err) + return containerClient +} + +func (self *ABS_Manager ) getBlobClient(containerName string, blobName string) *azblob.Client { + // From the Azure portal, get your Storage account blob service URL endpoint. + accountName, accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME"), os.Getenv("AZURE_STORAGE_ACCOUNT_KEY") + + blobURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", accountName, containerName, blobName) + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + handleError(err) + blobClient, err := blob.NewClientWithSharedKeyCredential(blobURL, credential, nil) + handleError(err) + return blobClient +} + +func (self *ABS_Manager) listBuckets() []string { + + client := self.getServiceClient() + + pager := client.NewListContainersPager(&azblob.ListContainersOptions{ + Include: azblob.ListContainersInclude{Metadata: true, Deleted: false}, + }) + + var buckets []string = [] + + for pager.More() { + resp, err := pager.NextPage(ctx) + handleError(err) // if err is not nil, break the loop. + for _, _container := range resp.ContainerItems { + buckets.append(_container) + } + } + return buckets +} + +func (self *ABS_Manager ) bucketExists(bucket string) (bool, error) { + client := self.getContainerClient(bucket) + _ err := client.getProperies() + if (ContainerNotFound == err ) { + return false, err + } else { + return true, nil + } +} + +func (self *ABS_Manager ) keyExists(bucket string, key string) (bool, error) { + client := self.getBlobClient(bucket, key) + _ err := client.getProperies() + if (BlobNotFound == err ) { + return false, err + } else { + return true, nil + } +} + +func (self *ABS_Manager ) readFile(bucket string, item string) ([] byte, error) { + + client := self.getServiceClient() + // Download the blob + downloadResponse, err := client.DownloadStream(ctx, bucket, item, nil) + handleError(err) + + // Assert that the content is correct + actualBlobData, err := io.ReadAll(downloadResponse.Body) + handleError(err) + err = reader.Close() + if err != nil { + return nil, err + } + return actualBlobData, nil +} + +func (self *ABS_Manager ) copyFile(bucket string, item string, other string) (error) { + + data, _ = self.readFile(bucket,item) + + client := self.getServiceClient() + + _, err = client.UploadBuffer(context.TODO(), path.Dir(other),, path.Base(other) data, &azblob.UploadBufferOptions{}) + handleError(err) +} + +func (self *ABS_Manager ) deleteFile(bucket string, item string) (error) { + client := self.getServiceClient() + // Delete the blob. + _, err = client.DeleteBlob(context.TODO(), bucket, item, nil) + handleError(err) + return err +} diff --git a/gcs.go b/gcs.go index ffea1a8..282f5ae 100644 --- a/gcs.go +++ b/gcs.go @@ -8,3 +8,53 @@ func (self *GCS_Manager ) keyExists(bucket string, key string) (bool, error) func (self *GCS_Manager ) readFile(bucket string, item string) ([] byte, error) func (self *GCS_Manager ) copyFile(bucket string, item string, other string) (error) func (self *GCS_Manager ) deleteFile(bucket string, item string) (error) + + + +import ( + "context" + "fmt" + "io" + "time" + + "cloud.google.com/go/storage" +) + +// copyFile copies an object into specified bucket. +func copyFile(w io.Writer, dstBucket, srcBucket, srcObject string) error { + // dstBucket := "bucket-1" + // srcBucket := "bucket-2" + // srcObject := "object" + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("storage.NewClient: %w", err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + + dstObject := srcObject + "-copy" + src := client.Bucket(srcBucket).Object(srcObject) + dst := client.Bucket(dstBucket).Object(dstObject) + + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to copy is aborted if the + // object's generation number does not match your precondition. + // For a dst object that does not yet exist, set the DoesNotExist precondition. + dst = dst.If(storage.Conditions{DoesNotExist: true}) + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + // attrs, err := dst.Attrs(ctx) + // if err != nil { + // return fmt.Errorf("object.Attrs: %w", err) + // } + // dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation}) + + if _, err := dst.CopierFrom(src).Run(ctx); err != nil { + return fmt.Errorf("Object(%q).CopierFrom(%q).Run: %w", dstObject, srcObject, err) + } + fmt.Fprintf(w, "Blob %v in bucket %v copied to blob %v in bucket %v.\n", srcObject, srcBucket, dstObject, dstBucket) + return nil +} \ No newline at end of file From 7844e6020e64e10cd55a75aff913b19eb277c7fc Mon Sep 17 00:00:00 2001 From: cloudina Date: Sun, 7 Jul 2024 22:45:34 +0100 Subject: [PATCH 08/11] update --- abs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abs.go b/abs.go index 2d0e702..9f91c8f 100644 --- a/abs.go +++ b/abs.go @@ -130,7 +130,7 @@ func (self *ABS_Manager ) copyFile(bucket string, item string, other string) (er client := self.getServiceClient() - _, err = client.UploadBuffer(context.TODO(), path.Dir(other),, path.Base(other) data, &azblob.UploadBufferOptions{}) + _, err = client.UploadBuffer(context.TODO(), path.Dir(other), path.Base(other), data, &azblob.UploadBufferOptions{}) handleError(err) } From 6444fa2eccd5934404c9cf4f58458c8112ae4679 Mon Sep 17 00:00:00 2001 From: cloudina Date: Sun, 7 Jul 2024 23:18:59 +0100 Subject: [PATCH 09/11] update --- gcs.go | 109 ++++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 88 insertions(+), 21 deletions(-) diff --git a/gcs.go b/gcs.go index 282f5ae..99b1c37 100644 --- a/gcs.go +++ b/gcs.go @@ -3,28 +3,67 @@ package main type GCS_Manager struct { *BucketMgr } -func (self *GCS_Manager ) bucketExists(bucket string) (bool, error) -func (self *GCS_Manager ) keyExists(bucket string, key string) (bool, error) -func (self *GCS_Manager ) readFile(bucket string, item string) ([] byte, error) -func (self *GCS_Manager ) copyFile(bucket string, item string, other string) (error) -func (self *GCS_Manager ) deleteFile(bucket string, item string) (error) +func (self *GCS_Manager ) bucketExists(bucket string) (bool, error) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("storage.NewClient: %w", err) + } + defer client.Close() + bucket := client.Bucket(bucket) + exists,err := bucket.Attrs(ctx) + if err != nil { + log.Fatalf("Message: %v",err) + return false, err + } else { + return true, nil + } +} -import ( - "context" - "fmt" - "io" - "time" +func (self *GCS_Manager ) keyExists(bucket string, key string) (bool, error) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("storage.NewClient: %w", err) + } + defer client.Close() - "cloud.google.com/go/storage" -) + bucket := client.Bucket(bucket) + object = bucket.Object(key) + exists,err := object.Attrs(ctx) + if err != nil { + log.Fatalf("Message: %v",err) + return false, err + } else { + return true, nil + } +} -// copyFile copies an object into specified bucket. -func copyFile(w io.Writer, dstBucket, srcBucket, srcObject string) error { - // dstBucket := "bucket-1" - // srcBucket := "bucket-2" - // srcObject := "object" +func (self *GCS_Manager ) readFile(bucket string, item string) ([] byte, error) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + return nil, fmt.Errorf("storage.NewClient: %w", err) + } + defer client.Close() + + rc, err := client.Bucket(bucket).Object(item).NewReader(ctx) + if err != nil { + return nil, fmt.Errorf("Object(%q).NewReader: %w", object, err) + } + defer rc.Close() + + data, err := ioutil.ReadAll(rc) + if err != nil { + return nil, fmt.Errorf("ioutil.ReadAll: %w", err) + } + fmt.Fprintf(w, "Blob %v downloaded.\n", object) + return data, nil +} + +func (self *GCS_Manager ) copyFile(srcBucket string, srcObject string, other string) (error) { ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { @@ -32,10 +71,9 @@ func copyFile(w io.Writer, dstBucket, srcBucket, srcObject string) error { } defer client.Close() - ctx, cancel := context.WithTimeout(ctx, time.Second*10) - defer cancel() + dstBucket := path.Dir(other) + dstObject := path.Base(other) - dstObject := srcObject + "-copy" src := client.Bucket(srcBucket).Object(srcObject) dst := client.Bucket(dstBucket).Object(dstObject) @@ -57,4 +95,33 @@ func copyFile(w io.Writer, dstBucket, srcBucket, srcObject string) error { } fmt.Fprintf(w, "Blob %v in bucket %v copied to blob %v in bucket %v.\n", srcObject, srcBucket, dstObject, dstBucket) return nil -} \ No newline at end of file +} + +func (self *GCS_Manager ) deleteFile(bucket string, item string) (error) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("storage.NewClient: %w", err) + } + defer client.Close() + + //ctx, cancel := context.WithTimeout(ctx, time.Second*10) + //defer cancel() + + o := client.Bucket(bucket).Object(item) + + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to delete the file is aborted + // if the object's generation number does not match your precondition. + attrs, err := o.Attrs(ctx) + if err != nil { + return fmt.Errorf("object.Attrs: %w", err) + } + o = o.If(storage.Conditions{GenerationMatch: attrs.Generation}) + + if err := o.Delete(ctx); err != nil { + return fmt.Errorf("Object(%q).Delete: %w", item, err) + } + fmt.Fprintf(w, "Blob %v deleted.\n", item) + return nil +} From c917343a6e77285d4dd8f1b308b1181edfcd3980 Mon Sep 17 00:00:00 2001 From: cloudina Date: Tue, 9 Jul 2024 07:17:09 +0100 Subject: [PATCH 10/11] fmt --- 404handler.go | 4 +- abs.go | 52 +++++------ bucket_interface.go | 6 +- bucket_interface_manager.go | 104 ++++++++++----------- bucket_manager.go | 1 - bucket_scan_object_handler.go | 43 +++++---- clamscanner.go | 64 ++++++------- custom_headers.go | 12 +-- custom_headers_middleware.go | 8 +- envhelper.go | 42 ++++----- gcs.go | 38 ++++---- healthcheckhandler.go | 2 +- indexhandler.go | 4 +- jsonresponse.go | 164 +++++++++++++++++----------------- main.go | 28 +++--- request.go | 8 +- response.go | 14 +-- rulelisthandler.go | 4 +- ruleset.go | 2 +- s3.go | 141 +++++++++++++++-------------- scanner.go | 57 ++++++------ scannerinterface.go | 10 +-- streamscanhandler.go | 2 +- yarascanner.go | 14 +-- 24 files changed, 410 insertions(+), 414 deletions(-) diff --git a/404handler.go b/404handler.go index c7b0866..8abf192 100644 --- a/404handler.go +++ b/404handler.go @@ -8,9 +8,9 @@ import ( func Handle404(helmet SimpleHelmet) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { for key, val := range helmet.headers { - w.Header().Set(key,val) + w.Header().Set(key, val) } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotFound) }) -} \ No newline at end of file +} diff --git a/abs.go b/abs.go index 9f91c8f..991e49b 100644 --- a/abs.go +++ b/abs.go @@ -3,27 +3,27 @@ package main import ( "context" "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "io" "log" "os" "strings" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" ) type ABS_Manager struct { *BucketMgr } -func (self *ABS_Manager ) handleError(err error) { +func (self *ABS_Manager) handleError(err error) { if err != nil { log.Fatal(err.Error()) } } -func (self *ABS_Manager ) getServiceClient() *azblob.Client { - // Create a new service client with token credential +func (self *ABS_Manager) getServiceClient() *azblob.Client { + // Create a new service client with token credential accountName, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") if !ok { panic("AZURE_STORAGE_ACCOUNT_NAME could not be found") @@ -31,15 +31,15 @@ func (self *ABS_Manager ) getServiceClient() *azblob.Client { serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) - credential, err := azidentity.NewDefaultAzureCredential(nil) - self.handleError(err) + credential, err := azidentity.NewDefaultAzureCredential(nil) + self.handleError(err) - client, err := azblob.NewClient(serviceURL, credential, nil) - self.handleError(err) - return client + client, err := azblob.NewClient(serviceURL, credential, nil) + self.handleError(err) + return client } -func (self *ABS_Manager ) getContainerClient(containerName string) *azblob.Client { +func (self *ABS_Manager) getContainerClient(containerName string) *azblob.Client { accountName, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") if !ok { panic("AZURE_STORAGE_ACCOUNT_NAME could not be found") @@ -55,7 +55,7 @@ func (self *ABS_Manager ) getContainerClient(containerName string) *azblob.Clien return containerClient } -func (self *ABS_Manager ) getBlobClient(containerName string, blobName string) *azblob.Client { +func (self *ABS_Manager) getBlobClient(containerName string, blobName string) *azblob.Client { // From the Azure portal, get your Storage account blob service URL endpoint. accountName, accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME"), os.Getenv("AZURE_STORAGE_ACCOUNT_KEY") @@ -75,7 +75,7 @@ func (self *ABS_Manager) listBuckets() []string { Include: azblob.ListContainersInclude{Metadata: true, Deleted: false}, }) - var buckets []string = [] + var buckets []string for pager.More() { resp, err := pager.NextPage(ctx) @@ -87,27 +87,27 @@ func (self *ABS_Manager) listBuckets() []string { return buckets } -func (self *ABS_Manager ) bucketExists(bucket string) (bool, error) { +func (self *ABS_Manager) bucketExists(bucket string) (bool, error) { client := self.getContainerClient(bucket) - _ err := client.getProperies() - if (ContainerNotFound == err ) { + _, err := client.getProperies() + if ContainerNotFound == err { return false, err } else { return true, nil } } -func (self *ABS_Manager ) keyExists(bucket string, key string) (bool, error) { +func (self *ABS_Manager) keyExists(bucket string, key string) (bool, error) { client := self.getBlobClient(bucket, key) - _ err := client.getProperies() - if (BlobNotFound == err ) { + _, err := client.getProperies() + if BlobNotFound == err { return false, err } else { return true, nil } } -func (self *ABS_Manager ) readFile(bucket string, item string) ([] byte, error) { +func (self *ABS_Manager) readFile(bucket string, item string) ([]byte, error) { client := self.getServiceClient() // Download the blob @@ -124,17 +124,17 @@ func (self *ABS_Manager ) readFile(bucket string, item string) ([] byte, error) return actualBlobData, nil } -func (self *ABS_Manager ) copyFile(bucket string, item string, other string) (error) { +func (self *ABS_Manager) copyFile(bucket string, item string, other string) error { + + data, _ = self.readFile(bucket, item) - data, _ = self.readFile(bucket,item) - client := self.getServiceClient() - + _, err = client.UploadBuffer(context.TODO(), path.Dir(other), path.Base(other), data, &azblob.UploadBufferOptions{}) handleError(err) } -func (self *ABS_Manager ) deleteFile(bucket string, item string) (error) { +func (self *ABS_Manager) deleteFile(bucket string, item string) error { client := self.getServiceClient() // Delete the blob. _, err = client.DeleteBlob(context.TODO(), bucket, item, nil) diff --git a/bucket_interface.go b/bucket_interface.go index 81b16c3..0a55b8d 100644 --- a/bucket_interface.go +++ b/bucket_interface.go @@ -4,7 +4,7 @@ package main type BucketInterface interface { bucketExists(bucket string) (bool, error) keyExists(bucket string, key string) (bool, error) - readFile(bucket string, item string) ([] byte, error) - copyFile(bucket string, item string, other string) (error) - deleteFile(bucket string, item string) (error) + readFile(bucket string, item string) ([]byte, error) + copyFile(bucket string, item string, other string) error + deleteFile(bucket string, item string) error } diff --git a/bucket_interface_manager.go b/bucket_interface_manager.go index e53bc7e..0f04af8 100644 --- a/bucket_interface_manager.go +++ b/bucket_interface_manager.go @@ -2,80 +2,80 @@ package main import ( // standard - "net/http" - "errors" "encoding/json" + "errors" "log" + "net/http" ) -func _getQurantineFilesBucket(qurantineFilesBucket string) string{ +func _getQurantineFilesBucket(qurantineFilesBucket string) string { // input has more priority - if (qurantineFilesBucket != "" ) { + if qurantineFilesBucket != "" { return qurantineFilesBucket - } - if (quarantine_files_bucket != "" ) { + } + if quarantine_files_bucket != "" { return quarantine_files_bucket } return "" } -func _getCleanFilesBucket(cleanFilesBucket string) string{ +func _getCleanFilesBucket(cleanFilesBucket string) string { // input has more priority - if (cleanFilesBucket != "" ) { + if cleanFilesBucket != "" { return cleanFilesBucket - } - if (clean_files_bucket != "" ) { + } + if clean_files_bucket != "" { return clean_files_bucket } return "" } func validateInputBucket(w http.ResponseWriter, bucket string, bucketInterface BucketInterface) error { - if (bucket == "") { + if bucket == "" { errorResponse(w, "Invalid input bucket", http.StatusUnprocessableEntity) return errors.New("Invalid input bucket") } bucketExists, err := bucketInterface.bucketExists(bucket) - if(err != nil) { - errorResponse(w, err.Error(), http.StatusInternalServerError) + if err != nil { + errorResponse(w, err.Error(), http.StatusInternalServerError) return err } - if (!bucketExists) { - errorResponse(w, "Bucket: "+bucket+" does not exists", http.StatusUnprocessableEntity) - return errors.New("Bucket: "+bucket+" does not exists") + if !bucketExists { + errorResponse(w, "Bucket: "+bucket+" does not exists", http.StatusUnprocessableEntity) + return errors.New("Bucket: " + bucket + " does not exists") } return nil } func validateInputKey(w http.ResponseWriter, bucket string, key string, bucketInterface BucketInterface) error { - if (key == "") { + if key == "" { errorResponse(w, "Invalid input key", http.StatusUnprocessableEntity) return errors.New("Invalid input key") } - keyExists, err := bucketInterface.keyExists(bucket,key) - if(err != nil) { - errorResponse(w, err.Error(), http.StatusInternalServerError) + keyExists, err := bucketInterface.keyExists(bucket, key) + if err != nil { + errorResponse(w, err.Error(), http.StatusInternalServerError) return err } - if (!keyExists) { - errorResponse(w, "Key: "+key+" does not exist in Bucket: "+bucket, http.StatusUnprocessableEntity) - return errors.New("Key: "+key+" does not exist in Bucket: "+bucket) + if !keyExists { + errorResponse(w, "Key: "+key+" does not exist in Bucket: "+bucket, http.StatusUnprocessableEntity) + return errors.New("Key: " + key + " does not exist in Bucket: " + bucket) } return nil } func validateQrantineFilesBucket(w http.ResponseWriter, qurantineFilesBucket string, bucketInterface BucketInterface) error { var bucket = _getQurantineFilesBucket(qurantineFilesBucket) - - if (bucket == "" ) { + + if bucket == "" { errorResponse(w, "Invalid qurantine files bucket", http.StatusBadRequest) return errors.New("Invalid qurantine files bucket") } else { - err := validateInputBucket(w,bucket, bucketInterface) + err := validateInputBucket(w, bucket, bucketInterface) if err != nil { return err } @@ -87,12 +87,12 @@ func validateCleanFilesBucket(w http.ResponseWriter, cleanFilesBucket string, bu var bucket = _getCleanFilesBucket(cleanFilesBucket) - if (bucket == "" ) { + if bucket == "" { errorResponse(w, "Invalid clean files bucket", http.StatusBadRequest) return errors.New("Invalid clean files bucket") } else { - err := validateInputBucket(w,bucket, bucketInterface) + err := validateInputBucket(w, bucket, bucketInterface) if err != nil { return err } @@ -103,22 +103,22 @@ func validateCleanFilesBucket(w http.ResponseWriter, cleanFilesBucket string, bu func validateInputData(w http.ResponseWriter, data *ScanObject, bucketInterface BucketInterface) error { - err := validateInputBucket(w,data.BucketName, bucketInterface) + err := validateInputBucket(w, data.BucketName, bucketInterface) if err != nil { return err } - err = validateInputKey(w,data.BucketName,data.Key, bucketInterface) + err = validateInputKey(w, data.BucketName, data.Key, bucketInterface) if err != nil { return err } - - err = validateQrantineFilesBucket(w,data.QurantineFilesBucket, bucketInterface) + + err = validateQrantineFilesBucket(w, data.QurantineFilesBucket, bucketInterface) if err != nil { return err } - - err = validateCleanFilesBucket(w,data.CleanFilesBucket, bucketInterface) + + err = validateCleanFilesBucket(w, data.CleanFilesBucket, bucketInterface) if err != nil { return err } @@ -126,22 +126,22 @@ func validateInputData(w http.ResponseWriter, data *ScanObject, bucketInterface return nil } -func ScanBucketObject(w http.ResponseWriter, r *http.Request, bucketInterface BucketInterface) () { +func ScanBucketObject(w http.ResponseWriter, r *http.Request, bucketInterface BucketInterface) { data := new(ScanObject) err := decodeJSONBody(w, r, &data) - if err != nil { - var mr *malformedRequest - if errors.As(err, &mr) { - errorResponse(w, mr.msg, mr.status) - } else { - log.Println(err.Error()) - errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - } - return - } - - err = validateInputData(w,data, bucketInterface) + if err != nil { + var mr *malformedRequest + if errors.As(err, &mr) { + errorResponse(w, mr.msg, mr.status) + } else { + log.Println(err.Error()) + errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + } + return + } + + err = validateInputData(w, data, bucketInterface) if err != nil { elog.Println(" validateInputData failed " + err.Error()) return @@ -149,7 +149,7 @@ func ScanBucketObject(w http.ResponseWriter, r *http.Request, bucketInterface Bu resp, _ := json.Marshal(data) info.Println(" Received ScanS3 request " + string(resp)) - + byteData, err := bucketInterface.readFile(data.BucketName, data.Key) if err != nil { elog.Println(err) @@ -169,9 +169,9 @@ func ScanBucketObject(w http.ResponseWriter, r *http.Request, bucketInterface Bu elog.Println(err) errorResponse(w, err.Error(), http.StatusInternalServerError) return - } else { + } else { if response.data.Status == "INFECTED" { - elog.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Infected") + elog.Println("Key " + data.Key + " from bucket " + data.BucketName + " is Infected") err = bucketInterface.copyFile(data.BucketName, data.Key, _getQurantineFilesBucket(data.QurantineFilesBucket)) if err != nil { elog.Println(err) @@ -185,12 +185,12 @@ func ScanBucketObject(w http.ResponseWriter, r *http.Request, bucketInterface Bu return } } else if response.data.Status == "CLEAN" { - info.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Clean") + info.Println("Key " + data.Key + " from bucket " + data.BucketName + " is Clean") err = bucketInterface.copyFile(data.BucketName, data.Key, _getCleanFilesBucket(data.CleanFilesBucket)) if err != nil { elog.Println(err) errorResponse(w, err.Error(), http.StatusInternalServerError) - return + return } err = bucketInterface.deleteFile(data.BucketName, data.Key) if err != nil { diff --git a/bucket_manager.go b/bucket_manager.go index cdb447f..f88b685 100644 --- a/bucket_manager.go +++ b/bucket_manager.go @@ -2,4 +2,3 @@ package main type BucketMgr struct { } - diff --git a/bucket_scan_object_handler.go b/bucket_scan_object_handler.go index e89da42..f534497 100644 --- a/bucket_scan_object_handler.go +++ b/bucket_scan_object_handler.go @@ -8,39 +8,38 @@ import ( ) type CloudProvider int + const ( - CloudProviderAWS = iota - CloudProviderAzure - CloudProviderGCP + CloudProviderAWS = iota + CloudProviderAzure + CloudProviderGCP ) -var CloudProviderMap = map[string] CloudProvider{ - "AWS": CloudProviderAWS , - "AZURE": CloudProviderAzure , - "GCP": CloudProviderGCP , +var CloudProviderMap = map[string]CloudProvider{ + "AWS": CloudProviderAWS, + "AZURE": CloudProviderAzure, + "GCP": CloudProviderGCP, } func ParseCloudProviderString(str string) (CloudProvider, bool) { - c, ok := CloudProviderMap[strings.ToUpper(str)] - return c, ok + c, ok := CloudProviderMap[strings.ToUpper(str)] + return c, ok } - func BucketScanObjectHandler(w http.ResponseWriter, r *http.Request) { switch cloud_provider { - case CloudProviderAWS: - s3_Mgr := &S3_Manager{} - ScanBucketObject(w,r, s3_Mgr) - case CloudProviderAzure: - abs_Mgr := &ABS_Manager{} - ScanBucketObject(w,r, abs_Mgr) - case CloudProviderGCP: - gcs_Mgr := &GCS_Manager{} - ScanBucketObject(w,r, gcs_Mgr) - default: - panic(fmt.Errorf("unwknown cloud_provider: %s", cloud_provider)) + case CloudProviderAWS: + s3_Mgr := &S3_Manager{} + ScanBucketObject(w, r, s3_Mgr) + case CloudProviderAzure: + abs_Mgr := &ABS_Manager{} + ScanBucketObject(w, r, abs_Mgr) + case CloudProviderGCP: + gcs_Mgr := &GCS_Manager{} + ScanBucketObject(w, r, gcs_Mgr) + default: + panic(fmt.Errorf("unwknown cloud_provider: %s", cloud_provider)) } } - diff --git a/clamscanner.go b/clamscanner.go index 5850a34..d910987 100644 --- a/clamscanner.go +++ b/clamscanner.go @@ -1,11 +1,11 @@ package main import ( - "github.com/dutchcoders/go-clamd" "bytes" "encoding/json" - "time" "fmt" + "github.com/dutchcoders/go-clamd" + "time" ) var eicar = []byte(`X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*`) @@ -20,7 +20,7 @@ func NewClamScanner(clamdaddr string) (*ClamScanner, error) { return scanner, nil } -func (self *ClamScanner) Scan(data [] byte) (*ScanReport,error) { +func (self *ClamScanner) Scan(data []byte) (*ScanReport, error) { var matches []ScanMatch response := new(ScanReport) response.Filename = "stream" @@ -36,7 +36,7 @@ func (self *ClamScanner) Scan(data [] byte) (*ScanReport,error) { return response, err } - r := (<-ch) //defer close(response) + r := (<-ch) //defer close(response) respJson, err := json.Marshal(&r) if err != nil { @@ -46,25 +46,25 @@ func (self *ClamScanner) Scan(data [] byte) (*ScanReport,error) { fmt.Printf(time.Now().Format(time.RFC3339)+" Scan result : %v\n", string(respJson)) switch r.Status { - case clamd.RES_OK: - response.Status = "CLEAN" - case clamd.RES_FOUND: - response.Status = "INFECTED" - var match ScanMatch - match.Namespace = "" - match.Tags = nil - match.Rule = r.Description - matches = append(matches, match) - case clamd.RES_ERROR: - case clamd.RES_PARSE_ERROR: - default: - response.Status = "ERROR" + case clamd.RES_OK: + response.Status = "CLEAN" + case clamd.RES_FOUND: + response.Status = "INFECTED" + var match ScanMatch + match.Namespace = "" + match.Tags = nil + match.Rule = r.Description + matches = append(matches, match) + case clamd.RES_ERROR: + case clamd.RES_PARSE_ERROR: + default: + response.Status = "ERROR" } if len(matches) <= 0 { - matches = [] ScanMatch{} + matches = []ScanMatch{} } - + response.Matches = matches fmt.Printf(time.Now().Format(time.RFC3339) + " Finished scanning: " + "\n") @@ -73,8 +73,8 @@ func (self *ClamScanner) Scan(data [] byte) (*ScanReport,error) { } // empty the channel so the goroutine from go-clamd/*CLAMDConn.readResponse() doesn't get stuck }() - return response,nil - + return response, nil + } func (self *ClamScanner) ping() error { @@ -97,22 +97,22 @@ func (self *ClamScanner) isClamdReady() bool { if err := self.ping(); err != nil { fmt.Printf("ClamD ping failed.. error [%v]\n", err) return false - } + } fmt.Printf("Connectted to ClamD Server\n") if response, err := self.version(); err != nil { - fmt.Printf("ClamD version check failed.. error [%v]\n", err) - return false + fmt.Printf("ClamD version check failed.. error [%v]\n", err) + return false } else { - fmt.Printf("ClamD version: %#v\n", response) + fmt.Printf("ClamD version: %#v\n", response) } - + return true - + } func (self *ClamScanner) runScanCheck() bool { - if ! self.isClamdReady() { + if !self.isClamdReady() { return false } @@ -124,18 +124,18 @@ func (self *ClamScanner) runScanCheck() bool { if _, err := self.Scan([]byte("hello world... how are you")); err != nil { fmt.Printf("ClamD sample text scan check failed.. error [%v]\n", err) return false - } + } return true } func (self *ClamScanner) warmUp() bool { - for i:=0; i < 24 ; i++ { - if ! self.runScanCheck() { + for i := 0; i < 24; i++ { + if !self.runScanCheck() { time.Sleep(time.Second * 5) } else { return true } } return false -} \ No newline at end of file +} diff --git a/custom_headers.go b/custom_headers.go index b81c8af..6515de7 100644 --- a/custom_headers.go +++ b/custom_headers.go @@ -1,6 +1,6 @@ package main -import( +import ( "github.com/MagnusFrater/helmet" ) @@ -9,19 +9,19 @@ func CustomHelmet() *helmet.Helmet { helmetObj := helmet.Empty() helmetObj.ContentSecurityPolicy = helmet.NewContentSecurityPolicy(map[helmet.CSPDirective][]helmet.CSPSource{ helmet.DirectiveFrameAncestors: {helmet.SourceNone}, - helmet.DirectiveDefaultSrc: {helmet.SourceNone}, + helmet.DirectiveDefaultSrc: {helmet.SourceNone}, }) helmetObj.XContentTypeOptions = helmet.XContentTypeOptionsNoSniff helmetObj.XDNSPrefetchControl = helmet.XDNSPrefetchControlOn helmetObj.XDownloadOptions = helmet.XDownloadOptionsNoOpen - helmetObj.ExpectCT = helmet.NewExpectCT(0,false,"") - helmetObj.FeaturePolicy = helmet.EmptyFeaturePolicy() + helmetObj.ExpectCT = helmet.NewExpectCT(0, false, "") + helmetObj.FeaturePolicy = helmet.EmptyFeaturePolicy() helmetObj.XFrameOptions = helmet.XFrameOptionsDeny helmetObj.XPermittedCrossDomainPolicies = helmet.PermittedCrossDomainPoliciesNone - helmetObj.XPoweredBy = helmet.NewXPoweredBy(true, "") + helmetObj.XPoweredBy = helmet.NewXPoweredBy(true, "") helmetObj.ReferrerPolicy = helmet.NewReferrerPolicy(helmet.DirectiveNoReferrer) helmetObj.StrictTransportSecurity = helmet.NewStrictTransportSecurity(31536000, true, false) helmetObj.XXSSProtection = helmet.NewXXSSProtection(true, helmet.DirectiveModeBlock, "") return helmetObj -} \ No newline at end of file +} diff --git a/custom_headers_middleware.go b/custom_headers_middleware.go index a2c7174..c12e5ce 100644 --- a/custom_headers_middleware.go +++ b/custom_headers_middleware.go @@ -27,10 +27,10 @@ func (helmet *SimpleHelmet) Default() { // Secure function, which will be called for each request func (helmet *SimpleHelmet) Secure(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { for key, val := range helmet.headers { - w.Header().Set(key,val) + w.Header().Set(key, val) } next.ServeHTTP(w, r) - }) -} \ No newline at end of file + }) +} diff --git a/envhelper.go b/envhelper.go index a964f38..cb742e7 100644 --- a/envhelper.go +++ b/envhelper.go @@ -2,48 +2,48 @@ package main import ( "os" - "strings" "strconv" + "strings" ) // Simple helper function to read an environment or return a default value func getEnv(key string, defaultVal string) string { - if value, exists := os.LookupEnv(key); exists { - return value - } + if value, exists := os.LookupEnv(key); exists { + return value + } - return defaultVal + return defaultVal } // Simple helper function to read an environment variable into integer or return a default value func getEnvAsInt(name string, defaultVal int) int { - valueStr := getEnv(name, "") - if value, err := strconv.Atoi(valueStr); err == nil { - return value - } + valueStr := getEnv(name, "") + if value, err := strconv.Atoi(valueStr); err == nil { + return value + } - return defaultVal + return defaultVal } // Helper to read an environment variable into a bool or return default value func getEnvAsBool(name string, defaultVal bool) bool { - valStr := getEnv(name, "") - if val, err := strconv.ParseBool(valStr); err == nil { - return val - } + valStr := getEnv(name, "") + if val, err := strconv.ParseBool(valStr); err == nil { + return val + } - return defaultVal + return defaultVal } // Helper to read an environment variable into a string slice or return default value func getEnvAsSlice(name string, defaultVal []string, sep string) []string { - valStr := getEnv(name, "") + valStr := getEnv(name, "") - if valStr == "" { - return defaultVal - } + if valStr == "" { + return defaultVal + } - val := strings.Split(valStr, sep) + val := strings.Split(valStr, sep) - return val + return val } diff --git a/gcs.go b/gcs.go index 99b1c37..928e528 100644 --- a/gcs.go +++ b/gcs.go @@ -4,70 +4,70 @@ type GCS_Manager struct { *BucketMgr } -func (self *GCS_Manager ) bucketExists(bucket string) (bool, error) { +func (self *GCS_Manager) bucketExists(bucket string) (bool, error) { ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { - return fmt.Errorf("storage.NewClient: %w", err) + return fmt.Errorf("storage.NewClient: %w", err) } defer client.Close() bucket := client.Bucket(bucket) - exists,err := bucket.Attrs(ctx) + exists, err := bucket.Attrs(ctx) if err != nil { - log.Fatalf("Message: %v",err) + log.Fatalf("Message: %v", err) return false, err } else { return true, nil } } -func (self *GCS_Manager ) keyExists(bucket string, key string) (bool, error) { +func (self *GCS_Manager) keyExists(bucket string, key string) (bool, error) { ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { - return fmt.Errorf("storage.NewClient: %w", err) + return fmt.Errorf("storage.NewClient: %w", err) } defer client.Close() bucket := client.Bucket(bucket) object = bucket.Object(key) - exists,err := object.Attrs(ctx) + exists, err := object.Attrs(ctx) if err != nil { - log.Fatalf("Message: %v",err) + log.Fatalf("Message: %v", err) return false, err } else { return true, nil } } -func (self *GCS_Manager ) readFile(bucket string, item string) ([] byte, error) { +func (self *GCS_Manager) readFile(bucket string, item string) ([]byte, error) { ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { - return nil, fmt.Errorf("storage.NewClient: %w", err) + return nil, fmt.Errorf("storage.NewClient: %w", err) } defer client.Close() rc, err := client.Bucket(bucket).Object(item).NewReader(ctx) if err != nil { - return nil, fmt.Errorf("Object(%q).NewReader: %w", object, err) + return nil, fmt.Errorf("Object(%q).NewReader: %w", object, err) } defer rc.Close() data, err := ioutil.ReadAll(rc) if err != nil { - return nil, fmt.Errorf("ioutil.ReadAll: %w", err) + return nil, fmt.Errorf("ioutil.ReadAll: %w", err) } fmt.Fprintf(w, "Blob %v downloaded.\n", object) return data, nil } -func (self *GCS_Manager ) copyFile(srcBucket string, srcObject string, other string) (error) { +func (self *GCS_Manager) copyFile(srcBucket string, srcObject string, other string) error { ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { - return fmt.Errorf("storage.NewClient: %w", err) + return fmt.Errorf("storage.NewClient: %w", err) } defer client.Close() @@ -91,17 +91,17 @@ func (self *GCS_Manager ) copyFile(srcBucket string, srcObject string, other str // dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation}) if _, err := dst.CopierFrom(src).Run(ctx); err != nil { - return fmt.Errorf("Object(%q).CopierFrom(%q).Run: %w", dstObject, srcObject, err) + return fmt.Errorf("Object(%q).CopierFrom(%q).Run: %w", dstObject, srcObject, err) } fmt.Fprintf(w, "Blob %v in bucket %v copied to blob %v in bucket %v.\n", srcObject, srcBucket, dstObject, dstBucket) return nil } -func (self *GCS_Manager ) deleteFile(bucket string, item string) (error) { +func (self *GCS_Manager) deleteFile(bucket string, item string) error { ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { - return fmt.Errorf("storage.NewClient: %w", err) + return fmt.Errorf("storage.NewClient: %w", err) } defer client.Close() @@ -115,12 +115,12 @@ func (self *GCS_Manager ) deleteFile(bucket string, item string) (error) { // if the object's generation number does not match your precondition. attrs, err := o.Attrs(ctx) if err != nil { - return fmt.Errorf("object.Attrs: %w", err) + return fmt.Errorf("object.Attrs: %w", err) } o = o.If(storage.Conditions{GenerationMatch: attrs.Generation}) if err := o.Delete(ctx); err != nil { - return fmt.Errorf("Object(%q).Delete: %w", item, err) + return fmt.Errorf("Object(%q).Delete: %w", item, err) } fmt.Fprintf(w, "Blob %v deleted.\n", item) return nil diff --git a/healthcheckhandler.go b/healthcheckhandler.go index 083659c..9f5e69d 100644 --- a/healthcheckhandler.go +++ b/healthcheckhandler.go @@ -7,7 +7,7 @@ import ( ) func HealthCheckHandler(w http.ResponseWriter, r *http.Request) { - validateContentType(w,r) + validateContentType(w, r) // send request for scanning newRequest := NewHealthCheckRequest() diff --git a/indexhandler.go b/indexhandler.go index 6270e6b..2ed1c9a 100644 --- a/indexhandler.go +++ b/indexhandler.go @@ -2,8 +2,8 @@ package main import ( // standard - "net/http" "encoding/json" + "net/http" ) func IndexHandler(w http.ResponseWriter, r *http.Request) { @@ -14,4 +14,4 @@ func IndexHandler(w http.ResponseWriter, r *http.Request) { return } sendJsonResponse(w, output) -} \ No newline at end of file +} diff --git a/jsonresponse.go b/jsonresponse.go index 39f5eb1..50425e4 100644 --- a/jsonresponse.go +++ b/jsonresponse.go @@ -1,118 +1,118 @@ package main + import ( // external "encoding/json" - "net/http" - "github.com/golang/gddo/httputil/header" + "errors" "fmt" + "github.com/golang/gddo/httputil/header" "io" - "errors" + "net/http" "strconv" "strings" ) -// errorResponse(w, "Content Type is not application/json", http.StatusUnsupportedMediaType) +// errorResponse(w, "Content Type is not application/json", http.StatusUnsupportedMediaType) func errorResponse(w http.ResponseWriter, message string, httpStatusCode int) { - w.Header().Set("Content-Type", "application/json") - // w.Header().Set("Cache-Control", "['no-cache','no-store','must-revalidate']") - // w.Header().Set("Pragma", "no-cache") + w.Header().Set("Content-Type", "application/json") + // w.Header().Set("Cache-Control", "['no-cache','no-store','must-revalidate']") + // w.Header().Set("Pragma", "no-cache") - w.WriteHeader(httpStatusCode) + w.WriteHeader(httpStatusCode) - scaErrorResp := new (ScanErrorResponse) - sanErrorData := ScanErrorData{message, strconv.Itoa(httpStatusCode)} - scaErrorResp.Error = sanErrorData + scaErrorResp := new(ScanErrorResponse) + sanErrorData := ScanErrorData{message, strconv.Itoa(httpStatusCode)} + scaErrorResp.Error = sanErrorData - // scaErrorResp := make(map[string]map[string]string) - // sanErrorData := map[string]string{"message": message, "code": strconv.Itoa(httpStatusCode)} - // scaErrorResp["error"] = sanErrorData - jsonResp, _ := json.Marshal(scaErrorResp) - w.Write(jsonResp) + // scaErrorResp := make(map[string]map[string]string) + // sanErrorData := map[string]string{"message": message, "code": strconv.Itoa(httpStatusCode)} + // scaErrorResp["error"] = sanErrorData + jsonResp, _ := json.Marshal(scaErrorResp) + w.Write(jsonResp) } func sendJsonResponse(w http.ResponseWriter, jsonResp []byte) { - w.Header().Set("Content-Type", "application/json") - // w.Header().Set("Cache-Control", "['no-cache','no-store','must-revalidate']") - // w.Header().Set("Pragma", "no-cache") - w.WriteHeader(200) - w.Write(jsonResp) + w.Header().Set("Content-Type", "application/json") + // w.Header().Set("Cache-Control", "['no-cache','no-store','must-revalidate']") + // w.Header().Set("Pragma", "no-cache") + w.WriteHeader(200) + w.Write(jsonResp) } type malformedRequest struct { - status int - msg string + status int + msg string } func (mr *malformedRequest) Error() string { - return mr.msg + return mr.msg } func validateContentType(w http.ResponseWriter, r *http.Request) error { - if r.Header.Get("Content-Type") != "" { - value, _ := header.ParseValueAndParams(r.Header, "Content-Type") - if value != "application/json" { - return &malformedRequest{status: http.StatusUnsupportedMediaType, msg: "Content-Type header is not application/json"} - } - } - return nil + if r.Header.Get("Content-Type") != "" { + value, _ := header.ParseValueAndParams(r.Header, "Content-Type") + if value != "application/json" { + return &malformedRequest{status: http.StatusUnsupportedMediaType, msg: "Content-Type header is not application/json"} + } + } + return nil } - func decodeJSONBody(w http.ResponseWriter, r *http.Request, dst interface{}) error { - if r.Header.Get("Content-Type") != "" { - value, _ := header.ParseValueAndParams(r.Header, "Content-Type") - if value != "application/json" { - return &malformedRequest{status: http.StatusUnsupportedMediaType, msg: "Content-Type header is not application/json"} - } - } - - r.Body = http.MaxBytesReader(w, r.Body, 1048576) - - dec := json.NewDecoder(r.Body) - dec.DisallowUnknownFields() - - err := dec.Decode(&dst) - if err != nil { - var syntaxError *json.SyntaxError - var unmarshalTypeError *json.UnmarshalTypeError - - switch { - case errors.As(err, &syntaxError): - msg := "Request body contains badly-formed JSON (at position "+ fmt.Sprintf("%d", syntaxError.Offset)+")" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - - case errors.Is(err, io.ErrUnexpectedEOF): - msg := "Request body contains badly-formed JSON" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - - case errors.As(err, &unmarshalTypeError): - msg := "Request body contains an invalid value for the "+unmarshalTypeError.Field+" field (at position "+ fmt.Sprintf("%d", syntaxError.Offset)+")" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - - case strings.HasPrefix(err.Error(), "json: unknown field "): - fieldName := strings.TrimPrefix(err.Error(), "json: unknown field ") - msg := "Request body contains unknown field "+fieldName - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - - case errors.Is(err, io.EOF): - msg := "Request body must not be empty" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - - case err.Error() == "http: request body too large": - msg := "Request body must not be larger than 1MB" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - default: - return err - } - } + if r.Header.Get("Content-Type") != "" { + value, _ := header.ParseValueAndParams(r.Header, "Content-Type") + if value != "application/json" { + return &malformedRequest{status: http.StatusUnsupportedMediaType, msg: "Content-Type header is not application/json"} + } + } + + r.Body = http.MaxBytesReader(w, r.Body, 1048576) + + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + + err := dec.Decode(&dst) + if err != nil { + var syntaxError *json.SyntaxError + var unmarshalTypeError *json.UnmarshalTypeError + + switch { + case errors.As(err, &syntaxError): + msg := "Request body contains badly-formed JSON (at position " + fmt.Sprintf("%d", syntaxError.Offset) + ")" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + + case errors.Is(err, io.ErrUnexpectedEOF): + msg := "Request body contains badly-formed JSON" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + + case errors.As(err, &unmarshalTypeError): + msg := "Request body contains an invalid value for the " + unmarshalTypeError.Field + " field (at position " + fmt.Sprintf("%d", syntaxError.Offset) + ")" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + + case strings.HasPrefix(err.Error(), "json: unknown field "): + fieldName := strings.TrimPrefix(err.Error(), "json: unknown field ") + msg := "Request body contains unknown field " + fieldName + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + + case errors.Is(err, io.EOF): + msg := "Request body must not be empty" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + + case err.Error() == "http: request body too large": + msg := "Request body must not be larger than 1MB" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + default: + return err + } + } err = dec.Decode(&struct{}{}) if err != io.EOF { - msg := "Request body must only contain a single JSON object" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - } + msg := "Request body must only contain a single JSON object" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + } - return nil + return nil } diff --git a/main.go b/main.go index 087a979..a559722 100644 --- a/main.go +++ b/main.go @@ -14,19 +14,19 @@ import ( var ( // config options - index_files StringArgs - address string - port string - addrport string - clamdaddr string - clean_files_bucket string + index_files StringArgs + address string + port string + addrport string + clamdaddr string + clean_files_bucket string quarantine_files_bucket string - cloud_provider CloudProvider + cloud_provider CloudProvider // channels healthcheckrequests chan *HealthCheckRequest - scanstreamrequests chan *ScanStreamRequest - namerequests chan *RuleSetRequest - rulerequests chan *RuleListRequest + scanstreamrequests chan *ScanStreamRequest + namerequests chan *RuleSetRequest + rulerequests chan *RuleListRequest // loggers info *log.Logger @@ -55,9 +55,9 @@ func init() { cloud_provider, _ = ParseCloudProviderString(cloud_provider_str) - info.Println("reading CLEAN_FILES_BUCKET value as " +clean_files_bucket) - info.Println("reading QUARANTINE_FILES_BUCKET value as " +quarantine_files_bucket) - info.Println("reading CLOUD_PROVIDER value as " +cloud_provider_str) + info.Println("reading CLEAN_FILES_BUCKET value as " + clean_files_bucket) + info.Println("reading QUARANTINE_FILES_BUCKET value as " + quarantine_files_bucket) + info.Println("reading CLOUD_PROVIDER value as " + cloud_provider_str) } @@ -112,7 +112,7 @@ func main() { ruleset_sub.HandleFunc("", RuleSetListHandler).Methods("GET") ruleset_sub.HandleFunc("/", RuleSetListHandler).Methods("GET") ruleset_sub.HandleFunc("/{ruleset}", RuleListHandler).Methods("GET") - + loggedRouter := handlers.CombinedLoggingHandler(os.Stdout, r) log.Fatal(http.ListenAndServe(addrport, loggedRouter)) //log.Fatal(http.ListenAndServe(addrport, r)) diff --git a/request.go b/request.go index cc12ddc..fa6374a 100644 --- a/request.go +++ b/request.go @@ -1,9 +1,9 @@ package main type ScanObject struct { - BucketName string `json:"bucketname"` - Key string `json:"key"` - CleanFilesBucket string `json:"clean_files_bucket,omitempty"` + BucketName string `json:"bucketname"` + Key string `json:"key"` + CleanFilesBucket string `json:"clean_files_bucket,omitempty"` QurantineFilesBucket string `json:"qurantine_files_bucket,omitempty"` } @@ -12,7 +12,7 @@ type HealthCheckRequest struct { } type ScanStreamRequest struct { - data [] byte + data []byte ResponseChan chan *ScanResponse } diff --git a/response.go b/response.go index 92907a4..1a2b030 100644 --- a/response.go +++ b/response.go @@ -1,13 +1,13 @@ package main type ScanResponse struct { - data *ScanReport - err error + data *ScanReport + err error } type HealthCheckResponse struct { - health string //OK,ERROR - err error + health string //OK,ERROR + err error } // struct to handle namespace requests @@ -17,7 +17,7 @@ type RuleSetResponseObject struct { type RuleSetResponse struct { data *RuleSetResponseObject - err error + err error } // sturc to handle @@ -27,5 +27,5 @@ type RuleListResponseObject struct { type RuleListResponse struct { data *RuleListResponseObject - err error -} \ No newline at end of file + err error +} diff --git a/rulelisthandler.go b/rulelisthandler.go index f02ffae..f289309 100644 --- a/rulelisthandler.go +++ b/rulelisthandler.go @@ -13,12 +13,12 @@ func RuleListHandler(w http.ResponseWriter, r *http.Request) { ruleset := vars["ruleset"] req := NewRuleListRequest(ruleset) - rulerequests<- req + rulerequests <- req response := <-req.ResponseChan var err error = response.err - + if err != nil { elog.Println(err) errorResponse(w, err.Error(), http.StatusInternalServerError) diff --git a/ruleset.go b/ruleset.go index d0744b1..051ea92 100644 --- a/ruleset.go +++ b/ruleset.go @@ -53,7 +53,7 @@ func NewRuleSet(indexpath string) (*RuleSet, error) { filename := fields[len(fields)-1] namespacestr := strings.Split(filename, "_")[0] - info.Println("NewRuleSet fields: " + strings.Join(fields,",")) + info.Println("NewRuleSet fields: " + strings.Join(fields, ",")) info.Println("NewRuleSet filename: " + filename) info.Println("NewRuleSet namespacestr: " + namespacestr) info.Println("NewRuleSet indexpath: " + indexpath) diff --git a/s3.go b/s3.go index 81e110b..fd3efc2 100644 --- a/s3.go +++ b/s3.go @@ -2,31 +2,30 @@ package main import ( "context" - "net/http" - "github.com/aws/smithy-go" + "errors" + "fmt" "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" - awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "net/http" "net/url" - "time" - "fmt" "os" "strconv" - "errors" + "time" ) - type S3_Manager struct { *BucketMgr } -func (self* S3_Manager) getPartSize() int64 { +func (self *S3_Manager) getPartSize() int64 { var partSize int64 strSizeInMb, err := os.LookupEnv("DOWNLOAD_PART_SIZE") - + if !err { elog.Println(time.Now().Format(time.RFC3339) + "DOWNLOAD_PART_SIZE is not present..using DefaultDownloadPartSize ") partSize = manager.DefaultDownloadPartSize @@ -42,7 +41,7 @@ func (self* S3_Manager) getPartSize() int64 { return partSize } -func (self* S3_Manager) getRegion() string { +func (self *S3_Manager) getRegion() string { region, err := os.LookupEnv("AWS_REGION") if !err { elog.Println(time.Now().Format(time.RFC3339) + " AWS_REGION is not present..using us-east-1") @@ -52,59 +51,59 @@ func (self* S3_Manager) getRegion() string { } // check if a bucket exists. -func (self *S3_Manager ) bucketExists(bucket string) (bool, error) { +func (self *S3_Manager) bucketExists(bucket string) (bool, error) { cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " bucketExists: Filed to load config for bucket "+bucket + " error : " + err.Error()) - return false, errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: Filed to load config for bucket " + bucket + " error : " + err.Error()) + return false, errors.New("Filed to load config") } s3client := s3.NewFromConfig(cfg) - _, err = s3client.HeadBucket(context.TODO(),&s3.HeadBucketInput{Bucket: aws.String(bucket)}) + _, err = s3client.HeadBucket(context.TODO(), &s3.HeadBucketInput{Bucket: aws.String(bucket)}) if err != nil { var apiErr smithy.APIError - if errors.As(err, &apiErr) { + if errors.As(err, &apiErr) { var httpResponseErr *awshttp.ResponseError - if errors.As(err, &httpResponseErr) { + if errors.As(err, &httpResponseErr) { switch httpResponseErr.HTTPStatusCode() { case http.StatusMovedPermanently: - elog.Println( time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket "+bucket + " error : " + err.Error()) - return false, errors.New("Bucket StatusMovedPermanently ") + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket " + bucket + " error : " + err.Error()) + return false, errors.New("Bucket StatusMovedPermanently ") case http.StatusForbidden: - elog.Println( time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket "+bucket + " error : " + err.Error()) - return false, errors.New("Bucket StatusForbidden") + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket " + bucket + " error : " + err.Error()) + return false, errors.New("Bucket StatusForbidden") case http.StatusNotFound: - elog.Println( time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket "+bucket + " error : " + err.Error()) - return false, nil + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket " + bucket + " error : " + err.Error()) + return false, nil default: - elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: ResponseError failed for bucket "+bucket + "with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: ResponseError failed for bucket " + bucket + "with error: " + err.Error()) return false, errors.New("Filed to find bucket") } } else { - elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: ApiError failed for bucket "+bucket + "with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: ApiError failed for bucket " + bucket + "with error: " + err.Error()) return false, errors.New("Filed to find bucket") } } else { - elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket "+bucket + "with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket " + bucket + "with error: " + err.Error()) return false, errors.New("Filed to find bucket") } } - return true,nil + return true, nil } -func (self* S3_Manager) getHeadObject(bucket string, key string) (*s3.HeadObjectOutput, error) { +func (self *S3_Manager) getHeadObject(bucket string, key string) (*s3.HeadObjectOutput, error) { cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " getHeadObject: Filed to load config for bucket "+bucket + " error : " + err.Error()) - return nil, errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: Filed to load config for bucket " + bucket + " error : " + err.Error()) + return nil, errors.New("Filed to load config") } s3client := s3.NewFromConfig(cfg) @@ -116,29 +115,29 @@ func (self* S3_Manager) getHeadObject(bucket string, key string) (*s3.HeadObject if err != nil { var apiErr smithy.APIError - if errors.As(err, &apiErr) { + if errors.As(err, &apiErr) { var httpResponseErr *awshttp.ResponseError - if errors.As(err, &httpResponseErr) { + if errors.As(err, &httpResponseErr) { switch httpResponseErr.HTTPStatusCode() { case http.StatusMovedPermanently: - elog.Println( time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket "+bucket +" key "+key+" error : " + err.Error()) - return nil, errors.New("Bucket StatusMovedPermanently ") + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket " + bucket + " key " + key + " error : " + err.Error()) + return nil, errors.New("Bucket StatusMovedPermanently ") case http.StatusForbidden: - elog.Println( time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket "+bucket +" key "+key+" error : " + err.Error()) - return nil, errors.New("Bucket StatusForbidden") + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket " + bucket + " key " + key + " error : " + err.Error()) + return nil, errors.New("Bucket StatusForbidden") case http.StatusNotFound: - elog.Println( time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket "+bucket +" key "+key+" error : " + err.Error()) - return nil, errors.New("Bucket StatusNotFound") + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket " + bucket + " key " + key + " error : " + err.Error()) + return nil, errors.New("Bucket StatusNotFound") default: - elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: ResponseError failed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: ResponseError failed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return nil, errors.New("Filed to find object") } } else { - elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: APIError failed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: APIError failed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return nil, errors.New("Filed to find object") } } else { - elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return nil, errors.New("Filed to find object") } } @@ -147,14 +146,14 @@ func (self* S3_Manager) getHeadObject(bucket string, key string) (*s3.HeadObject } // check if a file exists. -func (self* S3_Manager) keyExists(bucket string, key string) (bool, error) { +func (self *S3_Manager) keyExists(bucket string, key string) (bool, error) { cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + "keyExists: Filed to load config for bucket "+bucket +" key "+key+" error : " + err.Error()) - return false, errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + "keyExists: Filed to load config for bucket " + bucket + " key " + key + " error : " + err.Error()) + return false, errors.New("Filed to load config") } s3client := s3.NewFromConfig(cfg) @@ -166,23 +165,23 @@ func (self* S3_Manager) keyExists(bucket string, key string) (bool, error) { if err != nil { var apiErr smithy.APIError - if errors.As(err, &apiErr) { + if errors.As(err, &apiErr) { var httpResponseErr *awshttp.ResponseError - if errors.As(err, &httpResponseErr) { - switch httpResponseErr.HTTPStatusCode() { + if errors.As(err, &httpResponseErr) { + switch httpResponseErr.HTTPStatusCode() { case http.StatusNotFound: - elog.Println( time.Now().Format(time.RFC3339) + " keyExists: failed for bucket "+bucket +" key "+key+" error : " + err.Error()) - return false, nil + elog.Println(time.Now().Format(time.RFC3339) + " keyExists: failed for bucket " + bucket + " key " + key + " error : " + err.Error()) + return false, nil default: - elog.Println(time.Now().Format(time.RFC3339) + " keyExists: ResponseError failed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " keyExists: ResponseError failed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return false, errors.New("Filed to find key") } - } else { - elog.Println(time.Now().Format(time.RFC3339) + " keyExists: APIErrorfailed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + } else { + elog.Println(time.Now().Format(time.RFC3339) + " keyExists: APIErrorfailed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return false, errors.New("Filed to find key") } } else { - elog.Println(time.Now().Format(time.RFC3339) + " keyExists: failed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " keyExists: failed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return false, errors.New("Filed to find key") } } @@ -190,15 +189,15 @@ func (self* S3_Manager) keyExists(bucket string, key string) (bool, error) { return true, nil } -func (self *S3_Manager ) readFile(bucket string, item string) ([] byte, error) { +func (self *S3_Manager) readFile(bucket string, item string) ([]byte, error) { // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " readFile: Filed to load config to read file " +item+ " from bucket "+bucket + " error : " + err.Error()) - return nil, errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + " readFile: Filed to load config to read file " + item + " from bucket " + bucket + " error : " + err.Error()) + return nil, errors.New("Filed to load config") } // Create an S3 client using the loaded configuration @@ -209,10 +208,10 @@ func (self *S3_Manager ) readFile(bucket string, item string) ([] byte, error) { d.PartSize = self.getPartSize() }) - headObject, err := self.getHeadObject(bucket,item) + headObject, err := self.getHeadObject(bucket, item) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " readFile: getHeadObject failed " +item+ " from bucket "+bucket + " error : " + err.Error()) - return nil, errors.New("Filed to read file") + elog.Println(time.Now().Format(time.RFC3339) + " readFile: getHeadObject failed " + item + " from bucket " + bucket + " error : " + err.Error()) + return nil, errors.New("Filed to read file") } // pre-allocate in memory buffer, where headObject type is *s3.HeadObjectOutput buff := make([]byte, int(*headObject.ContentLength)) @@ -225,24 +224,24 @@ func (self *S3_Manager ) readFile(bucket string, item string) ([] byte, error) { }) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " Unable to read file " +item+ " from bucket "+bucket + " error : " + err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " Unable to read file " + item + " from bucket " + bucket + " error : " + err.Error()) return nil, errors.New("Unable to read file") } - - info.Println(time.Now().Format(time.RFC3339) +" Downloaded file "+item+ " from bucket "+bucket) + + info.Println(time.Now().Format(time.RFC3339) + " Downloaded file " + item + " from bucket " + bucket) return buff, nil } -func (self *S3_Manager ) copyFile(bucket string, item string, other string) (error){ +func (self *S3_Manager) copyFile(bucket string, item string, other string) error { // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " copyFile: Filed to load config to read file " +item+ " from bucket "+bucket + " error : " + err.Error()) - return errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + " copyFile: Filed to load config to read file " + item + " from bucket " + bucket + " error : " + err.Error()) + return errors.New("Filed to load config") } // Create an S3 client using the loaded configuration @@ -257,24 +256,24 @@ func (self *S3_Manager ) copyFile(bucket string, item string, other string) (err }) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " Unable to read file " +item+ " from bucket "+bucket+ " to bucket "+other+" error : " + err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " Unable to read file " + item + " from bucket " + bucket + " to bucket " + other + " error : " + err.Error()) return errors.New("Unable to copy file") } - info.Println( time.Now().Format(time.RFC3339) + " File "+ item+ " successfully copied from bucket "+bucket+ " to bucket "+other) + info.Println(time.Now().Format(time.RFC3339) + " File " + item + " successfully copied from bucket " + bucket + " to bucket " + other) return nil } -func (self *S3_Manager ) deleteFile(bucket string, item string) (error) { +func (self *S3_Manager) deleteFile(bucket string, item string) error { // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " deleteFile: Filed to load config to read file " +item+ " from bucket "+bucket + " error : " + err.Error()) - return errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + " deleteFile: Filed to load config to read file " + item + " from bucket " + bucket + " error : " + err.Error()) + return errors.New("Filed to load config") } // Create an S3 client using the loaded configuration @@ -286,7 +285,7 @@ func (self *S3_Manager ) deleteFile(bucket string, item string) (error) { }) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " Error occurred while deleting file " +item+ " from bucket "+bucket+" err: "+ fmt.Sprint(err)) + elog.Println(time.Now().Format(time.RFC3339) + " Error occurred while deleting file " + item + " from bucket " + bucket + " err: " + fmt.Sprint(err)) return errors.New("Error occurred while deleting file") } return nil diff --git a/scanner.go b/scanner.go index 4e36532..220426d 100644 --- a/scanner.go +++ b/scanner.go @@ -1,28 +1,27 @@ package main import ( - "time" "encoding/json" "os" "strconv" + "time" ) - // struct to hold compiler and channels type Scanner struct { - yarascanner YaraScanner - clamscanner ClamScanner + yarascanner YaraScanner + clamscanner ClamScanner healthcheckrequests chan *HealthCheckRequest - scanstreamrequests chan *ScanStreamRequest - namerequests chan *RuleSetRequest - rulerequests chan *RuleListRequest + scanstreamrequests chan *ScanStreamRequest + namerequests chan *RuleSetRequest + rulerequests chan *RuleListRequest } -func (self *Scanner) healthcheck() (*HealthCheckResponse) { +func (self *Scanner) healthcheck() *HealthCheckResponse { - healthCheckResponse := new (HealthCheckResponse) + healthCheckResponse := new(HealthCheckResponse) clamDHealth := self.clamscanner.isClamdReady() - + if clamDHealth { healthCheckResponse.health = "OK" } else { @@ -32,35 +31,35 @@ func (self *Scanner) healthcheck() (*HealthCheckResponse) { return healthCheckResponse } -func (self *Scanner) scanstream(data []byte) (*ScanResponse) { +func (self *Scanner) scanstream(data []byte) *ScanResponse { info.Println("Running yarascan") - scanResponse := new (ScanResponse) + scanResponse := new(ScanResponse) - yaraScannerResponse,yaraerr := ScanStream(&self.yarascanner, data) + yaraScannerResponse, yaraerr := ScanStream(&self.yarascanner, data) scanResponse.data = yaraScannerResponse scanResponse.err = yaraerr - + yaraRespJson, _ := json.Marshal(yaraScannerResponse) - info.Println( time.Now().Format(time.RFC3339) + " yarascan scan result " + string(yaraRespJson)) + info.Println(time.Now().Format(time.RFC3339) + " yarascan scan result " + string(yaraRespJson)) if (yaraerr == nil) && len(yaraScannerResponse.Matches) > 0 { - info.Println( time.Now().Format(time.RFC3339) + " Found matches with yara " + string(yaraRespJson)) + info.Println(time.Now().Format(time.RFC3339) + " Found matches with yara " + string(yaraRespJson)) } - info.Println("Running clamscan on addr: "+ clamdaddr) + info.Println("Running clamscan on addr: " + clamdaddr) - clamScannerResponse,clamerr := ScanStream(&self.clamscanner, data) + clamScannerResponse, clamerr := ScanStream(&self.clamscanner, data) clamRespJson, _ := json.Marshal(clamScannerResponse) - info.Println( time.Now().Format(time.RFC3339) + " clamav scan result " + string(clamRespJson)) + info.Println(time.Now().Format(time.RFC3339) + " clamav scan result " + string(clamRespJson)) if (clamerr == nil) && len(clamScannerResponse.Matches) > 0 { - info.Println( time.Now().Format(time.RFC3339) + " Found matches with clamav" + string(clamRespJson)) + info.Println(time.Now().Format(time.RFC3339) + " Found matches with clamav" + string(clamRespJson)) scanResponse.data = clamScannerResponse - } - + } + if clamerr != nil { scanResponse.err = clamerr } @@ -75,8 +74,8 @@ func (self *Scanner) warmUp() { var yaraHealth = bool(false) var clamDHealth = bool(false) - yaraScannerResponse,yaraerr := ScanStream(&self.yarascanner, eicar) - + yaraScannerResponse, yaraerr := ScanStream(&self.yarascanner, eicar) + if (yaraerr == nil) && len(yaraScannerResponse.Matches) > 0 { yaraHealth = true } @@ -86,7 +85,7 @@ func (self *Scanner) warmUp() { if yaraHealth && clamDHealth { info.Println("Warmed Up") } else { - info.Println( time.Now().Format(time.RFC3339) + " Warm up failed exiting.. Yara Health" + strconv.FormatBool(yaraHealth) + "ClamD Health" + strconv.FormatBool(clamDHealth)) + info.Println(time.Now().Format(time.RFC3339) + " Warm up failed exiting.. Yara Health" + strconv.FormatBool(yaraHealth) + "ClamD Health" + strconv.FormatBool(clamDHealth)) os.Exit(1) } } @@ -96,17 +95,17 @@ func (self *Scanner) LoadIndex(indexPath string) error { } func (self *Scanner) listRuleSets() *RuleSetResponse { - response,err := self.yarascanner.ListRuleSets() - ruleSetResponse := new (RuleSetResponse) + response, err := self.yarascanner.ListRuleSets() + ruleSetResponse := new(RuleSetResponse) ruleSetResponse.err = err ruleSetResponse.data = response return ruleSetResponse } -func (self *Scanner) listRules(rulesetname string) (*RuleListResponse) { +func (self *Scanner) listRules(rulesetname string) *RuleListResponse { response, err := self.yarascanner.ListRules(rulesetname) - ruleListResponse := new (RuleListResponse) + ruleListResponse := new(RuleListResponse) ruleListResponse.err = err ruleListResponse.data = response diff --git a/scannerinterface.go b/scannerinterface.go index d5a5f24..7aed6c3 100644 --- a/scannerinterface.go +++ b/scannerinterface.go @@ -2,13 +2,13 @@ package main // Defining an interface type ScannerInterface interface { - Scan(data [] byte) (*ScanReport, error) + Scan(data []byte) (*ScanReport, error) } // struct to handle matches type ScanMatch struct { - Rule string `json:"rule"` - Namespace string `json:"namespace"` + Rule string `json:"rule"` + Namespace string `json:"namespace"` Tags []string `json:"tags"` } @@ -17,9 +17,9 @@ type ListResponse struct { } type ScanReport struct { - Filename string `json:"filename"` + Filename string `json:"filename"` Matches []ScanMatch `json:"matches"` - Status string `json:"status"` + Status string `json:"status"` } type ScanErrorData struct { diff --git a/streamscanhandler.go b/streamscanhandler.go index 6f90234..c38eed6 100644 --- a/streamscanhandler.go +++ b/streamscanhandler.go @@ -3,8 +3,8 @@ package main import ( // standard "encoding/json" - "net/http" "io/ioutil" + "net/http" ) func ScanStreamHandler(w http.ResponseWriter, r *http.Request) { diff --git a/yarascanner.go b/yarascanner.go index 5c3a344..7e9fae0 100644 --- a/yarascanner.go +++ b/yarascanner.go @@ -11,7 +11,7 @@ type YaraScanner struct { } // To implement an interface in Go all you need to do is just define all the functions in the interface -func (self *YaraScanner) Scan(data [] byte) (*ScanReport, error) { +func (self *YaraScanner) Scan(data []byte) (*ScanReport, error) { var matches []ScanMatch response := new(ScanReport) response.Filename = "stream" @@ -22,7 +22,7 @@ func (self *YaraScanner) Scan(data [] byte) (*ScanReport, error) { err := ruleset.Rules.ScanMem(data, 0, 300, &m) if err != nil { response.Status = "ERROR" - return response,err + return response, err } for _, resp := range m { var match ScanMatch @@ -37,11 +37,11 @@ func (self *YaraScanner) Scan(data [] byte) (*ScanReport, error) { response.Status = "INFECTED" } else { response.Status = "CLEAN" - matches = [] ScanMatch{} + matches = []ScanMatch{} } - + response.Matches = matches - return response,nil + return response, nil } func (self *YaraScanner) LoadIndex(indexPath string) error { @@ -59,7 +59,7 @@ func (self *YaraScanner) ListRuleSets() (*RuleSetResponseObject, error) { response.Names = append(response.Names, ruleset.Name) } - return response,nil + return response, nil } @@ -80,4 +80,4 @@ func (self *YaraScanner) ListRules(rulesetname string) (*RuleListResponseObject, } return response, nil -} \ No newline at end of file +} From 0feb9b2619643ca908a11046fde67cc8abb422c8 Mon Sep 17 00:00:00 2001 From: cloudina Date: Sat, 13 Jul 2024 13:41:05 +0100 Subject: [PATCH 11/11] fix syntax errors --- abs.go | 68 +++++++++++++++++++++++++++++------------------------- gcs.go | 71 ++++++++++++++++++++++++++++++++++++++------------------- main.go | 3 --- 3 files changed, 85 insertions(+), 57 deletions(-) diff --git a/abs.go b/abs.go index 991e49b..9fa6d16 100644 --- a/abs.go +++ b/abs.go @@ -1,19 +1,21 @@ package main import ( + "bytes" "context" "fmt" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "io" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "path" "log" "os" - "strings" ) type ABS_Manager struct { - *BucketMgr + BucketMgr } func (self *ABS_Manager) handleError(err error) { @@ -39,12 +41,12 @@ func (self *ABS_Manager) getServiceClient() *azblob.Client { return client } -func (self *ABS_Manager) getContainerClient(containerName string) *azblob.Client { +func (self *ABS_Manager) getContainerClient(containerName string) *container.Client { accountName, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") if !ok { panic("AZURE_STORAGE_ACCOUNT_NAME could not be found") } - containerName := "testcontainer" + containerURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName) cred, err := azidentity.NewDefaultAzureCredential(nil) @@ -55,15 +57,15 @@ func (self *ABS_Manager) getContainerClient(containerName string) *azblob.Client return containerClient } -func (self *ABS_Manager) getBlobClient(containerName string, blobName string) *azblob.Client { +func (self *ABS_Manager) getBlobClient(containerName string, blobName string) *blob.Client { // From the Azure portal, get your Storage account blob service URL endpoint. accountName, accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME"), os.Getenv("AZURE_STORAGE_ACCOUNT_KEY") blobURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", accountName, containerName, blobName) credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) - handleError(err) + self.handleError(err) blobClient, err := blob.NewClientWithSharedKeyCredential(blobURL, credential, nil) - handleError(err) + self.handleError(err) return blobClient } @@ -78,10 +80,10 @@ func (self *ABS_Manager) listBuckets() []string { var buckets []string for pager.More() { - resp, err := pager.NextPage(ctx) - handleError(err) // if err is not nil, break the loop. + resp, err := pager.NextPage(context.TODO()) + self.handleError(err) // if err is not nil, break the loop. for _, _container := range resp.ContainerItems { - buckets.append(_container) + buckets = append(buckets, *_container.Name) } } return buckets @@ -89,8 +91,9 @@ func (self *ABS_Manager) listBuckets() []string { func (self *ABS_Manager) bucketExists(bucket string) (bool, error) { client := self.getContainerClient(bucket) - _, err := client.getProperies() - if ContainerNotFound == err { + _, err := client.GetProperties(context.TODO(), nil) + + if bloberror.HasCode(err, bloberror.ContainerNotFound) { return false, err } else { return true, nil @@ -99,8 +102,9 @@ func (self *ABS_Manager) bucketExists(bucket string) (bool, error) { func (self *ABS_Manager) keyExists(bucket string, key string) (bool, error) { client := self.getBlobClient(bucket, key) - _, err := client.getProperies() - if BlobNotFound == err { + _, err := client.GetProperties(context.TODO(), nil) + + if bloberror.HasCode(err, bloberror.BlobNotFound) { return false, err } else { return true, nil @@ -111,33 +115,35 @@ func (self *ABS_Manager) readFile(bucket string, item string) ([]byte, error) { client := self.getServiceClient() // Download the blob - downloadResponse, err := client.DownloadStream(ctx, bucket, item, nil) - handleError(err) + downloadResponse, err := client.DownloadStream(context.TODO(), bucket, item, nil) + self.handleError(err) - // Assert that the content is correct - actualBlobData, err := io.ReadAll(downloadResponse.Body) - handleError(err) - err = reader.Close() - if err != nil { - return nil, err - } - return actualBlobData, nil + downloadedData := bytes.Buffer{} + retryReader := downloadResponse.NewRetryReader(context.TODO(), &azblob.RetryReaderOptions{}) + _, err = downloadedData.ReadFrom(retryReader) + self.handleError(err) + + err = retryReader.Close() + self.handleError(err) + + return downloadedData.Bytes(), nil } func (self *ABS_Manager) copyFile(bucket string, item string, other string) error { - data, _ = self.readFile(bucket, item) + data, _ := self.readFile(bucket, item) client := self.getServiceClient() - _, err = client.UploadBuffer(context.TODO(), path.Dir(other), path.Base(other), data, &azblob.UploadBufferOptions{}) - handleError(err) + _, err := client.UploadBuffer(context.TODO(), path.Dir(other), path.Base(other), data, &azblob.UploadBufferOptions{}) + self.handleError(err) + return err } func (self *ABS_Manager) deleteFile(bucket string, item string) error { client := self.getServiceClient() // Delete the blob. - _, err = client.DeleteBlob(context.TODO(), bucket, item, nil) - handleError(err) + _, err := client.DeleteBlob(context.TODO(), bucket, item, nil) + self.handleError(err) return err } diff --git a/gcs.go b/gcs.go index 928e528..018b77e 100644 --- a/gcs.go +++ b/gcs.go @@ -1,22 +1,34 @@ package main +import ( + "context" + "errors" + "path" + "time" + "io/ioutil" + "cloud.google.com/go/storage" +) + type GCS_Manager struct { - *BucketMgr + BucketMgr } func (self *GCS_Manager) bucketExists(bucket string) (bool, error) { ctx := context.Background() client, err := storage.NewClient(ctx) + if err != nil { - return fmt.Errorf("storage.NewClient: %w", err) + elog.Println(time.Now().Format(time.RFC3339) + "bucketExists: storage.NewClient " + bucket + " error : " + err.Error()) + return false, errors.New("storage.NewClient Failed ") } defer client.Close() - bucket := client.Bucket(bucket) - exists, err := bucket.Attrs(ctx) + bucketObj := client.Bucket(bucket) + _, err = bucketObj.Attrs(ctx) + if err != nil { - log.Fatalf("Message: %v", err) - return false, err + elog.Println(time.Now().Format(time.RFC3339) + "bucketExists: bucketObj.Attrs " + bucket + " error : " + err.Error()) + return false, errors.New("bucketObj.Attrs Failed ") } else { return true, nil } @@ -25,17 +37,20 @@ func (self *GCS_Manager) bucketExists(bucket string) (bool, error) { func (self *GCS_Manager) keyExists(bucket string, key string) (bool, error) { ctx := context.Background() client, err := storage.NewClient(ctx) + if err != nil { - return fmt.Errorf("storage.NewClient: %w", err) + elog.Println(time.Now().Format(time.RFC3339) + "keyExists: storage.NewClient " + bucket + " error : " + err.Error()) + return false, errors.New("storage.NewClient Failed ") } defer client.Close() - bucket := client.Bucket(bucket) - object = bucket.Object(key) - exists, err := object.Attrs(ctx) + bucketObj := client.Bucket(bucket) + object := bucketObj.Object(key) + _, err = object.Attrs(ctx) + if err != nil { - log.Fatalf("Message: %v", err) - return false, err + elog.Println(time.Now().Format(time.RFC3339) + "keyExists: object.Attrs " + bucket + " error : " + err.Error()) + return false, errors.New("object.Attrs Failed ") } else { return true, nil } @@ -45,21 +60,26 @@ func (self *GCS_Manager) readFile(bucket string, item string) ([]byte, error) { ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { - return nil, fmt.Errorf("storage.NewClient: %w", err) + elog.Println(time.Now().Format(time.RFC3339) + "readFile: storage.NewClient " + bucket + " key " +item + " error : " + err.Error()) + return nil, errors.New("storage.NewClient Failed ") } defer client.Close() rc, err := client.Bucket(bucket).Object(item).NewReader(ctx) if err != nil { - return nil, fmt.Errorf("Object(%q).NewReader: %w", object, err) + elog.Println(time.Now().Format(time.RFC3339) + "readFile: client.Bucket " + bucket + " key " +item + " error : " + err.Error()) + return nil, errors.New("Bucket.Object.NewReader Failed ") } defer rc.Close() data, err := ioutil.ReadAll(rc) if err != nil { - return nil, fmt.Errorf("ioutil.ReadAll: %w", err) + elog.Println(time.Now().Format(time.RFC3339) + "readFile: client.Bucket " + bucket + " key " +item + " error : " + err.Error()) + return nil, errors.New("ioutil ReadAll Failed ") } - fmt.Fprintf(w, "Blob %v downloaded.\n", object) + + info.Println(time.Now().Format(time.RFC3339) + " Downloaded object " + item + " from bucket " + bucket) + return data, nil } @@ -67,7 +87,8 @@ func (self *GCS_Manager) copyFile(srcBucket string, srcObject string, other stri ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { - return fmt.Errorf("storage.NewClient: %w", err) + elog.Println(time.Now().Format(time.RFC3339) + "copyFile: storage.NewClient " + srcBucket + " error : " + err.Error()) + return errors.New("storage.NewClient Failed ") } defer client.Close() @@ -91,9 +112,10 @@ func (self *GCS_Manager) copyFile(srcBucket string, srcObject string, other stri // dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation}) if _, err := dst.CopierFrom(src).Run(ctx); err != nil { - return fmt.Errorf("Object(%q).CopierFrom(%q).Run: %w", dstObject, srcObject, err) + elog.Println(time.Now().Format(time.RFC3339) + " Unable to copy object " + srcObject + " from bucket " + srcBucket + " to bucket " + dstBucket + " error : " + err.Error()) + return errors.New("Unable to copy file") } - fmt.Fprintf(w, "Blob %v in bucket %v copied to blob %v in bucket %v.\n", srcObject, srcBucket, dstObject, dstBucket) + return nil } @@ -101,7 +123,8 @@ func (self *GCS_Manager) deleteFile(bucket string, item string) error { ctx := context.Background() client, err := storage.NewClient(ctx) if err != nil { - return fmt.Errorf("storage.NewClient: %w", err) + elog.Println(time.Now().Format(time.RFC3339) + "deleteFile: storage.NewClient " + bucket + " error : " + err.Error()) + return errors.New("storage.NewClient Failed ") } defer client.Close() @@ -115,13 +138,15 @@ func (self *GCS_Manager) deleteFile(bucket string, item string) error { // if the object's generation number does not match your precondition. attrs, err := o.Attrs(ctx) if err != nil { - return fmt.Errorf("object.Attrs: %w", err) + elog.Println(time.Now().Format(time.RFC3339) + "deleteFile: bucketObj.Attrs " + bucket + " error : " + err.Error()) + return errors.New("bucketObj.Attrs Failed ") } o = o.If(storage.Conditions{GenerationMatch: attrs.Generation}) if err := o.Delete(ctx); err != nil { - return fmt.Errorf("Object(%q).Delete: %w", item, err) + elog.Println(time.Now().Format(time.RFC3339) + "deleteFile: Delete " + bucket + " object "+item + " error : " + err.Error()) + return errors.New("Object Delete Failed ") } - fmt.Fprintf(w, "Blob %v deleted.\n", item) + info.Println(time.Now().Format(time.RFC3339) + " deleteFile " + item + " successfully deleted from bucket " + bucket) return nil } diff --git a/main.go b/main.go index a559722..b2624e0 100644 --- a/main.go +++ b/main.go @@ -92,8 +92,6 @@ func main() { // setup http server and begin serving traffic r := mux.NewRouter() - // helmet := CustomHelmet() - // r.Use(helmet.Secure) helmet := SimpleHelmet{} helmet.Default() @@ -115,5 +113,4 @@ func main() { loggedRouter := handlers.CombinedLoggingHandler(os.Stdout, r) log.Fatal(http.ListenAndServe(addrport, loggedRouter)) - //log.Fatal(http.ListenAndServe(addrport, r)) }