diff --git a/404handler.go b/404handler.go index c7b0866..8abf192 100644 --- a/404handler.go +++ b/404handler.go @@ -8,9 +8,9 @@ import ( func Handle404(helmet SimpleHelmet) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { for key, val := range helmet.headers { - w.Header().Set(key,val) + w.Header().Set(key, val) } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotFound) }) -} \ No newline at end of file +} diff --git a/abs.go b/abs.go new file mode 100644 index 0000000..9fa6d16 --- /dev/null +++ b/abs.go @@ -0,0 +1,149 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "path" + "log" + "os" +) + +type ABS_Manager struct { + BucketMgr +} + +func (self *ABS_Manager) handleError(err error) { + if err != nil { + log.Fatal(err.Error()) + } +} + +func (self *ABS_Manager) getServiceClient() *azblob.Client { + // Create a new service client with token credential + accountName, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") + if !ok { + panic("AZURE_STORAGE_ACCOUNT_NAME could not be found") + } + + serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) + + credential, err := azidentity.NewDefaultAzureCredential(nil) + self.handleError(err) + + client, err := azblob.NewClient(serviceURL, credential, nil) + self.handleError(err) + return client +} + +func (self *ABS_Manager) getContainerClient(containerName string) *container.Client { + accountName, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME") + if !ok { + panic("AZURE_STORAGE_ACCOUNT_NAME could not be found") + } + + containerURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName) + + cred, err := azidentity.NewDefaultAzureCredential(nil) + self.handleError(err) + + containerClient, err := container.NewClient(containerURL, cred, nil) + self.handleError(err) + return containerClient +} + +func (self *ABS_Manager) getBlobClient(containerName string, blobName string) *blob.Client { + // From the Azure portal, get your Storage account blob service URL endpoint. + accountName, accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME"), os.Getenv("AZURE_STORAGE_ACCOUNT_KEY") + + blobURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", accountName, containerName, blobName) + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + self.handleError(err) + blobClient, err := blob.NewClientWithSharedKeyCredential(blobURL, credential, nil) + self.handleError(err) + return blobClient +} + +func (self *ABS_Manager) listBuckets() []string { + + client := self.getServiceClient() + + pager := client.NewListContainersPager(&azblob.ListContainersOptions{ + Include: azblob.ListContainersInclude{Metadata: true, Deleted: false}, + }) + + var buckets []string + + for pager.More() { + resp, err := pager.NextPage(context.TODO()) + self.handleError(err) // if err is not nil, break the loop. + for _, _container := range resp.ContainerItems { + buckets = append(buckets, *_container.Name) + } + } + return buckets +} + +func (self *ABS_Manager) bucketExists(bucket string) (bool, error) { + client := self.getContainerClient(bucket) + _, err := client.GetProperties(context.TODO(), nil) + + if bloberror.HasCode(err, bloberror.ContainerNotFound) { + return false, err + } else { + return true, nil + } +} + +func (self *ABS_Manager) keyExists(bucket string, key string) (bool, error) { + client := self.getBlobClient(bucket, key) + _, err := client.GetProperties(context.TODO(), nil) + + if bloberror.HasCode(err, bloberror.BlobNotFound) { + return false, err + } else { + return true, nil + } +} + +func (self *ABS_Manager) readFile(bucket string, item string) ([]byte, error) { + + client := self.getServiceClient() + // Download the blob + downloadResponse, err := client.DownloadStream(context.TODO(), bucket, item, nil) + self.handleError(err) + + downloadedData := bytes.Buffer{} + retryReader := downloadResponse.NewRetryReader(context.TODO(), &azblob.RetryReaderOptions{}) + _, err = downloadedData.ReadFrom(retryReader) + self.handleError(err) + + err = retryReader.Close() + self.handleError(err) + + return downloadedData.Bytes(), nil +} + +func (self *ABS_Manager) copyFile(bucket string, item string, other string) error { + + data, _ := self.readFile(bucket, item) + + client := self.getServiceClient() + + _, err := client.UploadBuffer(context.TODO(), path.Dir(other), path.Base(other), data, &azblob.UploadBufferOptions{}) + self.handleError(err) + return err +} + +func (self *ABS_Manager) deleteFile(bucket string, item string) error { + client := self.getServiceClient() + // Delete the blob. + _, err := client.DeleteBlob(context.TODO(), bucket, item, nil) + self.handleError(err) + return err +} diff --git a/bucket_interface.go b/bucket_interface.go new file mode 100644 index 0000000..0a55b8d --- /dev/null +++ b/bucket_interface.go @@ -0,0 +1,10 @@ +package main + +// Defining an interface +type BucketInterface interface { + bucketExists(bucket string) (bool, error) + keyExists(bucket string, key string) (bool, error) + readFile(bucket string, item string) ([]byte, error) + copyFile(bucket string, item string, other string) error + deleteFile(bucket string, item string) error +} diff --git a/bucket_interface_manager.go b/bucket_interface_manager.go new file mode 100644 index 0000000..0f04af8 --- /dev/null +++ b/bucket_interface_manager.go @@ -0,0 +1,213 @@ +package main + +import ( + // standard + "encoding/json" + "errors" + "log" + "net/http" +) + +func _getQurantineFilesBucket(qurantineFilesBucket string) string { + // input has more priority + if qurantineFilesBucket != "" { + return qurantineFilesBucket + } + if quarantine_files_bucket != "" { + return quarantine_files_bucket + } + return "" +} + +func _getCleanFilesBucket(cleanFilesBucket string) string { + // input has more priority + if cleanFilesBucket != "" { + return cleanFilesBucket + } + if clean_files_bucket != "" { + return clean_files_bucket + } + return "" +} + +func validateInputBucket(w http.ResponseWriter, bucket string, bucketInterface BucketInterface) error { + if bucket == "" { + errorResponse(w, "Invalid input bucket", http.StatusUnprocessableEntity) + return errors.New("Invalid input bucket") + } + + bucketExists, err := bucketInterface.bucketExists(bucket) + + if err != nil { + errorResponse(w, err.Error(), http.StatusInternalServerError) + return err + } + if !bucketExists { + errorResponse(w, "Bucket: "+bucket+" does not exists", http.StatusUnprocessableEntity) + return errors.New("Bucket: " + bucket + " does not exists") + } + return nil +} + +func validateInputKey(w http.ResponseWriter, bucket string, key string, bucketInterface BucketInterface) error { + if key == "" { + errorResponse(w, "Invalid input key", http.StatusUnprocessableEntity) + return errors.New("Invalid input key") + } + + keyExists, err := bucketInterface.keyExists(bucket, key) + if err != nil { + errorResponse(w, err.Error(), http.StatusInternalServerError) + return err + } + if !keyExists { + errorResponse(w, "Key: "+key+" does not exist in Bucket: "+bucket, http.StatusUnprocessableEntity) + return errors.New("Key: " + key + " does not exist in Bucket: " + bucket) + } + return nil +} + +func validateQrantineFilesBucket(w http.ResponseWriter, qurantineFilesBucket string, bucketInterface BucketInterface) error { + var bucket = _getQurantineFilesBucket(qurantineFilesBucket) + + if bucket == "" { + errorResponse(w, "Invalid qurantine files bucket", http.StatusBadRequest) + return errors.New("Invalid qurantine files bucket") + + } else { + err := validateInputBucket(w, bucket, bucketInterface) + if err != nil { + return err + } + } + return nil +} + +func validateCleanFilesBucket(w http.ResponseWriter, cleanFilesBucket string, bucketInterface BucketInterface) error { + + var bucket = _getCleanFilesBucket(cleanFilesBucket) + + if bucket == "" { + errorResponse(w, "Invalid clean files bucket", http.StatusBadRequest) + return errors.New("Invalid clean files bucket") + + } else { + err := validateInputBucket(w, bucket, bucketInterface) + if err != nil { + return err + } + } + return nil + +} + +func validateInputData(w http.ResponseWriter, data *ScanObject, bucketInterface BucketInterface) error { + + err := validateInputBucket(w, data.BucketName, bucketInterface) + if err != nil { + return err + } + + err = validateInputKey(w, data.BucketName, data.Key, bucketInterface) + if err != nil { + return err + } + + err = validateQrantineFilesBucket(w, data.QurantineFilesBucket, bucketInterface) + if err != nil { + return err + } + + err = validateCleanFilesBucket(w, data.CleanFilesBucket, bucketInterface) + if err != nil { + return err + } + + return nil +} + +func ScanBucketObject(w http.ResponseWriter, r *http.Request, bucketInterface BucketInterface) { + + data := new(ScanObject) + err := decodeJSONBody(w, r, &data) + if err != nil { + var mr *malformedRequest + if errors.As(err, &mr) { + errorResponse(w, mr.msg, mr.status) + } else { + log.Println(err.Error()) + errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + } + return + } + + err = validateInputData(w, data, bucketInterface) + if err != nil { + elog.Println(" validateInputData failed " + err.Error()) + return + } + + resp, _ := json.Marshal(data) + info.Println(" Received ScanS3 request " + string(resp)) + + byteData, err := bucketInterface.readFile(data.BucketName, data.Key) + if err != nil { + elog.Println(err) + errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + // send request for scanning + newRequest := NewScanStreamRequest(byteData) + scanstreamrequests <- newRequest + + response := <-newRequest.ResponseChan + + err = response.err + + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } else { + if response.data.Status == "INFECTED" { + elog.Println("Key " + data.Key + " from bucket " + data.BucketName + " is Infected") + err = bucketInterface.copyFile(data.BucketName, data.Key, _getQurantineFilesBucket(data.QurantineFilesBucket)) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + err = bucketInterface.deleteFile(data.BucketName, data.Key) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + } else if response.data.Status == "CLEAN" { + info.Println("Key " + data.Key + " from bucket " + data.BucketName + " is Clean") + err = bucketInterface.copyFile(data.BucketName, data.Key, _getCleanFilesBucket(data.CleanFilesBucket)) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + err = bucketInterface.deleteFile(data.BucketName, data.Key) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + } + } + + output, err := json.Marshal(response.data) + if err != nil { + elog.Println(err) + errorResponse(w, err.Error(), http.StatusInternalServerError) + return + } + + sendJsonResponse(w, output) + //fmt.Fprintf(w, string(output)) +} diff --git a/bucket_manager.go b/bucket_manager.go new file mode 100644 index 0000000..f88b685 --- /dev/null +++ b/bucket_manager.go @@ -0,0 +1,4 @@ +package main + +type BucketMgr struct { +} diff --git a/bucket_scan_object_handler.go b/bucket_scan_object_handler.go new file mode 100644 index 0000000..f534497 --- /dev/null +++ b/bucket_scan_object_handler.go @@ -0,0 +1,45 @@ +package main + +import ( + // standard + "fmt" + "net/http" + "strings" +) + +type CloudProvider int + +const ( + CloudProviderAWS = iota + CloudProviderAzure + CloudProviderGCP +) + +var CloudProviderMap = map[string]CloudProvider{ + "AWS": CloudProviderAWS, + "AZURE": CloudProviderAzure, + "GCP": CloudProviderGCP, +} + +func ParseCloudProviderString(str string) (CloudProvider, bool) { + c, ok := CloudProviderMap[strings.ToUpper(str)] + return c, ok +} + +func BucketScanObjectHandler(w http.ResponseWriter, r *http.Request) { + + switch cloud_provider { + case CloudProviderAWS: + s3_Mgr := &S3_Manager{} + ScanBucketObject(w, r, s3_Mgr) + case CloudProviderAzure: + abs_Mgr := &ABS_Manager{} + ScanBucketObject(w, r, abs_Mgr) + case CloudProviderGCP: + gcs_Mgr := &GCS_Manager{} + ScanBucketObject(w, r, gcs_Mgr) + default: + panic(fmt.Errorf("unwknown cloud_provider: %s", cloud_provider)) + } + +} diff --git a/clamscanner.go b/clamscanner.go index 5850a34..d910987 100644 --- a/clamscanner.go +++ b/clamscanner.go @@ -1,11 +1,11 @@ package main import ( - "github.com/dutchcoders/go-clamd" "bytes" "encoding/json" - "time" "fmt" + "github.com/dutchcoders/go-clamd" + "time" ) var eicar = []byte(`X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*`) @@ -20,7 +20,7 @@ func NewClamScanner(clamdaddr string) (*ClamScanner, error) { return scanner, nil } -func (self *ClamScanner) Scan(data [] byte) (*ScanReport,error) { +func (self *ClamScanner) Scan(data []byte) (*ScanReport, error) { var matches []ScanMatch response := new(ScanReport) response.Filename = "stream" @@ -36,7 +36,7 @@ func (self *ClamScanner) Scan(data [] byte) (*ScanReport,error) { return response, err } - r := (<-ch) //defer close(response) + r := (<-ch) //defer close(response) respJson, err := json.Marshal(&r) if err != nil { @@ -46,25 +46,25 @@ func (self *ClamScanner) Scan(data [] byte) (*ScanReport,error) { fmt.Printf(time.Now().Format(time.RFC3339)+" Scan result : %v\n", string(respJson)) switch r.Status { - case clamd.RES_OK: - response.Status = "CLEAN" - case clamd.RES_FOUND: - response.Status = "INFECTED" - var match ScanMatch - match.Namespace = "" - match.Tags = nil - match.Rule = r.Description - matches = append(matches, match) - case clamd.RES_ERROR: - case clamd.RES_PARSE_ERROR: - default: - response.Status = "ERROR" + case clamd.RES_OK: + response.Status = "CLEAN" + case clamd.RES_FOUND: + response.Status = "INFECTED" + var match ScanMatch + match.Namespace = "" + match.Tags = nil + match.Rule = r.Description + matches = append(matches, match) + case clamd.RES_ERROR: + case clamd.RES_PARSE_ERROR: + default: + response.Status = "ERROR" } if len(matches) <= 0 { - matches = [] ScanMatch{} + matches = []ScanMatch{} } - + response.Matches = matches fmt.Printf(time.Now().Format(time.RFC3339) + " Finished scanning: " + "\n") @@ -73,8 +73,8 @@ func (self *ClamScanner) Scan(data [] byte) (*ScanReport,error) { } // empty the channel so the goroutine from go-clamd/*CLAMDConn.readResponse() doesn't get stuck }() - return response,nil - + return response, nil + } func (self *ClamScanner) ping() error { @@ -97,22 +97,22 @@ func (self *ClamScanner) isClamdReady() bool { if err := self.ping(); err != nil { fmt.Printf("ClamD ping failed.. error [%v]\n", err) return false - } + } fmt.Printf("Connectted to ClamD Server\n") if response, err := self.version(); err != nil { - fmt.Printf("ClamD version check failed.. error [%v]\n", err) - return false + fmt.Printf("ClamD version check failed.. error [%v]\n", err) + return false } else { - fmt.Printf("ClamD version: %#v\n", response) + fmt.Printf("ClamD version: %#v\n", response) } - + return true - + } func (self *ClamScanner) runScanCheck() bool { - if ! self.isClamdReady() { + if !self.isClamdReady() { return false } @@ -124,18 +124,18 @@ func (self *ClamScanner) runScanCheck() bool { if _, err := self.Scan([]byte("hello world... how are you")); err != nil { fmt.Printf("ClamD sample text scan check failed.. error [%v]\n", err) return false - } + } return true } func (self *ClamScanner) warmUp() bool { - for i:=0; i < 24 ; i++ { - if ! self.runScanCheck() { + for i := 0; i < 24; i++ { + if !self.runScanCheck() { time.Sleep(time.Second * 5) } else { return true } } return false -} \ No newline at end of file +} diff --git a/custom_headers.go b/custom_headers.go index b81c8af..6515de7 100644 --- a/custom_headers.go +++ b/custom_headers.go @@ -1,6 +1,6 @@ package main -import( +import ( "github.com/MagnusFrater/helmet" ) @@ -9,19 +9,19 @@ func CustomHelmet() *helmet.Helmet { helmetObj := helmet.Empty() helmetObj.ContentSecurityPolicy = helmet.NewContentSecurityPolicy(map[helmet.CSPDirective][]helmet.CSPSource{ helmet.DirectiveFrameAncestors: {helmet.SourceNone}, - helmet.DirectiveDefaultSrc: {helmet.SourceNone}, + helmet.DirectiveDefaultSrc: {helmet.SourceNone}, }) helmetObj.XContentTypeOptions = helmet.XContentTypeOptionsNoSniff helmetObj.XDNSPrefetchControl = helmet.XDNSPrefetchControlOn helmetObj.XDownloadOptions = helmet.XDownloadOptionsNoOpen - helmetObj.ExpectCT = helmet.NewExpectCT(0,false,"") - helmetObj.FeaturePolicy = helmet.EmptyFeaturePolicy() + helmetObj.ExpectCT = helmet.NewExpectCT(0, false, "") + helmetObj.FeaturePolicy = helmet.EmptyFeaturePolicy() helmetObj.XFrameOptions = helmet.XFrameOptionsDeny helmetObj.XPermittedCrossDomainPolicies = helmet.PermittedCrossDomainPoliciesNone - helmetObj.XPoweredBy = helmet.NewXPoweredBy(true, "") + helmetObj.XPoweredBy = helmet.NewXPoweredBy(true, "") helmetObj.ReferrerPolicy = helmet.NewReferrerPolicy(helmet.DirectiveNoReferrer) helmetObj.StrictTransportSecurity = helmet.NewStrictTransportSecurity(31536000, true, false) helmetObj.XXSSProtection = helmet.NewXXSSProtection(true, helmet.DirectiveModeBlock, "") return helmetObj -} \ No newline at end of file +} diff --git a/custom_headers_middleware.go b/custom_headers_middleware.go index a2c7174..c12e5ce 100644 --- a/custom_headers_middleware.go +++ b/custom_headers_middleware.go @@ -27,10 +27,10 @@ func (helmet *SimpleHelmet) Default() { // Secure function, which will be called for each request func (helmet *SimpleHelmet) Secure(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { for key, val := range helmet.headers { - w.Header().Set(key,val) + w.Header().Set(key, val) } next.ServeHTTP(w, r) - }) -} \ No newline at end of file + }) +} diff --git a/envhelper.go b/envhelper.go index a964f38..cb742e7 100644 --- a/envhelper.go +++ b/envhelper.go @@ -2,48 +2,48 @@ package main import ( "os" - "strings" "strconv" + "strings" ) // Simple helper function to read an environment or return a default value func getEnv(key string, defaultVal string) string { - if value, exists := os.LookupEnv(key); exists { - return value - } + if value, exists := os.LookupEnv(key); exists { + return value + } - return defaultVal + return defaultVal } // Simple helper function to read an environment variable into integer or return a default value func getEnvAsInt(name string, defaultVal int) int { - valueStr := getEnv(name, "") - if value, err := strconv.Atoi(valueStr); err == nil { - return value - } + valueStr := getEnv(name, "") + if value, err := strconv.Atoi(valueStr); err == nil { + return value + } - return defaultVal + return defaultVal } // Helper to read an environment variable into a bool or return default value func getEnvAsBool(name string, defaultVal bool) bool { - valStr := getEnv(name, "") - if val, err := strconv.ParseBool(valStr); err == nil { - return val - } + valStr := getEnv(name, "") + if val, err := strconv.ParseBool(valStr); err == nil { + return val + } - return defaultVal + return defaultVal } // Helper to read an environment variable into a string slice or return default value func getEnvAsSlice(name string, defaultVal []string, sep string) []string { - valStr := getEnv(name, "") + valStr := getEnv(name, "") - if valStr == "" { - return defaultVal - } + if valStr == "" { + return defaultVal + } - val := strings.Split(valStr, sep) + val := strings.Split(valStr, sep) - return val + return val } diff --git a/gcs.go b/gcs.go new file mode 100644 index 0000000..018b77e --- /dev/null +++ b/gcs.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "errors" + "path" + "time" + "io/ioutil" + "cloud.google.com/go/storage" +) + +type GCS_Manager struct { + BucketMgr +} + +func (self *GCS_Manager) bucketExists(bucket string) (bool, error) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + + if err != nil { + elog.Println(time.Now().Format(time.RFC3339) + "bucketExists: storage.NewClient " + bucket + " error : " + err.Error()) + return false, errors.New("storage.NewClient Failed ") + } + defer client.Close() + + bucketObj := client.Bucket(bucket) + _, err = bucketObj.Attrs(ctx) + + if err != nil { + elog.Println(time.Now().Format(time.RFC3339) + "bucketExists: bucketObj.Attrs " + bucket + " error : " + err.Error()) + return false, errors.New("bucketObj.Attrs Failed ") + } else { + return true, nil + } +} + +func (self *GCS_Manager) keyExists(bucket string, key string) (bool, error) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + + if err != nil { + elog.Println(time.Now().Format(time.RFC3339) + "keyExists: storage.NewClient " + bucket + " error : " + err.Error()) + return false, errors.New("storage.NewClient Failed ") + } + defer client.Close() + + bucketObj := client.Bucket(bucket) + object := bucketObj.Object(key) + _, err = object.Attrs(ctx) + + if err != nil { + elog.Println(time.Now().Format(time.RFC3339) + "keyExists: object.Attrs " + bucket + " error : " + err.Error()) + return false, errors.New("object.Attrs Failed ") + } else { + return true, nil + } +} + +func (self *GCS_Manager) readFile(bucket string, item string) ([]byte, error) { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + elog.Println(time.Now().Format(time.RFC3339) + "readFile: storage.NewClient " + bucket + " key " +item + " error : " + err.Error()) + return nil, errors.New("storage.NewClient Failed ") + } + defer client.Close() + + rc, err := client.Bucket(bucket).Object(item).NewReader(ctx) + if err != nil { + elog.Println(time.Now().Format(time.RFC3339) + "readFile: client.Bucket " + bucket + " key " +item + " error : " + err.Error()) + return nil, errors.New("Bucket.Object.NewReader Failed ") + } + defer rc.Close() + + data, err := ioutil.ReadAll(rc) + if err != nil { + elog.Println(time.Now().Format(time.RFC3339) + "readFile: client.Bucket " + bucket + " key " +item + " error : " + err.Error()) + return nil, errors.New("ioutil ReadAll Failed ") + } + + info.Println(time.Now().Format(time.RFC3339) + " Downloaded object " + item + " from bucket " + bucket) + + return data, nil +} + +func (self *GCS_Manager) copyFile(srcBucket string, srcObject string, other string) error { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + elog.Println(time.Now().Format(time.RFC3339) + "copyFile: storage.NewClient " + srcBucket + " error : " + err.Error()) + return errors.New("storage.NewClient Failed ") + } + defer client.Close() + + dstBucket := path.Dir(other) + dstObject := path.Base(other) + + src := client.Bucket(srcBucket).Object(srcObject) + dst := client.Bucket(dstBucket).Object(dstObject) + + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to copy is aborted if the + // object's generation number does not match your precondition. + // For a dst object that does not yet exist, set the DoesNotExist precondition. + dst = dst.If(storage.Conditions{DoesNotExist: true}) + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + // attrs, err := dst.Attrs(ctx) + // if err != nil { + // return fmt.Errorf("object.Attrs: %w", err) + // } + // dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation}) + + if _, err := dst.CopierFrom(src).Run(ctx); err != nil { + elog.Println(time.Now().Format(time.RFC3339) + " Unable to copy object " + srcObject + " from bucket " + srcBucket + " to bucket " + dstBucket + " error : " + err.Error()) + return errors.New("Unable to copy file") + } + + return nil +} + +func (self *GCS_Manager) deleteFile(bucket string, item string) error { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + elog.Println(time.Now().Format(time.RFC3339) + "deleteFile: storage.NewClient " + bucket + " error : " + err.Error()) + return errors.New("storage.NewClient Failed ") + } + defer client.Close() + + //ctx, cancel := context.WithTimeout(ctx, time.Second*10) + //defer cancel() + + o := client.Bucket(bucket).Object(item) + + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to delete the file is aborted + // if the object's generation number does not match your precondition. + attrs, err := o.Attrs(ctx) + if err != nil { + elog.Println(time.Now().Format(time.RFC3339) + "deleteFile: bucketObj.Attrs " + bucket + " error : " + err.Error()) + return errors.New("bucketObj.Attrs Failed ") + } + o = o.If(storage.Conditions{GenerationMatch: attrs.Generation}) + + if err := o.Delete(ctx); err != nil { + elog.Println(time.Now().Format(time.RFC3339) + "deleteFile: Delete " + bucket + " object "+item + " error : " + err.Error()) + return errors.New("Object Delete Failed ") + } + info.Println(time.Now().Format(time.RFC3339) + " deleteFile " + item + " successfully deleted from bucket " + bucket) + return nil +} diff --git a/healthcheckhandler.go b/healthcheckhandler.go index 083659c..9f5e69d 100644 --- a/healthcheckhandler.go +++ b/healthcheckhandler.go @@ -7,7 +7,7 @@ import ( ) func HealthCheckHandler(w http.ResponseWriter, r *http.Request) { - validateContentType(w,r) + validateContentType(w, r) // send request for scanning newRequest := NewHealthCheckRequest() diff --git a/indexhandler.go b/indexhandler.go index 6270e6b..2ed1c9a 100644 --- a/indexhandler.go +++ b/indexhandler.go @@ -2,8 +2,8 @@ package main import ( // standard - "net/http" "encoding/json" + "net/http" ) func IndexHandler(w http.ResponseWriter, r *http.Request) { @@ -14,4 +14,4 @@ func IndexHandler(w http.ResponseWriter, r *http.Request) { return } sendJsonResponse(w, output) -} \ No newline at end of file +} diff --git a/jsonresponse.go b/jsonresponse.go index 39f5eb1..50425e4 100644 --- a/jsonresponse.go +++ b/jsonresponse.go @@ -1,118 +1,118 @@ package main + import ( // external "encoding/json" - "net/http" - "github.com/golang/gddo/httputil/header" + "errors" "fmt" + "github.com/golang/gddo/httputil/header" "io" - "errors" + "net/http" "strconv" "strings" ) -// errorResponse(w, "Content Type is not application/json", http.StatusUnsupportedMediaType) +// errorResponse(w, "Content Type is not application/json", http.StatusUnsupportedMediaType) func errorResponse(w http.ResponseWriter, message string, httpStatusCode int) { - w.Header().Set("Content-Type", "application/json") - // w.Header().Set("Cache-Control", "['no-cache','no-store','must-revalidate']") - // w.Header().Set("Pragma", "no-cache") + w.Header().Set("Content-Type", "application/json") + // w.Header().Set("Cache-Control", "['no-cache','no-store','must-revalidate']") + // w.Header().Set("Pragma", "no-cache") - w.WriteHeader(httpStatusCode) + w.WriteHeader(httpStatusCode) - scaErrorResp := new (ScanErrorResponse) - sanErrorData := ScanErrorData{message, strconv.Itoa(httpStatusCode)} - scaErrorResp.Error = sanErrorData + scaErrorResp := new(ScanErrorResponse) + sanErrorData := ScanErrorData{message, strconv.Itoa(httpStatusCode)} + scaErrorResp.Error = sanErrorData - // scaErrorResp := make(map[string]map[string]string) - // sanErrorData := map[string]string{"message": message, "code": strconv.Itoa(httpStatusCode)} - // scaErrorResp["error"] = sanErrorData - jsonResp, _ := json.Marshal(scaErrorResp) - w.Write(jsonResp) + // scaErrorResp := make(map[string]map[string]string) + // sanErrorData := map[string]string{"message": message, "code": strconv.Itoa(httpStatusCode)} + // scaErrorResp["error"] = sanErrorData + jsonResp, _ := json.Marshal(scaErrorResp) + w.Write(jsonResp) } func sendJsonResponse(w http.ResponseWriter, jsonResp []byte) { - w.Header().Set("Content-Type", "application/json") - // w.Header().Set("Cache-Control", "['no-cache','no-store','must-revalidate']") - // w.Header().Set("Pragma", "no-cache") - w.WriteHeader(200) - w.Write(jsonResp) + w.Header().Set("Content-Type", "application/json") + // w.Header().Set("Cache-Control", "['no-cache','no-store','must-revalidate']") + // w.Header().Set("Pragma", "no-cache") + w.WriteHeader(200) + w.Write(jsonResp) } type malformedRequest struct { - status int - msg string + status int + msg string } func (mr *malformedRequest) Error() string { - return mr.msg + return mr.msg } func validateContentType(w http.ResponseWriter, r *http.Request) error { - if r.Header.Get("Content-Type") != "" { - value, _ := header.ParseValueAndParams(r.Header, "Content-Type") - if value != "application/json" { - return &malformedRequest{status: http.StatusUnsupportedMediaType, msg: "Content-Type header is not application/json"} - } - } - return nil + if r.Header.Get("Content-Type") != "" { + value, _ := header.ParseValueAndParams(r.Header, "Content-Type") + if value != "application/json" { + return &malformedRequest{status: http.StatusUnsupportedMediaType, msg: "Content-Type header is not application/json"} + } + } + return nil } - func decodeJSONBody(w http.ResponseWriter, r *http.Request, dst interface{}) error { - if r.Header.Get("Content-Type") != "" { - value, _ := header.ParseValueAndParams(r.Header, "Content-Type") - if value != "application/json" { - return &malformedRequest{status: http.StatusUnsupportedMediaType, msg: "Content-Type header is not application/json"} - } - } - - r.Body = http.MaxBytesReader(w, r.Body, 1048576) - - dec := json.NewDecoder(r.Body) - dec.DisallowUnknownFields() - - err := dec.Decode(&dst) - if err != nil { - var syntaxError *json.SyntaxError - var unmarshalTypeError *json.UnmarshalTypeError - - switch { - case errors.As(err, &syntaxError): - msg := "Request body contains badly-formed JSON (at position "+ fmt.Sprintf("%d", syntaxError.Offset)+")" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - - case errors.Is(err, io.ErrUnexpectedEOF): - msg := "Request body contains badly-formed JSON" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - - case errors.As(err, &unmarshalTypeError): - msg := "Request body contains an invalid value for the "+unmarshalTypeError.Field+" field (at position "+ fmt.Sprintf("%d", syntaxError.Offset)+")" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - - case strings.HasPrefix(err.Error(), "json: unknown field "): - fieldName := strings.TrimPrefix(err.Error(), "json: unknown field ") - msg := "Request body contains unknown field "+fieldName - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - - case errors.Is(err, io.EOF): - msg := "Request body must not be empty" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - - case err.Error() == "http: request body too large": - msg := "Request body must not be larger than 1MB" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - default: - return err - } - } + if r.Header.Get("Content-Type") != "" { + value, _ := header.ParseValueAndParams(r.Header, "Content-Type") + if value != "application/json" { + return &malformedRequest{status: http.StatusUnsupportedMediaType, msg: "Content-Type header is not application/json"} + } + } + + r.Body = http.MaxBytesReader(w, r.Body, 1048576) + + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + + err := dec.Decode(&dst) + if err != nil { + var syntaxError *json.SyntaxError + var unmarshalTypeError *json.UnmarshalTypeError + + switch { + case errors.As(err, &syntaxError): + msg := "Request body contains badly-formed JSON (at position " + fmt.Sprintf("%d", syntaxError.Offset) + ")" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + + case errors.Is(err, io.ErrUnexpectedEOF): + msg := "Request body contains badly-formed JSON" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + + case errors.As(err, &unmarshalTypeError): + msg := "Request body contains an invalid value for the " + unmarshalTypeError.Field + " field (at position " + fmt.Sprintf("%d", syntaxError.Offset) + ")" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + + case strings.HasPrefix(err.Error(), "json: unknown field "): + fieldName := strings.TrimPrefix(err.Error(), "json: unknown field ") + msg := "Request body contains unknown field " + fieldName + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + + case errors.Is(err, io.EOF): + msg := "Request body must not be empty" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + + case err.Error() == "http: request body too large": + msg := "Request body must not be larger than 1MB" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + default: + return err + } + } err = dec.Decode(&struct{}{}) if err != io.EOF { - msg := "Request body must only contain a single JSON object" - return &malformedRequest{status: http.StatusBadRequest, msg: msg} - } + msg := "Request body must only contain a single JSON object" + return &malformedRequest{status: http.StatusBadRequest, msg: msg} + } - return nil + return nil } diff --git a/main.go b/main.go index 3c825d5..b2624e0 100644 --- a/main.go +++ b/main.go @@ -14,18 +14,19 @@ import ( var ( // config options - index_files StringArgs - address string - port string - addrport string - clamdaddr string - clean_files_bucket string + index_files StringArgs + address string + port string + addrport string + clamdaddr string + clean_files_bucket string quarantine_files_bucket string + cloud_provider CloudProvider // channels healthcheckrequests chan *HealthCheckRequest - scanstreamrequests chan *ScanStreamRequest - namerequests chan *RuleSetRequest - rulerequests chan *RuleListRequest + scanstreamrequests chan *ScanStreamRequest + namerequests chan *RuleSetRequest + rulerequests chan *RuleListRequest // loggers info *log.Logger @@ -48,11 +49,16 @@ func init() { //build address string addrport = address + ":" + port - clean_files_bucket = getEnv("CLEAN_FILES_S3_BUCKET", "") - quarantine_files_bucket = getEnv("QUARANTINE_FILES_S3_BUCKET", "") + clean_files_bucket = getEnv("CLEAN_FILES_BUCKET", "") + quarantine_files_bucket = getEnv("QUARANTINE_FILES_BUCKET", "") + cloud_provider_str := getEnv("CLOUD_PROVIDER", "") + + cloud_provider, _ = ParseCloudProviderString(cloud_provider_str) + + info.Println("reading CLEAN_FILES_BUCKET value as " + clean_files_bucket) + info.Println("reading QUARANTINE_FILES_BUCKET value as " + quarantine_files_bucket) + info.Println("reading CLOUD_PROVIDER value as " + cloud_provider_str) - info.Println("reading CLEAN_FILES_S3_BUCKET value as " +clean_files_bucket) - info.Println("reading QUARANTINE_FILES_S3_BUCKET value as " +quarantine_files_bucket) } func main() { @@ -86,8 +92,6 @@ func main() { // setup http server and begin serving traffic r := mux.NewRouter() - // helmet := CustomHelmet() - // r.Use(helmet.Secure) helmet := SimpleHelmet{} helmet.Default() @@ -100,14 +104,13 @@ func main() { // Prometheus metrics r.Handle("/metrics", promhttp.Handler()) - s3_sub := r.PathPrefix("/s3").Subrouter() - s3_sub.HandleFunc("/scanfile", S3ScanFileHandler).Methods("POST") + bucket_sub := r.PathPrefix("/bucket").Subrouter() + bucket_sub.HandleFunc("/scanobject", BucketScanObjectHandler).Methods("POST") ruleset_sub := r.PathPrefix("/ruleset").Subrouter() ruleset_sub.HandleFunc("", RuleSetListHandler).Methods("GET") ruleset_sub.HandleFunc("/", RuleSetListHandler).Methods("GET") ruleset_sub.HandleFunc("/{ruleset}", RuleListHandler).Methods("GET") - + loggedRouter := handlers.CombinedLoggingHandler(os.Stdout, r) log.Fatal(http.ListenAndServe(addrport, loggedRouter)) - //log.Fatal(http.ListenAndServe(addrport, r)) } diff --git a/request.go b/request.go index f0fad10..fa6374a 100644 --- a/request.go +++ b/request.go @@ -1,9 +1,9 @@ package main -type ScanS3Object struct { - BucketName string `json:"bucketname"` - Key string `json:"key"` - CleanFilesBucket string `json:"clean_files_bucket,omitempty"` +type ScanObject struct { + BucketName string `json:"bucketname"` + Key string `json:"key"` + CleanFilesBucket string `json:"clean_files_bucket,omitempty"` QurantineFilesBucket string `json:"qurantine_files_bucket,omitempty"` } @@ -12,7 +12,7 @@ type HealthCheckRequest struct { } type ScanStreamRequest struct { - data [] byte + data []byte ResponseChan chan *ScanResponse } diff --git a/response.go b/response.go index 92907a4..1a2b030 100644 --- a/response.go +++ b/response.go @@ -1,13 +1,13 @@ package main type ScanResponse struct { - data *ScanReport - err error + data *ScanReport + err error } type HealthCheckResponse struct { - health string //OK,ERROR - err error + health string //OK,ERROR + err error } // struct to handle namespace requests @@ -17,7 +17,7 @@ type RuleSetResponseObject struct { type RuleSetResponse struct { data *RuleSetResponseObject - err error + err error } // sturc to handle @@ -27,5 +27,5 @@ type RuleListResponseObject struct { type RuleListResponse struct { data *RuleListResponseObject - err error -} \ No newline at end of file + err error +} diff --git a/rulelisthandler.go b/rulelisthandler.go index f02ffae..f289309 100644 --- a/rulelisthandler.go +++ b/rulelisthandler.go @@ -13,12 +13,12 @@ func RuleListHandler(w http.ResponseWriter, r *http.Request) { ruleset := vars["ruleset"] req := NewRuleListRequest(ruleset) - rulerequests<- req + rulerequests <- req response := <-req.ResponseChan var err error = response.err - + if err != nil { elog.Println(err) errorResponse(w, err.Error(), http.StatusInternalServerError) diff --git a/ruleset.go b/ruleset.go index d0744b1..051ea92 100644 --- a/ruleset.go +++ b/ruleset.go @@ -53,7 +53,7 @@ func NewRuleSet(indexpath string) (*RuleSet, error) { filename := fields[len(fields)-1] namespacestr := strings.Split(filename, "_")[0] - info.Println("NewRuleSet fields: " + strings.Join(fields,",")) + info.Println("NewRuleSet fields: " + strings.Join(fields, ",")) info.Println("NewRuleSet filename: " + filename) info.Println("NewRuleSet namespacestr: " + namespacestr) info.Println("NewRuleSet indexpath: " + indexpath) diff --git a/s3.go b/s3.go index 98b2d4e..fd3efc2 100644 --- a/s3.go +++ b/s3.go @@ -2,27 +2,30 @@ package main import ( "context" - "net/http" - "github.com/aws/smithy-go" + "errors" + "fmt" "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" - awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "net/http" "net/url" - "time" - "fmt" "os" "strconv" - "errors" + "time" ) +type S3_Manager struct { + *BucketMgr +} -func getPartSize() int64 { +func (self *S3_Manager) getPartSize() int64 { var partSize int64 strSizeInMb, err := os.LookupEnv("DOWNLOAD_PART_SIZE") - + if !err { elog.Println(time.Now().Format(time.RFC3339) + "DOWNLOAD_PART_SIZE is not present..using DefaultDownloadPartSize ") partSize = manager.DefaultDownloadPartSize @@ -38,7 +41,7 @@ func getPartSize() int64 { return partSize } -func getRegion() string { +func (self *S3_Manager) getRegion() string { region, err := os.LookupEnv("AWS_REGION") if !err { elog.Println(time.Now().Format(time.RFC3339) + " AWS_REGION is not present..using us-east-1") @@ -48,59 +51,59 @@ func getRegion() string { } // check if a bucket exists. -func bucketExists(bucket string) (bool, error) { +func (self *S3_Manager) bucketExists(bucket string) (bool, error) { cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " bucketExists: Filed to load config for bucket "+bucket + " error : " + err.Error()) - return false, errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: Filed to load config for bucket " + bucket + " error : " + err.Error()) + return false, errors.New("Filed to load config") } s3client := s3.NewFromConfig(cfg) - _, err = s3client.HeadBucket(context.TODO(),&s3.HeadBucketInput{Bucket: aws.String(bucket)}) + _, err = s3client.HeadBucket(context.TODO(), &s3.HeadBucketInput{Bucket: aws.String(bucket)}) if err != nil { var apiErr smithy.APIError - if errors.As(err, &apiErr) { + if errors.As(err, &apiErr) { var httpResponseErr *awshttp.ResponseError - if errors.As(err, &httpResponseErr) { + if errors.As(err, &httpResponseErr) { switch httpResponseErr.HTTPStatusCode() { case http.StatusMovedPermanently: - elog.Println( time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket "+bucket + " error : " + err.Error()) - return false, errors.New("Bucket StatusMovedPermanently ") + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket " + bucket + " error : " + err.Error()) + return false, errors.New("Bucket StatusMovedPermanently ") case http.StatusForbidden: - elog.Println( time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket "+bucket + " error : " + err.Error()) - return false, errors.New("Bucket StatusForbidden") + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket " + bucket + " error : " + err.Error()) + return false, errors.New("Bucket StatusForbidden") case http.StatusNotFound: - elog.Println( time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket "+bucket + " error : " + err.Error()) - return false, nil + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket " + bucket + " error : " + err.Error()) + return false, nil default: - elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: ResponseError failed for bucket "+bucket + "with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: ResponseError failed for bucket " + bucket + "with error: " + err.Error()) return false, errors.New("Filed to find bucket") } } else { - elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: ApiError failed for bucket "+bucket + "with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: ApiError failed for bucket " + bucket + "with error: " + err.Error()) return false, errors.New("Filed to find bucket") } } else { - elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket "+bucket + "with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " bucketExists: failed for bucket " + bucket + "with error: " + err.Error()) return false, errors.New("Filed to find bucket") } } - return true,nil + return true, nil } -func getHeadObject(bucket string, key string) (*s3.HeadObjectOutput, error) { +func (self *S3_Manager) getHeadObject(bucket string, key string) (*s3.HeadObjectOutput, error) { cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " getHeadObject: Filed to load config for bucket "+bucket + " error : " + err.Error()) - return nil, errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: Filed to load config for bucket " + bucket + " error : " + err.Error()) + return nil, errors.New("Filed to load config") } s3client := s3.NewFromConfig(cfg) @@ -112,29 +115,29 @@ func getHeadObject(bucket string, key string) (*s3.HeadObjectOutput, error) { if err != nil { var apiErr smithy.APIError - if errors.As(err, &apiErr) { + if errors.As(err, &apiErr) { var httpResponseErr *awshttp.ResponseError - if errors.As(err, &httpResponseErr) { + if errors.As(err, &httpResponseErr) { switch httpResponseErr.HTTPStatusCode() { case http.StatusMovedPermanently: - elog.Println( time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket "+bucket +" key "+key+" error : " + err.Error()) - return nil, errors.New("Bucket StatusMovedPermanently ") + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket " + bucket + " key " + key + " error : " + err.Error()) + return nil, errors.New("Bucket StatusMovedPermanently ") case http.StatusForbidden: - elog.Println( time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket "+bucket +" key "+key+" error : " + err.Error()) - return nil, errors.New("Bucket StatusForbidden") + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket " + bucket + " key " + key + " error : " + err.Error()) + return nil, errors.New("Bucket StatusForbidden") case http.StatusNotFound: - elog.Println( time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket "+bucket +" key "+key+" error : " + err.Error()) - return nil, errors.New("Bucket StatusNotFound") + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket " + bucket + " key " + key + " error : " + err.Error()) + return nil, errors.New("Bucket StatusNotFound") default: - elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: ResponseError failed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: ResponseError failed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return nil, errors.New("Filed to find object") } } else { - elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: APIError failed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: APIError failed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return nil, errors.New("Filed to find object") } } else { - elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " getHeadObject: failed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return nil, errors.New("Filed to find object") } } @@ -143,14 +146,14 @@ func getHeadObject(bucket string, key string) (*s3.HeadObjectOutput, error) { } // check if a file exists. -func keyExists(bucket string, key string) (bool, error) { +func (self *S3_Manager) keyExists(bucket string, key string) (bool, error) { cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + "keyExists: Filed to load config for bucket "+bucket +" key "+key+" error : " + err.Error()) - return false, errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + "keyExists: Filed to load config for bucket " + bucket + " key " + key + " error : " + err.Error()) + return false, errors.New("Filed to load config") } s3client := s3.NewFromConfig(cfg) @@ -162,23 +165,23 @@ func keyExists(bucket string, key string) (bool, error) { if err != nil { var apiErr smithy.APIError - if errors.As(err, &apiErr) { + if errors.As(err, &apiErr) { var httpResponseErr *awshttp.ResponseError - if errors.As(err, &httpResponseErr) { - switch httpResponseErr.HTTPStatusCode() { + if errors.As(err, &httpResponseErr) { + switch httpResponseErr.HTTPStatusCode() { case http.StatusNotFound: - elog.Println( time.Now().Format(time.RFC3339) + " keyExists: failed for bucket "+bucket +" key "+key+" error : " + err.Error()) - return false, nil + elog.Println(time.Now().Format(time.RFC3339) + " keyExists: failed for bucket " + bucket + " key " + key + " error : " + err.Error()) + return false, nil default: - elog.Println(time.Now().Format(time.RFC3339) + " keyExists: ResponseError failed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " keyExists: ResponseError failed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return false, errors.New("Filed to find key") } - } else { - elog.Println(time.Now().Format(time.RFC3339) + " keyExists: APIErrorfailed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + } else { + elog.Println(time.Now().Format(time.RFC3339) + " keyExists: APIErrorfailed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return false, errors.New("Filed to find key") } } else { - elog.Println(time.Now().Format(time.RFC3339) + " keyExists: failed for bucket "+bucket +" key "+key+" with error: "+err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " keyExists: failed for bucket " + bucket + " key " + key + " with error: " + err.Error()) return false, errors.New("Filed to find key") } } @@ -186,15 +189,15 @@ func keyExists(bucket string, key string) (bool, error) { return true, nil } -func readFile(bucket string, item string) ([] byte, error) { +func (self *S3_Manager) readFile(bucket string, item string) ([]byte, error) { // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " readFile: Filed to load config to read file " +item+ " from bucket "+bucket + " error : " + err.Error()) - return nil, errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + " readFile: Filed to load config to read file " + item + " from bucket " + bucket + " error : " + err.Error()) + return nil, errors.New("Filed to load config") } // Create an S3 client using the loaded configuration @@ -202,13 +205,13 @@ func readFile(bucket string, item string) ([] byte, error) { // Create a downloader with the client and custom downloader options downloader := manager.NewDownloader(s3client, func(d *manager.Downloader) { - d.PartSize = getPartSize() + d.PartSize = self.getPartSize() }) - headObject, err := getHeadObject(bucket,item) + headObject, err := self.getHeadObject(bucket, item) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " readFile: getHeadObject failed " +item+ " from bucket "+bucket + " error : " + err.Error()) - return nil, errors.New("Filed to read file") + elog.Println(time.Now().Format(time.RFC3339) + " readFile: getHeadObject failed " + item + " from bucket " + bucket + " error : " + err.Error()) + return nil, errors.New("Filed to read file") } // pre-allocate in memory buffer, where headObject type is *s3.HeadObjectOutput buff := make([]byte, int(*headObject.ContentLength)) @@ -221,24 +224,24 @@ func readFile(bucket string, item string) ([] byte, error) { }) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " Unable to read file " +item+ " from bucket "+bucket + " error : " + err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " Unable to read file " + item + " from bucket " + bucket + " error : " + err.Error()) return nil, errors.New("Unable to read file") } - - info.Println(time.Now().Format(time.RFC3339) +" Downloaded file "+item+ " from bucket "+bucket) + + info.Println(time.Now().Format(time.RFC3339) + " Downloaded file " + item + " from bucket " + bucket) return buff, nil } -func copyFile(bucket string, item string, other string) (error){ +func (self *S3_Manager) copyFile(bucket string, item string, other string) error { // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " copyFile: Filed to load config to read file " +item+ " from bucket "+bucket + " error : " + err.Error()) - return errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + " copyFile: Filed to load config to read file " + item + " from bucket " + bucket + " error : " + err.Error()) + return errors.New("Filed to load config") } // Create an S3 client using the loaded configuration @@ -253,24 +256,24 @@ func copyFile(bucket string, item string, other string) (error){ }) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " Unable to read file " +item+ " from bucket "+bucket+ " to bucket "+other+" error : " + err.Error()) + elog.Println(time.Now().Format(time.RFC3339) + " Unable to read file " + item + " from bucket " + bucket + " to bucket " + other + " error : " + err.Error()) return errors.New("Unable to copy file") } - info.Println( time.Now().Format(time.RFC3339) + " File "+ item+ " successfully copied from bucket "+bucket+ " to bucket "+other) + info.Println(time.Now().Format(time.RFC3339) + " File " + item + " successfully copied from bucket " + bucket + " to bucket " + other) return nil } -func deleteFile(bucket string, item string) (error) { +func (self *S3_Manager) deleteFile(bucket string, item string) error { // Load AWS Config cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(getRegion()), + config.WithRegion(self.getRegion()), ) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " deleteFile: Filed to load config to read file " +item+ " from bucket "+bucket + " error : " + err.Error()) - return errors.New("Filed to load config") + elog.Println(time.Now().Format(time.RFC3339) + " deleteFile: Filed to load config to read file " + item + " from bucket " + bucket + " error : " + err.Error()) + return errors.New("Filed to load config") } // Create an S3 client using the loaded configuration @@ -282,7 +285,7 @@ func deleteFile(bucket string, item string) (error) { }) if err != nil { - elog.Println( time.Now().Format(time.RFC3339) + " Error occurred while deleting file " +item+ " from bucket "+bucket+" err: "+ fmt.Sprint(err)) + elog.Println(time.Now().Format(time.RFC3339) + " Error occurred while deleting file " + item + " from bucket " + bucket + " err: " + fmt.Sprint(err)) return errors.New("Error occurred while deleting file") } return nil diff --git a/s3scanfilehandler.go b/s3scanfilehandler.go deleted file mode 100644 index 75fe888..0000000 --- a/s3scanfilehandler.go +++ /dev/null @@ -1,213 +0,0 @@ -package main - -import ( - // standard - "encoding/json" - "errors" - "log" - "net/http" - -) - -func validateInputBucket(w http.ResponseWriter, bucket string) error { - if (bucket == "") { - errorResponse(w, "Invalid input bucket", http.StatusUnprocessableEntity) - return errors.New("Invalid input bucket") - } - - bucketExists, err := bucketExists(bucket) - - if(err != nil) { - errorResponse(w, err.Error(), http.StatusInternalServerError) - return err - } - if (!bucketExists) { - errorResponse(w, "Bucket: "+bucket+" does not exists", http.StatusUnprocessableEntity) - return errors.New("Bucket: "+bucket+" does not exists") - } - return nil -} - -func validateInputKey(w http.ResponseWriter, bucket string, key string) error { - if (key == "") { - errorResponse(w, "Invalid input key", http.StatusUnprocessableEntity) - return errors.New("Invalid input key") - } - - keyExists, err := keyExists(bucket,key) - if(err != nil) { - errorResponse(w, err.Error(), http.StatusInternalServerError) - return err - } - if (!keyExists) { - errorResponse(w, "Key: "+key+" does not exist in Bucket: "+bucket, http.StatusUnprocessableEntity) - return errors.New("Key: "+key+" does not exist in Bucket: "+bucket) - } - return nil -} - -func getQurantineFilesBucket(qurantineFilesBucket string) string{ - // input has more priority - if (qurantineFilesBucket != "" ) { - return qurantineFilesBucket - } - if (quarantine_files_bucket != "" ) { - return quarantine_files_bucket - } - return "" -} - -func getCleanFilesBucket(cleanFilesBucket string) string{ - // input has more priority - if (cleanFilesBucket != "" ) { - return cleanFilesBucket - } - if (clean_files_bucket != "" ) { - return clean_files_bucket - } - return "" -} - -func validateQrantineFilesBucket(w http.ResponseWriter, qurantineFilesBucket string) error { - var bucket = getQurantineFilesBucket(qurantineFilesBucket) - - if (bucket == "" ) { - errorResponse(w, "Invalid qurantine files bucket", http.StatusBadRequest) - return errors.New("Invalid qurantine files bucket") - - } else { - err := validateInputBucket(w,bucket) - if err != nil { - return err - } - } - return nil -} - -func validateCleanFilesBucket(w http.ResponseWriter, cleanFilesBucket string) error { - - var bucket = getCleanFilesBucket(cleanFilesBucket) - - if (bucket == "" ) { - errorResponse(w, "Invalid clean files bucket", http.StatusBadRequest) - return errors.New("Invalid clean files bucket") - - } else { - err := validateInputBucket(w,bucket) - if err != nil { - return err - } - } - return nil - -} - -func validateInputData(w http.ResponseWriter, data *ScanS3Object) error { - - err := validateInputBucket(w,data.BucketName) - if err != nil { - return err - } - - err = validateInputKey(w,data.BucketName,data.Key) - if err != nil { - return err - } - - err = validateQrantineFilesBucket(w,data.QurantineFilesBucket) - if err != nil { - return err - } - - err = validateCleanFilesBucket(w,data.CleanFilesBucket) - if err != nil { - return err - } - - return nil -} - -func S3ScanFileHandler(w http.ResponseWriter, r *http.Request) { - data := new(ScanS3Object) - err := decodeJSONBody(w, r, &data) - if err != nil { - var mr *malformedRequest - if errors.As(err, &mr) { - errorResponse(w, mr.msg, mr.status) - } else { - log.Println(err.Error()) - errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - } - return - } - - err = validateInputData(w,data) - if err != nil { - elog.Println(" validateInputData failed " + err.Error()) - return - } - - resp, _ := json.Marshal(data) - info.Println(" Received ScanS3 request " + string(resp)) - - byteData, err := readFile(data.BucketName, data.Key) - if err != nil { - elog.Println(err) - errorResponse(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - // send request for scanning - newRequest := NewScanStreamRequest(byteData) - scanstreamrequests <- newRequest - - response := <-newRequest.ResponseChan - - err = response.err - - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } else { - if response.data.Status == "INFECTED" { - elog.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Infected") - err = copyFile(data.BucketName, data.Key, getQurantineFilesBucket(data.QurantineFilesBucket)) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - err = deleteFile(data.BucketName, data.Key) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - } else if response.data.Status == "CLEAN" { - info.Println("Key " +data.Key+ " from bucket "+data.BucketName+ " is Clean") - err = copyFile(data.BucketName, data.Key, getCleanFilesBucket(data.CleanFilesBucket)) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - err = deleteFile(data.BucketName, data.Key) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - } - } - - output, err := json.Marshal(response.data) - if err != nil { - elog.Println(err) - errorResponse(w, err.Error(), http.StatusInternalServerError) - return - } - - sendJsonResponse(w, output) - //fmt.Fprintf(w, string(output)) -} diff --git a/scanner.go b/scanner.go index 4e36532..220426d 100644 --- a/scanner.go +++ b/scanner.go @@ -1,28 +1,27 @@ package main import ( - "time" "encoding/json" "os" "strconv" + "time" ) - // struct to hold compiler and channels type Scanner struct { - yarascanner YaraScanner - clamscanner ClamScanner + yarascanner YaraScanner + clamscanner ClamScanner healthcheckrequests chan *HealthCheckRequest - scanstreamrequests chan *ScanStreamRequest - namerequests chan *RuleSetRequest - rulerequests chan *RuleListRequest + scanstreamrequests chan *ScanStreamRequest + namerequests chan *RuleSetRequest + rulerequests chan *RuleListRequest } -func (self *Scanner) healthcheck() (*HealthCheckResponse) { +func (self *Scanner) healthcheck() *HealthCheckResponse { - healthCheckResponse := new (HealthCheckResponse) + healthCheckResponse := new(HealthCheckResponse) clamDHealth := self.clamscanner.isClamdReady() - + if clamDHealth { healthCheckResponse.health = "OK" } else { @@ -32,35 +31,35 @@ func (self *Scanner) healthcheck() (*HealthCheckResponse) { return healthCheckResponse } -func (self *Scanner) scanstream(data []byte) (*ScanResponse) { +func (self *Scanner) scanstream(data []byte) *ScanResponse { info.Println("Running yarascan") - scanResponse := new (ScanResponse) + scanResponse := new(ScanResponse) - yaraScannerResponse,yaraerr := ScanStream(&self.yarascanner, data) + yaraScannerResponse, yaraerr := ScanStream(&self.yarascanner, data) scanResponse.data = yaraScannerResponse scanResponse.err = yaraerr - + yaraRespJson, _ := json.Marshal(yaraScannerResponse) - info.Println( time.Now().Format(time.RFC3339) + " yarascan scan result " + string(yaraRespJson)) + info.Println(time.Now().Format(time.RFC3339) + " yarascan scan result " + string(yaraRespJson)) if (yaraerr == nil) && len(yaraScannerResponse.Matches) > 0 { - info.Println( time.Now().Format(time.RFC3339) + " Found matches with yara " + string(yaraRespJson)) + info.Println(time.Now().Format(time.RFC3339) + " Found matches with yara " + string(yaraRespJson)) } - info.Println("Running clamscan on addr: "+ clamdaddr) + info.Println("Running clamscan on addr: " + clamdaddr) - clamScannerResponse,clamerr := ScanStream(&self.clamscanner, data) + clamScannerResponse, clamerr := ScanStream(&self.clamscanner, data) clamRespJson, _ := json.Marshal(clamScannerResponse) - info.Println( time.Now().Format(time.RFC3339) + " clamav scan result " + string(clamRespJson)) + info.Println(time.Now().Format(time.RFC3339) + " clamav scan result " + string(clamRespJson)) if (clamerr == nil) && len(clamScannerResponse.Matches) > 0 { - info.Println( time.Now().Format(time.RFC3339) + " Found matches with clamav" + string(clamRespJson)) + info.Println(time.Now().Format(time.RFC3339) + " Found matches with clamav" + string(clamRespJson)) scanResponse.data = clamScannerResponse - } - + } + if clamerr != nil { scanResponse.err = clamerr } @@ -75,8 +74,8 @@ func (self *Scanner) warmUp() { var yaraHealth = bool(false) var clamDHealth = bool(false) - yaraScannerResponse,yaraerr := ScanStream(&self.yarascanner, eicar) - + yaraScannerResponse, yaraerr := ScanStream(&self.yarascanner, eicar) + if (yaraerr == nil) && len(yaraScannerResponse.Matches) > 0 { yaraHealth = true } @@ -86,7 +85,7 @@ func (self *Scanner) warmUp() { if yaraHealth && clamDHealth { info.Println("Warmed Up") } else { - info.Println( time.Now().Format(time.RFC3339) + " Warm up failed exiting.. Yara Health" + strconv.FormatBool(yaraHealth) + "ClamD Health" + strconv.FormatBool(clamDHealth)) + info.Println(time.Now().Format(time.RFC3339) + " Warm up failed exiting.. Yara Health" + strconv.FormatBool(yaraHealth) + "ClamD Health" + strconv.FormatBool(clamDHealth)) os.Exit(1) } } @@ -96,17 +95,17 @@ func (self *Scanner) LoadIndex(indexPath string) error { } func (self *Scanner) listRuleSets() *RuleSetResponse { - response,err := self.yarascanner.ListRuleSets() - ruleSetResponse := new (RuleSetResponse) + response, err := self.yarascanner.ListRuleSets() + ruleSetResponse := new(RuleSetResponse) ruleSetResponse.err = err ruleSetResponse.data = response return ruleSetResponse } -func (self *Scanner) listRules(rulesetname string) (*RuleListResponse) { +func (self *Scanner) listRules(rulesetname string) *RuleListResponse { response, err := self.yarascanner.ListRules(rulesetname) - ruleListResponse := new (RuleListResponse) + ruleListResponse := new(RuleListResponse) ruleListResponse.err = err ruleListResponse.data = response diff --git a/scannerinterface.go b/scannerinterface.go index d5a5f24..7aed6c3 100644 --- a/scannerinterface.go +++ b/scannerinterface.go @@ -2,13 +2,13 @@ package main // Defining an interface type ScannerInterface interface { - Scan(data [] byte) (*ScanReport, error) + Scan(data []byte) (*ScanReport, error) } // struct to handle matches type ScanMatch struct { - Rule string `json:"rule"` - Namespace string `json:"namespace"` + Rule string `json:"rule"` + Namespace string `json:"namespace"` Tags []string `json:"tags"` } @@ -17,9 +17,9 @@ type ListResponse struct { } type ScanReport struct { - Filename string `json:"filename"` + Filename string `json:"filename"` Matches []ScanMatch `json:"matches"` - Status string `json:"status"` + Status string `json:"status"` } type ScanErrorData struct { diff --git a/streamscanhandler.go b/streamscanhandler.go index 6f90234..c38eed6 100644 --- a/streamscanhandler.go +++ b/streamscanhandler.go @@ -3,8 +3,8 @@ package main import ( // standard "encoding/json" - "net/http" "io/ioutil" + "net/http" ) func ScanStreamHandler(w http.ResponseWriter, r *http.Request) { diff --git a/yarascanner.go b/yarascanner.go index 5c3a344..7e9fae0 100644 --- a/yarascanner.go +++ b/yarascanner.go @@ -11,7 +11,7 @@ type YaraScanner struct { } // To implement an interface in Go all you need to do is just define all the functions in the interface -func (self *YaraScanner) Scan(data [] byte) (*ScanReport, error) { +func (self *YaraScanner) Scan(data []byte) (*ScanReport, error) { var matches []ScanMatch response := new(ScanReport) response.Filename = "stream" @@ -22,7 +22,7 @@ func (self *YaraScanner) Scan(data [] byte) (*ScanReport, error) { err := ruleset.Rules.ScanMem(data, 0, 300, &m) if err != nil { response.Status = "ERROR" - return response,err + return response, err } for _, resp := range m { var match ScanMatch @@ -37,11 +37,11 @@ func (self *YaraScanner) Scan(data [] byte) (*ScanReport, error) { response.Status = "INFECTED" } else { response.Status = "CLEAN" - matches = [] ScanMatch{} + matches = []ScanMatch{} } - + response.Matches = matches - return response,nil + return response, nil } func (self *YaraScanner) LoadIndex(indexPath string) error { @@ -59,7 +59,7 @@ func (self *YaraScanner) ListRuleSets() (*RuleSetResponseObject, error) { response.Names = append(response.Names, ruleset.Name) } - return response,nil + return response, nil } @@ -80,4 +80,4 @@ func (self *YaraScanner) ListRules(rulesetname string) (*RuleListResponseObject, } return response, nil -} \ No newline at end of file +}