Automated cherry pick of #1838: fix(aws): remove aws sdk

go.mod

@@ -13,10 +13,8 @@ require (
 	github.com/aws/aws-sdk-go-v2 v1.41.5
 	github.com/aws/aws-sdk-go-v2/config v1.31.12
 	github.com/aws/aws-sdk-go-v2/credentials v1.18.16
-	github.com/aws/aws-sdk-go-v2/service/organizations v1.51.1-0.20260401181752-9074b3ddace9
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.4
 	github.com/basgys/goxml2json v1.1.1-0.20181031222924-996d9fc8d313
-	github.com/coredns/coredns v1.3.0
 	github.com/fatih/color v1.13.0
 	github.com/golang-plus/errors v1.0.0
 	github.com/huaweicloud/huaweicloud-sdk-go v1.0.26
@@ -39,7 +37,7 @@ require (
 	moul.io/http2curl/v2 v2.3.0
 	yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1
 	yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
-	yunion.io/x/pkg v1.10.4-0.20260224071200-5c269fa62429
+	yunion.io/x/pkg v1.10.4-0.20260422030155-01b100134978
 	yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
 	yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
 )

go.sum

@@ -71,8 +71,6 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 h1:5r34CgVOD
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9/go.mod h1:dB12CEbNWPbzO2uC6QSWHteqOg4JfBVJOojbAoAUb5I=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9 h1:wuZ5uW2uhJR63zwNlqWH2W4aL4ZjeJP3o92/W+odDY4=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9/go.mod h1:/G58M2fGszCrOzvJUkDdY8O9kycodunH4VdT5oBAqls=
-github.com/aws/aws-sdk-go-v2/service/organizations v1.51.1-0.20260401181752-9074b3ddace9 h1:qi11riT4zVjV41Nh74zbBgK0EQbkbZG8rPm4shDFlJk=
-github.com/aws/aws-sdk-go-v2/service/organizations v1.51.1-0.20260401181752-9074b3ddace9/go.mod h1:urLFj1twuR/h5T0wN/2/kmY1gxBFa1tTKr+c60lZ2fA=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.4 h1:mUI3b885qJgfqKDUSj6RgbRqLdX0wGmg8ruM03zNfQA=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.4/go.mod h1:6v8ukAxc7z4x4oBjGUsLnH7KGLY9Uhcgij19UJNkiMg=
 github.com/aws/aws-sdk-go-v2/service/sso v1.29.6 h1:A1oRkiSQOWstGh61y4Wc/yQ04sqrQZr1Si/oAXj20/s=
@@ -96,8 +94,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ=
 github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM=
-github.com/coredns/coredns v1.3.0 h1:Ubjr8ZrOpEj95/A9dqQqQAG2k5DWEFdg2ZB3tUXuULA=
-github.com/coredns/coredns v1.3.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -451,8 +447,8 @@ yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXce
 yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 h1:inY5o3LDa/zgsIZuPN0HmpzKIsu/lLgsBmMttuDPGj4=
 yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v1.10.4-0.20260224071200-5c269fa62429 h1:yskc2HFnBZXEzfYeQBFdL2gXzVfdPQ8yT4qVDyuhajM=
-yunion.io/x/pkg v1.10.4-0.20260224071200-5c269fa62429/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
+yunion.io/x/pkg v1.10.4-0.20260422030155-01b100134978 h1:+BS9DValwi0xHa92xrB+RR6Cxp7bDbXVysa/KiC18rY=
+yunion.io/x/pkg v1.10.4-0.20260422030155-01b100134978/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
 yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4=
 yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ=

pkg/multicloud/aws/aws.go

@@ -16,19 +16,20 @@ package aws
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	sdk "github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
@@ -301,40 +302,33 @@ func (client *SAwsClient) fetchOwnerId() error {
 }
 
 func (client *SAwsClient) fetchBuckets() error {
-	s, err := client.getDefaultSession(true)
+	s3cli, err := client.GetS3Client()
 	if err != nil {
-		return errors.Wrap(err, "getDefaultSession")
+		return errors.Wrap(err, "GetS3Client")
 	}
-	s3cli := s3.New(s)
-	output, err := s3cli.ListBuckets(&s3.ListBucketsInput{})
+	output, err := s3cli.ListBuckets(context.Background(), &s3.ListBucketsInput{})
 	if err != nil {
-		if e, ok := err.(awserr.Error); ok && e.Code() == "AccessDenied" {
-			return errors.Wrapf(cloudprovider.ErrForbidden, "%s", e.Message())
+		if strings.Contains(err.Error(), "AccessDenied") {
+			return errors.Wrapf(cloudprovider.ErrForbidden, "%s", err.Error())
 		}
 		return errors.Wrap(err, "ListBuckets")
 	}
 
 	ret := make([]cloudprovider.ICloudBucket, 0)
 	for _, bInfo := range output.Buckets {
-		if err := FillZero(bInfo); err != nil {
-			log.Errorf("s3cli.Binfo.FillZero error %s", err)
+		if bInfo.Name == nil {
 			continue
 		}
 
-		input := &s3.GetBucketLocationInput{}
-		input.Bucket = bInfo.Name
-		output, err := s3cli.GetBucketLocation(input)
+		locOutput, err := s3cli.GetBucketLocation(context.Background(), &s3.GetBucketLocationInput{
+			Bucket: bInfo.Name,
+		})
 		if err != nil {
 			log.Errorf("s3cli.GetBucketLocation error %s", err)
 			continue
 		}
 
-		if err := FillZero(output); err != nil {
-			log.Errorf("s3cli.GetBucketLocation.FillZero error %s", err)
-			continue
-		}
-
-		location := *output.LocationConstraint
+		location := string(locOutput.LocationConstraint)
 		if len(location) == 0 {
 			// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html
 			// Buckets in Region us-east-1 have a LocationConstraint of null.
@@ -345,11 +339,15 @@ func (client *SAwsClient) fetchBuckets() error {
 			log.Errorf("client.getIRegionByRegionId %s fail %s", location, err)
 			continue
 		}
+		creationDate := time.Time{}
+		if bInfo.CreationDate != nil {
+			creationDate = *bInfo.CreationDate
+		}
 		b := SBucket{
 			region:       region.(*SRegion),
 			Name:         *bInfo.Name,
 			Location:     location,
-			CreationDate: *bInfo.CreationDate,
+			CreationDate: creationDate,
 		}
 		ret = append(ret, &b)
 	}
@@ -531,6 +529,14 @@ func (self *SAwsClient) stsRequest(apiName string, params map[string]string, ret
 	return self.request("", STS_SERVICE_NAME, STS_SERVICE_ID, "2011-06-15", apiName, params, retval, false)
 }
 
+func (self *SAwsClient) orgRequest(apiName string, params map[string]interface{}, retval interface{}) error {
+	return self.invoke("", ORG_SERVICE_NAME, ORG_SERVICE_ID, "2016-11-28", apiName, "", params, retval, true)
+}
+
+func (self *SAwsClient) ceRequest(apiName string, params map[string]interface{}, retval interface{}) error {
+	return self.invoke(self.getCostExplorerRegion(), CE_SERVICE_NAME, CE_SERVICE_ID, "2017-10-25", apiName, "", params, retval, true)
+}
+
 func (self *SAwsClient) GetCapabilities() []string {
 	caps := []string{
 		cloudprovider.CLOUD_CAPABILITY_COMPUTE,

pkg/multicloud/aws/aws_invoke.go

@@ -67,6 +67,22 @@ func (self *SAwsClient) invoke(regionId, serviceName, serviceId, apiVersion stri
 		metadata.TargetPrefix = "DynamoDB_20120810"
 		metadata.JSONVersion = "1.0"
 	}
+	if serviceName == ORG_SERVICE_NAME {
+		metadata.TargetPrefix = "AWSOrganizationsV20161128"
+		metadata.JSONVersion = "1.1"
+	}
+	if serviceName == CLOUD_TRAIL_SERVICE_NAME {
+		metadata.TargetPrefix = "com.amazonaws.cloudtrail.v20131101.CloudTrail_20131101"
+		metadata.JSONVersion = "1.1"
+	}
+	if serviceName == CE_SERVICE_NAME {
+		metadata.TargetPrefix = "AWSInsightsIndexService"
+		metadata.JSONVersion = "1.1"
+	}
+	if serviceName == WAF_SERVICE_NAME {
+		metadata.TargetPrefix = "AWSWAF_20190729"
+		metadata.JSONVersion = "1.1"
+	}
 
 	if self.debug {
 		logLevel := aws.LogLevelType(uint(aws.LogDebugWithRequestErrors) + uint(aws.LogDebugWithHTTPBody))
@@ -107,6 +123,10 @@ func jsonInvoke(cli *client.Client, apiName, path string, params map[string]inte
 		KINESIS_SERVICE_NAME,
 		DYNAMODB_SERVICE_NAME,
 		PRICING_SERVICE_NAME,
+		ORG_SERVICE_NAME,
+		CLOUD_TRAIL_SERVICE_NAME,
+		CE_SERVICE_NAME,
+		WAF_SERVICE_NAME,
 	}) {
 		method = "POST"
 	}

pkg/multicloud/aws/awsv2.go

@@ -73,3 +73,11 @@ func (r *SRegion) GetS3Client() (*s3.Client, error) {
 	}
 	return s3.NewFromConfig(cfg), nil
 }
+
+func (client *SAwsClient) GetS3Client() (*s3.Client, error) {
+	cfg, err := client.getConfig(context.Background(), client.getDefaultRegionId(), true)
+	if err != nil {
+		return nil, errors.Wrap(err, "getConfig")
+	}
+	return s3.NewFromConfig(cfg), nil
+}

pkg/multicloud/aws/business.go

@@ -0,0 +1,148 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aws
+
+import (
+	"sort"
+	"strconv"
+	"time"
+
+	api "yunion.io/x/cloudmux/pkg/apis/compute"
+	"yunion.io/x/pkg/errors"
+)
+
+type SMonthBill struct {
+	Month       string                 `json:"month"`
+	StartDate   string                 `json:"start_date"`
+	EndDate     string                 `json:"end_date"`
+	Currency    string                 `json:"currency"`
+	Total       float64                `json:"total"`
+	Metric      string                 `json:"metric"`
+	Granularity string                 `json:"granularity"`
+	Services    []SMonthBillServiceFee `json:"services"`
+}
+
+type SMonthBillServiceFee struct {
+	Service string `json:"service"`
+	Amount  string `json:"amount"`
+	Unit    string `json:"unit"`
+}
+
+type sMetricValue struct {
+	Amount string
+	Unit   string
+}
+
+type sCostGroup struct {
+	Keys    []string
+	Metrics map[string]sMetricValue
+}
+
+type sResultByTime struct {
+	Groups []sCostGroup
+}
+
+func getMonthDateRange(month string) (string, string, error) {
+	monthStart, err := time.Parse("2006-01", month)
+	if err != nil {
+		return "", "", errors.Wrapf(err, "invalid month %q, expected YYYY-MM", month)
+	}
+	monthEnd := monthStart.AddDate(0, 1, 0)
+	return monthStart.Format("2006-01-02"), monthEnd.Format("2006-01-02"), nil
+}
+
+func (self *SAwsClient) getCostExplorerRegion() string {
+	if self.GetAccessEnv() == api.CLOUD_ACCESS_ENV_AWS_CHINA {
+		return "cn-northwest-1"
+	}
+	return "us-east-1"
+}
+
+func (self *SAwsClient) GetMonthBill(month string) (*SMonthBill, error) {
+	startDate, endDate, err := getMonthDateRange(month)
+	if err != nil {
+		return nil, err
+	}
+
+	params := map[string]interface{}{
+		"TimePeriod": map[string]string{
+			"Start": startDate,
+			"End":   endDate,
+		},
+		"Granularity": "MONTHLY",
+		"Metrics":     []string{"UnblendedCost"},
+		"GroupBy": []map[string]string{
+			{
+				"Type": "DIMENSION",
+				"Key":  "SERVICE",
+			},
+		},
+	}
+	resp := struct {
+		ResultsByTime []sResultByTime
+	}{}
+	err = self.ceRequest("GetCostAndUsage", params, &resp)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetCostAndUsage")
+	}
+
+	ret := &SMonthBill{
+		Month:       month,
+		StartDate:   startDate,
+		EndDate:     endDate,
+		Metric:      "UnblendedCost",
+		Granularity: "MONTHLY",
+		Services:    make([]SMonthBillServiceFee, 0),
+	}
+
+	var total float64
+	for _, byTime := range resp.ResultsByTime {
+		for _, group := range byTime.Groups {
+			if len(group.Keys) == 0 {
+				continue
+			}
+			metric, ok := group.Metrics["UnblendedCost"]
+			if !ok {
+				continue
+			}
+			if len(ret.Currency) == 0 {
+				ret.Currency = metric.Unit
+			}
+			amount := metric.Amount
+			if len(amount) > 0 {
+				if val, parseErr := strconv.ParseFloat(amount, 64); parseErr == nil {
+					total += val
+				}
+			}
+			ret.Services = append(ret.Services, SMonthBillServiceFee{
+				Service: group.Keys[0],
+				Amount:  amount,
+				Unit:    metric.Unit,
+			})
+		}
+	}
+	ret.Total = total
+
+	sort.Slice(ret.Services, func(i, j int) bool {
+		iVal, iErr := strconv.ParseFloat(ret.Services[i].Amount, 64)
+		jVal, jErr := strconv.ParseFloat(ret.Services[j].Amount, 64)
+		if iErr != nil || jErr != nil {
+			return ret.Services[i].Amount > ret.Services[j].Amount
+		}
+		return iVal > jVal
+	})
+
+	return ret, nil
+}