diff --git a/go.mod b/go.mod index 2390561..5fa26aa 100644 --- a/go.mod +++ b/go.mod @@ -1 +1,3 @@ module codearts-ci-config + +go 1.23.1 diff --git a/resovler/core/escape.go b/resovler/core/escape.go new file mode 100644 index 0000000..e8c76b8 --- /dev/null +++ b/resovler/core/escape.go @@ -0,0 +1,42 @@ +// based on https://github.com/golang/go/blob/master/src/net/url/url.go +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package core + +func shouldEscape(c byte) bool { + if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '_' || c == '-' || c == '~' || c == '.' { + return false + } + return true +} +func escape(s string) string { + hexCount := 0 + for i := 0; i < len(s); i++ { + c := s[i] + if shouldEscape(c) { + hexCount++ + } + } + + if hexCount == 0 { + return s + } + + t := make([]byte, len(s)+2*hexCount) + j := 0 + for i := 0; i < len(s); i++ { + switch c := s[i]; { + case shouldEscape(c): + t[j] = '%' + t[j+1] = "0123456789ABCDEF"[c>>4] + t[j+2] = "0123456789ABCDEF"[c&15] + j += 3 + default: + t[j] = s[i] + j++ + } + } + return string(t) +} diff --git a/resovler/core/signer.go b/resovler/core/signer.go new file mode 100644 index 0000000..2179fbf --- /dev/null +++ b/resovler/core/signer.go @@ -0,0 +1,201 @@ +// HWS API Gateway Signature +// based on https://github.com/datastream/aws/blob/master/signv4.go +// Copyright (c) 2014, Xianjie + +package core + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "io" + + "fmt" + "net/http" + "sort" + "strings" + "time" +) + +const ( + DateFormat = "20060102T150405Z" + SignAlgorithm = "SDK-HMAC-SHA256" + HeaderXDateTime = "X-Sdk-Date" + HeaderXHost = "host" + HeaderXAuthorization = "Authorization" + HeaderXContentSha256 = "X-Sdk-Content-Sha256" +) + +func hmacsha256(keyByte []byte, dataStr string) ([]byte, error) { + hm := hmac.New(sha256.New, keyByte) + if _, err := hm.Write([]byte(dataStr)); err != nil { + return nil, err + } + return hm.Sum(nil), nil +} + +// Build a CanonicalRequest from a regular request string +func CanonicalRequest(request *http.Request, signedHeaders []string) (string, error) { + var hexencode string + var err error + if hex := request.Header.Get(HeaderXContentSha256); hex != "" { + hexencode = hex + } else { + bodyData, err := RequestPayload(request) + if err != nil { + return "", err + } + hexencode, err = HexEncodeSHA256Hash(bodyData) + if err != nil { + return "", err + } + } + return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", request.Method, CanonicalURI(request), CanonicalQueryString(request), CanonicalHeaders(request, signedHeaders), strings.Join(signedHeaders, ";"), hexencode), err +} + +// CanonicalURI returns request uri +func CanonicalURI(request *http.Request) string { + pattens := strings.Split(request.URL.Path, "/") + var uriSlice []string + for _, v := range pattens { + uriSlice = append(uriSlice, escape(v)) + } + urlpath := strings.Join(uriSlice, "/") + if len(urlpath) == 0 || urlpath[len(urlpath)-1] != '/' { + urlpath = urlpath + "/" + } + return urlpath +} + +// CanonicalQueryString +func CanonicalQueryString(request *http.Request) string { + var keys []string + queryMap := request.URL.Query() + for key := range queryMap { + keys = append(keys, key) + } + sort.Strings(keys) + var query []string + for _, key := range keys { + k := escape(key) + sort.Strings(queryMap[key]) + for _, v := range queryMap[key] { + kv := fmt.Sprintf("%s=%s", k, escape(v)) + query = append(query, kv) + } + } + queryStr := strings.Join(query, "&") + request.URL.RawQuery = queryStr + return queryStr +} + +// CanonicalHeaders +func CanonicalHeaders(request *http.Request, signerHeaders []string) string { + var canonicalHeaders []string + header := make(map[string][]string) + for k, v := range request.Header { + header[strings.ToLower(k)] = v + } + for _, key := range signerHeaders { + value := header[key] + if strings.EqualFold(key, HeaderXHost) { + value = []string{request.Host} + } + sort.Strings(value) + for _, v := range value { + canonicalHeaders = append(canonicalHeaders, key+":"+strings.TrimSpace(v)) + } + } + return fmt.Sprintf("%s\n", strings.Join(canonicalHeaders, "\n")) +} + +// SignedHeaders +func SignedHeaders(r *http.Request) []string { + var signedHeaders []string + for key := range r.Header { + signedHeaders = append(signedHeaders, strings.ToLower(key)) + } + sort.Strings(signedHeaders) + return signedHeaders +} + +// RequestPayload +func RequestPayload(request *http.Request) ([]byte, error) { + if request.Body == nil { + return []byte(""), nil + } + bodyByte, err := io.ReadAll(request.Body) + if err != nil { + return []byte(""), err + } + request.Body = io.NopCloser(bytes.NewBuffer(bodyByte)) + return bodyByte, err +} + +// Create a "String to Sign". +func StringToSign(canonicalRequest string, t time.Time) (string, error) { + hashStruct := sha256.New() + _, err := hashStruct.Write([]byte(canonicalRequest)) + if err != nil { + return "", err + } + return fmt.Sprintf("%s\n%s\n%x", + SignAlgorithm, t.UTC().Format(DateFormat), hashStruct.Sum(nil)), nil +} + +// Create the HWS Signature. +func SignStringToSign(stringToSign string, signingKey []byte) (string, error) { + hmsha, err := hmacsha256(signingKey, stringToSign) + return fmt.Sprintf("%x", hmsha), err +} + +// HexEncodeSHA256Hash returns hexcode of sha256 +func HexEncodeSHA256Hash(body []byte) (string, error) { + hashStruct := sha256.New() + if len(body) == 0 { + body = []byte("") + } + _, err := hashStruct.Write(body) + return fmt.Sprintf("%x", hashStruct.Sum(nil)), err +} + +// Get the finalized value for the "Authorization" header. The signature parameter is the output from SignStringToSign +func AuthHeaderValue(signatureStr, accessKeyStr string, signedHeaders []string) string { + return fmt.Sprintf("%s Access=%s, SignedHeaders=%s, Signature=%s", SignAlgorithm, accessKeyStr, strings.Join(signedHeaders, ";"), signatureStr) +} + +// Signature HWS meta +type Signer struct { + Key string + Secret string +} + +// SignRequest set Authorization header +func (s *Signer) Sign(request *http.Request) error { + var t time.Time + var err error + var date string + if date = request.Header.Get(HeaderXDateTime); date != "" { + t, err = time.Parse(DateFormat, date) + } + if err != nil || date == "" { + t = time.Now() + request.Header.Set(HeaderXDateTime, t.UTC().Format(DateFormat)) + } + signedHeaders := SignedHeaders(request) + canonicalRequest, err := CanonicalRequest(request, signedHeaders) + if err != nil { + return err + } + stringToSignStr, err := StringToSign(canonicalRequest, t) + if err != nil { + return err + } + signatureStr, err := SignStringToSign(stringToSignStr, []byte(s.Secret)) + if err != nil { + return err + } + authValueStr := AuthHeaderValue(signatureStr, s.Key, signedHeaders) + request.Header.Set(HeaderXAuthorization, authValueStr) + return nil +} diff --git a/resovler/go.mod b/resovler/go.mod new file mode 100644 index 0000000..87b57f9 --- /dev/null +++ b/resovler/go.mod @@ -0,0 +1,30 @@ +module resolver + +go 1.21 + +toolchain go1.23.1 + +require ( + github.com/xuri/excelize/v2 v2.9.0 + modernc.org/sqlite v1.34.5 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/richardlehane/mscfb v1.0.4 // indirect + github.com/richardlehane/msoleps v1.0.4 // indirect + github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect + github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.21.0 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect +) diff --git a/resovler/go.sum b/resovler/go.sum new file mode 100644 index 0000000..c2c2f23 --- /dev/null +++ b/resovler/go.sum @@ -0,0 +1,83 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= +github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= +github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d h1:llb0neMWDQe87IzJLS4Ci7psK/lVsjIS2otl+1WyRyY= +github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.9.0 h1:1tgOaEq92IOEumR1/JfYS/eR0KHOCsRv/rYXXh6YJQE= +github.com/xuri/excelize/v2 v2.9.0/go.mod h1:uqey4QBZ9gdMeWApPLdhm9x+9o2lq4iVmjiLfBS5hdE= +github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 h1:hPVCafDV85blFTabnqKgNhDCkJX25eik94Si9cTER4A= +github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= +golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/resovler/handler/SqlDbHandler.go b/resovler/handler/SqlDbHandler.go new file mode 100644 index 0000000..6239bb7 --- /dev/null +++ b/resovler/handler/SqlDbHandler.go @@ -0,0 +1,355 @@ +package handler + +import ( + "database/sql" + "encoding/json" + "fmt" + "log" + _ "modernc.org/sqlite" // 空导入驱动 + "reflect" + "strings" +) + +func initDb() *sql.DB { + // SQLite 数据库连接 + db, err := sql.Open("sqlite", "./codearts_scan.db?_busy_timeout=5000") + if err != nil { + log.Fatalf("Failed to connect to SQLite database: %v", err) + } + db.SetMaxOpenConns(20) + db.SetMaxIdleConns(20) + return db +} + +func QueryDatabase(sql string) ([]map[string]interface{}, error) { + db := initDb() + stmt, err := db.Prepare(sql) + rows, err := stmt.Query() + if err != nil { + _ = fmt.Errorf("failed to execute table query: %w", err) + return nil, err + } + var records = make([]map[string]interface{}, 0) + // 获取列名 + columns, err := rows.Columns() + if err != nil { + fmt.Errorf("failed to get columns: %w", err) + return nil, err + } + + for rows.Next() { + // 创建一个存储每列值的切片 + values := make([]interface{}, len(columns)) + valuePointers := make([]interface{}, len(columns)) + for i := range values { + valuePointers[i] = &values[i] + } + + // 扫描当前行 + if err := rows.Scan(valuePointers...); err != nil { + fmt.Errorf("failed to scan row: %w", err) + return nil, err + } + + // 将当前行数据存储为 map + rowMap := make(map[string]interface{}) + for i, col := range columns { + rowMap[col] = values[i] + } + records = append(records, rowMap) + } + log.Println("JSON data has been successfully queried from the SQLite database.", len(records)) + return records, nil +} + +func UpdateRecordsByWhere(tableName string, setClause, whereClause string) []map[string]interface{} { + db := initDb() + defer db.Close() + + query := fmt.Sprintf("UPDATE %s set %s where %s", tableName, setClause, whereClause) + stmt, err := db.Prepare(query) + _, err = stmt.Exec() + if err != nil { + _ = fmt.Errorf("failed to execute table query: %w", err) + return nil + } + var records = make([]map[string]interface{}, 0) + + log.Println("JSON data has been successfully updated from the SQLite database.", len(records)) + return records +} + +func UpdateRecordsByWhereAInBArr(tableName string, setClause, whereClause string, whereClauseColumn string, whereClauseIn []string) []map[string]interface{} { + db := initDb() + defer db.Close() + + strings.Join(whereClauseIn, ",") + query := fmt.Sprintf("UPDATE %s set %s where %s and %s in (%s)", tableName, setClause, whereClause, whereClauseColumn, whereClauseIn) + stmt, err := db.Prepare(query) + _, err = stmt.Exec() + if err != nil { + _ = fmt.Errorf("failed to execute table query: %w", err) + return nil + } + var records = make([]map[string]interface{}, 0) + + log.Println("JSON data has been successfully updated from the SQLite database.", len(records)) + return records +} + +func DeleteRecordsByWhere(tableName string, whereClause string) []map[string]interface{} { + db := initDb() + defer db.Close() + + query := fmt.Sprintf("DELETE FROM %s where %s", tableName, whereClause) + stmt, err := db.Prepare(query) + rows, err := stmt.Query() + if err != nil { + _ = fmt.Errorf("failed to execute table query: %w", err) + return nil + } + var records = make([]map[string]interface{}, 0) + // 获取列名 + columns, err := rows.Columns() + if err != nil { + fmt.Errorf("failed to get columns: %w", err) + return nil + } + + for rows.Next() { + // 创建一个存储每列值的切片 + values := make([]interface{}, len(columns)) + valuePointers := make([]interface{}, len(columns)) + for i := range values { + valuePointers[i] = &values[i] + } + + // 扫描当前行 + if err := rows.Scan(valuePointers...); err != nil { + fmt.Errorf("failed to scan row: %w", err) + return nil + } + + // 将当前行数据存储为 map + rowMap := make(map[string]interface{}) + for i, col := range columns { + rowMap[col] = values[i] + } + records = append(records, rowMap) + } + log.Println("JSON data has been successfully deleted from the SQLite database.", len(records)) + return records +} + +func QueryRecordsByWhere(tableName string, whereClause string) []map[string]interface{} { + db := initDb() + query := fmt.Sprintf("SELECT * FROM %s where %s", tableName, whereClause) + stmt, err := db.Prepare(query) + rows, err := stmt.Query() + if err != nil { + _ = fmt.Errorf("failed to execute table query: %w", err) + return nil + } + var records = make([]map[string]interface{}, 0) + // 获取列名 + columns, err := rows.Columns() + if err != nil { + fmt.Errorf("failed to get columns: %w", err) + return nil + } + + for rows.Next() { + // 创建一个存储每列值的切片 + values := make([]interface{}, len(columns)) + valuePointers := make([]interface{}, len(columns)) + for i := range values { + valuePointers[i] = &values[i] + } + + // 扫描当前行 + if err := rows.Scan(valuePointers...); err != nil { + fmt.Errorf("failed to scan row: %w", err) + return nil + } + + // 将当前行数据存储为 map + rowMap := make(map[string]interface{}) + for i, col := range columns { + rowMap[col] = values[i] + } + records = append(records, rowMap) + } + log.Println("JSON data has been successfully queried from the SQLite database.", len(records)) + return records +} + +func QueryAllRecords(tableName string) []map[string]interface{} { + db := initDb() + query := fmt.Sprintf("SELECT * FROM %s", tableName) + stmt, err := db.Prepare(query) + rows, err := stmt.Query() + if err != nil { + _ = fmt.Errorf("failed to execute table query: %w", err) + return nil + } + var records = make([]map[string]interface{}, 0) + // 获取列名 + columns, err := rows.Columns() + if err != nil { + fmt.Errorf("failed to get columns: %w", err) + return nil + } + + for rows.Next() { + // 创建一个存储每列值的切片 + values := make([]interface{}, len(columns)) + valuePointers := make([]interface{}, len(columns)) + for i := range values { + valuePointers[i] = &values[i] + } + + // 扫描当前行 + if err := rows.Scan(valuePointers...); err != nil { + fmt.Errorf("failed to scan row: %w", err) + return nil + } + + // 将当前行数据存储为 map + rowMap := make(map[string]interface{}) + for i, col := range columns { + rowMap[col] = values[i] + } + records = append(records, rowMap) + } + log.Println("JSON data has been successfully queried from the SQLite database.", len(records)) + return records +} + +func InsertJsonData2SqlLiteDb(tableName string, hr_Data []byte, primaryKeyConstraint ...string) { + db := initDb() + defer db.Close() + + // 自动创建表 + var records = make([]map[string]interface{}, 0) + err := json.Unmarshal(hr_Data, &records) + if err != nil { + return + } + if err := CreateTable(db, tableName, records, primaryKeyConstraint...); err != nil { + log.Fatalf("Failed to create table: %v", err) + } + + // 插入数据 + if err := InsertRecords(db, tableName, records); err != nil { + log.Fatalf("Failed to insert records: %v", err) + } + + log.Println("JSON data has been successfully inserted into the SQLite database.") +} + +func InsertJsonRecords2SqlLiteDb(tableName string, records []map[string]interface{}, primaryKeyConstraint ...string) { + db := initDb() + defer db.Close() + + if err := CreateTable(db, tableName, records, primaryKeyConstraint...); err != nil { + log.Fatalf("Failed to create table: %v", err) + } + + // 插入数据 + if err := InsertRecords(db, tableName, records); err != nil { + log.Fatalf("Failed to insert records: %v", err) + } + + log.Println("JSON data has been successfully inserted into the SQLite database.") +} + +// CreateTable 根据 JSON 动态创建表 +func CreateTable(db *sql.DB, tableName string, records []map[string]interface{}, primaryKeyConstraint ...string) error { + if len(records) == 0 { + log.Println("no records to create table from") + return nil + } + + columns := make([]string, 0) + for key, value := range records[0] { + sqlType := getSQLType(value) + columns = append(columns, fmt.Sprintf("%s %s", key, sqlType)) + } + + query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s,\n PRIMARY KEY (%s))", tableName, strings.Join(columns, ", "), strings.Join(primaryKeyConstraint, ", ")) + _, err := db.Exec(query) + if err != nil { + return fmt.Errorf("failed to execute create table query: %w", err) + } + + return nil +} + +// InsertRecords 插入 JSON 数据到表 +func InsertRecords(db *sql.DB, tableName string, records []map[string]interface{}) error { + if len(records) == 0 { + return nil + } + + columns := make([]string, 0) + placeholders := make([]string, 0) + for key := range records[0] { + columns = append(columns, fmt.Sprintf("%s", key)) + placeholders = append(placeholders, "?") + } + + query := fmt.Sprintf("INSERT OR IGNORE INTO %s (%s) VALUES (%s)", tableName, strings.Join(columns, ", "), strings.Join(placeholders, ", ")) + stmt, err := db.Prepare(query) + if err != nil { + return fmt.Errorf("failed to prepare insert statement: %w", err) + } + defer stmt.Close() + + for _, record := range records { + values := make([]interface{}, 0) + for _, column := range columns { + recordColum := record[strings.Trim(column, "")] + if v, ok := recordColum.(map[string]interface{}); ok { + jsonData, err := json.Marshal(recordColum) + if err != nil { + log.Println("Error marshalling map to JSON:", err, v) + continue + } + values = append(values, jsonData) + } else if v, ok := recordColum.([]interface{}); ok { + jsonData, err := json.Marshal(recordColum) + if err != nil { + log.Println("Error marshalling map to JSON:", err, v) + continue + } + values = append(values, jsonData) + } else { + values = append(values, record[strings.Trim(column, "")]) + } + } + if _, err := stmt.Exec(values...); err != nil { + return fmt.Errorf("failed to execute insert statement: %w", err) + } + } + + return nil +} + +// 根据值推断 SQL 数据类型 +func getSQLType(value interface{}) string { + if value == nil { + return "TEXT" + } + switch reflect.TypeOf(value).Kind() { + case reflect.String: + return "TEXT" + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return "INTEGER" + case reflect.Float32, reflect.Float64: + return "REAL" + case reflect.Bool: + return "INTEGER" + default: + return "TEXT" + } +} diff --git a/resovler/handler/excelHandler.go b/resovler/handler/excelHandler.go new file mode 100644 index 0000000..e2707a7 --- /dev/null +++ b/resovler/handler/excelHandler.go @@ -0,0 +1,113 @@ +package handler + +import ( + "encoding/json" + "fmt" + "github.com/xuri/excelize/v2" + "log" +) + +func CreateExcel(excelName string, bytes []byte) { + // 定义一个通用结构 + var records []map[string]interface{} + + // 解析 JSON + err := json.Unmarshal(bytes, &records) + if err != nil { + log.Println("Error decoding JSON:", err) + return + } + + // 创建 Excel 文件 + f := excelize.NewFile() + sheetName := "Sheet1" + _, _ = f.NewSheet(sheetName) + + // 写入表头 + if len(records) > 0 { + headers := make([]string, 0, len(records[0])) + col := 1 + for header := range records[0] { + headers = append(headers, header) + cell, _ := excelize.CoordinatesToCellName(col, 1) + f.SetCellValue(sheetName, cell, header) + col++ + } + + // 写入数据 + for row, record := range records { + col := 1 + for _, header := range headers { + cell, _ := excelize.CoordinatesToCellName(col, row+2) // 数据从第二行开始 + switch v := record[header].(type) { + case map[string]interface{}: + jsonData, err := json.Marshal(record[header]) + if err != nil { + log.Println("Error marshalling map to JSON:", err, v) + continue + } + // 将 JSON 字符串写入 Excel + f.SetCellValue(sheetName, cell, string(jsonData)) + default: + f.SetCellValue(sheetName, cell, record[header]) + } + col++ + } + } + } + + // 保存文件 + output := excelName + if err := f.SaveAs(output); err != nil { + log.Println("Error saving Excel file:", err) + return + } + + log.Println("Excel file created successfully:", output) +} + +func ReadExcel(filePath string, sheetName string) ([]map[string]interface{}, error) { + // 打开文件 + f, err := excelize.OpenFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open Excel file: %w", err) + } + defer f.Close() + + // 获取第一个工作表名称 + if sheetName == "" { + sheetName = f.GetSheetName(1) + } + + // 获取所有行 + rows, err := f.GetRows(sheetName) + if err != nil { + return nil, fmt.Errorf("failed to get rows: %w", err) + } + + if len(rows) < 1 { + return nil, fmt.Errorf("Excel sheet is empty") + } + + // 第一行为表头 + headers := rows[0] + + // 转换为 []map[string]interface{} + var result []map[string]interface{} + for _, row := range rows[1:] { // 从第二行开始读取数据 + rowMap := make(map[string]interface{}) + for j, cell := range row { + if j < len(headers) { // 确保不会越界 + rowMap[headers[j]] = cell + } + } + result = append(result, rowMap) + + // 如果某行少于表头列数,补充为空值 + for j := len(row); j < len(headers); j++ { + rowMap[headers[j]] = nil + } + } + + return result, nil +} diff --git a/resovler/hssVulResolver.go b/resovler/hssVulResolver.go new file mode 100644 index 0000000..656fa49 --- /dev/null +++ b/resovler/hssVulResolver.go @@ -0,0 +1,168 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2017-2023. All rights reserved. + */ + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "os" + "resolver/core" + "resolver/handler" +) + +var hssAssetList = []HssAsset{{ + hwsAccount: "freesky-edward", + region: "cn-north-4", + projectId: "25f40abeecb84d3e90731de258ca71ec", +}, {}, {}} + +type HssAsset struct { + hwsAccount string + region string + projectId string + AK string + SK string +} + +func main() { + for _, hssAsset := range hssAssetList { + if hssAsset.region == "" || hssAsset.projectId == "" { + continue + } + resolveRiskTypesData(hssAsset) + resolveWeakPasswordUserData(hssAsset) + resolvePasswordBaselineData(hssAsset) + resolveEventData(hssAsset) + resolveHostData(hssAsset) + resolveVulData(hssAsset) + } +} + +func resolveRiskTypesData(hssAsset HssAsset) { + hssList := getHssData(hssAsset, "https://hss.%s.myhuaweicloud.com/v5/%s/baseline/risk-configs?offset=%d&limit=%d&enterprise_project_id=all_granted_eps", "") + for _, hssData := range hssList { + resolveRiskConfigData(hssAsset, hssData["check_type"].(string)) + } +} + +func resolveRiskConfigHostData(hssAsset HssAsset, riskType string) { + hssList := getHssData(hssAsset, "https://hss.%s.myhuaweicloud.com/v5/%s/baseline/risk-config/"+riskType+"/check-rules?result_type=unhandled&standard=hw_standard&check_cce=true&enterprise_project_id=all_granted_eps&offset=%d&limit=%d", "") + handler.DeleteRecordsByWhere("hss_risk_config_detail_host", fmt.Sprintf("hwsAccount='%s' and projectId='%s' and region='%s' and check_type='%s'", hssAsset.hwsAccount, hssAsset.projectId, hssAsset.region, riskType)) + handler.InsertJsonRecords2SqlLiteDb("hss_risk_config_detail_host", hssList, "hwsAccount", "projectId", "region", "check_type", "host_id") +} + +func resolveRiskConfigData(hssAsset HssAsset, riskType string) { + hssList := getHssData(hssAsset, "https://hss.%s.myhuaweicloud.com/v5/%s/baseline/risk-config/"+riskType+"/check-rules?result_type=unhandled&standard=hw_standard&check_cce=true&enterprise_project_id=all_granted_eps&offset=%d&limit=%d", "") + handler.DeleteRecordsByWhere("hss_risk_config_detail", fmt.Sprintf("hwsAccount='%s' and projectId='%s' and region='%s' and check_type='%s'", hssAsset.hwsAccount, hssAsset.projectId, hssAsset.region, riskType)) + handler.InsertJsonRecords2SqlLiteDb("hss_risk_config_detail", hssList, "hwsAccount", "projectId", "region", "check_type", "check_rule_id") +} + +func resolveWeakPasswordUserData(hssAsset HssAsset) { + hssList := getHssData(hssAsset, "https://hss.%s.myhuaweicloud.com/v5/%s/baseline/weak-password-users?enterprise_project_id=all_granted_eps&offset=%d&limit=%d", "") + handler.DeleteRecordsByWhere("hss_weak_password_user", fmt.Sprintf("hwsAccount='%s' and projectId='%s' and region='%s'", hssAsset.hwsAccount, hssAsset.projectId, hssAsset.region)) + handler.InsertJsonRecords2SqlLiteDb("hss_weak_password_user", hssList, "hwsAccount", "projectId", "region", "host_id") +} + +func resolvePasswordBaselineData(hssAsset HssAsset) { + hssList := getHssData(hssAsset, "https://hss.%s.myhuaweicloud.com/v5/%s/baseline/password-complexity?result_type=unhandled&enterprise_project_id=all_granted_eps&offset=%d&limit=%d", "") + handler.DeleteRecordsByWhere("hss_password_config_baseline", fmt.Sprintf("hwsAccount='%s' and projectId='%s' and region='%s'", hssAsset.hwsAccount, hssAsset.projectId, hssAsset.region)) + handler.InsertJsonRecords2SqlLiteDb("hss_password_config_baseline", hssList, "hwsAccount", "projectId", "region", "host_id") +} + +func resolveEventData(hssAsset HssAsset) { + hssList := getHssData(hssAsset, "https://hss.%s.myhuaweicloud.com/v5/%s/event/events?offset=%d&limit=%d&handle_status=unhandled&category=host&enterprise_project_id=all_granted_eps", "") + handler.DeleteRecordsByWhere("hss_event", fmt.Sprintf("hwsAccount='%s' and projectId='%s' and region='%s'", hssAsset.hwsAccount, hssAsset.projectId, hssAsset.region)) + handler.InsertJsonRecords2SqlLiteDb("hss_event", hssList, "hwsAccount", "projectId", "region", "event_id") +} + +func getHssData(hssAsset HssAsset, url, vulType string) []map[string]interface{} { + var limit = 10 + var offset = 0 + var total = math.MaxInt + hssList := make([]map[string]interface{}, 0) + fmt.Println(hssList) + for total > offset { + var hssData []byte + if vulType != "" { + hssData = fetchData(hssAsset, fmt.Sprintf(url, hssAsset.region, hssAsset.projectId, offset, limit, vulType)) + } else { + hssData = fetchData(hssAsset, fmt.Sprintf(url, hssAsset.region, hssAsset.projectId, offset, limit)) + } + hssMap := make(map[string]interface{}) + err := json.Unmarshal(hssData, &hssMap) + if err != nil { + return nil + } + total = int(hssMap["total_num"].(float64)) + elems := hssMap["data_list"].([]interface{}) + for _, elem := range elems { + if v, ok := elem.(map[string]interface{}); ok { + hssList = append(hssList, v) + fmt.Println(v) + } else { + fmt.Println(11) + } + } + offset += limit + } + fmt.Println(len(hssList)) + for _, host := range hssList { + host["hwsAccount"] = hssAsset.hwsAccount + host["projectId"] = hssAsset.projectId + host["region"] = hssAsset.region + } + return hssList +} + +func resolveHostData(hssAsset HssAsset) { + hssList := getHssData(hssAsset, "https://hss.%s.myhuaweicloud.com/v5/%s/host-management/hosts?offset=%d&limit=%d&refresh=false&enterprise_project_id=all_granted_eps", "") + handler.DeleteRecordsByWhere("hss_host", fmt.Sprintf("hwsAccount='%s' and projectId='%s' and region='%s'", hssAsset.hwsAccount, hssAsset.projectId, hssAsset.region)) + handler.InsertJsonRecords2SqlLiteDb("hss_host", hssList, "hwsAccount", "projectId", "region", "host_id") +} + +func resolveVulData(hssAsset HssAsset) { + vulTypes := []string{"linux_vul", "windows_vul", "web_cms", "app_vul", "urgent_vul"} + for _, vulType := range vulTypes { + hssList := getHssData(hssAsset, "https://hss.%s.myhuaweicloud.com/v5/%s/vulnerability/vulnerabilities?handle_status=unhandled&repair_priority=Critical,High,Medium,Low&offset=%d&limit=%d&type=%s&enterprise_project_id=all_granted_eps", vulType) + handler.DeleteRecordsByWhere("hss_vuls", fmt.Sprintf("hwsAccount='%s' and projectId='%s' and region='%s' and type='%s'", hssAsset.hwsAccount, hssAsset.projectId, hssAsset.region, vulType)) + handler.InsertJsonRecords2SqlLiteDb("hss_vuls", hssList, "hwsAccount", "projectId", "region", "type", "vul_id") + } +} + +func fetchData(hssAsset HssAsset, url string) []byte { + // 认证用的ak和sk硬编码到代码中或者明文存储都有很大的安全风险,建议在配置文件或者环境变量中密文存放,使用时解密,确保安全; + // 本示例以ak和sk保存在环境变量中为例,运行本示例前请先在本地环境中设置环境变量HUAWEICLOUD_SDK_AK和HUAWEICLOUD_SDK_SK。 + var s = core.Signer{ + Key: os.Getenv(hssAsset.hwsAccount + "_" + hssAsset.region + "_AK"), + Secret: os.Getenv(hssAsset.hwsAccount + "_" + hssAsset.region + "_SK"), + } + r, err := http.NewRequest("GET", url, io.NopCloser(bytes.NewBuffer([]byte("foo=bar")))) + if err != nil { + fmt.Println(err) + return nil + } + + r.Header.Add("content-type", "application/json; charset=utf-8") + r.Header.Add("x-stage", "RELEASE") + r.Header.Add("region", hssAsset.region) + s.Sign(r) + client := http.DefaultClient + resp, err := client.Do(r) + if err != nil { + fmt.Println(err) + } + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println(err) + } + + return body +} diff --git a/resovler/licenses/license-datastream-aws b/resovler/licenses/license-datastream-aws new file mode 100644 index 0000000..e69de29 diff --git a/resovler/licenses/license-go b/resovler/licenses/license-go new file mode 100644 index 0000000..e69de29 diff --git a/resovler/osinraSecrets/osinfrasecrets.go b/resovler/osinraSecrets/osinfrasecrets.go new file mode 100644 index 0000000..932e949 --- /dev/null +++ b/resovler/osinraSecrets/osinfrasecrets.go @@ -0,0 +1,136 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "resolver/handler" + "strings" +) + +var k8sClusterName = "openEuler_hk" + +var xVaultToken = os.Getenv("xVaultToken") + +func main() { + url := "http://127.0.0.1:12345/v1/internal/metadata/" + secretGroupList := getSecretGroupList(url, "") + processSecretGroups(url, secretGroupList, "") +} + +func processSecretGroups(baseURL string, parentGroupList []interface{}, parentGroup string) { + for _, secretGroup := range parentGroupList { + groupPath := parentGroup + secretGroup.(string) + secretList := getSecretGroupList(baseURL, groupPath) + + // 如果secretList为空,说明当前groupPath是一个叶子节点,直接处理secrets + if len(secretList) == 0 { + processSecrets(groupPath) + } else { + // 否则,递归处理子group + processSecretGroups(baseURL, secretList, groupPath) + } + } +} + +func processSecrets(groupPath string) { + records := make([]map[string]interface{}, 0) + handler.DeleteRecordsByWhere(k8sClusterName+"_jenkins_vault_secrets", fmt.Sprintf("1=1 and 项目='%s'", groupPath)) + + log.Println("处理 ", groupPath) + split := strings.Split(groupPath, "/") + secretName := split[len(split)-1] + result := getSecretCurrentVersion(groupPath) + for key, value := range result { + record := make(map[string]interface{}) + record["项目"] = groupPath + record["服务"] = secretName + record["key"] = key + record["value"] = value + records = append(records, record) + } + handler.InsertJsonRecords2SqlLiteDb(k8sClusterName+"_jenkins_vault_secrets", records, "项目", "服务", "key") +} + +func getSecretCurrentVersion(path string) map[string]interface{} { + url := "http://127.0.0.1:12345/v1/internal/data/" + path + method := "GET" + + client := &http.Client{} + req, err := http.NewRequest(method, url, nil) + + if err != nil { + fmt.Println(err) + return nil + } + req.Header.Add("Accept", "*/*") + req.Header.Add("Accept-Language", "en,zh-CN;q=0.9,zh;q=0.8") + req.Header.Add("Connection", "keep-alive") + req.Header.Add("X-Vault-Token", xVaultToken) + + res, err := client.Do(req) + if err != nil { + fmt.Println(err) + return nil + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + fmt.Println(err) + return nil + } + result := make(map[string]interface{}) + err = json.Unmarshal(body, &result) + if err != nil { + return nil + } + return result["data"].(map[string]interface{})["data"].(map[string]interface{}) +} + +func getSecretGroupList(url string, appends string) []interface{} { + if appends != "" { + url += appends + } + url += "?list=true" + method := "GET" + + client := &http.Client{} + req, err := http.NewRequest(method, url, nil) + + if err != nil { + fmt.Println(err) + return nil + } + req.Header.Add("Accept", "*/*") + req.Header.Add("Accept-Language", "en,zh-CN;q=0.9,zh;q=0.8") + req.Header.Add("Connection", "keep-alive") + req.Header.Add("X-Vault-Token", xVaultToken) + req.Header.Add("Cookie", "HWWAFSESID=30063103fc2f5546483d; HWWAFSESTIME=1740471420298") + + res, err := client.Do(req) + if err != nil { + fmt.Println(err) + return nil + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + fmt.Println(err) + return nil + } + result := make(map[string]interface{}) + err = json.Unmarshal(body, &result) + if err != nil { + return nil + } + + if result["data"] == nil { + return make([]interface{}, 0) + } + return result["data"].(map[string]interface{})["keys"].([]interface{}) +} diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/K8s\347\216\257\345\242\203\350\277\236\346\216\245\345\207\206\345\244\207\345\222\214\344\275\277\347\224\250\346\226\207\344\273\266\346\235\203\351\231\220\346\216\222\346\237\245.md" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/K8s\347\216\257\345\242\203\350\277\236\346\216\245\345\207\206\345\244\207\345\222\214\344\275\277\347\224\250\346\226\207\344\273\266\346\235\203\351\231\220\346\216\222\346\237\245.md" new file mode 100644 index 0000000..2bdeb0a --- /dev/null +++ "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/K8s\347\216\257\345\242\203\350\277\236\346\216\245\345\207\206\345\244\207\345\222\214\344\275\277\347\224\250\346\226\207\344\273\266\346\235\203\351\231\220\346\216\222\346\237\245.md" @@ -0,0 +1,81 @@ +#### 一 安装下载 +- 1 下载k9s https://github.com/derailed/k9s/releases +下载链接:https://github.com/derailed/k9s/releases/download/v0.40.5/k9s_Windows_arm64.zip +- 2 下载kubectl http://kubernetes.p2hp.com/docs/tasks/tools/install-kubectl-windows.html +下载链接:https://dl.k8s.io/release/v1.25.0/bin/windows/amd64/kubectl.exe +- 3 下载配置文件以及.kube文件夹 +链接: https://pan.baidu.com/s/17Ilczj48XUy2uYWrajEUkw?pwd=q38q 提取码: q38q + +#### 二 配置修改 +- 1 将k9s、kubectl、.kube解压至C:\Users\Administrator 路径下 +- ![img_3.png](images/img_3.png) +- 2 将一.3下载的openubmc.txt中的内容,复制到.kube——config文件内(原有config文件内容全部清空) +- ![img.png](images/img_2025031005.png) +注:本文档是以openubmc为例编写,所以后续需要检视其他项目文件时,config文件的内容需要替换至相应项目的。 +- 3 百度搜索“查ip”,将公网ip,同步至运维(曹亚楠)加白 +- ![img_1.png](images/img_2025031004.png) + 4 添加系统环境变量,将C:\Users\Administrator ,添加至系统变量中 + ![img_2.png](images/img_2025031003.png) +三 文件权限检查 +1 打开cmd命令行工具 ,输入:k9s -n openubmc,配置成功后会打开k9s +![img_3.png](images/img_2025031002.png) + +2 选择需要检查的服务 按下enter 可选择vault日志和服务日志,再按下enter 可以查看日志 +![img_4.png](images/img_4.png) + +3 返回至k9s主界面,可以按S可以进入到容器内部,然后使用Linux命令,可查看文件权限。不同文件的权限,有不同的标准,具体需要参考团队/公司 对文件权限的管控标准,以下标准仅供参考 +文件类型 +设置值 + +用户主目录 +750(rwxr-x---) + +程序文件(含脚本文件、库文件等) +550(r-xr-x---) + +程序文件目录 +550(r-xr-x---) + +配置文件 +640(rw-r-----) + +配置文件目录 +750(rwxr-x---) + +日志文件(记录完毕或者已经归档) +440(r--r-----) + +日志文件(正在记录) +640(rw-r-----) + +日志文件目录 +750(rwxr-x---) + +Debug文件 +640(rw-r-----) + +Debug文件目录 +750(rwxr-x---) + +临时文件目录 +750(rwxr-x---) + +维护升级文件目录 +770(rwxrwx---) + +业务数据文件 +640(rw-r-----) + +业务数据文件目录 +750(rwxr-x---) + +密钥组件、私钥、证书、密文文件目录 +700(rwx------) + +密钥组件、私钥、证书、加密密文 +600(rw-------) + +加解密接口、加解密脚本 +500(r-x------) + + diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/gitleaks\346\211\253\346\217\217ECS\346\234\272\345\231\250Jenkins\346\227\245\345\277\227\346\226\207\344\273\266\346\225\217\346\204\237\344\277\241\346\201\257.md" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/gitleaks\346\211\253\346\217\217ECS\346\234\272\345\231\250Jenkins\346\227\245\345\277\227\346\226\207\344\273\266\346\225\217\346\204\237\344\277\241\346\201\257.md" new file mode 100644 index 0000000..d1aa98a --- /dev/null +++ "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/gitleaks\346\211\253\346\217\217ECS\346\234\272\345\231\250Jenkins\346\227\245\345\277\227\346\226\207\344\273\266\346\225\217\346\204\237\344\277\241\346\201\257.md" @@ -0,0 +1,65 @@ +### gitleaks扫描ECS机器Jenkins日志文件敏感信息 + +1. 准备环境 + 1. 找运维 申请临时ECS机器,需要扫描的磁盘备份, 并在安全组添加我们本地的IP。 +2. 扫描工具 gitleaks (github)---下载对应版本 + +![image-20250221141428136](images/image-20250221141428136.png) + +3.开发机登录ECS机器,并挂载数据磁盘 + + 先查到待挂载磁盘名称 lsblk + +![image-20250221142149678](images/image-20250221142149678.png) + +再新建一个挂载目录 /data (mkdir) + +再挂在磁盘 mount sdb /data +![img2025031001.png](images/img2025031001.png) +待扫描数据在 /data/jobs + +5.根目录新建 scan.sh +--- +```shell +#!/bin/bash + +# 定义根目录和结果目录 +ROOT_DIR="/data/jobs" +RESULT_DIR="/gitleaks" + +# 遍历 /data/jobs 下的所有子文件夹 +for dir in "$ROOT_DIR"/*/; do + # 去掉路径末尾的斜杠,获取文件夹名 + dir_name=$(basename "$dir") + if [ -f "$RESULT_DIR/result_${dir_name}.json" ];then + echo "文件已存在:${dir_name}" + continue + else + echo "$RESULT_DIR/result_${dir_name}.json" + echo "文件不存在:${dir_name}" + fi + + # 进入子文件夹 + cd "$dir" || { echo "无法进入目录: $dir"; continue; } + + # 执行 gitleaks 命令,结果保存到 /gitleaks/result(文件夹名).json + result_file="$RESULT_DIR/result_${dir_name}.json" + /gitleaks/gitleaks dir ./ --report-path "$result_file" + + # 打印日志 + if [ $? -eq 0 ]; then + echo "已完成: $dir -> $result_file" + else + echo "失败: $dir" + fi +done +``` + +6.执行扫描脚本 sh xx.sh + +7.扫描结果回传到开发机 + +8.结果分析 回传内网 内网分发整改 + + + diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/image-20250221141428136.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/image-20250221141428136.png" new file mode 100644 index 0000000..85490c4 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/image-20250221141428136.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/image-20250221142149678.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/image-20250221142149678.png" new file mode 100644 index 0000000..3f71448 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/image-20250221142149678.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img.png" new file mode 100644 index 0000000..0d4bea2 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img2025031001.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img2025031001.png" new file mode 100644 index 0000000..170251d Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img2025031001.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_1.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_1.png" new file mode 100644 index 0000000..24a906d Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_1.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_10.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_10.png" new file mode 100644 index 0000000..cc15653 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_10.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_11.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_11.png" new file mode 100644 index 0000000..a44b3c7 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_11.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_12.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_12.png" new file mode 100644 index 0000000..954c6e7 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_12.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_13.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_13.png" new file mode 100644 index 0000000..aef7897 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_13.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2.png" new file mode 100644 index 0000000..4ada475 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031002.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031002.png" new file mode 100644 index 0000000..64cbb75 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031002.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031003.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031003.png" new file mode 100644 index 0000000..07039f0 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031003.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031004.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031004.png" new file mode 100644 index 0000000..1d3e52b Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031004.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031005.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031005.png" new file mode 100644 index 0000000..b73c94b Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_2025031005.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_3.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_3.png" new file mode 100644 index 0000000..e145817 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_3.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_4.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_4.png" new file mode 100644 index 0000000..128c65f Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_4.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_5.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_5.png" new file mode 100644 index 0000000..5ae5bd0 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_5.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_6.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_6.png" new file mode 100644 index 0000000..af8f261 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_6.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_7.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_7.png" new file mode 100644 index 0000000..ecc5ba9 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_7.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_8.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_8.png" new file mode 100644 index 0000000..7df12a5 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_8.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_9.png" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_9.png" new file mode 100644 index 0000000..c7ea476 Binary files /dev/null and "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/images/img_9.png" differ diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/\345\215\216\344\270\272\344\272\221CodeArts\346\265\201\346\260\264\347\272\277\344\270\216SecGuard.md" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/\345\215\216\344\270\272\344\272\221CodeArts\346\265\201\346\260\264\347\272\277\344\270\216SecGuard.md" new file mode 100644 index 0000000..466927e --- /dev/null +++ "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/\345\215\216\344\270\272\344\272\221CodeArts\346\265\201\346\260\264\347\272\277\344\270\216SecGuard.md" @@ -0,0 +1,40 @@ + +#### 1 Seninfo 主要用于扫描日志文件 +把日志文件通过(邮件/git仓库等方式)在内网获取到之后 +通过Seninfo上传开扫即可 +#### 2 SecBinaryCheck +二进制包扫描工具 主要用来扫描开源漏洞 +#### 3 Virus Scan +病毒扫描工具 勾选全部的病毒库 上传包来扫描即可 +#### 4 BAS +是其中作用比较大的,集成了大量的脚本 + +#### 5 codearts流水线 +- (1)进入CICD示例 +![img_7.png](images/img_7.png) +- (2)点击编辑 +![img_5.png](images/img_5.png) +- (3)将待扫描的仓库和分支信息同步过去 +(3.1)新建服务扩展点 +![img_6.png](images/img_6.png) +(3.2)编辑扩展点信息(暂不支持gitcode) +连接名称自定义,一般为仓库名称 +用户名为gitee或github用户名 +access token为设置的access token(可在deepseek查询如何设置access token) +![img_8.png](images/img_8.png) +- 4 配置任务信息 +(4.1)漏洞扫描 +![img_9.png](images/img_9.png) +(4.2)敏感信息扫描 根据需要扫描的代码仓所属平台进行选择 +![img_10.png](images/img_10.png) +(4.3)安全编码扫描 +![img_11.png](images/img_11.png) +- (5) 任务执行配置 +(5.1) 点击保存并执行 +![img_12.png](images/img_12.png) +(5.2)执行配置编辑 + +repo为待扫描的仓库名称 +owner为待扫描的仓库的组织 +最后点击执行即可 +![img_13.png](images/img_13.png) \ No newline at end of file diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/\345\215\216\344\270\272\344\272\221\347\216\257\345\242\203vault\351\205\215\347\275\256\346\213\211\345\217\226\345\222\214\346\211\253\346\217\217.md" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/\345\215\216\344\270\272\344\272\221\347\216\257\345\242\203vault\351\205\215\347\275\256\346\213\211\345\217\226\345\222\214\346\211\253\346\217\217.md" new file mode 100644 index 0000000..3491ea8 --- /dev/null +++ "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/\345\215\216\344\270\272\344\272\221\347\216\257\345\242\203vault\351\205\215\347\275\256\346\213\211\345\217\226\345\222\214\346\211\253\346\217\217.md" @@ -0,0 +1,20 @@ +### 一、数据拉取 +#### 1.环境准备 + +- 准备kubectl环境 略 +- 登录CCE k8s 集群 下载kube 配置文件 + +![img.png](images/img.png) + +- 安全组增加自己的EIP +![img_1.png](images/img_1.png) + +- 完成端口映射 即可本地访问vault + +` .\kubectl.exe port-forward vault-0 12345:8200 -n vault --kubeconfig .\.kube\config` + +![img_2.png](images/img_2.png) + +### 二、 跑拉取任务完成数据拉取 + +### 三、 识别安全问题 弱密码 等安全配置问题 \ No newline at end of file diff --git "a/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/\345\256\271\345\231\250(\344\270\273\346\234\272)\346\274\217\346\264\236\346\243\200\346\265\213.md" "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/\345\256\271\345\231\250(\344\270\273\346\234\272)\346\274\217\346\264\236\346\243\200\346\265\213.md" new file mode 100644 index 0000000..2b8c5cb --- /dev/null +++ "b/\345\256\211\345\205\250\346\265\213\350\257\225\346\226\207\346\241\243/\345\256\271\345\231\250(\344\270\273\346\234\272)\346\274\217\346\264\236\346\243\200\346\265\213.md" @@ -0,0 +1,43 @@ +#### 一 准备工作: +需要与开发、运维沟通、协助,通过docker desktop搭建基本环境启动容器,并获得容器id + +#### 二 进入容器 +在cmd输入命令:docker exec -it -u root 容器id /bin/bash + +#### 三 执行检查 +##### 1 history检查: +- (1)进入容器后输入任意命令(如ll,ls)后,按键盘上下键,如果有历史记录,就说明有history X +- (2)直接输入history +##### 2 umask文件掩码检查 +- (1) 输入 umask 若输出为0027 则说明是符合标准的 +说明:从第二位起为027,用最高权限777减去027,得到750,表示在该容器内创建出的任和文件最大的权限只能是750,即:rwxr_x___ +##### 3 存疑组件/调试工具检查以及工具安装 +- (1)在二中进入的容器内输入: +yum install iputils -y --setopt=sslverify=false &yum install net-tools -y -setopt=sslverify=false &yum install openssl -y --setopt=sslverify=false &yum install file -y --setopt=sslverify=false +- (2)存在质疑组件/调试工具检查 +链接: +在“开源安全保姆”->“计算安全”->“计算生态型应用TOP问题专项排查指导”-> 6 +find xxxxxxxx +##### 4 ReDos检测工具 +内部工具 redoskiller +##### 5 服务通信是否启用TLS/HTTPS +内部工具--- C-spider +##### 6 防病毒软件扫描 +内部工具或者华为云流水线 +##### 7 检查是否存在已知不安全协议 +代码白盒搜索:TFTP,FTP ,Teinet ,SSL2.0 ,SSL3.0,TLS1.0 ,TLS1 ,SNMP v1/v2 ,SSHv1.x等 +##### 8 检查日志过期天数为30天 +cat /etc/login.defs |grep MAX_DAY + +##### 9 SSH服务安装/密钥生成/守护启动 +- ① 安装ssh服务:yum install openssh-server -y +- ② 生成ssh密钥:cd /etc/ssh/ & ssh-keygen-A +- ③ 以守护进程的方式启动SSH服务: /usr/sbin/sshd -D & +- ④ 配置ROOT 密码 echo "root:xxxxx" | chpasswd + +##### 10 主机漏洞扫描 + 内部工具 GSM 需要先把容器启动起来 +- (1) 配置资产 ->secguard->资产->新建 (几乎每次都要新建) +- (2) 点击配置好的资产,复制页面下方 curl开头的命令 +- (3) 在容器中执行复制的命令 +- (4) 新建任务->执行