Skip to content

init ec2 scanner #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jsbroks merged 7 commits into from
Apr 17, 2025
Merged

init ec2 scanner #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
30e99d1
init ec2 scanner
jsbroks Apr 14, 2025
8beca8c
Merge branch 'main' into ec2-scanner
jsbroks Apr 15, 2025
2e4fc0c
follow compute v1 format
jsbroks Apr 15, 2025
2ce7fca
add tag annotaitons
jsbroks Apr 15, 2025
a39b6d8
clean up conditions
jsbroks Apr 15, 2025
9598765
clean up
jsbroks Apr 15, 2025
0eb6b11
Update cmd/ctrlc/root/sync/aws/ec2/ec2.go
jsbroks Apr 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
273 changes: 273 additions & 0 deletions cmd/ctrlc/root/sync/aws/ec2/ec2.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,273 @@
package ec2

import (
"context"
"encoding/json"
"fmt"
"os"
"strconv"
"strings"
"time"

"github.com/MakeNowJust/heredoc/v2"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/charmbracelet/log"
"github.com/ctrlplanedev/cli/internal/api"
"github.com/ctrlplanedev/cli/internal/cliutil"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)

type ConnectionMethod struct {
Type string `json:"type"`
Region string `json:"region"`
InstanceID string `json:"instanceId"`
AccountID string `json:"accountId"`
}

type EC2Instance struct {
ID string `json:"id"`
Name string `json:"name"`
ConnectionMethod ConnectionMethod `json:"connectionMethod"`
}

func (t *EC2Instance) Struct() map[string]interface{} {
b, _ := json.Marshal(t)
var m map[string]interface{}
json.Unmarshal(b, &m)
return m
}

func NewSyncEC2Cmd() *cobra.Command {
var region string
var name string
cmd := &cobra.Command{
Use: "aws-ec2",
Short: "Sync AWS EC2 instances into Ctrlplane",
Example: heredoc.Doc(`
# Make sure AWS credentials are configured via environment variables or ~/.aws/credentials

# Sync all EC2 instances from a region
$ ctrlc sync ec2 --region us-west-2 --workspace 2a7c5560-75c9-4dbe-be74-04ee33bf8188
`),
PreRunE: func(cmd *cobra.Command, args []string) error {
if region == "" {
return fmt.Errorf("region is required")
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
log.Info("Syncing EC2 instances into Ctrlplane", "config-region", region)

ctx := context.Background()
cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region))
if err != nil {
return fmt.Errorf("failed to load AWS config: %w", err)
}

credentials, err := cfg.Credentials.Retrieve(ctx)
if err != nil {
return fmt.Errorf("failed to retrieve AWS credentials: %w", err)
}

log.Info("AWS credentials loaded successfully",
"provider", credentials.Source,
"region", region,
"access_key_id", credentials.AccessKeyID[:4]+"****",
"expiration", credentials.Expires,
"type", credentials.Source,
"profile", os.Getenv("AWS_PROFILE"),
)

// Create EC2 client with retry options
ec2Client := ec2.NewFromConfig(cfg, func(o *ec2.Options) {
o.RetryMaxAttempts = 3
o.RetryMode = aws.RetryModeStandard
})

apiURL := viper.GetString("url")
apiKey := viper.GetString("api-key")
workspaceId := viper.GetString("workspace")

Comment on lines +90 to +93
Copy link
Contributor

@coderabbitai coderabbitai bot Apr 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Add validation for required configuration values

There's no validation that apiURL, apiKey, and workspaceId are actually set. This could lead to cryptic errors when making API calls.

apiURL := viper.GetString("url")
apiKey := viper.GetString("api-key")
workspaceId := viper.GetString("workspace")

+if apiURL == "" || apiKey == "" || workspaceId == "" {
+	return fmt.Errorf("missing required configuration: url, api-key, or workspace")
+}
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
apiURL := viper.GetString("url")
apiKey := viper.GetString("api-key")
workspaceId := viper.GetString("workspace")
apiURL := viper.GetString("url")
apiKey := viper.GetString("api-key")
workspaceId := viper.GetString("workspace")
if apiURL == "" || apiKey == "" || workspaceId == "" {
return fmt.Errorf("missing required configuration: url, api-key, or workspace")
}

// Get EC2 instances
result, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{})
if err != nil {
return fmt.Errorf("failed to describe instances: %w", err)
}
Comment on lines +95 to +98
Copy link
Contributor

@coderabbitai coderabbitai bot Apr 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Implement pagination for DescribeInstances

The AWS EC2 API uses pagination for responses with many items. The current implementation fetches only the first page of results, which may lead to incomplete data if there are many instances.

// Get EC2 instances
-result, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{})
-if err != nil {
-	return fmt.Errorf("failed to describe instances: %w", err)
-}
+var resources []api.AgentResource
+var nextToken *string
+
+for {
+	result, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{
+		NextToken: nextToken,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to describe instances: %w", err)
+	}
+
+	// Process instances (existing code from lines 101-241)
+	// ...
+
+	// Check if there are more results
+	if result.NextToken == nil {
+		break
+	}
+	nextToken = result.NextToken
+}
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
result, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{})
if err != nil {
return fmt.Errorf("failed to describe instances: %w", err)
}
var resources []api.AgentResource
var nextToken *string
for {
result, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{
NextToken: nextToken,
})
if err != nil {
return fmt.Errorf("failed to describe instances: %w", err)
}
// Process instances (existing code from lines 101-241)
// ...
// Check if there are more results
if result.NextToken == nil {
break
}
nextToken = result.NextToken
}


resources := []api.AgentResource{}
for _, reservation := range result.Reservations {
accountId := *reservation.OwnerId
for _, instance := range reservation.Instances {
tags := make(map[string]string)

for _, tag := range instance.Tags {
tags[*tag.Key] = *tag.Value
}

// Get instance name from tags
name := tags["Name"]
if name == "" {
name = *instance.InstanceId
}

// Get EC2 region from instance availability zone
region := ""
if instance.Placement != nil && instance.Placement.AvailabilityZone != nil {
// Region is AZ without the last character
region = (*instance.Placement.AvailabilityZone)[:len(*instance.Placement.AvailabilityZone)-1]
}

instanceData := EC2Instance{
ID: *instance.InstanceId,
Name: name,
ConnectionMethod: ConnectionMethod{
Type: "aws",
Region: region,
InstanceID: *instance.InstanceId,
AccountID: accountId,
},
}

// Add AWS Console URL for the instance
consoleUrl := fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s",
region,
region,
*instance.InstanceId)

metadata := make(map[string]string)
for _, tag := range instance.Tags {
if tag.Key != nil && tag.Value != nil {
metadata[*tag.Key] = *tag.Value
metadata["compute/tag/"+*tag.Key] = *tag.Value
metadata["aws/tag/"+*tag.Key] = *tag.Value
}
}

metadata["compute/machine-type"] = string(instance.InstanceType)
metadata["compute/region"] = region
metadata["compute/type"] = "standard"
metadata["compute/architecture"] = strings.ReplaceAll(string(instance.Architecture), "_mac", "")
metadata["compute/boot-mode"] = string(instance.BootMode)

if instance.PlatformDetails != nil {
metadata["compute/platform"] = *instance.PlatformDetails
}

if instance.CpuOptions != nil && instance.CpuOptions.CoreCount != nil {
metadata["compute/cpu-cores"] = strconv.Itoa(int(*instance.CpuOptions.CoreCount))
if instance.CpuOptions.ThreadsPerCore != nil {
metadata["compute/cpu-threads-per-core"] = strconv.Itoa(int(*instance.CpuOptions.ThreadsPerCore))
metadata["compute/cpu-threads"] = strconv.Itoa(int(*instance.CpuOptions.ThreadsPerCore) * int(*instance.CpuOptions.CoreCount))
}
}
metadata["compute/hypervisor"] = string(instance.Hypervisor)

if instance.State != nil {
metadata["compute/state"] = string(instance.State.Name)
}

if instance.LaunchTime != nil {
metadata["compute/launch-time"] = instance.LaunchTime.Format(time.RFC3339)
}

if instance.PrivateIpAddress != nil {
metadata["network/private-ip"] = *instance.PrivateIpAddress
}

if instance.PublicIpAddress != nil {
metadata["network/public-ip"] = *instance.PublicIpAddress
}

if instance.PrivateDnsName != nil {
metadata["network/private-dns"] = *instance.PrivateDnsName
}

if instance.PublicDnsName != nil {
metadata["network/public-dns"] = *instance.PublicDnsName
}

metadata["aws/account-id"] = accountId
metadata["aws/region"] = region

if instance.VpcId != nil {
metadata["aws/vpc-id"] = *instance.VpcId
}
if instance.PlatformDetails != nil {
metadata["aws/platform-details"] = string(*instance.PlatformDetails)
}
if instance.InstanceId != nil {
metadata["aws/instance-id"] = *instance.InstanceId
}

if instance.ImageId != nil {
metadata["aws/ami-id"] = *instance.ImageId
}
if instance.AmiLaunchIndex != nil {
metadata["aws/ami-launch-index"] = strconv.Itoa(int(*instance.AmiLaunchIndex))
}

if instance.KeyName != nil {
metadata["aws/key-name"] = *instance.KeyName
}

if instance.EbsOptimized != nil {
metadata["aws/ebs-optimized"] = strconv.FormatBool(*instance.EbsOptimized)
}

if instance.EnaSupport != nil {
metadata["aws/ena-support"] = strconv.FormatBool(*instance.EnaSupport)
}

if instance.SubnetId != nil {
metadata["aws/subnet-id"] = *instance.SubnetId
}

metadata["ctrlplane/links"] = fmt.Sprintf("{ \"AWS Console\": \"%s\" }", consoleUrl)

// Get ARN for the instance
arn := fmt.Sprintf("arn:aws:ec2:%s:%s:instance/%s", region, accountId, *instance.InstanceId)
resources = append(resources, api.AgentResource{
Version: "compute/v1",
Kind: "Instance",
Name: name,
Identifier: arn,
Config: instanceData.Struct(),
Metadata: metadata,
})
}
}

// Create or update resource provider
if name == "" {
name = fmt.Sprintf("aws-ec2-region-%s", region)
}

ctrlplaneClient, err := api.NewAPIKeyClientWithResponses(apiURL, apiKey)
if err != nil {
return fmt.Errorf("failed to create API client: %w", err)
}

rp, err := api.NewResourceProvider(ctrlplaneClient, workspaceId, name)
if err != nil {
return fmt.Errorf("failed to create resource provider: %w", err)
}

upsertResp, err := rp.UpsertResource(ctx, resources)
log.Info("Response from upserting resources", "status", upsertResp.Status)
if err != nil {
return fmt.Errorf("failed to upsert resources: %w", err)
}
Comment on lines +258 to +262
Copy link
Contributor

@coderabbitai coderabbitai bot Apr 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Enhance error handling for API response

The code logs the response status but doesn't check if the status indicates an error. Consider adding more detailed response validation.

upsertResp, err := rp.UpsertResource(ctx, resources)
log.Info("Response from upserting resources", "status", upsertResp.Status)
if err != nil {
	return fmt.Errorf("failed to upsert resources: %w", err)
}
+
+// Check for non-success status codes
+if upsertResp.Status >= 400 {
+	return fmt.Errorf("failed to upsert resources: unexpected status code %d", upsertResp.Status)
+}
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
upsertResp, err := rp.UpsertResource(ctx, resources)
log.Info("Response from upserting resources", "status", upsertResp.Status)
if err != nil {
return fmt.Errorf("failed to upsert resources: %w", err)
}
upsertResp, err := rp.UpsertResource(ctx, resources)
log.Info("Response from upserting resources", "status", upsertResp.Status)
if err != nil {
return fmt.Errorf("failed to upsert resources: %w", err)
}
// Check for non-success status codes
if upsertResp.Status >= 400 {
return fmt.Errorf("failed to upsert resources: unexpected status code %d", upsertResp.Status)
}


return cliutil.HandleResponseOutput(cmd, upsertResp)
},
}

cmd.Flags().StringVarP(&name, "provider", "p", "", "Name of the resource provider")
cmd.Flags().StringVarP(&region, "region", "c", "", "AWS Region")
cmd.MarkFlagRequired("region")

return cmd
}
2 changes: 2 additions & 0 deletions cmd/ctrlc/root/sync/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package sync

import (
"github.com/MakeNowJust/heredoc/v2"
"github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/aws/ec2"
"github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/clickhouse"
"github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/tailscale"
"github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/terraform"
Expand All @@ -27,6 +28,7 @@ func NewSyncCmd() *cobra.Command {
cmd.AddCommand(cliutil.AddIntervalSupport(terraform.NewSyncTerraformCmd(), ""))
cmd.AddCommand(cliutil.AddIntervalSupport(tailscale.NewSyncTailscaleCmd(), ""))
cmd.AddCommand(cliutil.AddIntervalSupport(clickhouse.NewSyncClickhouseCmd(), ""))
cmd.AddCommand(cliutil.AddIntervalSupport(ec2.NewSyncEC2Cmd(), ""))

return cmd
}
14 changes: 14 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,20 @@ require (
require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect
github.com/aws/aws-sdk-go-v2/config v1.29.14 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.67 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
github.com/aws/aws-sdk-go-v2/service/ec2 v1.211.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
github.com/aws/smithy-go v1.22.2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/charmbracelet/lipgloss v0.10.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
Expand Down
28 changes: 28 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,34 @@ github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7D
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM=
github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g=
github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM=
github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.211.3 h1:4dPHqFVVvFG+ntkVUXrMrY55+E5dzFfEpjFWdkdSxnc=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.211.3/go.mod h1:ouvGEfHbLaIlWwpDpOVWPWR+YwO0HDv3vm5tYLq8ImY=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8=
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
Expand Down