From 7a0d5124d7decf1b20f14ebd0c0376395347100f Mon Sep 17 00:00:00 2001 From: jonathan meeks Date: Fri, 9 May 2025 19:14:42 -0500 Subject: [PATCH 1/7] initial work --- cmd/ctrlc/root/sync/aws/aws.go | 2 + cmd/ctrlc/root/sync/aws/networks/networks.go | 473 ++++++++++++++++++ .../root/sync/google/networks/networks.go | 4 +- 3 files changed, 478 insertions(+), 1 deletion(-) create mode 100644 cmd/ctrlc/root/sync/aws/networks/networks.go diff --git a/cmd/ctrlc/root/sync/aws/aws.go b/cmd/ctrlc/root/sync/aws/aws.go index 499eb01..b82c45c 100644 --- a/cmd/ctrlc/root/sync/aws/aws.go +++ b/cmd/ctrlc/root/sync/aws/aws.go @@ -4,6 +4,7 @@ import ( "github.com/MakeNowJust/heredoc/v2" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/aws/ec2" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/aws/eks" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/aws/networks" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/aws/rds" "github.com/ctrlplanedev/cli/internal/cliutil" "github.com/spf13/cobra" @@ -33,5 +34,6 @@ func NewAWSCmd() *cobra.Command { cmd.AddCommand(cliutil.AddIntervalSupport(ec2.NewSyncEC2Cmd(), "")) cmd.AddCommand(cliutil.AddIntervalSupport(eks.NewSyncEKSCmd(), "")) cmd.AddCommand(cliutil.AddIntervalSupport(rds.NewSyncRDSCmd(), "")) + cmd.AddCommand(cliutil.AddIntervalSupport(networks.NewSyncNetworksCmd(), "")) return cmd } diff --git a/cmd/ctrlc/root/sync/aws/networks/networks.go b/cmd/ctrlc/root/sync/aws/networks/networks.go new file mode 100644 index 0000000..530998c --- /dev/null +++ b/cmd/ctrlc/root/sync/aws/networks/networks.go @@ -0,0 +1,473 @@ +package networks + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "os" + "strconv" + "strings" + "time" + + "github.com/MakeNowJust/heredoc/v2" + "github.com/charmbracelet/log" + "github.com/ctrlplanedev/cli/internal/api" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// NewSyncNetworksCmd creates a new cobra command for syncing Google Networks +func NewSyncNetworksCmd() *cobra.Command { + var region string + var name string + + cmd := &cobra.Command{ + Use: "networks", + Short: "Sync AWS VPC networks and subnets into Ctrlplane", + Example: heredoc.Doc(` + # Make sure AWS credentials are configured via environment variables or application default credentials + + # Sync all VPC networks and subnets from a project + $ ctrlc sync aws networks --project my-project + `), + PreRunE: validateFlags(®ion), + RunE: runSync(®ion, &name), + } + + // Add command flags + cmd.Flags().StringVarP(&name, "provider", "p", "", "Name of the resource provider") + cmd.Flags().StringVarP(®ion, "region", "r", "", "AWS Region") + cmd.MarkFlagRequired("region") + + return cmd +} + +// validateFlags ensures required flags are set +func validateFlags(region *string) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + if *region == "" { + return fmt.Errorf("region is required") + } + return nil + } +} + +// runSync contains the main sync logic +func runSync(region *string, name *string) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + log.Info("Syncing Google Network resources into Ctrlplane", "region", *region) + + ctx := context.Background() + + apiURL := viper.GetString("url") + apiKey := viper.GetString("api-key") + workspaceId := viper.GetString("workspace") + + // Initialize compute client + ec2Client, err := initComputeClient(ctx, *region) + if err != nil { + return err + } + + // List and process networks + networkResources, err := processNetworks(ctx, ec2Client, *region) + if err != nil { + return err + } + + // List and process subnets + subnetResources, err := processSubnets(ctx, ec2Client, *region) + if err != nil { + return err + } + + // Combine all resources + resources := append(networkResources, subnetResources...) + // Upsert resources to Ctrlplane + return upsertToCtrlplane(ctx, resources, region, name) + + } +} + +// initComputeClient creates a new Compute Engine client +func initComputeClient(ctx context.Context, region string) (*ec2.Client, error) { + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + + credentials, err := cfg.Credentials.Retrieve(ctx) + if err != nil { + return nil, fmt.Errorf("failed to retrieve AWS credentials: %w", err) + } + + log.Info("AWS credentials loaded successfully", + "provider", credentials.Source, + "region", region, + "access_key_id", credentials.AccessKeyID[:4]+"****", + "expiration", credentials.Expires, + "type", credentials.Source, + "profile", os.Getenv("AWS_PROFILE"), + ) + + // Create EC2 client with retry options + ec2Client := ec2.NewFromConfig(cfg, func(o *ec2.Options) { + o.RetryMaxAttempts = 3 + o.RetryMode = aws.RetryModeStandard + }) + + return ec2Client, nil +} + +// processNetworks lists and processes all VPCs and subnets +func processNetworks(ctx context.Context, ec2Client *ec2.Client, project string) ([]api.AgentResource, error) { + var nextToken *string + vpcs := make([]types.Vpc, 0) + + for { + output, err := ec2Client.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{ + NextToken: nextToken, + }) + if err != nil { + return nil, fmt.Errorf("failed to list VPCs: %w", err) + } + + vpcs = append(vpcs, output.Vpcs...) + if output.NextToken == nil { + break + } + nextToken = output.NextToken + } + + log.Info("Found vpcOutput", "count", len(vpcs)) + + resources := []api.AgentResource{} + for _, vpc := range vpcs { + subnets, err := getSubnetsForVpc(ctx, ec2Client, *vpc.VpcId) + if err != nil { + log.Error("Failed to get subnets for VPC", "vpcId", *vpc.VpcId) + continue + } + subnetResources := processSubnets(ctx, ec2Client, *vpc.VpcId) + resource, err := processNetwork(vpc, project) + if err != nil { + log.Error("Failed to process vpc", "name", vpc.Name, "error", err) + continue + } + resources = append(resources, resource) + } + + return resources, nil +} + +// processNetwork handles processing of a single VPC network +func processNetwork(vpc *types.Vpc, region string) (api.AgentResource, error) { + metadata := initNetworkMetadata(vpc, region, subnetCount) + + // Build console URL + consoleUrl := fmt.Sprintf( + "https://%s.console.aws.amazon.com/vpcconsole/home?region=%s#VpcDetails:VpcId=%s", + region, *vpc.VpcId, *vpc.VpcId) + metadata["ctrlplane/links"] = fmt.Sprintf("{ \"AWS Console\": \"%s\" }", consoleUrl) + + // Determine subnet mode + subnetMode := "CUSTOM" + if vpc.AutoCreateSubnetworks { + subnetMode = "AUTO" + } + + // Create peering info for metadata + if vpc.Peerings != nil { + for i, peering := range vpc.Peerings { + metadata[fmt.Sprintf("vpc/peering/%d/name", i)] = peering.Name + metadata[fmt.Sprintf("vpc/peering/%d/vpc", i)] = getResourceName(peering.Network) + metadata[fmt.Sprintf("vpc/peering/%d/state", i)] = peering.State + metadata[fmt.Sprintf("vpc/peering/%d/auto-create-routes", i)] = strconv.FormatBool(peering.AutoCreateRoutes) + } + metadata["vpc/peering-count"] = strconv.Itoa(len(vpc.Peerings)) + } + + return api.AgentResource{ + Version: "ctrlplane.dev/vpc/v1", + Kind: "GoogleNetwork", + Name: vpc.Name, + Identifier: vpc.SelfLink, + Config: map[string]any{ + // Common cross-provider options + "name": vpc.Name, + "type": "vpc", + "id": strconv.FormatUint(vpc.Id, 10), + "mtu": vpc.Mtu, + + // Provider-specific implementation details + "googleNetwork": map[string]any{ + "project": project, + "selfLink": vpc.SelfLink, + "subnetMode": subnetMode, + "autoCreateSubnets": vpc.AutoCreateSubnetworks, + "subnetCount": subnetCount, + "routingMode": vpc.RoutingConfig.RoutingMode, + }, + }, + Metadata: metadata, + }, nil +} + +// initNetworkMetadata initializes the base metadata for a network +func initNetworkMetadata(vpc *types.Vpc, project string, subnetCount int) map[string]string { + var vpcName = vpc.VpcId // default to VPC ID + + metadata := map[string]string{ + "vpc/type": "vpc", + "vpc/name": *vpc.VpcId, + "vpc/subnet-mode": subnetMode, + "vpc/subnet-count": strconv.Itoa(subnetCount), + "vpc/id": strconv.FormatUint(vpc.Id, 10), + "vpc/mtu": strconv.FormatInt(vpc.Mtu, 10), + + "google/self-link": vpc.SelfLink, + "google/project": project, + "google/resource-type": "compute.googleapis.com/Network", + "google/console-url": consoleUrl, + "google/id": strconv.FormatUint(vpc.Id, 10), + } + + // Add creation timestamp + if vpc.CreationTimestamp != "" { + creationTime, err := time.Parse(time.RFC3339, vpc.CreationTimestamp) + if err == nil { + metadata["vpc/created"] = creationTime.Format(time.RFC3339) + } else { + metadata["vpc/created"] = vpc.CreationTimestamp + } + } + + // Add routing configuration + if vpc.RoutingConfig != nil && vpc.RoutingConfig.RoutingMode != "" { + metadata["vpc/routing-mode"] = vpc.RoutingConfig.RoutingMode + } + + return metadata +} + +// getSubnetsForVpc retrieves subnets as AWS SDK objects +// these objects are processed differently for VPC and subnet resources +func getRawAwsSubnets(ctx context.Context, ec2Client *ec2.Client, region string) ([]types.Subnet, error) { + var subnets []types.Subnet + var nextToken *string + + for { + subnetInput := &ec2.DescribeSubnetsInput{ + Filters: []types.Filter{ + { + Name: aws.String("region"), + Values: []string{region}, + }, + }, + NextToken: nextToken, + } + + subnetsOutput, err := ec2Client.DescribeSubnets(ctx, subnetInput) + if err != nil { + return nil, fmt.Errorf("failed to list subnets at region %s: %w", region, err) + } + + subnets = append(subnets, subnetsOutput.Subnets...) + if subnetsOutput.NextToken == nil { + break + } + nextToken = subnetsOutput.NextToken + } + + return subnets, nil +} + +// processSubnets lists and processes all subnetworks +func processSubnets(_ context.Context, subnets []types.Subnet, region string) ([]api.AgentResource, error) { + resources := []api.AgentResource{} + subnetCount := 0 + + // Process subnets from all regions + for subnet := range subnets { + resource, err := processSubnet(subnet, region) + if err != nil { + log.Error("Failed to process subnet", "name", subnet.Name, "error", err) + continue + } + resources = append(resources, resource) + subnetCount++ + } + + log.Info("Processed subnets", "count", subnetCount) + return resources, nil +} + +// processSubnet handles processing of a single subnet +func processSubnet(subnet types.Subnet, region string) (api.AgentResource, error) { + metadata := initSubnetMetadata(subnet, region) + + // Build console URL + consoleUrl := fmt.Sprintf("https://console.cloud.google.com/networking/subnetworks/details/%s/%s?project=%s", + region, subnet.Name, project) + metadata["ctrlplane/links"] = fmt.Sprintf("{ \"Google Cloud Console\": \"%s\" }", consoleUrl) + + // Extract network name from self link + networkName := getResourceName(subnet.Network) + + return api.AgentResource{ + Version: "ctrlplane.dev/network/subnet/v1", + Kind: "GoogleSubnet", + Name: subnet.Name, + Identifier: subnet.SelfLink, + Config: map[string]any{ + // Common cross-provider options + "name": subnet.Name, + "provider": "google", + "type": "subnet", + "cidr": subnet.IpCidrRange, + "region": region, + "id": strconv.FormatUint(subnet.Id, 10), + "gateway": subnet.GatewayAddress, + "networkName": networkName, + + // Provider-specific implementation details + "googleSubnet": map[string]any{ + "project": project, + "purpose": subnet.Purpose, + "role": subnet.Role, + "privateIpGoogleAccess": subnet.PrivateIpGoogleAccess, + "network": subnet.Network, + "stackType": subnet.StackType, + "ipv6AccessType": subnet.Ipv6AccessType, + "enableFlowLogs": subnet.EnableFlowLogs, + "secondaryIpRanges": subnet.SecondaryIpRanges, + }, + }, + Metadata: metadata, + }, nil +} + +// initSubnetMetadata initializes the base metadata for a subnet +func initSubnetMetadata(subnet *compute.Subnetwork, project string, region string) map[string]string { + consoleUrl := fmt.Sprintf("https://console.cloud.google.com/networking/subnetworks/details/%s/%s?project=%s", + region, subnet.Name, project) + + // Extract network name from self link + networkName := getResourceName(subnet.Network) + + metadata := map[string]string{ + "network/type": "subnet", + "network/name": subnet.Name, + "network/vpc": networkName, + "network/region": region, + "network/cidr": subnet.IpCidrRange, + "network/gateway": subnet.GatewayAddress, + "network/private-access": strconv.FormatBool(subnet.PrivateIpGoogleAccess), + + "google/project": project, + "google/resource-type": "compute.googleapis.com/Subnetwork", + "google/console-url": consoleUrl, + "google/region": region, + "google/id": strconv.FormatUint(subnet.Id, 10), + } + + // Add creation timestamp + if subnet.CreationTimestamp != "" { + creationTime, err := time.Parse(time.RFC3339, subnet.CreationTimestamp) + if err == nil { + metadata["network/created"] = creationTime.Format(time.RFC3339) + } else { + metadata["network/created"] = subnet.CreationTimestamp + } + } + + // Add purpose and role if set + if subnet.Purpose != "" { + metadata["network/purpose"] = subnet.Purpose + if subnet.Role != "" { + metadata["network/role"] = subnet.Role + } + } + + // Add secondary IP ranges if present + if subnet.SecondaryIpRanges != nil { + for i, secondaryRange := range subnet.SecondaryIpRanges { + metadata[fmt.Sprintf("network/secondary-range/%d/name", i)] = secondaryRange.RangeName + metadata[fmt.Sprintf("network/secondary-range/%d/cidr", i)] = secondaryRange.IpCidrRange + } + metadata["network/secondary-range-count"] = strconv.Itoa(len(subnet.SecondaryIpRanges)) + } + + // Add IP version details + if subnet.StackType != "" { + metadata["network/stack-type"] = subnet.StackType + } + if subnet.Ipv6AccessType != "" { + metadata["network/ipv6-access-type"] = subnet.Ipv6AccessType + } + if subnet.InternalIpv6Prefix != "" { + metadata["network/internal-ipv6-prefix"] = subnet.InternalIpv6Prefix + } + if subnet.ExternalIpv6Prefix != "" { + metadata["network/external-ipv6-prefix"] = subnet.ExternalIpv6Prefix + } + + // Add flow logs status + if subnet.EnableFlowLogs { + metadata["network/flow-logs"] = "enabled" + } else { + metadata["network/flow-logs"] = "disabled" + } + + return metadata +} + +// getRegionFromURL extracts the region name from a URL like "regions/us-central1" +func getRegionFromURL(regionURL string) string { + parts := strings.Split(regionURL, "/") + if len(parts) >= 2 { + return parts[1] + } + return regionURL +} + +// getResourceName extracts the resource name from its full path +func getResourceName(fullPath string) string { + if fullPath == "" { + return "" + } + parts := strings.Split(fullPath, "/") + return parts[len(parts)-1] +} + +// upsertToCtrlplane handles upserting resources to Ctrlplane +func upsertToCtrlplane(ctx context.Context, resources []api.AgentResource, project, name *string) error { + if *name == "" { + *name = fmt.Sprintf("google-networks-project-%s", *project) + } + + apiURL := viper.GetString("url") + apiKey := viper.GetString("api-key") + workspaceId := viper.GetString("workspace") + + ctrlplaneClient, err := api.NewAPIKeyClientWithResponses(apiURL, apiKey) + if err != nil { + return fmt.Errorf("failed to create API client: %w", err) + } + + rp, err := api.NewResourceProvider(ctrlplaneClient, workspaceId, *name) + if err != nil { + return fmt.Errorf("failed to create resource provider: %w", err) + } + + upsertResp, err := rp.UpsertResource(ctx, resources) + if err != nil { + return fmt.Errorf("failed to upsert resources: %w", err) + } + + log.Info("Response from upserting resources", "status", upsertResp.Status) + return nil +} diff --git a/cmd/ctrlc/root/sync/google/networks/networks.go b/cmd/ctrlc/root/sync/google/networks/networks.go index 2151965..eb1ab74 100644 --- a/cmd/ctrlc/root/sync/google/networks/networks.go +++ b/cmd/ctrlc/root/sync/google/networks/networks.go @@ -8,11 +8,13 @@ import ( "time" "github.com/MakeNowJust/heredoc/v2" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/charmbracelet/log" "github.com/ctrlplanedev/cli/internal/api" "github.com/spf13/cobra" "github.com/spf13/viper" - "google.golang.org/api/compute/v1" ) // NewSyncNetworksCmd creates a new cobra command for syncing Google Networks From 431150cf2d7fad8766f939fba15fd1d4d13ed20d Mon Sep 17 00:00:00 2001 From: jonathan meeks Date: Sun, 11 May 2025 10:59:20 -0500 Subject: [PATCH 2/7] fix bad google/networks change --- cmd/ctrlc/root/sync/google/networks/networks.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/ctrlc/root/sync/google/networks/networks.go b/cmd/ctrlc/root/sync/google/networks/networks.go index 9ca628a..fff9612 100644 --- a/cmd/ctrlc/root/sync/google/networks/networks.go +++ b/cmd/ctrlc/root/sync/google/networks/networks.go @@ -8,13 +8,11 @@ import ( "time" "github.com/MakeNowJust/heredoc/v2" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/charmbracelet/log" "github.com/ctrlplanedev/cli/internal/api" "github.com/spf13/cobra" "github.com/spf13/viper" + "google.golang.org/api/compute/v1" ) // NewSyncNetworksCmd creates a new cobra command for syncing Google Networks From cef2738dd13ac9e657b8c58c1a14872408fdb967 Mon Sep 17 00:00:00 2001 From: jonathan meeks Date: Mon, 12 May 2025 20:36:52 -0500 Subject: [PATCH 3/7] more aws/az network sync, wip --- cmd/ctrlc/root/sync/aws/common/provider.go | 39 ++ cmd/ctrlc/root/sync/aws/eks/eks.go | 47 +- cmd/ctrlc/root/sync/aws/networks/networks.go | 448 ++++++++---------- .../root/sync/azure/common/resourceGroup.go | 42 ++ .../root/sync/azure/networks/networks.go | 363 ++++++++++++++ go.mod | 25 +- go.sum | 21 +- 7 files changed, 690 insertions(+), 295 deletions(-) create mode 100644 cmd/ctrlc/root/sync/aws/common/provider.go create mode 100644 cmd/ctrlc/root/sync/azure/common/resourceGroup.go create mode 100644 cmd/ctrlc/root/sync/azure/networks/networks.go diff --git a/cmd/ctrlc/root/sync/aws/common/provider.go b/cmd/ctrlc/root/sync/aws/common/provider.go new file mode 100644 index 0000000..24db906 --- /dev/null +++ b/cmd/ctrlc/root/sync/aws/common/provider.go @@ -0,0 +1,39 @@ +package common + +import ( + "context" + "fmt" + "github.com/charmbracelet/log" + "strings" +) + +// ComputeProviderDetails generates a provider name and region string based on the provided parameters. +// The +func ComputeProviderDetails( + ctx context.Context, prefix string, regions []string, name *string, +) { + providerRegion := "all-regions" + // Use regions for name if none provided + if regions != nil && len(regions) > 0 { + providerRegion = strings.Join(regions, "-") + } + + // If name is not provided, try to get account ID to include in the provider name + if *name == "" { + // Get AWS account ID for provider name using common package + cfg, err := InitAWSConfig(ctx, regions[0]) + if err != nil { + log.Warn("Failed to load AWS config for account ID retrieval", "error", err) + *name = fmt.Sprintf("%s-%s", prefix, providerRegion) + } else { + accountID, err := GetAccountID(ctx, cfg) + if err == nil { + log.Info("Retrieved AWS account ID", "account_id", accountID) + *name = fmt.Sprintf("%s-%s-%s", prefix, accountID, providerRegion) + } else { + log.Warn("Failed to get AWS account ID", "error", err) + *name = fmt.Sprintf("%s-%s", prefix, providerRegion) + } + } + } +} diff --git a/cmd/ctrlc/root/sync/aws/eks/eks.go b/cmd/ctrlc/root/sync/aws/eks/eks.go index 2e95c20..930cabf 100644 --- a/cmd/ctrlc/root/sync/aws/eks/eks.go +++ b/cmd/ctrlc/root/sync/aws/eks/eks.go @@ -113,33 +113,10 @@ func runSync(regions *[]string, name *string) func(cmd *cobra.Command, args []st return nil } - // Use regions for name if none provided - providerRegion := "all-regions" - if regions != nil && len(*regions) > 0 { - providerRegion = strings.Join(*regions, "-") - } - - // If name is not provided, try to get account ID to include in the provider name - if *name == "" { - // Get AWS account ID for provider name using common package - cfg, err := common.InitAWSConfig(ctx, regionsToSync[0]) - if err != nil { - log.Warn("Failed to load AWS config for account ID retrieval", "error", err) - *name = fmt.Sprintf("aws-eks-%s", providerRegion) - } else { - accountID, err := common.GetAccountID(ctx, cfg) - if err == nil { - log.Info("Retrieved AWS account ID", "account_id", accountID) - *name = fmt.Sprintf("aws-eks-%s-%s", accountID, providerRegion) - } else { - log.Warn("Failed to get AWS account ID", "error", err) - *name = fmt.Sprintf("aws-eks-%s", providerRegion) - } - } - } + common.ComputeProviderDetails(ctx, "aws-eks", regionsToSync, name) // Upsert resources to Ctrlplane - return upsertToCtrlplane(ctx, allResources, &providerRegion, name) + return upsertToCtrlplane(ctx, allResources, name) } } @@ -239,18 +216,18 @@ func initClusterMetadata(cluster *types.Cluster, region string) map[string]strin log.Error("Failed to parse Kubernetes version", "version", *cluster.Version, "error", err) } - noramlizedStatus := "unknown" + normalizedStatus := "unknown" switch cluster.Status { case types.ClusterStatusActive: - noramlizedStatus = "running" + normalizedStatus = "running" case types.ClusterStatusUpdating: - noramlizedStatus = "updating" + normalizedStatus = "updating" case types.ClusterStatusCreating: - noramlizedStatus = "creating" + normalizedStatus = "creating" case types.ClusterStatusDeleting: - noramlizedStatus = "deleting" + normalizedStatus = "deleting" case types.ClusterStatusFailed: - noramlizedStatus = "failed" + normalizedStatus = "failed" } metadata := map[string]string{ @@ -264,7 +241,7 @@ func initClusterMetadata(cluster *types.Cluster, region string) map[string]strin kinds.K8SMetadataVersionMinor: strconv.FormatUint(uint64(version.Minor()), 10), kinds.K8SMetadataVersionPatch: strconv.FormatUint(uint64(version.Patch()), 10), kinds.K8SMetadataVersionPrerelease: version.Prerelease(), - kinds.K8SMetadataStatus: noramlizedStatus, + kinds.K8SMetadataStatus: normalizedStatus, "aws/region": region, "aws/resource-type": "eks:cluster", @@ -320,11 +297,7 @@ var relationshipRules = []api.CreateResourceRelationshipRule{ }, } -func upsertToCtrlplane(ctx context.Context, resources []api.CreateResource, region, name *string) error { - if *name == "" { - *name = fmt.Sprintf("aws-eks-%s", *region) - } - +func upsertToCtrlplane(ctx context.Context, resources []api.CreateResource, name *string) error { apiURL := viper.GetString("url") apiKey := viper.GetString("api-key") workspaceId := viper.GetString("workspace") diff --git a/cmd/ctrlc/root/sync/aws/networks/networks.go b/cmd/ctrlc/root/sync/aws/networks/networks.go index 530998c..7f93298 100644 --- a/cmd/ctrlc/root/sync/aws/networks/networks.go +++ b/cmd/ctrlc/root/sync/aws/networks/networks.go @@ -3,26 +3,25 @@ package networks import ( "context" "fmt" + "github.com/MakeNowJust/heredoc/v2" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "os" - "strconv" - "strings" - "time" - - "github.com/MakeNowJust/heredoc/v2" "github.com/charmbracelet/log" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/aws/common" "github.com/ctrlplanedev/cli/internal/api" "github.com/spf13/cobra" "github.com/spf13/viper" + "os" + "strconv" + "sync" ) // NewSyncNetworksCmd creates a new cobra command for syncing Google Networks func NewSyncNetworksCmd() *cobra.Command { - var region string var name string + var regions []string cmd := &cobra.Command{ Use: "networks", @@ -30,78 +29,126 @@ func NewSyncNetworksCmd() *cobra.Command { Example: heredoc.Doc(` # Make sure AWS credentials are configured via environment variables or application default credentials - # Sync all VPC networks and subnets from a project - $ ctrlc sync aws networks --project my-project + # Sync all VPC networks and subnets from a region + $ ctrlc sync aws networks --region my-region `), - PreRunE: validateFlags(®ion), - RunE: runSync(®ion, &name), + RunE: runSync(®ions, &name), } // Add command flags cmd.Flags().StringVarP(&name, "provider", "p", "", "Name of the resource provider") - cmd.Flags().StringVarP(®ion, "region", "r", "", "AWS Region") - cmd.MarkFlagRequired("region") + cmd.Flags().StringSliceVarP(®ions, "region", "r", []string{}, "AWS Region(s)") return cmd } -// validateFlags ensures required flags are set -func validateFlags(region *string) func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, args []string) error { - if *region == "" { - return fmt.Errorf("region is required") - } - return nil - } -} - // runSync contains the main sync logic -func runSync(region *string, name *string) func(cmd *cobra.Command, args []string) error { +func runSync(regions *[]string, name *string) func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { - log.Info("Syncing Google Network resources into Ctrlplane", "region", *region) - ctx := context.Background() - apiURL := viper.GetString("url") - apiKey := viper.GetString("api-key") - workspaceId := viper.GetString("workspace") - - // Initialize compute client - ec2Client, err := initComputeClient(ctx, *region) + // Get the regions to sync from using common package + regionsToSync, err := common.GetRegions(ctx, *regions) if err != nil { return err } - // List and process networks - networkResources, err := processNetworks(ctx, ec2Client, *region) - if err != nil { - return err + allResources := make([]api.CreateResource, 0) + + var mu sync.Mutex + var wg sync.WaitGroup + var syncErrors []error + + for _, region := range regionsToSync { + wg.Add(1) + go func(regionName string) { + defer wg.Done() + log.Info("Syncing AWS Network resources into Ctrlplane", "region", regionName) + + ctx := context.Background() + + //apiURL := viper.GetString("url") + //apiKey := viper.GetString("api-key") + //workspaceId := viper.GetString("workspace") + + // Initialize compute client + ec2Client, cfg, err := initComputeClient(ctx, regionName) + if err != nil { + log.Error("Failed to initialize EC2 client", "region", regionName, "error", err) + mu.Lock() + syncErrors = append(syncErrors, fmt.Errorf("region %s: %w", regionName, err)) + mu.Unlock() + return + } + + accountId, err := common.GetAccountID(ctx, cfg) + if err != nil { + log.Error("Failed get accountId", "region", regionName, "error", err) + mu.Lock() + syncErrors = append(syncErrors, fmt.Errorf("region %s: %w", regionName, err)) + mu.Unlock() + return + } + + awsSubnets, err := getAwsSubnets(ctx, ec2Client, regionName) + if err != nil { + log.Error("Failed to get subnets", "region", regionName, "error", err) + mu.Lock() + syncErrors = append(syncErrors, fmt.Errorf("region %s: %w", regionName, err)) + mu.Unlock() + return + } + + // List and process networks + vpcResources, err := processNetworks(ctx, ec2Client, awsSubnets, regionName, accountId) + if err != nil { + log.Error("Failed to process VPCs", "region", regionName, "error", err) + mu.Lock() + syncErrors = append(syncErrors, fmt.Errorf("region %s: %w", regionName, err)) + mu.Unlock() + return + } + + // List and process subnets + subnetResources, err := processSubnets(ctx, awsSubnets, regionName) + if err != nil { + log.Error("Failed to process subnets", "region", regionName, "error", err) + mu.Lock() + syncErrors = append(syncErrors, fmt.Errorf("region %s: %w", regionName, err)) + mu.Unlock() + return + } + + if len(vpcResources) > 0 { + mu.Lock() + allResources = append(allResources, vpcResources...) + mu.Unlock() + } + if len(subnetResources) > 0 { + mu.Lock() + allResources = append(allResources, subnetResources...) + mu.Unlock() + } + }(region) } - // List and process subnets - subnetResources, err := processSubnets(ctx, ec2Client, *region) - if err != nil { - return err - } + common.ComputeProviderDetails(ctx, "aws-networks", regionsToSync, name) - // Combine all resources - resources := append(networkResources, subnetResources...) // Upsert resources to Ctrlplane - return upsertToCtrlplane(ctx, resources, region, name) - + return upsertToCtrlplane(ctx, allResources, name) } } // initComputeClient creates a new Compute Engine client -func initComputeClient(ctx context.Context, region string) (*ec2.Client, error) { +func initComputeClient(ctx context.Context, region string) (*ec2.Client, aws.Config, error) { cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) if err != nil { - return nil, fmt.Errorf("failed to load AWS config: %w", err) + return nil, cfg, fmt.Errorf("failed to load AWS config: %w", err) } credentials, err := cfg.Credentials.Retrieve(ctx) if err != nil { - return nil, fmt.Errorf("failed to retrieve AWS credentials: %w", err) + return nil, cfg, fmt.Errorf("failed to retrieve AWS credentials: %w", err) } log.Info("AWS credentials loaded successfully", @@ -119,13 +166,26 @@ func initComputeClient(ctx context.Context, region string) (*ec2.Client, error) o.RetryMode = aws.RetryModeStandard }) - return ec2Client, nil + return ec2Client, cfg, nil } // processNetworks lists and processes all VPCs and subnets -func processNetworks(ctx context.Context, ec2Client *ec2.Client, project string) ([]api.AgentResource, error) { +func processNetworks( + ctx context.Context, ec2Client *ec2.Client, awsSubnets []types.Subnet, region string, accountId string, +) ([]api.CreateResource, error) { var nextToken *string vpcs := make([]types.Vpc, 0) + subnetsByVpc := make(map[string][]types.Subnet) + var awsVpcSubnets []types.Subnet + var exists bool + + for _, sn := range awsSubnets { + vpcId := *sn.VpcId + if _, exists = subnetsByVpc[vpcId]; !exists { + subnetsByVpc[vpcId] = []types.Subnet{} + } + subnetsByVpc[vpcId] = append(subnetsByVpc[vpcId], sn) + } for { output, err := ec2Client.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{ @@ -144,17 +204,14 @@ func processNetworks(ctx context.Context, ec2Client *ec2.Client, project string) log.Info("Found vpcOutput", "count", len(vpcs)) - resources := []api.AgentResource{} + resources := make([]api.CreateResource, 0) for _, vpc := range vpcs { - subnets, err := getSubnetsForVpc(ctx, ec2Client, *vpc.VpcId) - if err != nil { - log.Error("Failed to get subnets for VPC", "vpcId", *vpc.VpcId) - continue + if awsVpcSubnets, exists = subnetsByVpc[*vpc.VpcId]; !exists { + awsVpcSubnets = []types.Subnet{} } - subnetResources := processSubnets(ctx, ec2Client, *vpc.VpcId) - resource, err := processNetwork(vpc, project) + resource, err := processNetwork(vpc, awsVpcSubnets, region, accountId) if err != nil { - log.Error("Failed to process vpc", "name", vpc.Name, "error", err) + log.Error("Failed to process vpc", "vpcId", vpc.VpcId, "error", err) continue } resources = append(resources, resource) @@ -164,52 +221,33 @@ func processNetworks(ctx context.Context, ec2Client *ec2.Client, project string) } // processNetwork handles processing of a single VPC network -func processNetwork(vpc *types.Vpc, region string) (api.AgentResource, error) { - metadata := initNetworkMetadata(vpc, region, subnetCount) +func processNetwork( + vpc types.Vpc, subnets []types.Subnet, region string, accountId string, +) (api.CreateResource, error) { + metadata := initNetworkMetadata(vpc, region, len(subnets)) + vpcName := getVpcName(vpc) // Build console URL - consoleUrl := fmt.Sprintf( - "https://%s.console.aws.amazon.com/vpcconsole/home?region=%s#VpcDetails:VpcId=%s", - region, *vpc.VpcId, *vpc.VpcId) + consoleUrl := getVpcConsoleUrl(vpc, region) metadata["ctrlplane/links"] = fmt.Sprintf("{ \"AWS Console\": \"%s\" }", consoleUrl) - // Determine subnet mode - subnetMode := "CUSTOM" - if vpc.AutoCreateSubnetworks { - subnetMode = "AUTO" - } - - // Create peering info for metadata - if vpc.Peerings != nil { - for i, peering := range vpc.Peerings { - metadata[fmt.Sprintf("vpc/peering/%d/name", i)] = peering.Name - metadata[fmt.Sprintf("vpc/peering/%d/vpc", i)] = getResourceName(peering.Network) - metadata[fmt.Sprintf("vpc/peering/%d/state", i)] = peering.State - metadata[fmt.Sprintf("vpc/peering/%d/auto-create-routes", i)] = strconv.FormatBool(peering.AutoCreateRoutes) - } - metadata["vpc/peering-count"] = strconv.Itoa(len(vpc.Peerings)) - } - - return api.AgentResource{ + return api.CreateResource{ Version: "ctrlplane.dev/vpc/v1", - Kind: "GoogleNetwork", - Name: vpc.Name, - Identifier: vpc.SelfLink, + Kind: "AWSNetwork", + Name: vpcName, + Identifier: *vpc.VpcId, Config: map[string]any{ // Common cross-provider options - "name": vpc.Name, + "name": vpcName, "type": "vpc", - "id": strconv.FormatUint(vpc.Id, 10), - "mtu": vpc.Mtu, + "id": vpc.VpcId, // Provider-specific implementation details - "googleNetwork": map[string]any{ - "project": project, - "selfLink": vpc.SelfLink, - "subnetMode": subnetMode, - "autoCreateSubnets": vpc.AutoCreateSubnetworks, - "subnetCount": subnetCount, - "routingMode": vpc.RoutingConfig.RoutingMode, + "awsVpc": map[string]any{ + "accountId": accountId, + "region": region, + "state": string(vpc.State), + "subnetCount": len(subnets), }, }, Metadata: metadata, @@ -217,37 +255,22 @@ func processNetwork(vpc *types.Vpc, region string) (api.AgentResource, error) { } // initNetworkMetadata initializes the base metadata for a network -func initNetworkMetadata(vpc *types.Vpc, project string, subnetCount int) map[string]string { - var vpcName = vpc.VpcId // default to VPC ID +func initNetworkMetadata(vpc types.Vpc, region string, subnetCount int) map[string]string { + var vpcName = getVpcName(vpc) + var consoleUrl = getVpcConsoleUrl(vpc, region) metadata := map[string]string{ "vpc/type": "vpc", - "vpc/name": *vpc.VpcId, - "vpc/subnet-mode": subnetMode, + "vpc/name": vpcName, "vpc/subnet-count": strconv.Itoa(subnetCount), - "vpc/id": strconv.FormatUint(vpc.Id, 10), - "vpc/mtu": strconv.FormatInt(vpc.Mtu, 10), - - "google/self-link": vpc.SelfLink, - "google/project": project, - "google/resource-type": "compute.googleapis.com/Network", - "google/console-url": consoleUrl, - "google/id": strconv.FormatUint(vpc.Id, 10), - } - - // Add creation timestamp - if vpc.CreationTimestamp != "" { - creationTime, err := time.Parse(time.RFC3339, vpc.CreationTimestamp) - if err == nil { - metadata["vpc/created"] = creationTime.Format(time.RFC3339) - } else { - metadata["vpc/created"] = vpc.CreationTimestamp - } - } - - // Add routing configuration - if vpc.RoutingConfig != nil && vpc.RoutingConfig.RoutingMode != "" { - metadata["vpc/routing-mode"] = vpc.RoutingConfig.RoutingMode + "vpc/id": *vpc.VpcId, + "vpc/tenancy": string(vpc.InstanceTenancy), + + "aws/region": region, + "aws/resource-type": "vpc", + "aws/status": string(vpc.State), + "aws/console-url": consoleUrl, + "aws/id": *vpc.VpcId, } return metadata @@ -255,18 +278,13 @@ func initNetworkMetadata(vpc *types.Vpc, project string, subnetCount int) map[st // getSubnetsForVpc retrieves subnets as AWS SDK objects // these objects are processed differently for VPC and subnet resources -func getRawAwsSubnets(ctx context.Context, ec2Client *ec2.Client, region string) ([]types.Subnet, error) { +func getAwsSubnets(ctx context.Context, ec2Client *ec2.Client, region string) ([]types.Subnet, error) { var subnets []types.Subnet var nextToken *string for { subnetInput := &ec2.DescribeSubnetsInput{ - Filters: []types.Filter{ - { - Name: aws.String("region"), - Values: []string{region}, - }, - }, + Filters: []types.Filter{}, NextToken: nextToken, } @@ -286,15 +304,15 @@ func getRawAwsSubnets(ctx context.Context, ec2Client *ec2.Client, region string) } // processSubnets lists and processes all subnetworks -func processSubnets(_ context.Context, subnets []types.Subnet, region string) ([]api.AgentResource, error) { - resources := []api.AgentResource{} +func processSubnets(_ context.Context, subnets []types.Subnet, region string) ([]api.CreateResource, error) { + resources := make([]api.CreateResource, 0) subnetCount := 0 // Process subnets from all regions - for subnet := range subnets { + for _, subnet := range subnets { resource, err := processSubnet(subnet, region) if err != nil { - log.Error("Failed to process subnet", "name", subnet.Name, "error", err) + log.Error("Failed to process subnet", "subnetId", subnet.SubnetId, "error", err) continue } resources = append(resources, resource) @@ -306,149 +324,89 @@ func processSubnets(_ context.Context, subnets []types.Subnet, region string) ([ } // processSubnet handles processing of a single subnet -func processSubnet(subnet types.Subnet, region string) (api.AgentResource, error) { +func processSubnet(subnet types.Subnet, region string) (api.CreateResource, error) { metadata := initSubnetMetadata(subnet, region) + subnetName := getSubnetName(subnet) + consoleUrl := getSubnetConsoleUrl(subnet, region) + metadata["ctrlplane/links"] = fmt.Sprintf("{ \"AWS Console\": \"%s\" }", consoleUrl) - // Build console URL - consoleUrl := fmt.Sprintf("https://console.cloud.google.com/networking/subnetworks/details/%s/%s?project=%s", - region, subnet.Name, project) - metadata["ctrlplane/links"] = fmt.Sprintf("{ \"Google Cloud Console\": \"%s\" }", consoleUrl) - - // Extract network name from self link - networkName := getResourceName(subnet.Network) - - return api.AgentResource{ + return api.CreateResource{ Version: "ctrlplane.dev/network/subnet/v1", - Kind: "GoogleSubnet", - Name: subnet.Name, - Identifier: subnet.SelfLink, + Kind: "AWSSubnet", + Name: subnetName, + Identifier: *subnet.SubnetArn, Config: map[string]any{ // Common cross-provider options - "name": subnet.Name, - "provider": "google", - "type": "subnet", - "cidr": subnet.IpCidrRange, - "region": region, - "id": strconv.FormatUint(subnet.Id, 10), - "gateway": subnet.GatewayAddress, - "networkName": networkName, - - // Provider-specific implementation details - "googleSubnet": map[string]any{ - "project": project, - "purpose": subnet.Purpose, - "role": subnet.Role, - "privateIpGoogleAccess": subnet.PrivateIpGoogleAccess, - "network": subnet.Network, - "stackType": subnet.StackType, - "ipv6AccessType": subnet.Ipv6AccessType, - "enableFlowLogs": subnet.EnableFlowLogs, - "secondaryIpRanges": subnet.SecondaryIpRanges, - }, + "name": subnetName, + "provider": "aws", + "type": "subnet", + "cidr": subnet.CidrBlock, + "region": region, + "id": subnet.SubnetId, + "vpcId": subnet.VpcId, }, Metadata: metadata, }, nil } // initSubnetMetadata initializes the base metadata for a subnet -func initSubnetMetadata(subnet *compute.Subnetwork, project string, region string) map[string]string { - consoleUrl := fmt.Sprintf("https://console.cloud.google.com/networking/subnetworks/details/%s/%s?project=%s", - region, subnet.Name, project) - - // Extract network name from self link - networkName := getResourceName(subnet.Network) +func initSubnetMetadata(subnet types.Subnet, region string) map[string]string { + consoleUrl := getSubnetConsoleUrl(subnet, region) + subnetName := getSubnetName(subnet) metadata := map[string]string{ - "network/type": "subnet", - "network/name": subnet.Name, - "network/vpc": networkName, - "network/region": region, - "network/cidr": subnet.IpCidrRange, - "network/gateway": subnet.GatewayAddress, - "network/private-access": strconv.FormatBool(subnet.PrivateIpGoogleAccess), - - "google/project": project, + "network/type": "subnet", + "network/name": subnetName, + "network/vpc": *subnet.VpcId, + "network/region": region, + "network/cidr": *subnet.CidrBlock, + "network/block-public-access": string(subnet.BlockPublicAccessStates.InternetGatewayBlockMode), + "google/resource-type": "compute.googleapis.com/Subnetwork", "google/console-url": consoleUrl, "google/region": region, - "google/id": strconv.FormatUint(subnet.Id, 10), + "google/id": *subnet.SubnetId, } - // Add creation timestamp - if subnet.CreationTimestamp != "" { - creationTime, err := time.Parse(time.RFC3339, subnet.CreationTimestamp) - if err == nil { - metadata["network/created"] = creationTime.Format(time.RFC3339) - } else { - metadata["network/created"] = subnet.CreationTimestamp - } - } + return metadata +} - // Add purpose and role if set - if subnet.Purpose != "" { - metadata["network/purpose"] = subnet.Purpose - if subnet.Role != "" { - metadata["network/role"] = subnet.Role - } - } +func getVpcConsoleUrl(vpc types.Vpc, region string) string { + return fmt.Sprintf( + "https://%s.console.aws.amazon.com/vpcconsole/home?region=%s#VpcDetails:VpcId=%s", + region, region, *vpc.VpcId) +} - // Add secondary IP ranges if present - if subnet.SecondaryIpRanges != nil { - for i, secondaryRange := range subnet.SecondaryIpRanges { - metadata[fmt.Sprintf("network/secondary-range/%d/name", i)] = secondaryRange.RangeName - metadata[fmt.Sprintf("network/secondary-range/%d/cidr", i)] = secondaryRange.IpCidrRange +func getVpcName(vpc types.Vpc) string { + vpcName := *vpc.VpcId + for _, tag := range vpc.Tags { + if *tag.Key == "Name" { + vpcName = *tag.Value + break } - metadata["network/secondary-range-count"] = strconv.Itoa(len(subnet.SecondaryIpRanges)) - } - - // Add IP version details - if subnet.StackType != "" { - metadata["network/stack-type"] = subnet.StackType } - if subnet.Ipv6AccessType != "" { - metadata["network/ipv6-access-type"] = subnet.Ipv6AccessType - } - if subnet.InternalIpv6Prefix != "" { - metadata["network/internal-ipv6-prefix"] = subnet.InternalIpv6Prefix - } - if subnet.ExternalIpv6Prefix != "" { - metadata["network/external-ipv6-prefix"] = subnet.ExternalIpv6Prefix - } - - // Add flow logs status - if subnet.EnableFlowLogs { - metadata["network/flow-logs"] = "enabled" - } else { - metadata["network/flow-logs"] = "disabled" - } - - return metadata + return vpcName } -// getRegionFromURL extracts the region name from a URL like "regions/us-central1" -func getRegionFromURL(regionURL string) string { - parts := strings.Split(regionURL, "/") - if len(parts) >= 2 { - return parts[1] - } - return regionURL +func getSubnetConsoleUrl(subnet types.Subnet, region string) string { + return fmt.Sprintf( + "https://%s.console.aws.amazon.com/vpcconsole/home?region=%s#SubnetDetails:subnetId=%s", + region, region, *subnet.VpcId) } -// getResourceName extracts the resource name from its full path -func getResourceName(fullPath string) string { - if fullPath == "" { - return "" +func getSubnetName(subnet types.Subnet) string { + subnetName := *subnet.VpcId + for _, tag := range subnet.Tags { + if *tag.Key == "Name" { + subnetName = *tag.Value + break + } } - parts := strings.Split(fullPath, "/") - return parts[len(parts)-1] + return subnetName } // upsertToCtrlplane handles upserting resources to Ctrlplane -func upsertToCtrlplane(ctx context.Context, resources []api.AgentResource, project, name *string) error { - if *name == "" { - *name = fmt.Sprintf("google-networks-project-%s", *project) - } - +func upsertToCtrlplane(ctx context.Context, resources []api.CreateResource, name *string) error { apiURL := viper.GetString("url") apiKey := viper.GetString("api-key") workspaceId := viper.GetString("workspace") diff --git a/cmd/ctrlc/root/sync/azure/common/resourceGroup.go b/cmd/ctrlc/root/sync/azure/common/resourceGroup.go new file mode 100644 index 0000000..4baa8a7 --- /dev/null +++ b/cmd/ctrlc/root/sync/azure/common/resourceGroup.go @@ -0,0 +1,42 @@ +package common + +import ( + "context" + "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" +) + +type ResourceGroupInfo struct { + Name string + Location string +} + +func GetResourceGroupInfo( + ctx context.Context, cred azcore.TokenCredential, subscriptionID string, +) ([]ResourceGroupInfo, error) { + + results := make([]ResourceGroupInfo, 0) + + client, err := armresources.NewResourceGroupsClient(subscriptionID, cred, nil) + if err != nil { + return nil, fmt.Errorf("failed to create Resource Group client: %w", err) + } + pager := client.NewListPager(nil) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get next page of resource groups: %v\n", err) + } + + // Iterate through the resource groups in the current page + for _, rg := range page.Value { + results = append(results, ResourceGroupInfo{ + Name: *rg.Name, + Location: *rg.Location, + }) + } + } + + return results, nil +} diff --git a/cmd/ctrlc/root/sync/azure/networks/networks.go b/cmd/ctrlc/root/sync/azure/networks/networks.go new file mode 100644 index 0000000..ada636f --- /dev/null +++ b/cmd/ctrlc/root/sync/azure/networks/networks.go @@ -0,0 +1,363 @@ +package aks + +import ( + "context" + "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription" + "github.com/MakeNowJust/heredoc/v2" + "github.com/charmbracelet/log" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/azure/common" + "github.com/ctrlplanedev/cli/internal/api" + "github.com/ctrlplanedev/cli/internal/kinds" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "os" + "sync" +) + +// NewSyncAKSCmd creates a new cobra command for syncing AKS clusters +func NewSyncAKSCmd() *cobra.Command { + var subscriptionID string + var name string + + cmd := &cobra.Command{ + Use: "aks", + Short: "Sync Azure Kubernetes Service networks into Ctrlplane", + Example: heredoc.Doc(` + # Make sure Azure credentials are configured via environment variables or Azure CLI + + # Sync all AKS VPCs and subnets from the subscription + $ ctrlc sync azure networks + + # Sync all AKS VPCs and subnets from a specific subscription + $ ctrlc sync azure networks --subscription-id 00000000-0000-0000-0000-000000000000 + + # Sync all AKS VPCs and subnets every 5 minutes + $ ctrlc sync azure networks --interval 5m + `), + RunE: runSync(&subscriptionID, &name), + } + + cmd.Flags().StringVarP(&name, "provider", "p", "", "Name of the resource provider") + cmd.Flags().StringVarP(&subscriptionID, "subscription-id", "s", "", "Azure Subscription ID") + + return cmd +} + +func runSync(subscriptionID, name *string) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + + // Initialize Azure credential from environment or CLI + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain Azure credential: %w", err) + } + + // If subscription ID is not provided, get the default one + if *subscriptionID == "" { + defaultSubscriptionID, err := getDefaultSubscriptionID(ctx, cred) + if err != nil { + return fmt.Errorf("failed to get default subscription ID: %w", err) + } + *subscriptionID = defaultSubscriptionID + log.Info("Using default subscription ID", "subscriptionID", *subscriptionID) + } + + // Get tenant ID from the subscription + tenantID, err := getTenantIDFromSubscription(ctx, cred, *subscriptionID) + if err != nil { + log.Warn("Failed to get tenant ID from subscription, falling back to environment variables", "error", err) + tenantID = getTenantIDFromEnv() + } + + log.Info("Syncing all AKS clusters", "subscriptionID", *subscriptionID, "tenantID", tenantID) + + resources, err := processNetworks(ctx, cred, *subscriptionID, tenantID) + if err != nil { + return err + } + + if len(resources) == 0 { + log.Info("No AKS clusters found") + return nil + } + + // If name is not provided, use subscription ID + if *name == "" { + *name = fmt.Sprintf("azure-aks-%s", *subscriptionID) + } + + // Upsert resources to Ctrlplane + return upsertToCtrlplane(ctx, resources, subscriptionID, name) + } +} + +func getTenantIDFromSubscription(ctx context.Context, cred azcore.TokenCredential, subscriptionID string) (string, error) { + // Create a subscriptions client + subsClient, err := armsubscriptions.NewClient(cred, nil) + if err != nil { + return "", fmt.Errorf("failed to create subscriptions client: %w", err) + } + + // Get the subscription details + resp, err := subsClient.Get(ctx, subscriptionID, nil) + if err != nil { + return "", fmt.Errorf("failed to get subscription details: %w", err) + } + + // Extract tenant ID from subscription + if resp.TenantID == nil || *resp.TenantID == "" { + return "", fmt.Errorf("subscription doesn't have a tenant ID") + } + + return *resp.TenantID, nil +} + +func getTenantIDFromEnv() string { + // Check environment variables + if tenantID := os.Getenv("AZURE_TENANT_ID"); tenantID != "" { + return tenantID + } + + // Check viper config + if tenantID := viper.GetString("azure.tenant-id"); tenantID != "" { + return tenantID + } + + return "" +} + +func getDefaultSubscriptionID(ctx context.Context, cred azcore.TokenCredential) (string, error) { + subClient, err := armsubscription.NewSubscriptionsClient(cred, nil) + if err != nil { + return "", fmt.Errorf("failed to create subscription client: %w", err) + } + + pager := subClient.NewListPager(nil) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return "", fmt.Errorf("failed to list subscriptions: %w", err) + } + + // Return the first subscription as default + if len(page.Value) > 0 && page.Value[0].SubscriptionID != nil { + return *page.Value[0].SubscriptionID, nil + } + } + + return "", fmt.Errorf("no subscriptions found") +} + +func processNetworks( + ctx context.Context, cred azcore.TokenCredential, subscriptionID string, tenantID string, +) ([]api.CreateResource, error) { + var resources []api.CreateResource + var resourceGroups []common.ResourceGroupInfo + var mu sync.Mutex + var wg sync.WaitGroup + var err error + var syncErrors []error + + if resourceGroups, err = common.GetResourceGroupInfo(ctx, cred, subscriptionID); err != nil { + return nil, err + } + + // Create virtual network client + client, err := armnetwork.NewVirtualNetworksClient(subscriptionID, cred, nil) + if err != nil { + return nil, fmt.Errorf("failed to create Virtual Network client: %w", err) + } + + for _, rg := range resourceGroups { + wg.Add(1) + go func(resourceGroup string) { + defer wg.Done() + + pager := client.NewListPager(rg.Name, nil) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + mu.Lock() + syncErrors = append(syncErrors, fmt.Errorf("failed to list AKS clusters: %w", err)) + mu.Unlock() + } + for _, network := range page.Value { + resource, err := processNetwork(ctx, network, resourceGroup, subscriptionID, tenantID) + if err != nil { + log.Error("Failed to process AKS cluster", "name", *network.Name, "error", err) + mu.Lock() + syncErrors = append(syncErrors, fmt.Errorf("cluster %s: %w", *network.Name, err)) + mu.Unlock() + return + } + mu.Lock() + resources = append(resources, resource) + mu.Unlock() + } + } + }(rg.Name) + } + + wg.Wait() + + if len(syncErrors) > 0 { + log.Warn("Some clusters failed to sync", "errors", len(syncErrors)) + // Continue with the clusters that succeeded + } + + log.Info("Found AKS clusters", "count", len(resources)) + return resources, nil +} + +func processNetwork( + _ context.Context, network *armnetwork.VirtualNetwork, resourceGroup string, subscriptionID string, tenantID string, +) ([]api.CreateResource, error) { + resources := make([]api.CreateResource, 0) + networkName := network.Name + metadata := initNetworkMetadata(network, subscriptionID, resourceGroup, tenantID) + + // Build console URL + consoleUrl := getVirtualNetworkConsoleUrl(subscriptionID, resourceGroup, *networkName) + metadata[kinds.CtrlplaneMetadataLinks] = fmt.Sprintf("{ \"Azure Portal\": \"%s\" }", consoleUrl) + + resources = append(resources, api.CreateResource{ + Version: "ctrlplane.dev/vpc/v1", + Kind: "AzureNetwork", + Name: *networkName, + Identifier: *network.ID, + Config: map[string]any{ + // Common cross-provider options + "name": networkName, + "type": "vpc", + "id": network.ID, + + // Provider-specific implementation details + "azureVirtualNetwork": map[string]any{ + "type": network.Type, + "region": network.Location, + "state": network.Properties.ProvisioningState, + "subnetCount": len(network.Properties.Subnets), + }, + }, + Metadata: metadata, + }) + for _, subnet := range network.Properties.Subnets { + if res, err := processSubnet(networkName, subnet, resourceGroup, subscriptionID, tenantID); err != nil { + return nil, err + } else { + resources = append(resources, res) + } + } +} + +func processSubnet( + networkName *string, subnet *armnetwork.Subnet, resourceGroup string, subscriptionID string, tenantID string, +) (api.CreateResource, error) { + +} + +func initNetworkMetadata( + network *armnetwork.VirtualNetwork, subscriptionID, resourceGroup string, tenantID string, +) map[string]string { + + metadata := map[string]string{ + "azure/subscription": subscriptionID, + "azure/tenant": tenantID, + "azure/resource-group": resourceGroup, + "azure/resource-type": "Microsoft.Network/virtualNetworks/subnets", + "azure/location": *network.Location, + "azure/status": string(*network.Properties.ProvisioningState), + "azure/id": *network.ID, + "azure/console-url": getVirtualNetworkConsoleUrl(subscriptionID, resourceGroup, *network.Name), + } + + // Tags + if network.Tags != nil { + for key, value := range network.Tags { + if value != nil { + metadata[fmt.Sprintf("tags/%s", key)] = *value + } + } + } + + return metadata +} + +func initSubnetMetadata( + network *armnetwork.VirtualNetwork, subnet *armnetwork.Subnet, subscriptionID, resourceGroup string, tenantID string, +) map[string]string { + + metadata := map[string]string{ + "azure/subscription": subscriptionID, + "azure/tenant": tenantID, + "azure/resource-group": resourceGroup, + "azure/resource-type": "Microsoft.Network/virtualNetworks/subnets", + "azure/location": *network.Location, + "azure/status": string(*subnet.Properties.ProvisioningState), + "azure/id": *subnet.ID, + "azure/console-url": getSubnetConsoleUrl(subscriptionID, resourceGroup, *network.Name), + } + + // Tags + if network.Tags != nil { + for key, value := range network.Tags { + if value != nil { + metadata[fmt.Sprintf("tags/%s", key)] = *value + } + } + } + + return metadata +} + +func getVirtualNetworkConsoleUrl(subscriptionID, resourceGroup, networkName string) string { + return fmt.Sprintf( + "https://portal.azure.com/#@/resource/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s", + subscriptionID, + resourceGroup, + networkName, + ) +} + +func getSubnetConsoleUrl(subscriptionID, resourceGroup, networkName string) string { + return fmt.Sprintf( + "https://portal.azure.com/#@/resource/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets", + subscriptionID, + resourceGroup, + networkName, + ) +} + +func upsertToCtrlplane(ctx context.Context, resources []api.CreateResource, subscriptionID, name *string) error { + if *name == "" { + *name = fmt.Sprintf("azure-aks-%s", *subscriptionID) + } + + apiURL := viper.GetString("url") + apiKey := viper.GetString("api-key") + workspaceId := viper.GetString("workspace") + + ctrlplaneClient, err := api.NewAPIKeyClientWithResponses(apiURL, apiKey) + if err != nil { + return fmt.Errorf("failed to create API client: %w", err) + } + + rp, err := api.NewResourceProvider(ctrlplaneClient, workspaceId, *name) + if err != nil { + return fmt.Errorf("failed to create resource provider: %w", err) + } + + upsertResp, err := rp.UpsertResource(ctx, resources) + if err != nil { + return fmt.Errorf("failed to upsert resources: %w", err) + } + + log.Info("Response from upserting resources", "status", upsertResp.Status) + return nil +} diff --git a/go.mod b/go.mod index c7bd946..36d0841 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,25 @@ module github.com/ctrlplanedev/cli go 1.24.2 require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.2.0 github.com/MakeNowJust/heredoc/v2 v2.0.1 github.com/Masterminds/semver v1.5.0 github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.14 + github.com/aws/aws-sdk-go-v2/credentials v1.17.67 github.com/aws/aws-sdk-go-v2/service/ec2 v1.211.3 + github.com/aws/aws-sdk-go-v2/service/eks v1.64.0 + github.com/aws/aws-sdk-go-v2/service/rds v1.95.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 github.com/charmbracelet/log v0.4.0 github.com/creack/pty v1.1.24 + github.com/google/go-github/v57 v57.0.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/hashicorp/go-tfe v1.73.1 @@ -20,8 +31,10 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/tailscale/tailscale-client-go/v2 v2.0.0-20241217012816-8143c7dc1766 + golang.org/x/oauth2 v0.29.0 google.golang.org/api v0.230.0 gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.32.1 k8s.io/apimachinery v0.32.1 k8s.io/client-go v0.32.1 @@ -31,27 +44,18 @@ require ( cloud.google.com/go/auth v0.16.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.6.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.2.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.67 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/service/eks v1.64.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect - github.com/aws/aws-sdk-go-v2/service/rds v1.95.0 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect github.com/aws/smithy-go v1.22.2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/charmbracelet/lipgloss v0.10.0 // indirect @@ -73,7 +77,6 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/go-github/v57 v57.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.9 // indirect @@ -130,7 +133,6 @@ require ( golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.39.0 // indirect - golang.org/x/oauth2 v0.29.0 // indirect golang.org/x/sync v0.13.0 // indirect golang.org/x/sys v0.32.0 // indirect golang.org/x/term v0.31.0 // indirect @@ -144,7 +146,6 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect diff --git a/go.sum b/go.sum index e38ff60..8995603 100644 --- a/go.sum +++ b/go.sum @@ -8,16 +8,28 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 h1:figxyQZXzZQIcP3njhC68bYUiTw45J8/SsHaLW8Ax0M= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0/go.mod h1:TmlMW4W5OvXOmOyKNnor8nlMMiO1ctIyzmHme/VHsrA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0 h1:lMW1lD/17LUA5z1XTURo7LcVG2ICBPlyMHjIUrcFZNQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0/go.mod h1:ceIuwmxDWptoW3eCqSXlnPsZFKh4X+R38dWPv7GS9Vs= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 h1:QM6sE5k2ZT/vI5BEe0r7mqjsUSnhVBFbOsVkEuaEfiA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0/go.mod h1:243D9iHbcQXoFUtgHJwL7gl2zx1aDuDMjvBZVGr2uW0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 h1:wxQx2Bt4xzPIKvW59WQf1tJNx/ZZKPfN+EhPX3Z6CYY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.2.0 h1:UrGzkHueDwAWDdjQxC+QaXHd4tVCkISYE9j7fSSXF8k= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.2.0/go.mod h1:qskvSQeW+cxEE2bcKYyKimB1/KiQ9xpJ99bcHY0BX6c= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A= @@ -64,6 +76,8 @@ github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxY github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMtwg/AFW3s= github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE= github.com/charmbracelet/log v0.4.0 h1:G9bQAcx8rWA2T3pWvx7YtPTPwgqpk7D68BX21IRW8ZM= @@ -79,6 +93,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 h1:PRxIJD8XjimM5aTknUK9w6DHLDox2r2M3DI4i2pnd3w= github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936/go.mod h1:ttYvX5qlB+mlV1okblJqcSMtR4c52UKxDiX9GRBS8+Q= @@ -121,7 +137,6 @@ github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -194,6 +209,8 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -271,6 +288,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= +github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= From 9f2a5d597ea06601a5098e45baa85b805215fd15 Mon Sep 17 00:00:00 2001 From: jonathan meeks Date: Tue, 13 May 2025 09:05:17 -0500 Subject: [PATCH 4/7] aws/az networks sync successfully --- cmd/ctrlc/root/sync/aws/networks/networks.go | 2 +- cmd/ctrlc/root/sync/azure/azure.go | 2 + .../root/sync/azure/networks/networks.go | 87 ++++++++++++------- go.mod | 1 + go.sum | 6 +- 5 files changed, 63 insertions(+), 35 deletions(-) diff --git a/cmd/ctrlc/root/sync/aws/networks/networks.go b/cmd/ctrlc/root/sync/aws/networks/networks.go index 7f93298..85b68f6 100644 --- a/cmd/ctrlc/root/sync/aws/networks/networks.go +++ b/cmd/ctrlc/root/sync/aws/networks/networks.go @@ -232,7 +232,7 @@ func processNetwork( metadata["ctrlplane/links"] = fmt.Sprintf("{ \"AWS Console\": \"%s\" }", consoleUrl) return api.CreateResource{ - Version: "ctrlplane.dev/vpc/v1", + Version: "ctrlplane.dev/network/v1", Kind: "AWSNetwork", Name: vpcName, Identifier: *vpc.VpcId, diff --git a/cmd/ctrlc/root/sync/azure/azure.go b/cmd/ctrlc/root/sync/azure/azure.go index 76e76a0..117a7b8 100644 --- a/cmd/ctrlc/root/sync/azure/azure.go +++ b/cmd/ctrlc/root/sync/azure/azure.go @@ -3,6 +3,7 @@ package azure import ( "github.com/MakeNowJust/heredoc/v2" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/azure/aks" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/azure/networks" "github.com/ctrlplanedev/cli/internal/cliutil" "github.com/spf13/cobra" ) @@ -32,6 +33,7 @@ func NewAzureCmd() *cobra.Command { // Add all Azure sync subcommands cmd.AddCommand(cliutil.AddIntervalSupport(aks.NewSyncAKSCmd(), "")) + cmd.AddCommand(cliutil.AddIntervalSupport(networks.NewSyncNetworksCmd(), "")) return cmd } diff --git a/cmd/ctrlc/root/sync/azure/networks/networks.go b/cmd/ctrlc/root/sync/azure/networks/networks.go index ada636f..bf7c2d4 100644 --- a/cmd/ctrlc/root/sync/azure/networks/networks.go +++ b/cmd/ctrlc/root/sync/azure/networks/networks.go @@ -1,4 +1,4 @@ -package aks +package networks import ( "context" @@ -19,14 +19,13 @@ import ( "sync" ) -// NewSyncAKSCmd creates a new cobra command for syncing AKS clusters -func NewSyncAKSCmd() *cobra.Command { +func NewSyncNetworksCmd() *cobra.Command { var subscriptionID string var name string cmd := &cobra.Command{ - Use: "aks", - Short: "Sync Azure Kubernetes Service networks into Ctrlplane", + Use: "networks", + Short: "Sync Azure Virtual Networks into Ctrlplane", Example: heredoc.Doc(` # Make sure Azure credentials are configured via environment variables or Azure CLI @@ -75,7 +74,7 @@ func runSync(subscriptionID, name *string) func(cmd *cobra.Command, args []strin tenantID = getTenantIDFromEnv() } - log.Info("Syncing all AKS clusters", "subscriptionID", *subscriptionID, "tenantID", tenantID) + log.Info("Syncing all Networks", "subscriptionID", *subscriptionID, "tenantID", tenantID) resources, err := processNetworks(ctx, cred, *subscriptionID, tenantID) if err != nil { @@ -83,13 +82,13 @@ func runSync(subscriptionID, name *string) func(cmd *cobra.Command, args []strin } if len(resources) == 0 { - log.Info("No AKS clusters found") + log.Info("No Networks found") return nil } // If name is not provided, use subscription ID if *name == "" { - *name = fmt.Sprintf("azure-aks-%s", *subscriptionID) + *name = fmt.Sprintf("azure-networks-%s", *subscriptionID) } // Upsert resources to Ctrlplane @@ -157,7 +156,7 @@ func getDefaultSubscriptionID(ctx context.Context, cred azcore.TokenCredential) func processNetworks( ctx context.Context, cred azcore.TokenCredential, subscriptionID string, tenantID string, ) ([]api.CreateResource, error) { - var resources []api.CreateResource + var allResources []api.CreateResource var resourceGroups []common.ResourceGroupInfo var mu sync.Mutex var wg sync.WaitGroup @@ -184,20 +183,20 @@ func processNetworks( page, err := pager.NextPage(ctx) if err != nil { mu.Lock() - syncErrors = append(syncErrors, fmt.Errorf("failed to list AKS clusters: %w", err)) + syncErrors = append(syncErrors, fmt.Errorf("failed to list networks: %w", err)) mu.Unlock() } for _, network := range page.Value { - resource, err := processNetwork(ctx, network, resourceGroup, subscriptionID, tenantID) + resources, err := processNetwork(ctx, network, resourceGroup, subscriptionID, tenantID) if err != nil { - log.Error("Failed to process AKS cluster", "name", *network.Name, "error", err) + log.Error("Failed to process network", "name", *network.Name, "error", err) mu.Lock() - syncErrors = append(syncErrors, fmt.Errorf("cluster %s: %w", *network.Name, err)) + syncErrors = append(syncErrors, fmt.Errorf("network %s: %w", *network.Name, err)) mu.Unlock() return } mu.Lock() - resources = append(resources, resource) + allResources = append(allResources, resources...) mu.Unlock() } } @@ -211,8 +210,8 @@ func processNetworks( // Continue with the clusters that succeeded } - log.Info("Found AKS clusters", "count", len(resources)) - return resources, nil + log.Info("Found network resources", "count", len(allResources)) + return allResources, nil } func processNetwork( @@ -220,14 +219,14 @@ func processNetwork( ) ([]api.CreateResource, error) { resources := make([]api.CreateResource, 0) networkName := network.Name - metadata := initNetworkMetadata(network, subscriptionID, resourceGroup, tenantID) + metadata := initNetworkMetadata(network, resourceGroup, subscriptionID, tenantID) // Build console URL - consoleUrl := getVirtualNetworkConsoleUrl(subscriptionID, resourceGroup, *networkName) + consoleUrl := getVirtualNetworkConsoleUrl(resourceGroup, subscriptionID, *networkName) metadata[kinds.CtrlplaneMetadataLinks] = fmt.Sprintf("{ \"Azure Portal\": \"%s\" }", consoleUrl) resources = append(resources, api.CreateResource{ - Version: "ctrlplane.dev/vpc/v1", + Version: "ctrlplane.dev/network/v1", Kind: "AzureNetwork", Name: *networkName, Identifier: *network.ID, @@ -248,23 +247,49 @@ func processNetwork( Metadata: metadata, }) for _, subnet := range network.Properties.Subnets { - if res, err := processSubnet(networkName, subnet, resourceGroup, subscriptionID, tenantID); err != nil { + if res, err := processSubnet(network, subnet, resourceGroup, subscriptionID, tenantID); err != nil { return nil, err } else { resources = append(resources, res) } } + return resources, nil } func processSubnet( - networkName *string, subnet *armnetwork.Subnet, resourceGroup string, subscriptionID string, tenantID string, + network *armnetwork.VirtualNetwork, subnet *armnetwork.Subnet, resourceGroup string, subscriptionID string, tenantID string, ) (api.CreateResource, error) { + metadata := initSubnetMetadata(network, subnet, resourceGroup, subscriptionID, tenantID) + networkName := network.Name + subnetName := subnet.Name + // Build console URL + consoleUrl := getSubnetConsoleUrl(resourceGroup, subscriptionID, *networkName) + metadata[kinds.CtrlplaneMetadataLinks] = fmt.Sprintf("{ \"Azure Portal\": \"%s\" }", consoleUrl) + + return api.CreateResource{ + Version: "ctrlplane.dev/network/subnet/v1", + Kind: "AzureSubnet", + Name: *subnetName, + Identifier: *subnet.ID, + Config: map[string]any{ + // Common cross-provider options + "name": subnetName, + "type": "subnet", + "id": subnet.ID, + + // Provider-specific implementation details + "azureSubnet": map[string]any{ + "type": subnet.Type, + "purpose": subnet.Properties.Purpose, + "state": subnet.Properties.ProvisioningState, + }, + }, + Metadata: metadata, + }, nil } -func initNetworkMetadata( - network *armnetwork.VirtualNetwork, subscriptionID, resourceGroup string, tenantID string, -) map[string]string { +func initNetworkMetadata(network *armnetwork.VirtualNetwork, resourceGroup, subscriptionID, tenantID string) map[string]string { metadata := map[string]string{ "azure/subscription": subscriptionID, @@ -274,7 +299,7 @@ func initNetworkMetadata( "azure/location": *network.Location, "azure/status": string(*network.Properties.ProvisioningState), "azure/id": *network.ID, - "azure/console-url": getVirtualNetworkConsoleUrl(subscriptionID, resourceGroup, *network.Name), + "azure/console-url": getVirtualNetworkConsoleUrl(resourceGroup, subscriptionID, *network.Name), } // Tags @@ -289,9 +314,7 @@ func initNetworkMetadata( return metadata } -func initSubnetMetadata( - network *armnetwork.VirtualNetwork, subnet *armnetwork.Subnet, subscriptionID, resourceGroup string, tenantID string, -) map[string]string { +func initSubnetMetadata(network *armnetwork.VirtualNetwork, subnet *armnetwork.Subnet, resourceGroup, subscriptionID, tenantID string) map[string]string { metadata := map[string]string{ "azure/subscription": subscriptionID, @@ -301,7 +324,7 @@ func initSubnetMetadata( "azure/location": *network.Location, "azure/status": string(*subnet.Properties.ProvisioningState), "azure/id": *subnet.ID, - "azure/console-url": getSubnetConsoleUrl(subscriptionID, resourceGroup, *network.Name), + "azure/console-url": getSubnetConsoleUrl(resourceGroup, subscriptionID, *network.Name), } // Tags @@ -316,7 +339,7 @@ func initSubnetMetadata( return metadata } -func getVirtualNetworkConsoleUrl(subscriptionID, resourceGroup, networkName string) string { +func getVirtualNetworkConsoleUrl(resourceGroup, subscriptionID, networkName string) string { return fmt.Sprintf( "https://portal.azure.com/#@/resource/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s", subscriptionID, @@ -325,7 +348,7 @@ func getVirtualNetworkConsoleUrl(subscriptionID, resourceGroup, networkName stri ) } -func getSubnetConsoleUrl(subscriptionID, resourceGroup, networkName string) string { +func getSubnetConsoleUrl(resourceGroup, subscriptionID, networkName string) string { return fmt.Sprintf( "https://portal.azure.com/#@/resource/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets", subscriptionID, @@ -336,7 +359,7 @@ func getSubnetConsoleUrl(subscriptionID, resourceGroup, networkName string) stri func upsertToCtrlplane(ctx context.Context, resources []api.CreateResource, subscriptionID, name *string) error { if *name == "" { - *name = fmt.Sprintf("azure-aks-%s", *subscriptionID) + *name = fmt.Sprintf("azure-networks-%s", *subscriptionID) } apiURL := viper.GetString("url") diff --git a/go.mod b/go.mod index 36d0841..9b7faa7 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.2.0 github.com/MakeNowJust/heredoc/v2 v2.0.1 diff --git a/go.sum b/go.sum index 8995603..7cb7294 100644 --- a/go.sum +++ b/go.sum @@ -14,10 +14,12 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8U github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 h1:figxyQZXzZQIcP3njhC68bYUiTw45J8/SsHaLW8Ax0M= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0/go.mod h1:TmlMW4W5OvXOmOyKNnor8nlMMiO1ctIyzmHme/VHsrA= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0 h1:lMW1lD/17LUA5z1XTURo7LcVG2ICBPlyMHjIUrcFZNQ= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.0.0/go.mod h1:ceIuwmxDWptoW3eCqSXlnPsZFKh4X+R38dWPv7GS9Vs= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2/go.mod h1:FbdwsQ2EzwvXxOPcMFYO8ogEc9uMMIj3YkmCdXdAFmk= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 h1:QM6sE5k2ZT/vI5BEe0r7mqjsUSnhVBFbOsVkEuaEfiA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0/go.mod h1:243D9iHbcQXoFUtgHJwL7gl2zx1aDuDMjvBZVGr2uW0= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= From 88931bcd52b874764e5b26ba97427c22a1c2de19 Mon Sep 17 00:00:00 2001 From: jonathan meeks Date: Tue, 13 May 2025 10:37:20 -0500 Subject: [PATCH 5/7] aws/az networks: fixed nil protection & race conditions, added network tags --- cmd/ctrlc/root/sync/aws/common/provider.go | 7 +- cmd/ctrlc/root/sync/aws/eks/eks.go | 2 +- cmd/ctrlc/root/sync/aws/networks/networks.go | 37 ++++++--- .../root/sync/azure/networks/networks.go | 81 +++++++++++++------ 4 files changed, 87 insertions(+), 40 deletions(-) diff --git a/cmd/ctrlc/root/sync/aws/common/provider.go b/cmd/ctrlc/root/sync/aws/common/provider.go index 24db906..a44274f 100644 --- a/cmd/ctrlc/root/sync/aws/common/provider.go +++ b/cmd/ctrlc/root/sync/aws/common/provider.go @@ -7,9 +7,9 @@ import ( "strings" ) -// ComputeProviderDetails generates a provider name and region string based on the provided parameters. +// EnsureProviderDetails generates a provider name and region string based on the provided parameters. // The -func ComputeProviderDetails( +func EnsureProviderDetails( ctx context.Context, prefix string, regions []string, name *string, ) { providerRegion := "all-regions" @@ -19,6 +19,9 @@ func ComputeProviderDetails( } // If name is not provided, try to get account ID to include in the provider name + if name == nil { + name = new(string) + } if *name == "" { // Get AWS account ID for provider name using common package cfg, err := InitAWSConfig(ctx, regions[0]) diff --git a/cmd/ctrlc/root/sync/aws/eks/eks.go b/cmd/ctrlc/root/sync/aws/eks/eks.go index 930cabf..8a7a32b 100644 --- a/cmd/ctrlc/root/sync/aws/eks/eks.go +++ b/cmd/ctrlc/root/sync/aws/eks/eks.go @@ -113,7 +113,7 @@ func runSync(regions *[]string, name *string) func(cmd *cobra.Command, args []st return nil } - common.ComputeProviderDetails(ctx, "aws-eks", regionsToSync, name) + common.EnsureProviderDetails(ctx, "aws-eks", regionsToSync, name) // Upsert resources to Ctrlplane return upsertToCtrlplane(ctx, allResources, name) diff --git a/cmd/ctrlc/root/sync/aws/networks/networks.go b/cmd/ctrlc/root/sync/aws/networks/networks.go index 85b68f6..a2027ce 100644 --- a/cmd/ctrlc/root/sync/aws/networks/networks.go +++ b/cmd/ctrlc/root/sync/aws/networks/networks.go @@ -18,7 +18,7 @@ import ( "sync" ) -// NewSyncNetworksCmd creates a new cobra command for syncing Google Networks +// NewSyncNetworksCmd creates a new cobra command for syncing AWS Networks func NewSyncNetworksCmd() *cobra.Command { var name string var regions []string @@ -131,8 +131,9 @@ func runSync(regions *[]string, name *string) func(cmd *cobra.Command, args []st } }(region) } + wg.Wait() - common.ComputeProviderDetails(ctx, "aws-networks", regionsToSync, name) + common.EnsureProviderDetails(ctx, "aws-networks", regionsToSync, name) // Upsert resources to Ctrlplane return upsertToCtrlplane(ctx, allResources, name) @@ -233,14 +234,14 @@ func processNetwork( return api.CreateResource{ Version: "ctrlplane.dev/network/v1", - Kind: "AWSNetwork", + Kind: "AmazonNetwork", Name: vpcName, Identifier: *vpc.VpcId, Config: map[string]any{ // Common cross-provider options "name": vpcName, "type": "vpc", - "id": vpc.VpcId, + "id": *vpc.VpcId, // Provider-specific implementation details "awsVpc": map[string]any{ @@ -273,6 +274,13 @@ func initNetworkMetadata(vpc types.Vpc, region string, subnetCount int) map[stri "aws/id": *vpc.VpcId, } + // Tags + if vpc.Tags != nil { + for _, tag := range vpc.Tags { + metadata[fmt.Sprintf("tags/%s", *tag.Key)] = *tag.Value + } + } + return metadata } @@ -332,7 +340,7 @@ func processSubnet(subnet types.Subnet, region string) (api.CreateResource, erro return api.CreateResource{ Version: "ctrlplane.dev/network/subnet/v1", - Kind: "AWSSubnet", + Kind: "AmazonSubnet", Name: subnetName, Identifier: *subnet.SubnetArn, Config: map[string]any{ @@ -342,8 +350,8 @@ func processSubnet(subnet types.Subnet, region string) (api.CreateResource, erro "type": "subnet", "cidr": subnet.CidrBlock, "region": region, - "id": subnet.SubnetId, - "vpcId": subnet.VpcId, + "id": *subnet.SubnetId, + "vpcId": *subnet.VpcId, }, Metadata: metadata, }, nil @@ -362,10 +370,17 @@ func initSubnetMetadata(subnet types.Subnet, region string) map[string]string { "network/cidr": *subnet.CidrBlock, "network/block-public-access": string(subnet.BlockPublicAccessStates.InternetGatewayBlockMode), - "google/resource-type": "compute.googleapis.com/Subnetwork", - "google/console-url": consoleUrl, - "google/region": region, - "google/id": *subnet.SubnetId, + "aws/resource-type": "aws/Subnet", + "aws/console-url": consoleUrl, + "aws/region": region, + "aws/id": *subnet.SubnetId, + } + + // Tags + if subnet.Tags != nil { + for _, tag := range subnet.Tags { + metadata[fmt.Sprintf("tags/%s", *tag.Key)] = *tag.Value + } } return metadata diff --git a/cmd/ctrlc/root/sync/azure/networks/networks.go b/cmd/ctrlc/root/sync/azure/networks/networks.go index bf7c2d4..c586a31 100644 --- a/cmd/ctrlc/root/sync/azure/networks/networks.go +++ b/cmd/ctrlc/root/sync/azure/networks/networks.go @@ -175,10 +175,11 @@ func processNetworks( for _, rg := range resourceGroups { wg.Add(1) + rgName := rg.Name go func(resourceGroup string) { defer wg.Done() - pager := client.NewListPager(rg.Name, nil) + pager := client.NewListPager(resourceGroup, nil) for pager.More() { page, err := pager.NextPage(ctx) if err != nil { @@ -200,7 +201,7 @@ func processNetworks( mu.Unlock() } } - }(rg.Name) + }(rgName) } wg.Wait() @@ -222,7 +223,7 @@ func processNetwork( metadata := initNetworkMetadata(network, resourceGroup, subscriptionID, tenantID) // Build console URL - consoleUrl := getVirtualNetworkConsoleUrl(resourceGroup, subscriptionID, *networkName) + consoleUrl := getNetworkConsoleUrl(resourceGroup, subscriptionID, *networkName) metadata[kinds.CtrlplaneMetadataLinks] = fmt.Sprintf("{ \"Azure Portal\": \"%s\" }", consoleUrl) resources = append(resources, api.CreateResource{ @@ -240,17 +241,19 @@ func processNetwork( "azureVirtualNetwork": map[string]any{ "type": network.Type, "region": network.Location, - "state": network.Properties.ProvisioningState, - "subnetCount": len(network.Properties.Subnets), + "state": getNetworkState(network), + "subnetCount": getNetworkSubnetCount(network), }, }, Metadata: metadata, }) - for _, subnet := range network.Properties.Subnets { - if res, err := processSubnet(network, subnet, resourceGroup, subscriptionID, tenantID); err != nil { - return nil, err - } else { - resources = append(resources, res) + if network.Properties != nil && network.Properties.Subnets != nil { + for _, subnet := range network.Properties.Subnets { + if res, err := processSubnet(network, subnet, resourceGroup, subscriptionID, tenantID); err != nil { + return nil, err + } else { + resources = append(resources, res) + } } } return resources, nil @@ -281,8 +284,8 @@ func processSubnet( // Provider-specific implementation details "azureSubnet": map[string]any{ "type": subnet.Type, - "purpose": subnet.Properties.Purpose, - "state": subnet.Properties.ProvisioningState, + "purpose": getSubnetPurpose(subnet), + "state": getSubnetState(subnet), }, }, Metadata: metadata, @@ -297,9 +300,9 @@ func initNetworkMetadata(network *armnetwork.VirtualNetwork, resourceGroup, subs "azure/resource-group": resourceGroup, "azure/resource-type": "Microsoft.Network/virtualNetworks/subnets", "azure/location": *network.Location, - "azure/status": string(*network.Properties.ProvisioningState), + "azure/status": getNetworkState(network), "azure/id": *network.ID, - "azure/console-url": getVirtualNetworkConsoleUrl(resourceGroup, subscriptionID, *network.Name), + "azure/console-url": getNetworkConsoleUrl(resourceGroup, subscriptionID, *network.Name), } // Tags @@ -322,24 +325,20 @@ func initSubnetMetadata(network *armnetwork.VirtualNetwork, subnet *armnetwork.S "azure/resource-group": resourceGroup, "azure/resource-type": "Microsoft.Network/virtualNetworks/subnets", "azure/location": *network.Location, - "azure/status": string(*subnet.Properties.ProvisioningState), - "azure/id": *subnet.ID, - "azure/console-url": getSubnetConsoleUrl(resourceGroup, subscriptionID, *network.Name), - } - - // Tags - if network.Tags != nil { - for key, value := range network.Tags { - if value != nil { - metadata[fmt.Sprintf("tags/%s", key)] = *value + "azure/status": func() string { + if network.Properties != nil { + return string(*subnet.Properties.ProvisioningState) } - } + return "" + }(), + "azure/id": *subnet.ID, + "azure/console-url": getSubnetConsoleUrl(resourceGroup, subscriptionID, *network.Name), } return metadata } -func getVirtualNetworkConsoleUrl(resourceGroup, subscriptionID, networkName string) string { +func getNetworkConsoleUrl(resourceGroup, subscriptionID, networkName string) string { return fmt.Sprintf( "https://portal.azure.com/#@/resource/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s", subscriptionID, @@ -348,6 +347,22 @@ func getVirtualNetworkConsoleUrl(resourceGroup, subscriptionID, networkName stri ) } +func getNetworkState(network *armnetwork.VirtualNetwork) string { + return func() string { + if network.Properties != nil { + return string(*network.Properties.ProvisioningState) + } + return "" + }() +} + +func getNetworkSubnetCount(network *armnetwork.VirtualNetwork) int { + if network.Properties != nil && network.Properties.Subnets != nil { + return len(network.Properties.Subnets) + } + return 0 +} + func getSubnetConsoleUrl(resourceGroup, subscriptionID, networkName string) string { return fmt.Sprintf( "https://portal.azure.com/#@/resource/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets", @@ -357,6 +372,20 @@ func getSubnetConsoleUrl(resourceGroup, subscriptionID, networkName string) stri ) } +func getSubnetPurpose(subnet *armnetwork.Subnet) *string { + if subnet.Properties != nil { + return subnet.Properties.Purpose + } + return nil +} + +func getSubnetState(subnet *armnetwork.Subnet) string { + if subnet.Properties != nil { + return string(*subnet.Properties.ProvisioningState) + } + return "" +} + func upsertToCtrlplane(ctx context.Context, resources []api.CreateResource, subscriptionID, name *string) error { if *name == "" { *name = fmt.Sprintf("azure-networks-%s", *subscriptionID) From e2b7f19f21388d4da6a9dc1df8509181d571b8c9 Mon Sep 17 00:00:00 2001 From: jonathan meeks Date: Tue, 13 May 2025 11:15:15 -0500 Subject: [PATCH 6/7] better metadata, more nil fixes --- cmd/ctrlc/root/sync/aws/networks/networks.go | 14 +++---- .../root/sync/azure/networks/networks.go | 38 +++++++++++++++---- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/cmd/ctrlc/root/sync/aws/networks/networks.go b/cmd/ctrlc/root/sync/aws/networks/networks.go index a2027ce..218c706 100644 --- a/cmd/ctrlc/root/sync/aws/networks/networks.go +++ b/cmd/ctrlc/root/sync/aws/networks/networks.go @@ -257,15 +257,15 @@ func processNetwork( // initNetworkMetadata initializes the base metadata for a network func initNetworkMetadata(vpc types.Vpc, region string, subnetCount int) map[string]string { - var vpcName = getVpcName(vpc) - var consoleUrl = getVpcConsoleUrl(vpc, region) + vpcName := getVpcName(vpc) + consoleUrl := getVpcConsoleUrl(vpc, region) metadata := map[string]string{ - "vpc/type": "vpc", - "vpc/name": vpcName, - "vpc/subnet-count": strconv.Itoa(subnetCount), - "vpc/id": *vpc.VpcId, - "vpc/tenancy": string(vpc.InstanceTenancy), + "network/type": "vpc", + "network/name": vpcName, + "network/subnet-count": strconv.Itoa(subnetCount), + "network/id": *vpc.VpcId, + "network/tenancy": string(vpc.InstanceTenancy), "aws/region": region, "aws/resource-type": "vpc", diff --git a/cmd/ctrlc/root/sync/azure/networks/networks.go b/cmd/ctrlc/root/sync/azure/networks/networks.go index c586a31..44d88e5 100644 --- a/cmd/ctrlc/root/sync/azure/networks/networks.go +++ b/cmd/ctrlc/root/sync/azure/networks/networks.go @@ -16,6 +16,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "os" + "strconv" "sync" ) @@ -186,6 +187,7 @@ func processNetworks( mu.Lock() syncErrors = append(syncErrors, fmt.Errorf("failed to list networks: %w", err)) mu.Unlock() + return } for _, network := range page.Value { resources, err := processNetwork(ctx, network, resourceGroup, subscriptionID, tenantID) @@ -293,8 +295,16 @@ func processSubnet( } func initNetworkMetadata(network *armnetwork.VirtualNetwork, resourceGroup, subscriptionID, tenantID string) map[string]string { + subnetCount := 0 + if network.Properties != nil && network.Properties.Subnets != nil { + subnetCount = len(network.Properties.Subnets) + } metadata := map[string]string{ + "network/type": "vpc", + "network/name": *network.Name, + "network/subnet-count": strconv.Itoa(subnetCount), + "network/id": *network.ID, "azure/subscription": subscriptionID, "azure/tenant": tenantID, "azure/resource-group": resourceGroup, @@ -319,12 +329,24 @@ func initNetworkMetadata(network *armnetwork.VirtualNetwork, resourceGroup, subs func initSubnetMetadata(network *armnetwork.VirtualNetwork, subnet *armnetwork.Subnet, resourceGroup, subscriptionID, tenantID string) map[string]string { + privateAccess := false + if subnet.Properties != nil { + if subnet.Properties.PrivateEndpoints != nil && len(subnet.Properties.PrivateEndpoints) > 0 { + privateAccess = true + } + } + metadata := map[string]string{ - "azure/subscription": subscriptionID, - "azure/tenant": tenantID, - "azure/resource-group": resourceGroup, - "azure/resource-type": "Microsoft.Network/virtualNetworks/subnets", - "azure/location": *network.Location, + "network/type": "subnet", + "network/name": *subnet.Name, + "network/vpc": *network.Name, + "network/region": *network.Location, + "network/private-access": strconv.FormatBool(privateAccess), + "azure/subscription": subscriptionID, + "azure/tenant": tenantID, + "azure/resource-group": resourceGroup, + "azure/resource-type": "Microsoft.Network/virtualNetworks/subnets", + "azure/location": *network.Location, "azure/status": func() string { if network.Properties != nil { return string(*subnet.Properties.ProvisioningState) @@ -349,7 +371,7 @@ func getNetworkConsoleUrl(resourceGroup, subscriptionID, networkName string) str func getNetworkState(network *armnetwork.VirtualNetwork) string { return func() string { - if network.Properties != nil { + if network.Properties != nil && network.Properties.ProvisioningState != nil { return string(*network.Properties.ProvisioningState) } return "" @@ -373,14 +395,14 @@ func getSubnetConsoleUrl(resourceGroup, subscriptionID, networkName string) stri } func getSubnetPurpose(subnet *armnetwork.Subnet) *string { - if subnet.Properties != nil { + if subnet.Properties != nil && subnet.Properties.Purpose != nil { return subnet.Properties.Purpose } return nil } func getSubnetState(subnet *armnetwork.Subnet) string { - if subnet.Properties != nil { + if subnet.Properties != nil && subnet.Properties.ProvisioningState != nil { return string(*subnet.Properties.ProvisioningState) } return "" From a9cd890a00aef294d6b93f3a0b9c9b1d433ad222 Mon Sep 17 00:00:00 2001 From: jonathan meeks Date: Tue, 13 May 2025 11:55:20 -0500 Subject: [PATCH 7/7] more nil fixes --- cmd/ctrlc/root/sync/azure/networks/networks.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cmd/ctrlc/root/sync/azure/networks/networks.go b/cmd/ctrlc/root/sync/azure/networks/networks.go index 44d88e5..90fb581 100644 --- a/cmd/ctrlc/root/sync/azure/networks/networks.go +++ b/cmd/ctrlc/root/sync/azure/networks/networks.go @@ -347,14 +347,9 @@ func initSubnetMetadata(network *armnetwork.VirtualNetwork, subnet *armnetwork.S "azure/resource-group": resourceGroup, "azure/resource-type": "Microsoft.Network/virtualNetworks/subnets", "azure/location": *network.Location, - "azure/status": func() string { - if network.Properties != nil { - return string(*subnet.Properties.ProvisioningState) - } - return "" - }(), - "azure/id": *subnet.ID, - "azure/console-url": getSubnetConsoleUrl(resourceGroup, subscriptionID, *network.Name), + "azure/status": getSubnetState(subnet), + "azure/id": *subnet.ID, + "azure/console-url": getSubnetConsoleUrl(resourceGroup, subscriptionID, *network.Name), } return metadata