From f82549298117878f0e3e1cf931cbbdf36824903b Mon Sep 17 00:00:00 2001 From: Alex Ilies Date: Sat, 20 Dec 2025 23:10:38 +0200 Subject: [PATCH 1/6] infra(gsi): gsi changes --- README.md | 1 - cmd/app/internal/database/client_test.go | 4 +- .../database/user_skill_repository.go | 6 +- .../user_skill_repository_dynamodb.go | 22 +- .../database/user_skill_repository_mock.go | 12 +- cmd/app/internal/handler/user_handler.go | 12 +- cmd/app/internal/models/user_skill.go | 2 - cmd/app/internal/service/skill_service.go | 20 +- deployments/app/cdk.go | 19 +- docs/api-testing/api-test.http | 37 +- docs/dynamodb_table_design.md | 1044 +++++++++++++++++ 11 files changed, 1124 insertions(+), 55 deletions(-) create mode 100644 docs/dynamodb_table_design.md diff --git a/README.md b/README.md index 9bbb695..9551115 100644 --- a/README.md +++ b/README.md @@ -474,7 +474,6 @@ Deployed resources (via AWS CDK in Go): - **Name**: `glad-entities` - **Optimized single table** with 1 Global Secondary Index - **Table Keys**: `EntityType` (PK) + `entity_id` (SK) -- **DynamoDB Streams**: Enabled (NEW_AND_OLD_IMAGES) - **Capacity**: On-demand billing mode - **Point-in-time recovery**: Disabled (dev-friendly) - **Removal policy**: DESTROY (dev-friendly) diff --git a/cmd/app/internal/database/client_test.go b/cmd/app/internal/database/client_test.go index b7cfc88..2b367f8 100644 --- a/cmd/app/internal/database/client_test.go +++ b/cmd/app/internal/database/client_test.go @@ -371,8 +371,8 @@ func TestMockRepository_ListUsersBySkill(t *testing.T) { repo.CreateSkill(skill2) repo.CreateSkill(skill3) - // Test list users with Go skill - skills, err := repo.ListUsersBySkill("Go") + // Test list users with Go skill in Programming category + skills, err := repo.ListUsersBySkill("Programming", "Go") if err != nil { t.Errorf("Expected no error, got %v", err) } diff --git a/cmd/app/internal/database/user_skill_repository.go b/cmd/app/internal/database/user_skill_repository.go index 4a95740..fd0a84a 100644 --- a/cmd/app/internal/database/user_skill_repository.go +++ b/cmd/app/internal/database/user_skill_repository.go @@ -9,6 +9,8 @@ type SkillRepository interface { UpdateSkill(skill *models.UserSkill) error DeleteSkill(username, skillID string) error ListSkillsForUser(username string) ([]*models.UserSkill, error) - ListUsersBySkill(skillName string) ([]*models.UserSkill, error) - ListUsersBySkillAndLevel(skillName string, proficiencyLevel models.ProficiencyLevel) ([]*models.UserSkill, error) + // ListUsersBySkill queries the BySkill GSI with Category + SkillName + ListUsersBySkill(category, skillName string) ([]*models.UserSkill, error) + // ListUsersBySkillAndLevel queries the BySkill GSI with Category + SkillName + ProficiencyLevel + ListUsersBySkillAndLevel(category, skillName string, proficiencyLevel models.ProficiencyLevel) ([]*models.UserSkill, error) } diff --git a/cmd/app/internal/database/user_skill_repository_dynamodb.go b/cmd/app/internal/database/user_skill_repository_dynamodb.go index 1e192b5..558fb0f 100644 --- a/cmd/app/internal/database/user_skill_repository_dynamodb.go +++ b/cmd/app/internal/database/user_skill_repository_dynamodb.go @@ -34,7 +34,6 @@ func (r *DynamoDBRepository) CreateSkill(skill *models.UserSkill) error { ConditionExpression: aws.String("attribute_not_exists(entity_id)"), } - _, err = r.client.PutItem(input) _, err = r.client.PutItem(input) if err != nil { log.Error("Failed to create skill in DynamoDB", "error", err.Error(), "duration", time.Since(start)) @@ -182,8 +181,9 @@ func (r *DynamoDBRepository) ListSkillsForUser(username string) ([]*models.UserS } // ListUsersBySkill retrieves all users who have a specific skill using GSI BySkill -func (r *DynamoDBRepository) ListUsersBySkill(skillName string) ([]*models.UserSkill, error) { - log := logger.WithComponent("database").With("operation", "ListUsersBySkill", "skill", skillName) +// GSI BySkill structure: PK=Category, SK=SkillName+ProficiencyLevel+YearsOfExperience+Username +func (r *DynamoDBRepository) ListUsersBySkill(category, skillName string) ([]*models.UserSkill, error) { + log := logger.WithComponent("database").With("operation", "ListUsersBySkill", "category", category, "skill", skillName) start := time.Now() log.Debug("Starting users list retrieval by skill") @@ -191,8 +191,9 @@ func (r *DynamoDBRepository) ListUsersBySkill(skillName string) ([]*models.UserS input := &dynamodb.QueryInput{ TableName: aws.String(TableName), IndexName: aws.String(GSIBySkill), - KeyConditionExpression: aws.String("SkillName = :skillName"), + KeyConditionExpression: aws.String("Category = :category AND SkillName = :skillName"), ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ + ":category": {S: aws.String(category)}, ":skillName": {S: aws.String(skillName)}, }, } @@ -213,13 +214,15 @@ func (r *DynamoDBRepository) ListUsersBySkill(skillName string) ([]*models.UserS skills = append(skills, &skill) } - log.Info("Users with skill retrieved successfully", "skill", skillName, "count", len(skills), "duration", time.Since(start)) + log.Info("Users with skill retrieved successfully", "category", category, "skill", skillName, "count", len(skills), "duration", time.Since(start)) return skills, nil } // ListUsersBySkillAndLevel retrieves users with a specific skill at a specific proficiency level -func (r *DynamoDBRepository) ListUsersBySkillAndLevel(skillName string, proficiencyLevel models.ProficiencyLevel) ([]*models.UserSkill, error) { - log := logger.WithComponent("database").With("operation", "ListUsersBySkillAndLevel", "skill", skillName, "level", proficiencyLevel) +// GSI BySkill structure: PK=Category, SK=SkillName+ProficiencyLevel+YearsOfExperience+Username +// Uses composite sort key matching: Category + SkillName + ProficiencyLevel (left-to-right) +func (r *DynamoDBRepository) ListUsersBySkillAndLevel(category, skillName string, proficiencyLevel models.ProficiencyLevel) ([]*models.UserSkill, error) { + log := logger.WithComponent("database").With("operation", "ListUsersBySkillAndLevel", "category", category, "skill", skillName, "level", proficiencyLevel) start := time.Now() log.Debug("Starting users list retrieval by skill and level") @@ -227,8 +230,9 @@ func (r *DynamoDBRepository) ListUsersBySkillAndLevel(skillName string, proficie input := &dynamodb.QueryInput{ TableName: aws.String(TableName), IndexName: aws.String(GSIBySkill), - KeyConditionExpression: aws.String("SkillName = :skillName AND ProficiencyLevel = :level"), + KeyConditionExpression: aws.String("Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level"), ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ + ":category": {S: aws.String(category)}, ":skillName": {S: aws.String(skillName)}, ":level": {S: aws.String(string(proficiencyLevel))}, }, @@ -250,6 +254,6 @@ func (r *DynamoDBRepository) ListUsersBySkillAndLevel(skillName string, proficie skills = append(skills, &skill) } - log.Info("Users with skill and level retrieved successfully", "skill", skillName, "level", proficiencyLevel, "count", len(skills), "duration", time.Since(start)) + log.Info("Users with skill and level retrieved successfully", "category", category, "skill", skillName, "level", proficiencyLevel, "count", len(skills), "duration", time.Since(start)) return skills, nil } diff --git a/cmd/app/internal/database/user_skill_repository_mock.go b/cmd/app/internal/database/user_skill_repository_mock.go index f57da15..634fcbf 100644 --- a/cmd/app/internal/database/user_skill_repository_mock.go +++ b/cmd/app/internal/database/user_skill_repository_mock.go @@ -114,8 +114,8 @@ func (m *MockRepository) ListSkillsForUser(username string) ([]*models.UserSkill } // ListUsersBySkill retrieves all users with a specific skill from memory -func (m *MockRepository) ListUsersBySkill(skillName string) ([]*models.UserSkill, error) { - log := logger.WithComponent("database").With("operation", "ListUsersBySkill", "skill", skillName, "repository", "mock") +func (m *MockRepository) ListUsersBySkill(category, skillName string) ([]*models.UserSkill, error) { + log := logger.WithComponent("database").With("operation", "ListUsersBySkill", "category", category, "skill", skillName, "repository", "mock") start := time.Now() log.Debug("Starting users list retrieval by skill from mock repository") @@ -125,7 +125,7 @@ func (m *MockRepository) ListUsersBySkill(skillName string) ([]*models.UserSkill var skills []*models.UserSkill for _, skill := range m.skills { - if skill.SkillName == skillName { + if skill.Category == category && skill.SkillName == skillName { skills = append(skills, skill) } } @@ -135,8 +135,8 @@ func (m *MockRepository) ListUsersBySkill(skillName string) ([]*models.UserSkill } // ListUsersBySkillAndLevel retrieves all users with a specific skill and proficiency level from memory -func (m *MockRepository) ListUsersBySkillAndLevel(skillName string, proficiencyLevel models.ProficiencyLevel) ([]*models.UserSkill, error) { - log := logger.WithComponent("database").With("operation", "ListUsersBySkillAndLevel", "skill", skillName, "level", proficiencyLevel, "repository", "mock") +func (m *MockRepository) ListUsersBySkillAndLevel(category, skillName string, proficiencyLevel models.ProficiencyLevel) ([]*models.UserSkill, error) { + log := logger.WithComponent("database").With("operation", "ListUsersBySkillAndLevel", "category", category, "skill", skillName, "level", proficiencyLevel, "repository", "mock") start := time.Now() log.Debug("Starting users list retrieval by skill and level from mock repository") @@ -146,7 +146,7 @@ func (m *MockRepository) ListUsersBySkillAndLevel(skillName string, proficiencyL var skills []*models.UserSkill for _, skill := range m.skills { - if skill.SkillName == skillName && skill.ProficiencyLevel == proficiencyLevel { + if skill.Category == category && skill.SkillName == skillName && skill.ProficiencyLevel == proficiencyLevel { skills = append(skills, skill) } } diff --git a/cmd/app/internal/handler/user_handler.go b/cmd/app/internal/handler/user_handler.go index 3f1fc7d..9ff97ad 100644 --- a/cmd/app/internal/handler/user_handler.go +++ b/cmd/app/internal/handler/user_handler.go @@ -313,7 +313,7 @@ func (h *Handler) DeleteSkill(request events.APIGatewayProxyRequest) (events.API } // ListUsersBySkill handles finding all users with a specific skill -// GET /skills/{skillName}/users +// GET /skills/{skillName}/users?category=&level= func (h *Handler) ListUsersBySkill(request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { // Get skill name from path parameter skillName, ok := request.PathParameters["skillName"] @@ -321,12 +321,18 @@ func (h *Handler) ListUsersBySkill(request events.APIGatewayProxyRequest) (event return errorResponse(http.StatusBadRequest, "Skill name is required"), nil } + // Get category from query parameters (required for multi-key GSI) + category, ok := request.QueryStringParameters["category"] + if !ok || category == "" { + return errorResponse(http.StatusBadRequest, "Category is required"), nil + } + // Check for proficiency level filter in query parameters proficiencyLevel, ok := request.QueryStringParameters["level"] if ok && proficiencyLevel != "" { // Query with level filter level := models.ProficiencyLevel(proficiencyLevel) - users, err := h.skillService.ListUsersBySkillAndLevel(skillName, level) + users, err := h.skillService.ListUsersBySkillAndLevel(category, skillName, level) if err != nil { return h.handleServiceError(err), nil } @@ -334,7 +340,7 @@ func (h *Handler) ListUsersBySkill(request events.APIGatewayProxyRequest) (event } // Query all users with skill - users, err := h.skillService.ListUsersBySkill(skillName) + users, err := h.skillService.ListUsersBySkill(category, skillName) if err != nil { return h.handleServiceError(err), nil } diff --git a/cmd/app/internal/models/user_skill.go b/cmd/app/internal/models/user_skill.go index bf3f99f..0a9b320 100644 --- a/cmd/app/internal/models/user_skill.go +++ b/cmd/app/internal/models/user_skill.go @@ -100,8 +100,6 @@ func (s *UserSkill) SetKeys() { // Base table key: Unique identifier s.EntityID = BuildUserSkillEntityID(s.Username, s.SkillID) s.EntityType = "UserSkill" - // Composite sort key: ProficiencyLevel#Username - s.SkillCompositeSort = string(s.ProficiencyLevel) + "#" + s.Username } // UpdateProficiency updates the skill proficiency level diff --git a/cmd/app/internal/service/skill_service.go b/cmd/app/internal/service/skill_service.go index 7e14021..52fdd28 100644 --- a/cmd/app/internal/service/skill_service.go +++ b/cmd/app/internal/service/skill_service.go @@ -179,14 +179,14 @@ func (s *SkillService) ListSkillsForUser(username string) ([]dto.SkillResponse, return result, nil } -// ListUsersBySkill retrieves all users who have a specific skill -func (s *SkillService) ListUsersBySkill(skillName string) ([]dto.UserSkillResponse, error) { - log := logger.WithComponent("service").With("operation", "ListUsersBySkill", "skill", skillName) +// ListUsersBySkill retrieves all users who have a specific skill in a category +func (s *SkillService) ListUsersBySkill(category, skillName string) ([]dto.UserSkillResponse, error) { + log := logger.WithComponent("service").With("operation", "ListUsersBySkill", "category", category, "skill", skillName) start := time.Now() log.Info("Retrieving users by skill") - skills, err := s.repo.ListUsersBySkill(skillName) + skills, err := s.repo.ListUsersBySkill(category, skillName) if err != nil { log.Error("Failed to retrieve users by skill", "error", err.Error(), "duration", time.Since(start)) return nil, err @@ -205,18 +205,18 @@ func (s *SkillService) ListUsersBySkill(skillName string) ([]dto.UserSkillRespon } } - log.Info("Users with skill retrieved successfully", "skill", skillName, "count", len(result), "duration", time.Since(start)) + log.Info("Users with skill retrieved successfully", "category", category, "skill", skillName, "count", len(result), "duration", time.Since(start)) return result, nil } -// ListUsersBySkillAndLevel retrieves users with a skill at a specific proficiency level -func (s *SkillService) ListUsersBySkillAndLevel(skillName string, proficiencyLevel models.ProficiencyLevel) ([]dto.UserSkillResponse, error) { - log := logger.WithComponent("service").With("operation", "ListUsersBySkillAndLevel", "skill", skillName, "level", proficiencyLevel) +// ListUsersBySkillAndLevel retrieves users with a skill at a specific proficiency level in a category +func (s *SkillService) ListUsersBySkillAndLevel(category, skillName string, proficiencyLevel models.ProficiencyLevel) ([]dto.UserSkillResponse, error) { + log := logger.WithComponent("service").With("operation", "ListUsersBySkillAndLevel", "category", category, "skill", skillName, "level", proficiencyLevel) start := time.Now() log.Info("Retrieving users by skill and level") - skills, err := s.repo.ListUsersBySkillAndLevel(skillName, proficiencyLevel) + skills, err := s.repo.ListUsersBySkillAndLevel(category, skillName, proficiencyLevel) if err != nil { log.Error("Failed to retrieve users by skill and level", "error", err.Error(), "duration", time.Since(start)) return nil, err @@ -235,6 +235,6 @@ func (s *SkillService) ListUsersBySkillAndLevel(skillName string, proficiencyLev } } - log.Info("Users with skill and level retrieved successfully", "skill", skillName, "level", proficiencyLevel, "count", len(result), "duration", time.Since(start)) + log.Info("Users with skill and level retrieved successfully", "category", category, "skill", skillName, "level", proficiencyLevel, "count", len(result), "duration", time.Since(start)) return result, nil } diff --git a/deployments/app/cdk.go b/deployments/app/cdk.go index 8b77cb5..56849df 100644 --- a/deployments/app/cdk.go +++ b/deployments/app/cdk.go @@ -31,18 +31,33 @@ func createEntitiesTable(stack awscdk.Stack, id *string, environment string) aws }, GlobalSecondaryIndexes: &[]*awsdynamodb.GlobalSecondaryIndexPropsV2{ - // GSI for skill-only queries + // GSI for flexible category/skill/proficiency queries + // Single PK: Category (allows broad queries) + // Composite SK: SkillName + ProficiencyLevel + YearsOfExperience + Username + // This design provides maximum query flexibility: + // - Query by Category alone + // - Query by Category + SkillName + // - Query by Category + SkillName + ProficiencyLevel + // - Query by Category + SkillName + ProficiencyLevel + YearsOfExperience (with range) { IndexName: jsii.String("BySkill"), PartitionKey: &awsdynamodb.Attribute{ - Name: jsii.String("SkillName"), + Name: jsii.String("Category"), Type: awsdynamodb.AttributeType_STRING, }, SortKeys: &[]*awsdynamodb.Attribute{ + { + Name: jsii.String("SkillName"), + Type: awsdynamodb.AttributeType_STRING, + }, { Name: jsii.String("ProficiencyLevel"), Type: awsdynamodb.AttributeType_STRING, }, + { + Name: jsii.String("YearsOfExperience"), + Type: awsdynamodb.AttributeType_NUMBER, + }, { Name: jsii.String("Username"), Type: awsdynamodb.AttributeType_STRING, diff --git a/docs/api-testing/api-test.http b/docs/api-testing/api-test.http index e91426d..f7ab068 100644 --- a/docs/api-testing/api-test.http +++ b/docs/api-testing/api-test.http @@ -66,7 +66,7 @@ Content-Type: application/json { "username": "jane_doe", - "password": "secure123" + "password": "newPassword123" } ### 1.7 Login - Alice @@ -331,7 +331,7 @@ Authorization: Bearer {{token}} Content-Type: application/json { - "skill_name": "JavaScripdt", + "skill_name": "JavaScript", "proficiency_level": "Advanced", "years_of_experience": 5, "notes": "Frontend development with React" @@ -421,50 +421,51 @@ Authorization: Bearer {{token}} ############################################################################### ### 5. CROSS-USER SKILL QUERIES (Protected Routes) +### NOTE: category parameter is REQUIRED for multi-key GSI queries ############################################################################### -### 5.1 Find All Users with Python -GET {{API_URL}}/skills/Python/users +### 5.1 Find All Users with Python (Programming category) +GET {{API_URL}}/skills/Python/users?category=Programming Authorization: Bearer {{token}} -### 5.2 Find All Users with JavaScript -GET {{API_URL}}/skills/JavaScript/users +### 5.2 Find All Users with JavaScript (Programming category) +GET {{API_URL}}/skills/JavaScript/users?category=Programming Authorization: Bearer {{token}} -### 5.3 Find All Users with Go -GET {{API_URL}}/skills/Go/users +### 5.3 Find All Users with Go (Programming category) +GET {{API_URL}}/skills/Go/users?category=Programming Authorization: Bearer {{token}} ### 5.4 Find Expert Python Developers -GET {{API_URL}}/skills/Python/users?level=Expert +GET {{API_URL}}/skills/Python/users?category=Programming&level=Expert Authorization: Bearer {{token}} ### 5.5 Find Intermediate Python Developers -GET {{API_URL}}/skills/Python/users?level=Intermediate +GET {{API_URL}}/skills/Python/users?category=Programming&level=Intermediate Authorization: Bearer {{token}} ### 5.6 Find Advanced Python Developers -GET {{API_URL}}/skills/Python/users?level=Advanced +GET {{API_URL}}/skills/Python/users?category=Programming&level=Advanced Authorization: Bearer {{token}} ### 5.7 Find Beginner Python Developers -GET {{API_URL}}/skills/Python/users?level=Beginner +GET {{API_URL}}/skills/Python/users?category=Programming&level=Beginner Authorization: Bearer {{token}} ### 5.8 Find Expert JavaScript Developers -GET {{API_URL}}/skills/JavaScript/users?level=Expert +GET {{API_URL}}/skills/JavaScript/users?category=Programming&level=Expert Authorization: Bearer {{token}} ### 5.9 Find Advanced JavaScript Developers -GET {{API_URL}}/skills/JavaScript/users?level=Advanced +GET {{API_URL}}/skills/JavaScript/users?category=Programming&level=Advanced Authorization: Bearer {{token}} ### 5.10 Find Expert Go Developers -GET {{API_URL}}/skills/Go/users?level=Expert +GET {{API_URL}}/skills/Go/users?category=Programming&level=Expert Authorization: Bearer {{token}} ### 5.11 Find Intermediate Go Developers -GET {{API_URL}}/skills/Go/users?level=Intermediate +GET {{API_URL}}/skills/Go/users?category=Programming&level=Intermediate Authorization: Bearer {{token}} ############################################################################### @@ -703,8 +704,8 @@ Content-Type: application/json "notes": "Making good progress with Rust" } -### 8.8 WORKFLOW Step 8 - Find All Rust Developers -GET {{API_URL}}/skills/Rust/users +### 8.8 WORKFLOW Step 8 - Find All Rust Developers (Programming category) +GET {{API_URL}}/skills/Rust/users?category=Programming Authorization: Bearer {{token}} ### 8.9 WORKFLOW Step 9 - Update User Profile diff --git a/docs/dynamodb_table_design.md b/docs/dynamodb_table_design.md new file mode 100644 index 0000000..a2ea66b --- /dev/null +++ b/docs/dynamodb_table_design.md @@ -0,0 +1,1044 @@ +# DynamoDB Test Queries - AWS CLI Commands + +This document contains executable AWS CLI commands to test all query patterns for the GLAD entities table. + +--- + +## Table Overview + +### Main Table: `glad-entities` + +**Table Structure (from CDK):** +- **Partition Key (PK):** `EntityType` (String) - Entity type discriminator +- **Sort Key (SK):** `entity_id` (String) - Unique identifier for each entity +- **Billing Mode:** PAY_PER_REQUEST (on-demand) +- **Point-in-Time Recovery:** Disabled +- **Deletion Protection:** Enabled + +**Entity Types:** +- `User` - User profiles +- `Skill` - Master skill definitions (catalog) +- `UserSkill` - User-to-skill relationships with proficiency data + +--- + +## The Power of Multi-Key Composite GSI + +**With just ONE Global Secondary Index using composite keys, you can create 15+ different query patterns!** + +This is the power of DynamoDB's multi-key GSI design. Instead of creating multiple indexes for different query patterns, you strategically design a single index with composite keys that enables flexible querying at multiple levels of granularity. + +### Single GSI: `BySkill` + +**Composite Partition Key (1 attribute):** +- `Category` (String) - Skill category for broad partitioning + +**Composite Sort Keys (4 attributes):** +1. `SkillName` (String) - Specific skill name +2. `ProficiencyLevel` (String) - Skill proficiency level +3. `YearsOfExperience` (Number) - Years of experience with the skill +4. `Username` (String) - User identifier (ensures uniqueness) + +**Why This Design is Powerful:** + +✅ **Hierarchical Querying:** Query from broad (Category) to specific (down to individual Username) +✅ **Partial Sort Keys:** Use just SK1, or SK1+SK2, or SK1+SK2+SK3, etc. - DynamoDB allows left-to-right querying +✅ **Range Operations:** Apply `>=`, `<=`, `BETWEEN` on the last sort key in your query +✅ **Efficient Sorting:** Data is automatically sorted by the composite key order +✅ **Single Index Cost:** Pay for only one GSI instead of multiple indexes + +**Query Flexibility Examples:** +- Query 1: `Category = "Programming"` → All programming skills +- Query 2: `Category = "Programming" AND SkillName = "Python"` → All Python users +- Query 3: `Category = "Programming" AND SkillName = "Python" AND ProficiencyLevel = "Expert"` → Python experts +- Query 4: `... AND YearsOfExperience >= 5` → Experienced Python experts +- Query 5: `... AND Username = "john"` → Specific user check + +**All from ONE index!** + +--- + +## Sample Data Structure + +### Main Table Sample Items + +| EntityType | entity_id | Additional Attributes | Description | +|-------------|-----------------------------|---------------------------------------------------------------------------------------------------------|-------------------------------| +| `User` | `USER#john_doe` | Username, Name, Email, CreatedAt, UpdatedAt | User profile | +| `Skill` | `SKILL#python` | SkillID, SkillName, Category, Description, Tags | Master skill catalog | +| `UserSkill` | `USERSKILL#john_doe#python` | Username, SkillID, SkillName, Category, ProficiencyLevel, YearsOfExperience, Endorsements, LastUsedDate | User's skill with proficiency | + +### GSI `BySkill` Sample Items + +Here's how UserSkill items appear in the GSI (sorted by composite sort key): + +| Category | SkillName | ProficiencyLevel | YearsOfExperience | Username | Endorsements | LastUsedDate | +|-------------|------------|------------------|-------------------|---------------|--------------|--------------| +| Programming | Python | Beginner | 1 | jane_smith | 5 | 2025-12-10 | +| Programming | Python | Intermediate | 3 | mike_wilson | 25 | 2025-11-15 | +| Programming | Python | Advanced | 5 | alice_johnson | 65 | 2025-12-15 | +| Programming | Python | Expert | 7 | bob_smith | 120 | 2025-12-18 | +| Programming | Python | Expert | 10 | diana_evans | 200 | 2025-12-20 | +| Frontend | TypeScript | Advanced | 4 | alex_chen | 30 | 2025-12-15 | +| Frontend | TypeScript | Expert | 6 | betty_wang | 75 | 2025-12-18 | +| Backend | Go | Expert | 7 | alice_smith | 80 | 2025-12-18 | +| Cloud | AWS | Expert | 9 | charlie_brown | 150 | 2025-11-20 | +| DevOps | Docker | Beginner | 1 | tom_davis | 8 | 2025-10-10 | + +**Notice:** Items are naturally sorted by `Category` → `SkillName` → `ProficiencyLevel` → `YearsOfExperience` → `Username`. This sorting enables efficient range queries and pagination. + +--- + +## Key Attributes Explained + +### Main Table Keys +- **EntityType:** Discriminator to separate Users, Skills, and UserSkills +- **entity_id:** Hierarchical identifier pattern: + - `USER#` + - `SKILL#` + - `USERSKILL##` + +### GSI Keys (BySkill) +- **Category (PK):** Broad partitioning (Programming, Frontend, Backend, Cloud, DevOps, Database, Mobile, Data, Security, Other) +- **SkillName (SK1):** Specific skill within category +- **ProficiencyLevel (SK2):** Beginner, Intermediate, Advanced, Expert +- **YearsOfExperience (SK3):** NUMBER type for range queries and sorting +- **Username (SK4):** User identifier, ensures uniqueness + +### Non-Key Attributes (Available for FilterExpression) +- **Endorsements:** NUMBER - Peer endorsements count +- **LastUsedDate:** STRING (ISO 8601) - When skill was last used +- **Notes:** STRING - Additional skill notes +- **SkillID:** STRING - Immutable skill identifier +- **CreatedAt/UpdatedAt:** STRING (ISO 8601) - Timestamps + +--- + +## GSI Design (Option 1 - Maximum Flexibility) + +**Index Name:** `BySkill` + +**Partition Key (1 attribute):** +- `Category` (String) + +**Sort Keys (4 attributes):** +1. `SkillName` (String) +2. `ProficiencyLevel` (String) +3. `YearsOfExperience` (Number) +4. `Username` (String) + +**Note:** Endorsements (Number) and LastUsedDate (String) are not in the GSI keys, but can be used with FilterExpression. + +--- + +## Access Patterns Summary + +### Main Table Access Patterns + +| # | Pattern Name | Table/Index | Key Condition | Use Case | API Endpoint | +|---|---------------------------|-------------|--------------------------------------------------------------------------------|-----------------------------------|--------------------------------------------| +| 1 | Get All Users | Main Table | `EntityType = "User"` | List all users in system | `GET /users` | +| 2 | Get All Master Skills | Main Table | `EntityType = "Skill"` | List all master skill definitions | `GET /master-skills` | +| 3 | Get All User Skills | Main Table | `EntityType = "UserSkill"` | List all user skill records | - | +| 4 | Get Specific User | Main Table | `EntityType = "User" AND entity_id = "USER#"` | Get user profile by username | `GET /users/{username}` | +| 5 | Get Specific Master Skill | Main Table | `EntityType = "Skill" AND entity_id = "SKILL#"` | Get master skill details | `GET /master-skills/{skillID}` | +| 6 | Get Specific User Skill | Main Table | `EntityType = "UserSkill" AND entity_id = "USERSKILL##"` | Get user's specific skill | `GET /users/{username}/skills/{skillName}` | +| 7 | Get All Skills for User | Main Table | `EntityType = "UserSkill" AND begins_with(entity_id, "USERSKILL##")` | List all skills for a user | `GET /users/{username}/skills` | + +### GSI Access Patterns (BySkill Index) + +| # | Pattern Name | PK | SK Condition | Filter | Use Case | API Endpoint | +|----|-------------------------------|-------------------|------------------------------------------------------------------------------------------------|--------------------------------------------------|---------------------------------------------------------------------|----------------------------------------------------------------------------------------------| +| 8 | All Skills in Category | `Category = :cat` | - | - | All skills in Programming category | `GET /skills?category=Programming` | +| 9 | All Users with Skill | `Category = :cat` | `SkillName = :skill` | - | Everyone who knows Python | `GET /skills/python/users` | +| 10 | Users at Skill Level | `Category = :cat` | `SkillName = :skill AND ProficiencyLevel = :level` | - | All Python experts | `GET /skills/python/users?level=Expert` | +| 11 | Users with Min Experience | `Category = :cat` | `SkillName = :skill AND ProficiencyLevel = :level AND YearsOfExperience >= :years` | - | Python experts with 5+ years | `GET /skills/python/users?level=Expert&minYears=5` | +| 12 | Users by Experience (Desc) | `Category = :cat` | `SkillName = :skill AND ProficiencyLevel = :level` | `--scan-index-forward false` | Most experienced Python experts | `GET /skills/python/users?level=Expert&sort=experience_desc` | +| 13 | Experience Range | `Category = :cat` | `SkillName = :skill AND ProficiencyLevel = :level AND YearsOfExperience BETWEEN :min AND :max` | - | Intermediate Python devs (2-5 years) | `GET /skills/python/users?level=Intermediate&minYears=2&maxYears=5` | +| 14 | Check User Skill Level | `Category = :cat` | `SkillName = :skill AND ProficiencyLevel = :level AND Username = :user` | - | Check if john_doe is Python expert | `GET /users/john_doe/skills/python?checkLevel=Expert` | +| 15 | Filter by Endorsements | `Category = :cat` | `SkillName = :skill AND ProficiencyLevel = :level` | `Endorsements >= :min` | Python experts with 50+ endorsements | `GET /skills/python/users?level=Expert&minEndorsements=50` | +| 16 | Filter by Last Used | `Category = :cat` | `SkillName = :skill AND ProficiencyLevel = :level` | `LastUsedDate >= :date` | Python experts active in last 6 months | `GET /skills/python/users?level=Expert&activeSince=2024-06-20` | +| 17 | Multi-Criteria Filter | `Category = :cat` | `SkillName = :skill AND ProficiencyLevel = :level AND YearsOfExperience >= :years` | `Endorsements >= :min AND LastUsedDate >= :date` | Senior Python experts (5+ years, 50+ endorsements, recently active) | `GET /skills/python/users?level=Expert&minYears=5&minEndorsements=50&activeSince=2024-06-20` | +| 18 | All Expert Skills in Category | `Category = :cat` | - | `ProficiencyLevel = :level` | All expert-level skills in Programming | `GET /skills?category=Programming&level=Expert` | +| 19 | Skills with Prefix | `Category = :cat` | `begins_with(SkillName, :prefix)` | - | All Java-related skills (Java, JavaScript) | `GET /skills?category=Programming&skillPrefix=Java` | +| 20 | Proficiency Distribution | `Category = :cat` | `SkillName = :skill AND ProficiencyLevel = :level` | `--select COUNT` | Count Python users by proficiency level | `GET /skills/python/distribution` | +| 21 | Top N by Experience | `Category = :cat` | `SkillName = :skill AND ProficiencyLevel = :level` | `--scan-index-forward false --limit N` | Top 10 most experienced Go experts | `GET /skills/go/users/top?limit=10&level=Expert` | +| 22 | Users Above Min Level | `Category = :cat` | `SkillName = :skill` | `ProficiencyLevel IN (:level1, :level2)` | TypeScript users at Advanced or Expert | `GET /skills/typescript/users?minLevel=Advanced` | +| 23 | Pagination | `Category = :cat` | - | `--limit N --exclusive-start-key` | Paginated results for category | `GET /skills?category=Programming&page=2&limit=20` | + +**Total Access Patterns:** 23 (7 Main Table + 16 GSI) + +--- + +## Prerequisites + +```bash +# AWS Profile +export AWS_PROFILE=passbrains-ilisa-amplify + +# Table and Index names +TABLE_NAME="glad-entities" +GSI_NAME="BySkill" +``` + +--- + +## Main Table Query Patterns + +### Pattern 1: Get All Users + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --key-condition-expression "EntityType = :entityType" \ + --expression-attribute-values '{ + ":entityType": {"S": "User"} + }' +``` + +--- + +### Pattern 2: Get All Master Skills + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --key-condition-expression "EntityType = :entityType" \ + --expression-attribute-values '{ + ":entityType": {"S": "Skill"} + }' +``` + +--- + +### Pattern 3: Get All User Skills + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --key-condition-expression "EntityType = :entityType" \ + --expression-attribute-values '{ + ":entityType": {"S": "UserSkill"} + }' +``` + +--- + +### Pattern 4: Get Specific User + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --key-condition-expression "EntityType = :entityType AND entity_id = :entityId" \ + --expression-attribute-values '{ + ":entityType": {"S": "User"}, + ":entityId": {"S": "USER#john_doe"} + }' +``` + +--- + +### Pattern 5: Get Specific Master Skill + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --key-condition-expression "EntityType = :entityType AND entity_id = :entityId" \ + --expression-attribute-values '{ + ":entityType": {"S": "Skill"}, + ":entityId": {"S": "SKILL#python"} + }' +``` + +--- + +### Pattern 6: Get Specific User Skill + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --key-condition-expression "EntityType = :entityType AND entity_id = :entityId" \ + --expression-attribute-values '{ + ":entityType": {"S": "UserSkill"}, + ":entityId": {"S": "USERSKILL#jane_doe#python"} + }' +``` + +--- + +### Pattern 7: Get All Skills for a User + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --key-condition-expression "EntityType = :entityType AND begins_with(entity_id, :prefix)" \ + --expression-attribute-values '{ + ":entityType": {"S": "UserSkill"}, + ":prefix": {"S": "USERSKILL#john_doe#"} + }' +``` + +--- + +## GSI Query Patterns (BySkill) - Maximum Flexibility! + +### GSI Pattern 1: All Skills in a Category + +**Use Case:** List all user skills in a specific category (e.g., all Programming skills) + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category" \ + --expression-attribute-values '{ + ":category": {"S": "Programming"} + }' +``` + +**Returns:** All UserSkill items in the Programming category + +--- + +### GSI Pattern 2: All Users with a Specific Skill (Any Level) + +**Use Case:** Find everyone who has Python, regardless of proficiency level + +//todo +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName" \ + --expression-attribute-values '{ + ":category": {"S": "Programming"}, + ":skillName": {"S": "JavaScript"} + }' +``` + +**Returns:** All UserSkill items for Python (all proficiency levels) + +--- + +### GSI Pattern 3: Users with Skill at Specific Proficiency Level + +**Use Case:** Find all Python experts + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ + --expression-attribute-values '{ + ":category": {"S": "DevOps"}, + ":skillName": {"S": "Docker"}, + ":level": {"S": "Beginner"} + }' +``` + +**Returns:** All Python experts + +--- + +### GSI Pattern 4: Users with Minimum Years of Experience + +**Use Case:** Find Python experts with at least 5 years of experience + +**Option A: Using Sort Key Condition (Efficient)** + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level AND YearsOfExperience >= :minYears" \ + --expression-attribute-values '{ + ":category": {"S": "DevOps"}, + ":skillName": {"S": "Docker"}, + ":level": {"S": "Beginner"}, + ":minYears": {"N": "1"} + }' +``` + +**Returns:** Python experts with 5+ years of experience, sorted by experience (ascending) + +--- + +### GSI Pattern 5: Users Sorted by Experience (Descending) + +**Use Case:** Find Python experts ordered by most experienced first + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ + --no-scan-index-forward \ + --expression-attribute-values '{ + ":category": {"S": "DevOps"}, + ":skillName": {"S": "Docker"}, + ":level": {"S": "Beginner"} + }' +``` + +**Returns:** Python experts sorted by experience (most experienced first) + +--- + +### GSI Pattern 6: Experience Range Query + +**Use Case:** Find intermediate Python developers with 2-5 years of experience + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level AND YearsOfExperience BETWEEN :minYears AND :maxYears" \ + --expression-attribute-values '{ + ":category": {"S": "DevOps"}, + ":skillName": {"S": "Docker"}, + ":level": {"S": "Beginner"}, + ":minYears": {"N": "1"}, + ":maxYears": {"N": "10"} + }' +``` + +**Returns:** Intermediate Python developers with 2-5 years experience + +--- + +### GSI Pattern 7: Check if Specific User Has Skill at Level + +**Use Case:** Check if user "john_doe" is a Python expert + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --key-condition-expression "EntityType = :entityType AND entity_id = :entityId" \ + --expression-attribute-values '{ + ":entityType": {"S": "UserSkill"}, + ":entityId": {"S": "USERSKILL#jane_doe#python"} + }' + +``` + +**Returns:** Single UserSkill item if exists, empty if not + +--- + +### GSI Pattern 8: Filter by Endorsements (Using FilterExpression) + +**Use Case:** Find Python experts with at least 50 endorsements + +**Note:** Endorsements is NOT in the GSI keys, so we use FilterExpression + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ + --filter-expression "Endorsements >= :minEndorsements" \ + --expression-attribute-values '{ + ":category": {"S": "DevOps"}, + ":skillName": {"S": "Docker"}, + ":level": {"S": "Beginner"}, + ":minEndorsements": {"N": "2"} + }' +``` + +**Returns:** Python experts with 50+ endorsements (filtered after query) + +--- + +### GSI Pattern 9: Filter by Last Used Date (Using FilterExpression) + +**Use Case:** Find Beginners who used Docker in the last 6 months + +**Note:** LastUsedDate is NOT in the GSI keys, so we use FilterExpression + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ + --filter-expression "LastUsedDate >= :recentDate" \ + --expression-attribute-values '{ + ":category": {"S": "DevOps"}, + ":skillName": {"S": "Docker"}, + ":level": {"S": "Beginner"}, + ":recentDate": {"S": "2024-06-20"} + }' +``` + +**Returns:** Beginners who used Docker since 2024-06-20 + +--- + +### GSI Pattern 10: Complex Multi-Criteria Filter + +**Use Case:** Find senior Python experts (5+ years, 50+ endorsements, used recently) + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level AND YearsOfExperience >= :minYears" \ + --filter-expression "Endorsements >= :minEndorsements AND LastUsedDate >= :minDate" \ + --expression-attribute-values '{ + ":category": {"S": "Programming"}, + ":skillName": {"S": "Python 3"}, + ":level": {"S": "Expert"}, + ":minYears": {"N": "5"}, + ":minEndorsements": {"N": "50"}, + ":minDate": {"S": "2024-06-20"} + }' +``` + +**Returns:** Highly qualified, active Python experts + +--- + +### GSI Pattern 11: All Skills at Specific Proficiency (Across All Skills) + +**Use Case:** Find all Expert-level skills in Programming category + +**⚠️ WARNING:** This query will FAIL because `ProficiencyLevel` is a GSI key attribute (SK2) and cannot be used in FilterExpression. + +**Error:** `Filter Expression can only contain non-primary key attributes: Primary key attribute: ProficiencyLevel` + +**Workaround:** Query each proficiency level separately or fetch all skills in the category and filter in application code. + +```bash +# This query will return an ERROR - kept for reference +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category" \ + --filter-expression "ProficiencyLevel = :level" \ + --expression-attribute-values '{ + ":category": {"S": "Programming"}, + ":level": {"S": "Expert"} + }' +``` + +**Alternative - Fetch all and filter in code:** +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category" \ + --expression-attribute-values '{ + ":category": {"S": "Programming"} + }' +# Then filter ProficiencyLevel = "Expert" in your application code +``` + +--- + +### GSI Pattern 12: Skills Starting with Prefix + +**Use Case:** Find all users with skills starting with "Java" (Java, JavaScript, etc.) + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND begins_with(SkillName, :skillPrefix)" \ + --expression-attribute-values '{ + ":category": {"S": "Programming"}, + ":skillPrefix": {"S": "Java"} + }' +``` + +**Returns:** All UserSkills with skills starting with "Java" + +--- + +### GSI Pattern 13: Proficiency Distribution for a Skill (Count by Level) + +**Use Case:** Get count of Python users at each proficiency level + +**Query 1 - Beginner:** + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ + --select COUNT \ + --expression-attribute-values '{ + ":category": {"S": "Programming"}, + ":skillName": {"S": "Python"}, + ":level": {"S": "Beginner"} + }' +``` + +**Query 2 - Intermediate:** + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ + --select COUNT \ + --expression-attribute-values '{ + ":category": {"S": "Programming"}, + ":skillName": {"S": "Python"}, + ":level": {"S": "Intermediate"} + }' +``` + +**Query 3 - Advanced:** + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ + --select COUNT \ + --expression-attribute-values '{ + ":category": {"S": "Programming"}, + ":skillName": {"S": "Python"}, + ":level": {"S": "Advanced"} + }' +``` + +**Query 4 - Expert:** + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ + --select COUNT \ + --expression-attribute-values '{ + ":category": {"S": "DevOps"}, + ":skillName": {"S": "Docker"}, + ":level": {"S": "Beginner"} + }' +``` + +**Returns:** Count for each proficiency level (4 queries total) + +--- + +### GSI Pattern 14: Top N Users by Experience + +**Use Case:** Get top 10 most experienced Go experts + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ + --limit 10 \ + --expression-attribute-values '{ + ":category": {"S": "Backend"}, + ":skillName": {"S": "Go"}, + ":level": {"S": "Expert"} + }' +``` + +**Returns:** Top 10 Go experts by experience + +--- + +### GSI Pattern 15: Users Above Minimum Level (Multiple Levels) + +**Use Case:** Find TypeScript users at Advanced OR Expert level + +**Note:** This pattern returns ALL proficiency levels, so filter in application code or run separate queries for each level. + +```bash + aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName" \ + --expression-attribute-values '{ + ":category": {"S": "Frontend"}, + ":skillName": {"S": "TypeScript"} + }' + +# Then filter for ProficiencyLevel IN ("Advanced", "Expert") in application code +``` + +**Alternative - Run two separate queries:** +```bash +# Query 1: Advanced +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ + --expression-attribute-values '{ + ":category": {"S": "Frontend"}, + ":skillName": {"S": "TypeScript"}, + ":level": {"S": "Advanced"} + }' + +# Query 2: Expert +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ + --expression-attribute-values '{ + ":category": {"S": "Frontend"}, + ":skillName": {"S": "TypeScript"}, + ":level": {"S": "Expert"} + }' +# Merge results in application code +``` + +**Returns:** TypeScript users at Advanced or Expert level + +--- + +### GSI Pattern 16: Pagination Example + +**Use Case:** Get first 20 skills in Programming category, then get next page + +**Page 1:** + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category" \ + --limit 20 \ + --expression-attribute-values '{ + ":category": {"S": "Programming"} + }' +``` + +**Page 2 (use LastEvaluatedKey from previous response):** + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category" \ + --limit 20 \ + --exclusive-start-key '{"Category":{"S":"Programming"},"SkillName":{"S":"Python"},"ProficiencyLevel":{"S":"Expert"},"YearsOfExperience":{"N":"5"},"Username":{"S":"alice"},"EntityType":{"S":"UserSkill"},"entity_id":{"S":"USERSKILL#alice#python"}}' \ + --expression-attribute-values '{ + ":category": {"S": "Programming"} + }' +``` + +--- + +## Utility Commands + +### Count Total Users + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --key-condition-expression "EntityType = :entityType" \ + --select COUNT \ + --expression-attribute-values '{ + ":entityType": {"S": "User"} + }' +``` + +--- + +### Count Users with Specific Skill + +```bash +aws dynamodb query \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --index-name BySkill \ + --key-condition-expression "Category = :category AND SkillName = :skillName" \ + --select COUNT \ + --expression-attribute-values '{ + ":category": {"S": "Programming"}, + ":skillName": {"S": "Python"} + }' +``` + +--- + +### Get Item by Primary Key (GetItem - faster than Query) + +```bash +aws dynamodb get-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --key '{ + "EntityType": {"S": "User"}, + "entity_id": {"S": "USER#john_doe"} + }' +``` + +--- + +## Sample Data for Testing + +### Create Test Users + +```bash +aws dynamodb put-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --item '{ + "EntityType": {"S": "User"}, + "entity_id": {"S": "USER#john_doe"}, + "Username": {"S": "john_doe"}, + "Name": {"S": "John Doe"}, + "Email": {"S": "john@example.com"}, + "CreatedAt": {"S": "2025-01-01T00:00:00Z"}, + "UpdatedAt": {"S": "2025-01-01T00:00:00Z"} + }' + +aws dynamodb put-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --item '{ + "EntityType": {"S": "User"}, + "entity_id": {"S": "USER#jane_doe"}, + "Username": {"S": "jane_doe"}, + "Name": {"S": "Jane Doe"}, + "Email": {"S": "jane@example.com"}, + "CreatedAt": {"S": "2025-01-01T00:00:00Z"}, + "UpdatedAt": {"S": "2025-01-01T00:00:00Z"} + }' + +aws dynamodb put-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --item '{ + "EntityType": {"S": "User"}, + "entity_id": {"S": "USER#alice_smith"}, + "Username": {"S": "alice_smith"}, + "Name": {"S": "Alice Smith"}, + "Email": {"S": "alice@example.com"}, + "CreatedAt": {"S": "2025-01-01T00:00:00Z"}, + "UpdatedAt": {"S": "2025-01-01T00:00:00Z"} + }' +``` + +--- + +### Create Test Master Skills + +```bash +aws dynamodb put-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --item '{ + "EntityType": {"S": "Skill"}, + "entity_id": {"S": "SKILL#python"}, + "SkillID": {"S": "python"}, + "SkillName": {"S": "Python"}, + "Category": {"S": "Programming"}, + "Description": {"S": "Python programming language"}, + "CreatedAt": {"S": "2025-01-01T00:00:00Z"}, + "UpdatedAt": {"S": "2025-01-01T00:00:00Z"} + }' + +aws dynamodb put-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --item '{ + "EntityType": {"S": "Skill"}, + "entity_id": {"S": "SKILL#go"}, + "SkillID": {"S": "go"}, + "SkillName": {"S": "Go"}, + "Category": {"S": "Backend"}, + "Description": {"S": "Go programming language"}, + "CreatedAt": {"S": "2025-01-01T00:00:00Z"}, + "UpdatedAt": {"S": "2025-01-01T00:00:00Z"} + }' + +aws dynamodb put-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --item '{ + "EntityType": {"S": "Skill"}, + "entity_id": {"S": "SKILL#react"}, + "SkillID": {"S": "react"}, + "SkillName": {"S": "React"}, + "Category": {"S": "Frontend"}, + "Description": {"S": "React JavaScript library"}, + "CreatedAt": {"S": "2025-01-01T00:00:00Z"}, + "UpdatedAt": {"S": "2025-01-01T00:00:00Z"} + }' +``` + +--- + +### Create Test UserSkills + +```bash +# John - Python Expert (10 years, 150 endorsements) +aws dynamodb put-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --item '{ + "EntityType": {"S": "UserSkill"}, + "entity_id": {"S": "USERSKILL#john_doe#python"}, + "Username": {"S": "john_doe"}, + "SkillID": {"S": "python"}, + "SkillName": {"S": "Python"}, + "Category": {"S": "Programming"}, + "ProficiencyLevel": {"S": "Expert"}, + "YearsOfExperience": {"N": "10"}, + "Endorsements": {"N": "150"}, + "LastUsedDate": {"S": "2025-12-15"}, + "Notes": {"S": "Experienced Python developer"}, + "CreatedAt": {"S": "2025-01-01T00:00:00Z"}, + "UpdatedAt": {"S": "2025-12-15T00:00:00Z"} + }' + +# Jane - Python Advanced (5 years, 42 endorsements) +aws dynamodb put-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --item '{ + "EntityType": {"S": "UserSkill"}, + "entity_id": {"S": "USERSKILL#jane_doe#python"}, + "Username": {"S": "jane_doe"}, + "SkillID": {"S": "python"}, + "SkillName": {"S": "Python"}, + "Category": {"S": "Programming"}, + "ProficiencyLevel": {"S": "Advanced"}, + "YearsOfExperience": {"N": "5"}, + "Endorsements": {"N": "42"}, + "LastUsedDate": {"S": "2025-11-20"}, + "Notes": {"S": ""}, + "CreatedAt": {"S": "2025-01-01T00:00:00Z"}, + "UpdatedAt": {"S": "2025-11-20T00:00:00Z"} + }' + +# Alice - Go Expert (7 years, 80 endorsements) +aws dynamodb put-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --item '{ + "EntityType": {"S": "UserSkill"}, + "entity_id": {"S": "USERSKILL#alice_smith#go"}, + "Username": {"S": "alice_smith"}, + "SkillID": {"S": "go"}, + "SkillName": {"S": "Go"}, + "Category": {"S": "Backend"}, + "ProficiencyLevel": {"S": "Expert"}, + "YearsOfExperience": {"N": "7"}, + "Endorsements": {"N": "80"}, + "LastUsedDate": {"S": "2025-12-18"}, + "Notes": {"S": ""}, + "CreatedAt": {"S": "2025-01-01T00:00:00Z"}, + "UpdatedAt": {"S": "2025-12-18T00:00:00Z"} + }' + +# John - React Intermediate (3 years, 25 endorsements) +aws dynamodb put-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --item '{ + "EntityType": {"S": "UserSkill"}, + "entity_id": {"S": "USERSKILL#john_doe#react"}, + "Username": {"S": "john_doe"}, + "SkillID": {"S": "react"}, + "SkillName": {"S": "React"}, + "Category": {"S": "Frontend"}, + "ProficiencyLevel": {"S": "Intermediate"}, + "YearsOfExperience": {"N": "3"}, + "Endorsements": {"N": "25"}, + "LastUsedDate": {"S": "2025-12-10"}, + "Notes": {"S": ""}, + "CreatedAt": {"S": "2025-01-01T00:00:00Z"}, + "UpdatedAt": {"S": "2025-12-10T00:00:00Z"} + }' + +# Jane - Python Beginner (1 year, 5 endorsements) +aws dynamodb put-item \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --item '{ + "EntityType": {"S": "UserSkill"}, + "entity_id": {"S": "USERSKILL#jane_doe#go"}, + "Username": {"S": "jane_doe"}, + "SkillID": {"S": "go"}, + "SkillName": {"S": "Go"}, + "Category": {"S": "Backend"}, + "ProficiencyLevel": {"S": "Beginner"}, + "YearsOfExperience": {"N": "1"}, + "Endorsements": {"N": "5"}, + "LastUsedDate": {"S": "2025-10-15"}, + "Notes": {"S": ""}, + "CreatedAt": {"S": "2025-01-01T00:00:00Z"}, + "UpdatedAt": {"S": "2025-10-15T00:00:00Z"} + }' +``` + +--- + +## Troubleshooting + +### Check if GSI Exists + +```bash +aws dynamodb describe-table \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --query "Table.GlobalSecondaryIndexes[?IndexName=='BySkill']" +``` + +--- + +### Check GSI Status + +```bash +aws dynamodb describe-table \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --query "Table.GlobalSecondaryIndexes[?IndexName=='BySkill'].IndexStatus" +``` + +--- + +### List All Attributes in Table + +```bash +aws dynamodb describe-table \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --query "Table.AttributeDefinitions" +``` + +--- + +### Scan Entire Table (Use Sparingly!) + +```bash +aws dynamodb scan \ + --table-name glad-entities --profile passbrains-ilisa-amplify \ + --limit 10 +``` + +--- + +## Query Pattern Summary + +| Pattern | PK | SK1 | SK2 | SK3 | SK4 | Use Case | +|---------|----------|-----------|------------------|---------------------------|----------|------------------------| +| 1 | Category | - | - | - | - | All skills in category | +| 2 | Category | SkillName | - | - | - | All users with skill | +| 3 | Category | SkillName | ProficiencyLevel | - | - | Users at skill level | +| 4 | Category | SkillName | ProficiencyLevel | YearsOfExperience >= | - | Min experience filter | +| 5 | Category | SkillName | ProficiencyLevel | YearsOfExperience (desc) | - | Most experienced first | +| 6 | Category | SkillName | ProficiencyLevel | YearsOfExperience BETWEEN | - | Experience range | +| 7 | Category | SkillName | ProficiencyLevel | YearsOfExperience | Username | Specific user check | + +--- + +## Important Notes + +1. **Data Types:** + - `YearsOfExperience`: NUMBER type (e.g., {"N": "5"}) + - `Endorsements`: NUMBER type (e.g., {"N": "42"}) + - No zero-padding needed since both are stored as numbers + +2. **Sort Key Hierarchy:** + - Must query from left to right + - Can use partial sort keys (SK1, SK1+SK2, SK1+SK2+SK3, etc.) + - Cannot skip sort keys (e.g., cannot query SK1+SK3 without SK2) + +3. **Range Queries:** + - Only on the LAST sort key specified + - Operators: `<`, `>`, `<=`, `>=`, `BETWEEN`, `begins_with` + +4. **FilterExpression:** + - For attributes NOT in GSI keys (Endorsements, LastUsedDate) + - Applied AFTER query (less efficient than KeyConditionExpression) + +5. **Sorting:** + - Default: Ascending (`--scan-index-forward true`) + - Descending: `--scan-index-forward false` + +6. **Pagination:** + - Use `--limit` to control page size + - Use `--exclusive-start-key` with LastEvaluatedKey for next page \ No newline at end of file From 80929ba7d9bc8af113546716b056f20c32590028 Mon Sep 17 00:00:00 2001 From: Alex Ilies Date: Sat, 20 Dec 2025 23:14:38 +0200 Subject: [PATCH 2/6] infra(gsi): gsi changes --- docs/dynamodb_table_design.md | 98 +++++++++++++++++------------------ 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/docs/dynamodb_table_design.md b/docs/dynamodb_table_design.md index a2ea66b..221b314 100644 --- a/docs/dynamodb_table_design.md +++ b/docs/dynamodb_table_design.md @@ -174,7 +174,7 @@ Here's how UserSkill items appear in the GSI (sorted by composite sort key): ```bash # AWS Profile -export AWS_PROFILE=passbrains-ilisa-amplify +export AWS_PROFILE={YOUR_AWS_PROFILE_NAME} # Table and Index names TABLE_NAME="glad-entities" @@ -189,7 +189,7 @@ GSI_NAME="BySkill" ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --key-condition-expression "EntityType = :entityType" \ --expression-attribute-values '{ ":entityType": {"S": "User"} @@ -202,7 +202,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --key-condition-expression "EntityType = :entityType" \ --expression-attribute-values '{ ":entityType": {"S": "Skill"} @@ -215,7 +215,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --key-condition-expression "EntityType = :entityType" \ --expression-attribute-values '{ ":entityType": {"S": "UserSkill"} @@ -228,7 +228,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --key-condition-expression "EntityType = :entityType AND entity_id = :entityId" \ --expression-attribute-values '{ ":entityType": {"S": "User"}, @@ -242,7 +242,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --key-condition-expression "EntityType = :entityType AND entity_id = :entityId" \ --expression-attribute-values '{ ":entityType": {"S": "Skill"}, @@ -256,7 +256,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --key-condition-expression "EntityType = :entityType AND entity_id = :entityId" \ --expression-attribute-values '{ ":entityType": {"S": "UserSkill"}, @@ -270,7 +270,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --key-condition-expression "EntityType = :entityType AND begins_with(entity_id, :prefix)" \ --expression-attribute-values '{ ":entityType": {"S": "UserSkill"}, @@ -288,7 +288,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category" \ --expression-attribute-values '{ @@ -307,7 +307,7 @@ aws dynamodb query \ //todo ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName" \ --expression-attribute-values '{ @@ -326,7 +326,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ --expression-attribute-values '{ @@ -348,7 +348,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level AND YearsOfExperience >= :minYears" \ --expression-attribute-values '{ @@ -369,7 +369,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ --no-scan-index-forward \ @@ -390,7 +390,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level AND YearsOfExperience BETWEEN :minYears AND :maxYears" \ --expression-attribute-values '{ @@ -412,7 +412,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --key-condition-expression "EntityType = :entityType AND entity_id = :entityId" \ --expression-attribute-values '{ ":entityType": {"S": "UserSkill"}, @@ -433,7 +433,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ --filter-expression "Endorsements >= :minEndorsements" \ @@ -457,7 +457,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ --filter-expression "LastUsedDate >= :recentDate" \ @@ -479,7 +479,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level AND YearsOfExperience >= :minYears" \ --filter-expression "Endorsements >= :minEndorsements AND LastUsedDate >= :minDate" \ @@ -510,7 +510,7 @@ aws dynamodb query \ ```bash # This query will return an ERROR - kept for reference aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category" \ --filter-expression "ProficiencyLevel = :level" \ @@ -523,7 +523,7 @@ aws dynamodb query \ **Alternative - Fetch all and filter in code:** ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category" \ --expression-attribute-values '{ @@ -540,7 +540,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND begins_with(SkillName, :skillPrefix)" \ --expression-attribute-values '{ @@ -561,7 +561,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ --select COUNT \ @@ -576,7 +576,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ --select COUNT \ @@ -591,7 +591,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ --select COUNT \ @@ -606,7 +606,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ --select COUNT \ @@ -627,7 +627,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ --limit 10 \ @@ -650,7 +650,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName" \ --expression-attribute-values '{ @@ -665,7 +665,7 @@ aws dynamodb query \ ```bash # Query 1: Advanced aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ --expression-attribute-values '{ @@ -676,7 +676,7 @@ aws dynamodb query \ # Query 2: Expert aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName AND ProficiencyLevel = :level" \ --expression-attribute-values '{ @@ -699,7 +699,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category" \ --limit 20 \ @@ -712,7 +712,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category" \ --limit 20 \ @@ -730,7 +730,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --key-condition-expression "EntityType = :entityType" \ --select COUNT \ --expression-attribute-values '{ @@ -744,7 +744,7 @@ aws dynamodb query \ ```bash aws dynamodb query \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --index-name BySkill \ --key-condition-expression "Category = :category AND SkillName = :skillName" \ --select COUNT \ @@ -760,7 +760,7 @@ aws dynamodb query \ ```bash aws dynamodb get-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --key '{ "EntityType": {"S": "User"}, "entity_id": {"S": "USER#john_doe"} @@ -775,7 +775,7 @@ aws dynamodb get-item \ ```bash aws dynamodb put-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --item '{ "EntityType": {"S": "User"}, "entity_id": {"S": "USER#john_doe"}, @@ -787,7 +787,7 @@ aws dynamodb put-item \ }' aws dynamodb put-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --item '{ "EntityType": {"S": "User"}, "entity_id": {"S": "USER#jane_doe"}, @@ -799,7 +799,7 @@ aws dynamodb put-item \ }' aws dynamodb put-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --item '{ "EntityType": {"S": "User"}, "entity_id": {"S": "USER#alice_smith"}, @@ -817,7 +817,7 @@ aws dynamodb put-item \ ```bash aws dynamodb put-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --item '{ "EntityType": {"S": "Skill"}, "entity_id": {"S": "SKILL#python"}, @@ -830,7 +830,7 @@ aws dynamodb put-item \ }' aws dynamodb put-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --item '{ "EntityType": {"S": "Skill"}, "entity_id": {"S": "SKILL#go"}, @@ -843,7 +843,7 @@ aws dynamodb put-item \ }' aws dynamodb put-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --item '{ "EntityType": {"S": "Skill"}, "entity_id": {"S": "SKILL#react"}, @@ -863,7 +863,7 @@ aws dynamodb put-item \ ```bash # John - Python Expert (10 years, 150 endorsements) aws dynamodb put-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --item '{ "EntityType": {"S": "UserSkill"}, "entity_id": {"S": "USERSKILL#john_doe#python"}, @@ -882,7 +882,7 @@ aws dynamodb put-item \ # Jane - Python Advanced (5 years, 42 endorsements) aws dynamodb put-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --item '{ "EntityType": {"S": "UserSkill"}, "entity_id": {"S": "USERSKILL#jane_doe#python"}, @@ -901,7 +901,7 @@ aws dynamodb put-item \ # Alice - Go Expert (7 years, 80 endorsements) aws dynamodb put-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --item '{ "EntityType": {"S": "UserSkill"}, "entity_id": {"S": "USERSKILL#alice_smith#go"}, @@ -920,7 +920,7 @@ aws dynamodb put-item \ # John - React Intermediate (3 years, 25 endorsements) aws dynamodb put-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --item '{ "EntityType": {"S": "UserSkill"}, "entity_id": {"S": "USERSKILL#john_doe#react"}, @@ -939,7 +939,7 @@ aws dynamodb put-item \ # Jane - Python Beginner (1 year, 5 endorsements) aws dynamodb put-item \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --item '{ "EntityType": {"S": "UserSkill"}, "entity_id": {"S": "USERSKILL#jane_doe#go"}, @@ -965,7 +965,7 @@ aws dynamodb put-item \ ```bash aws dynamodb describe-table \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --query "Table.GlobalSecondaryIndexes[?IndexName=='BySkill']" ``` @@ -975,7 +975,7 @@ aws dynamodb describe-table \ ```bash aws dynamodb describe-table \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --query "Table.GlobalSecondaryIndexes[?IndexName=='BySkill'].IndexStatus" ``` @@ -985,7 +985,7 @@ aws dynamodb describe-table \ ```bash aws dynamodb describe-table \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --query "Table.AttributeDefinitions" ``` @@ -995,7 +995,7 @@ aws dynamodb describe-table \ ```bash aws dynamodb scan \ - --table-name glad-entities --profile passbrains-ilisa-amplify \ + --table-name glad-entities --profile AWS_PROFILE \ --limit 10 ``` From 4834286bd51842518668e87436833c13c8b65fa3 Mon Sep 17 00:00:00 2001 From: Alex Ilies Date: Sun, 21 Dec 2025 00:42:36 +0200 Subject: [PATCH 3/6] tests(api): run .http tests in Docker --- cmd/app/integration_test.go | 2 +- .../master_skill_repository_dynamodb.go | 6 +- .../user_skill_repository_dynamodb.go | 3 +- cmd/app/internal/errors/user.go | 6 + cmd/app/internal/handler/error_mapper.go | 18 + cmd/app/internal/handler/user_handler_test.go | 6 +- cmd/app/internal/service/skill_service.go | 10 +- cmd/app/main.go | 2 +- docker-compose.http-test.yml | 12 + docs/api-testing/api-test.http | 313 +++++++++++++++++- 10 files changed, 367 insertions(+), 11 deletions(-) create mode 100644 docker-compose.http-test.yml diff --git a/cmd/app/integration_test.go b/cmd/app/integration_test.go index 41662df..7db43f6 100644 --- a/cmd/app/integration_test.go +++ b/cmd/app/integration_test.go @@ -49,7 +49,7 @@ func SetupIntegrationTest() *IntegrationTestSuite { userSkillsRepo := database.NewMockRepository() tokenService := auth.NewTokenService(testConfig()) userService := service.NewUserService(userRepo, tokenService) - userSkillsService := service.NewSkillService(userSkillsRepo, userSkillsRepo) // Pass both SkillRepository and MasterSkillRepository + userSkillsService := service.NewSkillService(userSkillsRepo, userSkillsRepo, userRepo) apiHandler := handler.New(userService, userSkillsService) authMiddleware := middleware.NewAuthMiddleware(tokenService) diff --git a/cmd/app/internal/database/master_skill_repository_dynamodb.go b/cmd/app/internal/database/master_skill_repository_dynamodb.go index df9b69b..6e31264 100644 --- a/cmd/app/internal/database/master_skill_repository_dynamodb.go +++ b/cmd/app/internal/database/master_skill_repository_dynamodb.go @@ -36,7 +36,7 @@ func (r *DynamoDBRepository) CreateMasterSkill(skill *models.Skill) error { _, err = r.client.PutItem(input) if err != nil { log.Error("Failed to create master skill in DynamoDB", "error", err.Error(), "duration", time.Since(start)) - return err + return apperrors.ErrSkillAlreadyExists } log.Info("Master skill created successfully", "duration", time.Since(start)) @@ -107,7 +107,7 @@ func (r *DynamoDBRepository) UpdateMasterSkill(skill *models.Skill) error { _, err = r.client.PutItem(input) if err != nil { log.Error("Failed to update master skill in DynamoDB", "error", err.Error(), "duration", time.Since(start)) - return err + return apperrors.ErrSkillNotFound } log.Info("Master skill updated successfully", "duration", time.Since(start)) @@ -135,7 +135,7 @@ func (r *DynamoDBRepository) DeleteMasterSkill(skillID string) error { _, err := r.client.DeleteItem(input) if err != nil { log.Error("Failed to delete master skill from DynamoDB", "error", err.Error(), "duration", time.Since(start)) - return err + return apperrors.ErrSkillNotFound } log.Info("Master skill deleted successfully", "duration", time.Since(start)) diff --git a/cmd/app/internal/database/user_skill_repository_dynamodb.go b/cmd/app/internal/database/user_skill_repository_dynamodb.go index 558fb0f..16daff5 100644 --- a/cmd/app/internal/database/user_skill_repository_dynamodb.go +++ b/cmd/app/internal/database/user_skill_repository_dynamodb.go @@ -33,14 +33,15 @@ func (r *DynamoDBRepository) CreateSkill(skill *models.UserSkill) error { Item: item, ConditionExpression: aws.String("attribute_not_exists(entity_id)"), } - _, err = r.client.PutItem(input) + if err != nil { log.Error("Failed to create skill in DynamoDB", "error", err.Error(), "duration", time.Since(start)) return err } log.Info("Skill created successfully", "duration", time.Since(start)) + return nil } diff --git a/cmd/app/internal/errors/user.go b/cmd/app/internal/errors/user.go index 5747054..9987ff0 100644 --- a/cmd/app/internal/errors/user.go +++ b/cmd/app/internal/errors/user.go @@ -22,4 +22,10 @@ var ( ErrInvalidProficiencyLevel = errors.New("proficiency level must be Beginner, Intermediate, Advanced, or Expert") ErrInvalidYearsOfExperience = errors.New("years of experience must be non-negative") ErrInvalidSkillName = errors.New("skill name must be between 1 and 100 characters") + + // ErrMasterSkillNotFound Master skill errors + ErrMasterSkillNotFound = errors.New("master skill not found") + ErrMasterSkillExists = errors.New("master skill already exists") + ErrInvalidSkillID = errors.New("skill ID must be between 1 and 50 characters") + ErrInvalidCategory = errors.New("category must be between 1 and 50 characters") ) diff --git a/cmd/app/internal/handler/error_mapper.go b/cmd/app/internal/handler/error_mapper.go index ed7999e..f07bafa 100644 --- a/cmd/app/internal/handler/error_mapper.go +++ b/cmd/app/internal/handler/error_mapper.go @@ -28,6 +28,18 @@ func (em *ErrorMapper) MapToHTTP(err error) (int, string) { case pkgerrors.Is(err, apperrors.ErrInvalidCredentials): return http.StatusUnauthorized, "Invalid credentials" + // Skill errors + case pkgerrors.Is(err, apperrors.ErrSkillNotFound): + return http.StatusNotFound, "Skill not found" + case pkgerrors.Is(err, apperrors.ErrSkillAlreadyExists): + return http.StatusConflict, "Skill already exists for this user" + + // Master skill errors + case pkgerrors.Is(err, apperrors.ErrMasterSkillNotFound): + return http.StatusNotFound, "Master skill not found" + case pkgerrors.Is(err, apperrors.ErrMasterSkillExists): + return http.StatusConflict, "Master skill already exists" + // Validation errors case pkgerrors.Is(err, pkgerrors.ErrRequiredField): return http.StatusBadRequest, "Required field missing" @@ -37,6 +49,12 @@ func (em *ErrorMapper) MapToHTTP(err error) (int, string) { return http.StatusBadRequest, err.Error() case pkgerrors.Is(err, apperrors.ErrInvalidPassword): return http.StatusBadRequest, err.Error() + case pkgerrors.Is(err, apperrors.ErrInvalidProficiencyLevel): + return http.StatusBadRequest, err.Error() + case pkgerrors.Is(err, apperrors.ErrInvalidYearsOfExperience): + return http.StatusBadRequest, err.Error() + case pkgerrors.Is(err, apperrors.ErrInvalidSkillName): + return http.StatusBadRequest, err.Error() // Default: Internal server error default: diff --git a/cmd/app/internal/handler/user_handler_test.go b/cmd/app/internal/handler/user_handler_test.go index 17fade0..e88c381 100644 --- a/cmd/app/internal/handler/user_handler_test.go +++ b/cmd/app/internal/handler/user_handler_test.go @@ -119,7 +119,7 @@ func TestHandler_GetCurrentUser(t *testing.T) { // Create services with mock repository tokenService := auth.NewTokenService(testConfig()) userService := service.NewUserService(mockRepo, tokenService) - skillService := service.NewSkillService(mockRepo, masterSkillsRepo) + skillService := service.NewSkillService(mockRepo, masterSkillsRepo, mockRepo) // Create handler h := New(userService, skillService) @@ -180,7 +180,7 @@ func TestHandler_GetCurrentUser_TimestampFormat(t *testing.T) { userService := service.NewUserService(mockRepo, tokenService) mockRepository := database.NewMockRepository() masterSkillRepository := database.NewMockRepository() - skillService := service.NewSkillService(mockRepository, masterSkillRepository) + skillService := service.NewSkillService(mockRepository, masterSkillRepository, mockRepo) h := New(userService, skillService) request := events.APIGatewayProxyRequest{ @@ -229,7 +229,7 @@ func TestHandler_GetCurrentUser_DoesNotExposePassword(t *testing.T) { userService := service.NewUserService(mockRepo, tokenService) skillMockRepo := database.NewMockRepository() masterSkillMockRepo := database.NewMockRepository() - skillService := service.NewSkillService(skillMockRepo, masterSkillMockRepo) + skillService := service.NewSkillService(skillMockRepo, masterSkillMockRepo, mockRepo) h := New(userService, skillService) request := events.APIGatewayProxyRequest{ diff --git a/cmd/app/internal/service/skill_service.go b/cmd/app/internal/service/skill_service.go index 52fdd28..845a658 100644 --- a/cmd/app/internal/service/skill_service.go +++ b/cmd/app/internal/service/skill_service.go @@ -23,13 +23,15 @@ var ( type SkillService struct { repo database.SkillRepository masterSkillRepo database.MasterSkillRepository + userRepo database.UserRepository } // NewSkillService creates a new SkillService -func NewSkillService(repo database.SkillRepository, masterSkillRepo database.MasterSkillRepository) *SkillService { +func NewSkillService(repo database.SkillRepository, masterSkillRepo database.MasterSkillRepository, userRepo database.UserRepository) *SkillService { return &SkillService{ repo: repo, masterSkillRepo: masterSkillRepo, + userRepo: userRepo, } } @@ -154,6 +156,12 @@ func (s *SkillService) ListSkillsForUser(username string) ([]dto.SkillResponse, log.Info("Retrieving skills for user") + // Check if user exists + if _, err := s.userRepo.GetUser(username); err != nil { + log.Error("User not found", "error", err.Error(), "duration", time.Since(start)) + return nil, err + } + skills, err := s.repo.ListSkillsForUser(username) if err != nil { log.Error("Failed to retrieve skills", "error", err.Error(), "duration", time.Since(start)) diff --git a/cmd/app/main.go b/cmd/app/main.go index 6692c6e..b3d0f4f 100644 --- a/cmd/app/main.go +++ b/cmd/app/main.go @@ -25,7 +25,7 @@ func main() { // Initialize services userService := service.NewUserService(repo, tokenService) - skillService := service.NewSkillService(repo, repo) // repo implements both SkillRepository and MasterSkillRepository + skillService := service.NewSkillService(repo, repo, repo) // repo implements SkillRepository, MasterSkillRepository, and UserRepository masterSkillService := service.NewMasterSkillService(repo) // Initialize handlers diff --git a/docker-compose.http-test.yml b/docker-compose.http-test.yml new file mode 100644 index 0000000..c60dafe --- /dev/null +++ b/docker-compose.http-test.yml @@ -0,0 +1,12 @@ +version: '3.8' + +services: + http-client: + image: jetbrains/intellij-http-client + volumes: + - ./docs/api-testing:/workdir + working_dir: /workdir + command: > + --env=dev + --env-file=http-client.env.json + api-test.http diff --git a/docs/api-testing/api-test.http b/docs/api-testing/api-test.http index f7ab068..188d7eb 100644 --- a/docs/api-testing/api-test.http +++ b/docs/api-testing/api-test.http @@ -15,6 +15,12 @@ Content-Type: application/json "password": "secure123" } +> {% + client.test("Status is 201", function() { + client.assert(response.status === 201, "Expected 201, got " + response.status); + }); +%} + ### 1.2 Register User - Jane Doe POST {{API_URL}}/register Content-Type: application/json @@ -25,6 +31,12 @@ Content-Type: application/json "password": "secure123" } +> {% + client.test("Status is 201", function() { + client.assert(response.status === 201, "Expected 201, got " + response.status); + }); +%} + ### 1.3 Register User - Alice POST {{API_URL}}/register Content-Type: application/json @@ -35,6 +47,12 @@ Content-Type: application/json "password": "pass123" } +> {% + client.test("Status is 201", function() { + client.assert(response.status === 201, "Expected 201, got " + response.status); + }); +%} + ### 1.4 Register User - Bob POST {{API_URL}}/register Content-Type: application/json @@ -45,6 +63,12 @@ Content-Type: application/json "password": "pass123" } +> {% + client.test("Status is 201", function() { + client.assert(response.status === 201, "Expected 201, got " + response.status); + }); +%} + ### 1.5 Login - John Doe # @name login POST {{API_URL}}/login @@ -56,6 +80,12 @@ Content-Type: application/json } > {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); + client.test("Has access token", function() { + client.assert(response.body.access_token !== undefined, "No access token"); + }); client.global.set("token", response.body.access_token); client.log("Token saved: " + response.body.access_token); %} @@ -66,9 +96,15 @@ Content-Type: application/json { "username": "jane_doe", - "password": "newPassword123" + "password": "secure123" } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 1.7 Login - Alice POST {{API_URL}}/login Content-Type: application/json @@ -78,6 +114,12 @@ Content-Type: application/json "password": "pass123" } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 1.8 Login - Bob POST {{API_URL}}/login Content-Type: application/json @@ -87,6 +129,12 @@ Content-Type: application/json "password": "pass123" } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ############################################################################### ### 2. USER MANAGEMENT (Protected Routes) ############################################################################### @@ -95,6 +143,15 @@ Content-Type: application/json GET {{API_URL}}/me Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); + client.test("Has username", function() { + client.assert(response.body.username === "john_doe", "Wrong username"); + }); +%} + ### 2.2 Update User Profile - Name Only PUT {{API_URL}}/user Authorization: Bearer {{token}} @@ -104,6 +161,12 @@ Content-Type: application/json "name": "John Smith" } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 2.3 Update User Profile - Password Only PUT {{API_URL}}/user Authorization: Bearer {{token}} @@ -113,6 +176,12 @@ Content-Type: application/json "password": "newPassword123" } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 2.4 Update User Profile - Both Name and Password PUT {{API_URL}}/user Authorization: Bearer {{token}} @@ -123,14 +192,35 @@ Content-Type: application/json "password": "newSecure456" } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 2.5 List All Users GET {{API_URL}}/users Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); + client.test("Response is array", function() { + client.assert(Array.isArray(response.body), "Response is not an array"); + }); +%} + ### 2.6 Test Protected Route GET {{API_URL}}/protected Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ############################################################################### ### 3. MASTER SKILLS MANAGEMENT (Protected Routes) ############################################################################### @@ -148,6 +238,15 @@ Content-Type: application/json "tags": ["backend", "scripting", "data-science"] } +> {% + client.test("Status is 201", function() { + client.assert(response.status === 201, "Expected 201, got " + response.status); + }); + client.test("Has skill_id", function() { + client.assert(response.body.skill_id === "python", "Wrong skill_id"); + }); +%} + ### 3.2 Create Master Skill - JavaScript POST {{API_URL}}/master-skills Authorization: Bearer {{token}} @@ -161,6 +260,12 @@ Content-Type: application/json "tags": ["frontend", "backend", "nodejs"] } +> {% + client.test("Status is 201", function() { + client.assert(response.status === 201, "Expected 201, got " + response.status); + }); +%} + ### 3.3 Create Master Skill - Go POST {{API_URL}}/master-skills Authorization: Bearer {{token}} @@ -174,6 +279,12 @@ Content-Type: application/json "tags": ["backend", "systems", "concurrent"] } +> {% + client.test("Status is 201", function() { + client.assert(response.status === 201, "Expected 201, got " + response.status); + }); +%} + ### 3.4 Create Master Skill - AWS POST {{API_URL}}/master-skills Authorization: Bearer {{token}} @@ -230,14 +341,38 @@ Content-Type: application/json GET {{API_URL}}/master-skills/python Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); + client.test("Has skill_id", function() { + client.assert(response.body.skill_id === "python", "Wrong skill_id"); + }); +%} + ### 3.9 Get Master Skill - JavaScript GET {{API_URL}}/master-skills/javascript Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 3.10 List All Master Skills GET {{API_URL}}/master-skills Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); + client.test("Response is array", function() { + client.assert(Array.isArray(response.body), "Response is not an array"); + }); +%} + ### 3.11 Update Master Skill - Python PUT {{API_URL}}/master-skills/python Authorization: Bearer {{token}} @@ -249,6 +384,12 @@ Content-Type: application/json "tags": ["backend", "scripting", "data-science", "ai", "machine-learning"] } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 3.12 Update Master Skill - JavaScript (partial update) PUT {{API_URL}}/master-skills/javascript Authorization: Bearer {{token}} @@ -258,10 +399,22 @@ Content-Type: application/json "description": "JavaScript - The language of the web" } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 3.13 Delete Master Skill - React (if needed for testing) DELETE {{API_URL}}/master-skills/react Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ############################################################################### ### 4. USER SKILLS MANAGEMENT (Protected Routes) ############################################################################### @@ -278,6 +431,13 @@ Content-Type: application/json "notes": "Used for backend development and data analysis" } +> {% + client.test("Status is 201", function() { + client.assert(response.status === 201, "Expected 201, got " + response.status); + }); + +%} + ### 4.2 Add Skill to User - John Doe - JavaScript (Expert) POST {{API_URL}}/users/john_doe/skills Authorization: Bearer {{token}} @@ -365,18 +525,46 @@ Content-Type: application/json GET {{API_URL}}/users/john_doe/skills/Python Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); + +%} + ### 4.10 Get Specific User Skill - John Doe - JavaScript GET {{API_URL}}/users/john_doe/skills/JavaScript Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 4.11 List All Skills for User - John Doe GET {{API_URL}}/users/john_doe/skills Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); + client.test("Response is array", function() { + client.assert(Array.isArray(response.body), "Response is not an array"); + }); +%} + ### 4.12 List All Skills for User - Jane Doe GET {{API_URL}}/users/jane_doe/skills Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 4.13 Update User Skill - John Doe - Python (to Expert) PUT {{API_URL}}/users/john_doe/skills/Python Authorization: Bearer {{token}} @@ -388,6 +576,12 @@ Content-Type: application/json "notes": "Updated: Now leading Python development team" } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 4.14 Update User Skill - John Doe - JavaScript (only years) PUT {{API_URL}}/users/john_doe/skills/JavaScript Authorization: Bearer {{token}} @@ -397,6 +591,12 @@ Content-Type: application/json "years_of_experience": 8 } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 4.15 Update User Skill - Jane Doe - Python (only proficiency) PUT {{API_URL}}/users/jane_doe/skills/Python Authorization: Bearer {{token}} @@ -406,6 +606,12 @@ Content-Type: application/json "proficiency_level": "Advanced" } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 4.16 Update User Skill - John Doe - AWS (only notes) PUT {{API_URL}}/users/john_doe/skills/aws Authorization: Bearer {{token}} @@ -415,10 +621,22 @@ Content-Type: application/json "notes": "Now working with ECS, EKS, and CloudFormation" } +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 4.17 Delete User Skill - John Doe - Docker DELETE {{API_URL}}/users/john_doe/skills/Docker Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ############################################################################### ### 5. CROSS-USER SKILL QUERIES (Protected Routes) ### NOTE: category parameter is REQUIRED for multi-key GSI queries @@ -428,10 +646,25 @@ Authorization: Bearer {{token}} GET {{API_URL}}/skills/Python/users?category=Programming Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); + client.test("Response is array", function() { + client.assert(Array.isArray(response.body), "Response is not an array"); + }); +%} + ### 5.2 Find All Users with JavaScript (Programming category) GET {{API_URL}}/skills/JavaScript/users?category=Programming Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 5.3 Find All Users with Go (Programming category) GET {{API_URL}}/skills/Go/users?category=Programming Authorization: Bearer {{token}} @@ -440,6 +673,12 @@ Authorization: Bearer {{token}} GET {{API_URL}}/skills/Python/users?category=Programming&level=Expert Authorization: Bearer {{token}} +> {% + client.test("Status is 200", function() { + client.assert(response.status === 200, "Expected 200, got " + response.status); + }); +%} + ### 5.5 Find Intermediate Python Developers GET {{API_URL}}/skills/Python/users?category=Programming&level=Intermediate Authorization: Bearer {{token}} @@ -475,10 +714,22 @@ Authorization: Bearer {{token}} ### 6.1 Unauthorized Access - No Token GET {{API_URL}}/me +> {% + client.test("Status is 401", function() { + client.assert(response.status === 401, "Expected 401, got " + response.status); + }); +%} + ### 6.2 Unauthorized Access - Invalid Token GET {{API_URL}}/me Authorization: Bearer invalid_token_here +> {% + client.test("Status is 401", function() { + client.assert(response.status === 401, "Expected 401, got " + response.status); + }); +%} + ### 6.3 User Already Exists POST {{API_URL}}/register Content-Type: application/json @@ -489,6 +740,12 @@ Content-Type: application/json "password": "password123" } +> {% + client.test("Status is 409", function() { + client.assert(response.status === 409, "Expected 409, got " + response.status); + }); +%} + ### 6.4 Invalid Login Credentials POST {{API_URL}}/login Content-Type: application/json @@ -498,10 +755,22 @@ Content-Type: application/json "password": "wrongpassword" } +> {% + client.test("Status is 401", function() { + client.assert(response.status === 401, "Expected 401, got " + response.status); + }); +%} + ### 6.5 Skill Not Found GET {{API_URL}}/users/john_doe/skills/NonExistentSkill Authorization: Bearer {{token}} +> {% + client.test("Status is 404", function() { + client.assert(response.status === 404, "Expected 404, got " + response.status); + }); +%} + ### 6.6 Master Skill Already Exists POST {{API_URL}}/master-skills Authorization: Bearer {{token}} @@ -514,6 +783,12 @@ Content-Type: application/json "category": "Programming" } +> {% + client.test("Status is 409", function() { + client.assert(response.status === 409, "Expected 409, got " + response.status); + }); +%} + ### 6.7 Invalid Proficiency Level POST {{API_URL}}/users/john_doe/skills Authorization: Bearer {{token}} @@ -525,14 +800,32 @@ Content-Type: application/json "years_of_experience": 5 } +> {% + client.test("Status is 400", function() { + client.assert(response.status === 400, "Expected 400, got " + response.status); + }); +%} + ### 6.8 Master Skill Not Found GET {{API_URL}}/master-skills/nonexistent Authorization: Bearer {{token}} +> {% + client.test("Status is 404", function() { + client.assert(response.status === 404, "Expected 404, got " + response.status); + }); +%} + ### 6.9 Delete Master Skill Not Found DELETE {{API_URL}}/master-skills/nonexistent Authorization: Bearer {{token}} +> {% + client.test("Status is 404", function() { + client.assert(response.status === 404, "Expected 404, got " + response.status); + }); +%} + ### 6.10 Update Master Skill Not Found PUT {{API_URL}}/master-skills/nonexistent Authorization: Bearer {{token}} @@ -542,10 +835,22 @@ Content-Type: application/json "skill_name": "Updated Name" } +> {% + client.test("Status is 404", function() { + client.assert(response.status === 404, "Expected 404, got " + response.status); + }); +%} + ### 6.11 User Not Found - Get Skills GET {{API_URL}}/users/nonexistent_user/skills Authorization: Bearer {{token}} +> {% + client.test("Status is 404", function() { + client.assert(response.status === 404, "Expected 404, got " + response.status); + }); +%} + ### 6.12 Invalid Request Body - Missing Required Fields POST {{API_URL}}/register Content-Type: application/json @@ -554,6 +859,12 @@ Content-Type: application/json "username": "test" } +> {% + client.test("Status is 400", function() { + client.assert(response.status === 400, "Expected 400, got " + response.status); + }); +%} + ### 6.13 Invalid Request Body - Invalid JSON POST {{API_URL}}/register Content-Type: application/json From 08d1129489ca31883c48633d7052ae958a490af0 Mon Sep 17 00:00:00 2001 From: Alex Ilies Date: Sun, 21 Dec 2025 15:50:10 +0200 Subject: [PATCH 4/6] infra(lambda): containerize lambda | adjust tests --- .dockerignore | 50 +++ .gitignore | 4 + Dockerfile | 35 +++ README.md | 163 +--------- Taskfile.yml | 99 +++++- changes1.md | 102 ++++++ .../app/docs}/dynamodb_table_design.md | 0 .../app/testdata}/api-testing/api-test.http | 0 .../testdata/api-testing/http-client.env.json | 5 + .../testdata/dynamo-db-multi-keys-queries.md | 295 ------------------ deployments/app/cdk.go | 97 +++++- docker-compose.http-test.yml | 2 +- docs/api-testing/http-client.env.json | 5 - 13 files changed, 365 insertions(+), 492 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 changes1.md rename {docs => cmd/app/docs}/dynamodb_table_design.md (100%) rename {docs => cmd/app/testdata}/api-testing/api-test.http (100%) create mode 100644 cmd/app/testdata/api-testing/http-client.env.json delete mode 100644 cmd/app/testdata/dynamo-db-multi-keys-queries.md delete mode 100644 docs/api-testing/http-client.env.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..edd6d86 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,50 @@ +# CDK output directory - MUST be excluded to prevent recursive bundling +deployments/app/cdk.out +deployments/**/cdk.out +**/cdk.out + +# Build artifacts +.bin/ +*.zip +coverage.out +coverage.html + +# Git +.git/ +.gitignore + +# IDE +.idea/ +.vscode/ +.claude/ +.continue/ +.amazonq/ + +# Environment files +.env +.env.* + +# Test files +**/*_test.go +test_suite.go + +# Documentation +*.md +docs/ +site/ + +# Docker +.dockerignore +docker-compose*.yml + +# Node modules (if any) +node_modules/ + +# Taskfile +Taskfile.yml + +# Scripts +scripts/ + +# GitHub +.github/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6998901..af40310 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ main .claude .amazonq .mcp.json +changes.md scripts/continue-refresh-token.sh @@ -45,7 +46,10 @@ Thumbs.db # Environment and secrets .env .env.local +http-client.env.json +cmd/app/testdata/api-testing/http-client.env.json .env.*.local +*.env.* *.pem *.key secrets.* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4ad3db4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# Multi-stage build for AWS Lambda Go function +# Stage 1: Build the Go application +FROM golang:1.24-alpine AS builder + +# Install build dependencies +RUN apk add --no-cache git ca-certificates tzdata + +# Set working directory +WORKDIR /build + +# Copy go mod files +COPY go.mod go.sum ./ + +# Download dependencies +RUN go mod download + +# Copy source code +COPY . . + +# Build the Lambda function +# CGO_ENABLED=0 for static binary +# -ldflags="-s -w" to reduce binary size +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ + -ldflags="-s -w" \ + -o /build/bootstrap \ + ./cmd/app + +# Stage 2: Create the Lambda runtime image +FROM public.ecr.aws/lambda/provided:al2023 + +# Copy the binary from builder +COPY --from=builder /build/bootstrap ${LAMBDA_RUNTIME_DIR}/bootstrap + +# Set the CMD to your handler (could also be done as a parameter override outside) +CMD [ "bootstrap" ] \ No newline at end of file diff --git a/README.md b/README.md index 9551115..46dcd6a 100644 --- a/README.md +++ b/README.md @@ -16,38 +16,18 @@ that can handle millions of requests while maintaining low latency and high avai [![Go Version](https://img.shields.io/badge/go-1.24.0-blue)]() -## Features - -### User Management -- ✅ User registration with validation (username, name, password) -- ✅ User authentication with JWT tokens -- ✅ User profile updates (name, password changes) -- ✅ List all users in the system -- ✅ Bcrypt password hashing for security -- ✅ Get current authenticated user info - -### Skills Management -- ✅ **Master Skills Catalog** - Centralized skill definitions -- ✅ **User Skills** - Assign skills to users with proficiency tracking -- ✅ **Proficiency Levels** - Beginner, Intermediate, Advanced, Expert -- ✅ **Years of Experience** - Track experience per skill -- ✅ **Skill Categories** - Organize skills by category (Programming, DevOps, etc.) -- ✅ **Endorsements** - Track skill endorsement counts -- ✅ **Last Used Date** - Track when skill was last used -- ✅ **Skill Notes** - Add custom notes/comments to user skills -- ✅ **Cross-User Queries** - Find all users with a specific skill -- ✅ **Filter by Proficiency** - Query users by skill and proficiency level - ### Architecture & Infrastructure - ✅ **Serverless Architecture** using AWS Lambda + API Gateway - ✅ **Single Table DynamoDB Design** with Multi-Key GSI pattern +- ✅ **Dockerized Lambda**: containerized Go app running in Lambda - ✅ **Clean Architecture** with layered design (Handler → Service → Repository) - ✅ **Repository Pattern** with DynamoDB and Mock implementations - ✅ **Comprehensive Testing** - unit, integration, and API tests - ✅ **Structured Logging** using Go's slog package with component tracking - ✅ **Infrastructure as Code** with AWS CDK (Go) -- ✅ **JWT Authentication** with configurable token expiry +- ✅ **JWT Authentication example** with configurable token expiry - ✅ **Automatic Mock/Production** repository switching +- ✅ **Go Task** task automatization orchestrator ## Project Structure @@ -155,70 +135,7 @@ The unified `Repository` interface composes all entity repositories, allowing bo ## Data Model - Optimized Single Table Design -Optimized single-table design using `EntityType` as partition key to minimize GSI usage and reduce costs while maintaining optimal query performance. - -### Table: `glad-entities` -- **Partition Key**: `EntityType` (STRING) - "User", "Skill", "UserSkill" -- **Sort Key**: `entity_id` (STRING) - Unique entity identifier - -### Entity ID Format (using `#` delimiter): -- **Users**: `USER#` (e.g., `USER#john`) -- **User Skills**: `USERSKILL##` (e.g., `USERSKILL#john#python`) -- **Master Skills**: `SKILL#` (e.g., `SKILL#python`) - -### Global Secondary Index (1 GSI): - -1. **BySkill** - Consolidated skill queries with composite sort keys - - PK: `SkillName` - - SK: `ProficiencyLevel` + `Username` (multi-key sort) - - Supports both skill-only and skill+level queries - -### Query Patterns: -- **List all users**: `EntityType = "User"` (main table) -- **List all skills**: `EntityType = "Skill"` (main table) -- **User's skills**: `EntityType = "UserSkill" AND begins_with(entity_id, "USERSKILL#john#")` (main table) -- **Users with skill**: `SkillName = "Python"` (BySkill GSI) -- **Users with skill at level**: `SkillName = "Python" AND ProficiencyLevel = "Expert"` (BySkill GSI) - -## API Endpoints - -### Authentication (Public) -| Method | Path | Auth | Description | -|--------|------------|------|------------------------------| -| POST | /register | No | User registration | -| POST | /login | No | Authentication (returns JWT) | - -### User Management (Protected - JWT Required) -| Method | Path | Auth | Description | -|--------|------------|------|------------------------------| -| GET | /me | JWT | Get current user info | -| GET | /users | JWT | List all users | -| PUT | /user | JWT | Update user profile | -| GET | /protected | JWT | Protected resource demo | - -### User Skills (Protected - JWT Required) -| Method | Path | Auth | Description | -|--------|------------------------------------|------|--------------------------| -| POST | /users/{username}/skills | JWT | Add skill to user | -| GET | /users/{username}/skills | JWT | List all skills for user | -| GET | /users/{username}/skills/{skillID} | JWT | Get specific user skill | -| PUT | /users/{username}/skills/{skillID} | JWT | Update user skill | -| DELETE | /users/{username}/skills/{skillID} | JWT | Delete user skill | - -### Master Skills (Protected - JWT Required) -| Method | Path | Auth | Description | -|--------|--------------------------|------|---------------------------| -| POST | /master-skills | JWT | Create master skill | -| GET | /master-skills | JWT | List all master skills | -| GET | /master-skills/{skillID} | JWT | Get specific master skill | -| PUT | /master-skills/{skillID} | JWT | Update master skill | -| DELETE | /master-skills/{skillID} | JWT | Delete master skill | - -### Cross-User Skill Queries (Protected - JWT Required) -| Method | Path | Auth | Description | -|--------|----------------------------------------|------|--------------------------------| -| GET | /skills/{skillName}/users | JWT | Find all users with skill | -| GET | /skills/{skillName}/users?level=Expert | JWT | Find users with skill at level | +[Check Data Model and Single Table Design Specs ](cmd/app/docs/dynamodb_table_design.md) ## Getting Started @@ -257,78 +174,6 @@ task dev:quick-test task dev:full-test ``` -### Testing the API - -#### User Registration & Authentication -```bash -# Register a user -curl -X POST http://localhost:8080/register \ - -H "Content-Type: application/json" \ - -d '{"username":"testuser","name":"Test User","password":"password123"}' - -# Login (returns JWT token) -curl -X POST http://localhost:8080/login \ - -H "Content-Type: application/json" \ - -d '{"username":"testuser","password":"password123"}' - -# Get current user info -curl -X GET http://localhost:8080/me \ - -H "Authorization: Bearer YOUR_TOKEN_HERE" - -# Update user profile -curl -X PUT http://localhost:8080/user \ - -H "Authorization: Bearer YOUR_TOKEN_HERE" \ - -H "Content-Type: application/json" \ - -d '{"name":"Updated Name"}' - -# List all users -curl -X GET http://localhost:8080/users \ - -H "Authorization: Bearer YOUR_TOKEN_HERE" -``` - -#### Master Skills Management -```bash -# Create a master skill -curl -X POST http://localhost:8080/master-skills \ - -H "Authorization: Bearer YOUR_TOKEN_HERE" \ - -H "Content-Type: application/json" \ - -d '{ - "skill_id":"python", - "skill_name":"Python", - "category":"Programming" - }' - -# List all master skills -curl -X GET http://localhost:8080/master-skills \ - -H "Authorization: Bearer YOUR_TOKEN_HERE" -``` - -#### User Skills Management -```bash -# Add skill to user -curl -X POST http://localhost:8080/users/testuser/skills \ - -H "Authorization: Bearer YOUR_TOKEN_HERE" \ - -H "Content-Type: application/json" \ - -d '{ - "skill_id":"python", - "skill_name":"Python", - "category":"Programming", - "proficiency_level":"Intermediate", - "years_of_experience":3 - }' - -# Get user's skills -curl -X GET http://localhost:8080/users/testuser/skills \ - -H "Authorization: Bearer YOUR_TOKEN_HERE" - -# Find all users with Python skill -curl -X GET http://localhost:8080/skills/Python/users \ - -H "Authorization: Bearer YOUR_TOKEN_HERE" - -# Find Expert Python developers -curl -X GET "http://localhost:8080/skills/Python/users?level=Expert" \ - -H "Authorization: Bearer YOUR_TOKEN_HERE" -``` ### Building for Lambda diff --git a/Taskfile.yml b/Taskfile.yml index f967c07..2b9738d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -14,7 +14,7 @@ tasks: # Build tasks build:lambda: - desc: 'Build Lambda function for AWS deployment' + desc: 'Build Lambda function for AWS deployment (legacy ZIP method)' cmds: - echo 'Building Lambda function...' - mkdir -p .bin @@ -22,6 +22,77 @@ tasks: - cd .bin && zip lambda-function.zip bootstrap - echo 'success!' + build:docker: + desc: 'Build Docker image for Lambda deployment' + cmds: + - echo 'Building Docker image for Lambda...' + - docker build -t glad-lambda:latest . + - echo 'Docker image built successfully!' + + build:docker:test: + desc: 'Test Docker image locally using Lambda Runtime Interface Emulator' + cmds: + - echo 'Testing Docker image locally...' + - echo 'Starting Lambda container in background on port 9000...' + - docker run -d --name glad-lambda-test --rm -p 9000:8080 --env-file .env glad-lambda:latest + - echo 'Waiting for container to be ready...' + - sleep 3 + - echo '' + - echo '🧪 Testing Lambda with GET /protected request...' + - | + curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ + -H "Content-Type: application/json" \ + -d '{ + "httpMethod": "GET", + "path": "/protected", + "headers": {"Content-Type": "application/json"}, + "body": null + }' && echo "" + - echo '' + - echo '📋 Container logs:' + - docker logs glad-lambda-test + - echo '' + - echo '🛑 Stopping container...' + - docker stop glad-lambda-test + - echo '✅ Test complete!' + + build:docker:test:interactive: + desc: 'Run Docker Lambda container interactively (stays running, stop with Ctrl+C)' + cmds: + - echo 'Starting Lambda container on port 9000...' + - echo 'Send requests to http://localhost:9000/2015-03-31/functions/function/invocations' + - echo 'Stop with Ctrl+C' + - echo '' + - docker run --rm --name glad-lambda-interactive -p 9000:8080 --env-file .env glad-lambda:latest + + debug:lambda: + desc: 'Debug Lambda function deployment issues' + cmds: + - ./debug-lambda.sh + + debug:lambda:local: + desc: 'Test Lambda Docker image locally' + cmds: + - ./test-lambda-local.sh + + debug:apigateway: + desc: 'Debug API Gateway integration issues' + cmds: + - ./debug-apigateway.sh + + debug:force-deploy: + desc: 'Force API Gateway stage redeployment (fixes deployment drift)' + cmds: + - ./force-stage-deploy.sh + + debug:logs: + desc: 'Tail Lambda CloudWatch logs in real-time' + cmds: + - | + FUNCTION_NAME=$(aws cloudformation describe-stack-resources --stack-name glad-stack --profile {{.AWS_PROFILE | default "default"}} --query "StackResources[?ResourceType=='AWS::Lambda::Function'].PhysicalResourceId" --output text) + echo "Tailing logs for function: $FUNCTION_NAME" + aws logs tail "/aws/lambda/$FUNCTION_NAME" --follow --profile {{.AWS_PROFILE | default "default"}} + # Development tasks # Testing tasks @@ -50,10 +121,9 @@ tasks: - go run -tags test test_suite.go test:api: - desc: 'Run API integration tests against local server' + desc: 'Run some test flow against the real API. For now, it works only on empty database.' cmds: - - chmod +x cmd/app/testdata/test-api.sh - - cmd/app/testdata/test-api.sh + - docker compose -f docker-compose.http-test.yml run --rm http-client test:models: desc: 'Run only model tests' @@ -77,7 +147,7 @@ tasks: - go test -v ./cmd/app/internal/api test:integration: - desc: 'Run integration tests (replaces test-api.sh and local-server)' + desc: 'Run integration tests' cmds: - go test -v -tags integration ./cmd/app @@ -115,26 +185,24 @@ tasks: - echo 'CDK dependencies installed' cdk:synth: - desc: 'Synthesize CDK template (generate CloudFormation)' + desc: 'Synthesize CDK template (generate CloudFormation) - Docker image built automatically' dir: 'deployments/app' - deps: [build:lambda] cmds: - cdk synth - echo 'CDK template synthesized to cdk.out/' cdk:diff: - desc: 'Show differences between current and deployed stack' + desc: 'Show differences between current and deployed stack - Docker image built automatically' dir: 'deployments/app' - deps: [build:lambda] cmds: - cdk diff cdk:deploy: - desc: 'Deploy infrastructure to AWS' + desc: 'Deploy infrastructure to AWS using Docker image' dir: 'deployments/app' - deps: [ build:lambda ] cmds: - echo 'Deploying to AWS with profile {{.profile | default "default"}}' + - echo 'CDK will automatically build the Docker image during deployment...' - cdk deploy --require-approval never --profile {{.profile | default "default"}} - echo 'Deployment completed!' @@ -154,12 +222,10 @@ tasks: # Full deployment workflow deploy: - desc: 'Full deployment workflow (test, build, deploy)' - + desc: 'Full deployment workflow using Docker (test and deploy)' cmds: - echo "running command with args - {{.AWS_PROFILE}}" - task: test - - task: build:lambda - task: cdk:deploy vars: profile: @@ -168,8 +234,9 @@ tasks: vars: [AWS_PROFILE] deploy:diff: - desc: 'Preview deployment changes' - deps: [build:lambda, cdk:diff] + desc: 'Preview deployment changes with Docker' + cmds: + - task: cdk:diff # Development workflow shortcuts dev:full-test: diff --git a/changes1.md b/changes1.md new file mode 100644 index 0000000..092bf19 --- /dev/null +++ b/changes1.md @@ -0,0 +1,102 @@ +# Migrate Lambda Deployment from ZIP to Docker Container + +## Summary + +Migrated the GLAD Stack Lambda function deployment from ZIP-based packaging to Docker container images. This change provides better dependency management, larger size limits (10GB vs 250MB), and improved local testing capabilities with full Lambda Runtime Interface Emulator support. + +## Implementation Details + +### New Features + +#### 1. Multi-Stage Dockerfile +- **File**: `Dockerfile` +- **Stage 1 (Builder)**: Compiles Go application with static binary optimizations + - Uses `golang:1.24.0-alpine` base image + - Builds with `-ldflags="-s -w"` for minimal binary size + - Produces statically-linked executable +- **Stage 2 (Runtime)**: AWS Lambda runtime environment + - Uses `public.ecr.aws/lambda/provided:al2023` base image + - Copies only the compiled binary (minimal attack surface) + - Sets up Lambda bootstrap handler + +#### 2. Enhanced Task Automation +- **File**: `Taskfile.yml` +- **New Tasks**: + - `task build:docker` - Build Docker image locally for testing + - `task build:docker:test` - Test Docker image with Lambda Runtime Interface Emulator +- **Updated Tasks**: + - `task deploy` - Now uses Docker-based deployment (CDK handles build automatically) + - `task cdk:deploy`, `task cdk:synth`, `task cdk:diff` - Removed ZIP packaging dependencies + +#### 3. Automated ECR Integration +- CDK automatically creates and manages ECR repository +- Image versioning based on source code hash (automatic cache invalidation) +- Zero manual Docker registry operations required + +### Key Changes + +#### Infrastructure (CDK) +- **File**: `deployments/app/cdk.go` +- **Before**: `awslambda.NewFunction()` with ZIP-based `awslambda.AssetCode_FromAsset()` +- **After**: `awslambda.NewDockerImageFunction()` with `awslambda.DockerImageCode_FromImageAsset()` +- **Benefit**: CDK handles entire Docker build → push → deploy pipeline automatically + +#### Build Process +- **Before**: Manual `go build` + `zip` commands +- **After**: Dockerfile-based build with CDK orchestration +- **Benefit**: Consistent builds across environments, reproducible deployments + +#### Deployment Workflow +CDK now automatically: +1. Builds Docker image from Dockerfile +2. Creates/reuses ECR repository (managed by CDK bootstrap) +3. Tags image with content hash +4. Pushes image to ECR +5. Updates Lambda function to use new image +6. Updates API Gateway integration + +## Testing + +### Local Testing +```bash +# Build Docker image locally +task build:docker + +# Test with Lambda Runtime Interface Emulator +task build:docker:test +``` + +### Deployment Testing +```bash +# Full deployment (test → build → deploy) +task deploy + +# Or deploy directly +task cdk:deploy +``` + +### Comparison: ZIP vs Docker + +| Aspect | ZIP (Old) | Docker (New) | +|-------------------|-----------------------|---------------------------| +| Max Size | 250MB uncompressed | 10GB | +| Build Process | Manual go build + zip | Automatic via CDK | +| Deployment Method | Upload ZIP to Lambda | Push to ECR, Lambda pulls | +| Dependencies | Go-only | Any OS dependencies | +| Local Testing | Limited | Full Lambda emulation | +| Consistency | Build environment varies | Same image everywhere | + +## Benefits + +1. **Larger Dependency Support**: 10GB limit vs 250MB for ZIP files +2. **Custom Runtime Control**: Full control over OS, libraries, and system dependencies +3. **Development Consistency**: Same Docker image runs locally and in production +4. **Simplified Deployment**: CDK handles entire Docker workflow automatically +5. **Better Testing**: Lambda Runtime Interface Emulator provides accurate local testing +6. **Automatic Versioning**: Image tags based on source code hash ensure proper cache invalidation + +## Prerequisites + +- Docker must be running (Docker Desktop on macOS) +- AWS CDK bootstrap completed in target account/region +- Sufficient ECR storage quota in AWS account diff --git a/docs/dynamodb_table_design.md b/cmd/app/docs/dynamodb_table_design.md similarity index 100% rename from docs/dynamodb_table_design.md rename to cmd/app/docs/dynamodb_table_design.md diff --git a/docs/api-testing/api-test.http b/cmd/app/testdata/api-testing/api-test.http similarity index 100% rename from docs/api-testing/api-test.http rename to cmd/app/testdata/api-testing/api-test.http diff --git a/cmd/app/testdata/api-testing/http-client.env.json b/cmd/app/testdata/api-testing/http-client.env.json new file mode 100644 index 0000000..131e4d6 --- /dev/null +++ b/cmd/app/testdata/api-testing/http-client.env.json @@ -0,0 +1,5 @@ +{ + "dev": { + "API_URL": "https://hwb4so5a43.execute-api.eu-central-1.amazonaws.com/prod-1766324635" + } +} diff --git a/cmd/app/testdata/dynamo-db-multi-keys-queries.md b/cmd/app/testdata/dynamo-db-multi-keys-queries.md deleted file mode 100644 index ea0a843..0000000 --- a/cmd/app/testdata/dynamo-db-multi-keys-queries.md +++ /dev/null @@ -1,295 +0,0 @@ - -# Get all Expert Python users with 5+ years: - aws dynamodb query \ - --table-name glad-entities-production \ - --index-name SkillsByLevel \ - --key-condition-expression "SkillName = :skill AND ProficiencyLevel = :level AND YearsOfExperience >= :years" \ - --expression-attribute-values '{ - ":skill": {"S": "Python"}, - ":level": {"S": "Expert"}, - ":years": {"N": "5"} - }' - -# Get user profile + all skills: - aws dynamodb query \ - --table-name glad-entities-production \ - --index-name ByUser \ - --profile passbrains-ilisa-amplify \ - --key-condition-expression "Username = :user" \ - --expression-attribute-values '{":user": {"S": "john"}}' - -# Get just user profile: - aws dynamodb query \ - --table-name glad-entities-production \ - --profile passbrains-ilisa-amplify \ - --index-name ByUser \ - --key-condition-expression "Username = :user AND EntityType = :type" \ - --expression-attribute-values '{ - ":user": {"S": "john"}, - ":type": {"S": "User"} - }' - - -# User 1: John Doe -aws dynamodb put-item \ ---table-name glad-entities-production \ ---profile passbrains-ilisa-amplify \ ---item '{ -"entity_id": {"S": "USER-john"}, -"EntityType": {"S": "User"}, -"Username": {"S": "john"}, -"Name": {"S": "John Doe"}, -"Email": {"S": "john@example.com"}, -"PasswordHash": {"S": "$2a$10$examplehash"}, -"CreatedAt": {"S": "2025-01-01T10:00:00Z"}, -"UpdatedAt": {"S": "2025-01-01T10:00:00Z"} -}' - -# User 2: Jane Smith -aws dynamodb put-item \ ---table-name glad-entities-production \ ---profile passbrains-ilisa-amplify \ ---item '{ -"entity_id": {"S": "USER-jane"}, -"EntityType": {"S": "User"}, -"Username": {"S": "jane"}, -"Name": {"S": "Jane Smith"}, -"Email": {"S": "jane@example.com"}, -"PasswordHash": {"S": "$2a$10$examplehash"}, -"CreatedAt": {"S": "2025-01-15T09:00:00Z"}, -"UpdatedAt": {"S": "2025-01-15T09:00:00Z"} -}' - -# User 3: Bob Wilson -aws dynamodb put-item \ ---table-name glad-entities-production \ ---profile passbrains-ilisa-amplify \ ---item '{ -"entity_id": {"S": "USER-bob"}, -"EntityType": {"S": "User"}, -"Username": {"S": "bob"}, -"Name": {"S": "Bob Wilson"}, -"Email": {"S": "bob@example.com"}, -"PasswordHash": {"S": "$2a$10$examplehash"}, -"CreatedAt": {"S": "2025-02-01T08:00:00Z"}, -"UpdatedAt": {"S": "2025-02-01T08:00:00Z"} -}' - - -# Create User Skills -# John's Skills -# Python - Expert, 7 years - -aws dynamodb put-item \ ---table-name glad-entities-production \ ---profile passbrains-ilisa-amplify \ ---item '{ -"entity_id": {"S": "SKILL-john-Python"}, -"EntityType": {"S": "UserSkill"}, -"Username": {"S": "john"}, -"SkillName": {"S": "Python"}, -"ProficiencyLevel": {"S": "Expert"}, -"YearsOfExperience": {"N": "7"}, -"Endorsements": {"N": "15"}, -"LastUsedDate": {"S": "2025-12-10"}, -"Notes": {"S": "Specialized in data science and ML"}, -"CreatedAt": {"S": "2025-01-02T10:00:00Z"}, -"UpdatedAt": {"S": "2025-12-10T10:00:00Z"} -}' - -# JavaScript - Advanced, 5 years -aws dynamodb put-item \ ---table-name glad-entities-production \ ---profile passbrains-ilisa-amplify \ ---item '{ -"entity_id": {"S": "SKILL-john-JavaScript"}, -"EntityType": {"S": "UserSkill"}, -"Username": {"S": "john"}, -"SkillName": {"S": "JavaScript"}, -"ProficiencyLevel": {"S": "Advanced"}, -"YearsOfExperience": {"N": "5"}, -"Endorsements": {"N": "8"}, -"LastUsedDate": {"S": "2025-12-13"}, -"Notes": {"S": "React and Node.js expert"}, -"CreatedAt": {"S": "2025-01-02T10:00:00Z"}, -"UpdatedAt": {"S": "2025-12-13T10:00:00Z"} -}' - -# Jane's Skills -# Python - Expert, 10 years -aws dynamodb put-item \ ---table-name glad-entities-production \ ---profile passbrains-ilisa-amplify \ ---item '{ -"entity_id": {"S": "SKILL-jane-Python"}, -"EntityType": {"S": "UserSkill"}, -"Username": {"S": "jane"}, -"SkillName": {"S": "Python"}, -"ProficiencyLevel": {"S": "Expert"}, -"YearsOfExperience": {"S": "10"}, -"Endorsements": {"N": "25"}, -"LastUsedDate": {"S": "2025-12-12"}, -"Notes": {"S": "Backend systems and APIs"}, -"CreatedAt": {"S": "2025-01-16T09:00:00Z"}, -"UpdatedAt": {"S": "2025-12-12T09:00:00Z"} -}' - -# Go - Advanced, 6 years - -aws dynamodb put-item \ ---profile passbrains-ilisa-amplify \ ---table-name glad-entities-production \ ---item '{ -"entity_id": {"S": "SKILL-jane-Go"}, -"EntityType": {"S": "UserSkill"}, -"Username": {"S": "jane"}, -"SkillName": {"S": "Go"}, -"ProficiencyLevel": {"S": "Advanced"}, -"YearsOfExperience": {"S": "6"}, -"Endorsements": {"N": "12"}, -"LastUsedDate": {"S": "2025-12-11"}, -"Notes": {"S": "Microservices and cloud infrastructure"}, -"CreatedAt": {"S": "2025-01-16T09:00:00Z"}, -"UpdatedAt": {"S": "2025-12-11T09:00:00Z"} -}' - -# Bob's Skills -# Python - Intermediate, 3 years -aws dynamodb put-item \ ---table-name glad-entities-production \ ---item '{ -"entity_id": {"S": "SKILL-bob-Python"}, -"EntityType": {"S": "UserSkill"}, -"Username": {"S": "bob"}, -"SkillName": {"S": "Python"}, -"ProficiencyLevel": {"S": "Intermediate"}, -"YearsOfExperience": {"N": "3"}, -"Endorsements": {"N": "5"}, -"LastUsedDate": {"S": "2025-12-09"}, -"Notes": {"S": "Learning Django framework"}, -"CreatedAt": {"S": "2025-02-02T08:00:00Z"}, -"UpdatedAt": {"S": "2025-12-09T08:00:00Z"} -}' - -# JavaScript - Expert, 8 years -aws dynamodb put-item \ ---table-name glad-entities-production \ ---item '{ -"entity_id": {"S": "SKILL-bob-JavaScript"}, -"EntityType": {"S": "UserSkill"}, -"Username": {"S": "bob"}, -"SkillName": {"S": "JavaScript"}, -"ProficiencyLevel": {"S": "Expert"}, -"YearsOfExperience": {"N": "8"}, -"Endorsements": {"N": "20"}, -"LastUsedDate": {"S": "2025-12-13"}, -"Notes": {"S": "Full-stack JavaScript developer"}, -"CreatedAt": {"S": "2025-02-02T08:00:00Z"}, -"UpdatedAt": {"S": "2025-12-13T08:00:00Z"} -}' - -# TypeScript - Advanced, 4 years -aws dynamodb put-item \ ---table-name glad-entities-production \ ---item '{ -"entity_id": {"S": "SKILL-bob-TypeScript"}, -"EntityType": {"S": "UserSkill"}, -"Username": {"S": "bob"}, -"SkillName": {"S": "TypeScript"}, -"ProficiencyLevel": {"S": "Advanced"}, -"YearsOfExperience": {"N": "4"}, -"Endorsements": {"N": "10"}, -"LastUsedDate": {"S": "2025-12-13"}, -"Notes": {"S": "Building enterprise applications"}, -"CreatedAt": {"S": "2025-02-02T08:00:00Z"}, -"UpdatedAt": {"S": "2025-12-13T08:00:00Z"} -}' - -# Query Examples Using Composite Keys - -# Find all Expert Python users -aws dynamodb query \ ---profile passbrains-ilisa-amplify \ ---table-name glad-entities-production \ ---index-name SkillsByLevel \ ---key-condition-expression "SkillName = :skill AND ProficiencyLevel = :level" \ ---expression-attribute-values '{ -":skill": {"S": "Python"}, -":level": {"S": "Expert"} -}' - -# Find Expert Python users with 5+ years experience -aws dynamodb query \ ---table-name glad-entities-production \ ---index-name SkillsByLevel \ ---key-condition-expression "SkillName = :skill AND ProficiencyLevel = :level AND YearsOfExperience >= :years" \ ---expression-attribute-values '{ -":skill": {"S": "Python"}, -":level": {"S": "Expert"}, -":years": {"N": "5"} -}' - -# Get all of John's profile and skills -aws dynamodb query \ ---table-name glad-entities-production \ ---index-name ByUser \ ---key-condition-expression "Username = :user" \ ---expression-attribute-values '{":user": {"S": "john"}}' - -# Get only John's profile -aws dynamodb query \ ---table-name glad-entities-production \ ---index-name ByUser \ ---key-condition-expression "Username = :user AND EntityType = :type" \ ---expression-attribute-values '{ -":user": {"S": "john"}, -":type": {"S": "User"} -}' - -# Get only John's skills -aws dynamodb query \ ---table-name glad-entities-production \ ---index-name ByUser \ ---key-condition-expression "Username = :user AND EntityType = :type" \ ---expression-attribute-values '{ -":user": {"S": "john"}, -":type": {"S": "UserSkill"} -}' - -aws dynamodb put-item \ ---table-name glad-entities-production \ ---profile passbrains-ilisa-amplify \ ---item '{ -"entity_id": {"S": "SKILL-Python"}, -"EntityType": {"S": "Skill"}, -"SkillName": {"S": "Python"}, -"Description": {"S": "High-level programming language for general-purpose programming"}, -"Category": {"S": "Programming"}, -"CreatedAt": {"S": "2025-01-01T00:00:00Z"}, -"UpdatedAt": {"S": "2025-01-01T00:00:00Z"} -}' - -# Get all available skills (not user-specific) -aws dynamodb query \ ---table-name glad-entities-production \ ---index-name ByEntityType \ ---profile passbrains-ilisa-amplify \ ---key-condition-expression "EntityType = :type" \ ---expression-attribute-values '{":type": {"S": "Skill"}}' - -# Get a specific skill -aws dynamodb get-item \ ---table-name glad-entities-production \ ---profile passbrains-ilisa-amplify \ ---key '{"entity_id": {"S": "SKILL-Python"}}' - -# Get all skills in Programming category -aws dynamodb query \ ---table-name glad-entities-production \ ---profile passbrains-ilisa-amplify \ ---index-name SkillsByCategory \ ---key-condition-expression "EntityType = :type AND Category = :cat" \ ---expression-attribute-values '{ -":type": {"S": "Skill"}, -":cat": {"S": "Programming"} -}' diff --git a/deployments/app/cdk.go b/deployments/app/cdk.go index 56849df..474c039 100644 --- a/deployments/app/cdk.go +++ b/deployments/app/cdk.go @@ -1,11 +1,13 @@ package main import ( + "fmt" "os" "github.com/aws/aws-cdk-go/awscdk/v2" "github.com/aws/aws-cdk-go/awscdk/v2/awsapigateway" "github.com/aws/aws-cdk-go/awscdk/v2/awsdynamodb" + "github.com/aws/aws-cdk-go/awscdk/v2/awsiam" "github.com/aws/aws-cdk-go/awscdk/v2/awslambda" // "github.com/aws/aws-cdk-go/awscdk/v2/awssqs" @@ -97,27 +99,44 @@ func NewCdkStack(scope constructs.Construct, id string, props *CdkStackProps) aw // Add environment tag awscdk.Tags_Of(stack).Add(jsii.String("Environment"), jsii.String(ENVIRONMENT), nil) - // Create Lambda - myFunc := awslambda.NewFunction(stack, jsii.String(id+"-go-func"), &awslambda.FunctionProps{ - Runtime: awslambda.Runtime_PROVIDED_AL2023(), - Code: awslambda.AssetCode_FromAsset(jsii.String("../../.bin/lambda-function.zip"), nil), - Handler: jsii.String("main"), + // Create Lambda using Docker image + myFunc := awslambda.NewDockerImageFunction(stack, jsii.String(id+"-go-func"), &awslambda.DockerImageFunctionProps{ + Code: awslambda.DockerImageCode_FromImageAsset(jsii.String("../../"), &awslambda.AssetImageCodeProps{ + File: jsii.String("Dockerfile"), + }), + Timeout: awscdk.Duration_Seconds(jsii.Number(30)), + MemorySize: jsii.Number(512), + Description: jsii.String("GLAD Lambda function using Docker image"), + Architecture: awslambda.Architecture_X86_64(), }) myFunc.AddEnvironment(jsii.String("ENVIRONMENT"), jsii.String(ENVIRONMENT), nil) //// Create table | Grant Lambda read/write access to DynamoDB table entitiesTable := createEntitiesTable(stack, jsii.String(id+"-entities-table-"+ENVIRONMENT), ENVIRONMENT) - entitiesTable.GrantReadWriteData(myFunc) + // Grant access to table and all GSIs with wildcard to avoid policy size issues + myFunc.AddToRolePolicy(awsiam.NewPolicyStatement(&awsiam.PolicyStatementProps{ + Effect: awsiam.Effect_ALLOW, + Actions: jsii.Strings( + "dynamodb:PutItem", + "dynamodb:GetItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:Query", + "dynamodb:Scan", + ), + Resources: jsii.Strings( + *entitiesTable.TableArn(), + *entitiesTable.TableArn()+"/index/*", + ), + })) api := awsapigateway.NewRestApi(stack, jsii.String(id+"-api-gateway"), &awsapigateway.RestApiProps{ RestApiName: jsii.String("glad-api gateway"), Description: jsii.String("GLAD Stack API"), - DeployOptions: &awsapigateway.StageOptions{ - StageName: jsii.String("prod"), - ThrottlingBurstLimit: jsii.Number(200), - ThrottlingRateLimit: jsii.Number(100), - }, + // Don't use DeployOptions - we manage deployment explicitly below + Deploy: jsii.Bool(false), // Disable automatic deployment + CloudWatchRole: jsii.Bool(true), // Auto-create IAM role for CloudWatch Logs DefaultCorsPreflightOptions: &awsapigateway.CorsOptions{ AllowOrigins: jsii.Strings("*"), AllowCredentials: jsii.Bool(true), @@ -126,14 +145,31 @@ func NewCdkStack(scope constructs.Construct, id string, props *CdkStackProps) aw }, }) - // Create Lambda integration - integration := awsapigateway.NewLambdaIntegration(myFunc, nil) + // Create Lambda integration with explicit proxy configuration + // AWS_PROXY mode passes the entire request to Lambda and expects Lambda to return proper API Gateway response + integration := awsapigateway.NewLambdaIntegration(myFunc, &awsapigateway.LambdaIntegrationOptions{ + Proxy: jsii.Bool(true), // Explicitly enable AWS_PROXY mode + }) + + // Add single wildcard permission for all API Gateway methods to avoid policy size limit + myFunc.AddPermission(jsii.String("ApiGatewayInvoke"), &awslambda.Permission{ + Principal: awsiam.NewServicePrincipal(jsii.String("apigateway.amazonaws.com"), nil), + Action: jsii.String("lambda:InvokeFunction"), + SourceArn: jsii.String(fmt.Sprintf("arn:aws:execute-api:%s:%s:%s/*/*", + *stack.Region(), + *stack.Account(), + *api.RestApiId())), + }) registerResource := api.Root().AddResource(jsii.String("register"), nil) - registerResource.AddMethod(jsii.String("POST"), integration, nil) + registerResource.AddMethod(jsii.String("POST"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) loginResource := api.Root().AddResource(jsii.String("login"), nil) - loginResource.AddMethod(jsii.String("POST"), integration, nil) + loginResource.AddMethod(jsii.String("POST"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) protectedResource := api.Root().AddResource(jsii.String("protected"), nil) protectedResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ @@ -233,6 +269,28 @@ func NewCdkStack(scope constructs.Construct, id string, props *CdkStackProps) aw AuthorizationType: awsapigateway.AuthorizationType_NONE, }) + // Force API Gateway to create new deployment when Lambda changes + // This prevents deployment drift issues when switching between ZIP and Docker images + deployment := awsapigateway.NewDeployment(stack, jsii.String(id+"-api-deployment"), &awsapigateway.DeploymentProps{ + Api: api, + Description: jsii.String("Deployment triggered by Lambda changes"), + }) + + // Add dependency on Lambda function to trigger redeployment when Lambda changes + deployment.Node().AddDependency(myFunc) + + // Update stage to use the explicit deployment + // Use fixed logical ID for stable updates + stage := awsapigateway.NewStage(stack, jsii.String(id+"-api-stage"), &awsapigateway.StageProps{ + Deployment: deployment, + StageName: jsii.String("prod"), + ThrottlingBurstLimit: jsii.Number(200), + ThrottlingRateLimit: jsii.Number(100), + LoggingLevel: awsapigateway.MethodLoggingLevel_INFO, + DataTraceEnabled: jsii.Bool(true), + MetricsEnabled: jsii.Bool(true), + }) + // Create UsagePlan AFTER all methods are defined awsapigateway.NewUsagePlan(stack, jsii.String(id+"-api-gateway-usage-plan"), &awsapigateway.UsagePlanProps{ Name: jsii.String(id + "-api-gateway-usage-plan"), @@ -244,11 +302,18 @@ func NewCdkStack(scope constructs.Construct, id string, props *CdkStackProps) aw ApiStages: &[]*awsapigateway.UsagePlanPerApiStage{ { Api: api, - Stage: api.DeploymentStage(), + Stage: stage, // Use explicit stage instead of api.DeploymentStage() }, }, }) + // Output the API URL + awscdk.NewCfnOutput(stack, jsii.String("ApiUrl"), &awscdk.CfnOutputProps{ + Value: jsii.String(fmt.Sprintf("https://%s.execute-api.%s.amazonaws.com/%s", *api.RestApiId(), *stack.Region(), *stage.StageName())), + Description: jsii.String("API Gateway endpoint URL"), + ExportName: jsii.String("GladApiUrl"), + }) + return stack } diff --git a/docker-compose.http-test.yml b/docker-compose.http-test.yml index c60dafe..e7159e7 100644 --- a/docker-compose.http-test.yml +++ b/docker-compose.http-test.yml @@ -4,7 +4,7 @@ services: http-client: image: jetbrains/intellij-http-client volumes: - - ./docs/api-testing:/workdir + - ./cmd/app/testdata/api-testing:/workdir working_dir: /workdir command: > --env=dev diff --git a/docs/api-testing/http-client.env.json b/docs/api-testing/http-client.env.json deleted file mode 100644 index d1a88c0..0000000 --- a/docs/api-testing/http-client.env.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dev": { - "API_URL": "https://ame7n22jj9.execute-api.eu-central-1.amazonaws.com/prod" - } -} \ No newline at end of file From f83d2a6dc7a78e6449b6c1751548b87095edcab0 Mon Sep 17 00:00:00 2001 From: Alex Ilies Date: Sun, 21 Dec 2025 16:21:02 +0200 Subject: [PATCH 5/6] Remove http-client.env.json from git tracking --- .../testdata/api-testing/http-client.env.json | 5 - deployments/app/app_stack.go | 214 ++++++++++++++++++ deployments/app/database_stack.go | 91 ++++++++ deployments/app/main.go | 39 ++++ 4 files changed, 344 insertions(+), 5 deletions(-) delete mode 100644 cmd/app/testdata/api-testing/http-client.env.json create mode 100644 deployments/app/app_stack.go create mode 100644 deployments/app/database_stack.go create mode 100644 deployments/app/main.go diff --git a/cmd/app/testdata/api-testing/http-client.env.json b/cmd/app/testdata/api-testing/http-client.env.json deleted file mode 100644 index 131e4d6..0000000 --- a/cmd/app/testdata/api-testing/http-client.env.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dev": { - "API_URL": "https://hwb4so5a43.execute-api.eu-central-1.amazonaws.com/prod-1766324635" - } -} diff --git a/deployments/app/app_stack.go b/deployments/app/app_stack.go new file mode 100644 index 0000000..f5bcf95 --- /dev/null +++ b/deployments/app/app_stack.go @@ -0,0 +1,214 @@ +package main + +import ( + "fmt" + "time" + + "github.com/aws/aws-cdk-go/awscdk/v2" + "github.com/aws/aws-cdk-go/awscdk/v2/awsapigateway" + "github.com/aws/aws-cdk-go/awscdk/v2/awsiam" + "github.com/aws/aws-cdk-go/awscdk/v2/awslambda" + "github.com/aws/constructs-go/constructs/v10" + "github.com/aws/jsii-runtime-go" +) + +type AppStackProps struct { + awscdk.StackProps +} + +func NewAppStack(scope constructs.Construct, id string, props *AppStackProps) awscdk.Stack { + var sprops awscdk.StackProps + if props != nil { + sprops = props.StackProps + } + stack := awscdk.NewStack(scope, &id, &sprops) + + ENVIRONMENT := "production" + awscdk.Tags_Of(stack).Add(jsii.String("Environment"), jsii.String(ENVIRONMENT), nil) + + // Import table from database stack + tableName := awscdk.Fn_ImportValue(jsii.String("GladTableName")) + tableArn := awscdk.Fn_ImportValue(jsii.String("GladTableArn")) + + // Create Lambda using Docker image + myFunc := awslambda.NewDockerImageFunction(stack, jsii.String(id+"-go-func"), &awslambda.DockerImageFunctionProps{ + Code: awslambda.DockerImageCode_FromImageAsset(jsii.String("../../"), &awslambda.AssetImageCodeProps{ + File: jsii.String("Dockerfile"), + }), + Timeout: awscdk.Duration_Seconds(jsii.Number(30)), + MemorySize: jsii.Number(512), + Description: jsii.String("GLAD Lambda function using Docker image"), + Architecture: awslambda.Architecture_X86_64(), + }) + + myFunc.AddEnvironment(jsii.String("ENVIRONMENT"), jsii.String(ENVIRONMENT), nil) + myFunc.AddEnvironment(jsii.String("DYNAMODB_TABLE"), tableName, nil) + + // Grant Lambda access to DynamoDB table + myFunc.AddToRolePolicy(awsiam.NewPolicyStatement(&awsiam.PolicyStatementProps{ + Effect: awsiam.Effect_ALLOW, + Actions: jsii.Strings( + "dynamodb:PutItem", + "dynamodb:GetItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:Query", + "dynamodb:Scan", + ), + Resources: jsii.Strings( + *tableArn, + *tableArn+"/index/*", + ), + })) + + api := awsapigateway.NewRestApi(stack, jsii.String(id+"-api-gateway"), &awsapigateway.RestApiProps{ + RestApiName: jsii.String("glad-api gateway"), + Description: jsii.String("GLAD Stack API"), + Deploy: jsii.Bool(false), + CloudWatchRole: jsii.Bool(true), + DefaultCorsPreflightOptions: &awsapigateway.CorsOptions{ + AllowOrigins: jsii.Strings("*"), + AllowCredentials: jsii.Bool(true), + AllowHeaders: jsii.Strings("Content-Type", "Authorization"), + AllowMethods: jsii.Strings("GET", "POST", "DELETE", "PUT", "OPTIONS"), + }, + }) + + integration := awsapigateway.NewLambdaIntegration(myFunc, &awsapigateway.LambdaIntegrationOptions{ + Proxy: jsii.Bool(true), + }) + + // Add single wildcard permission for all API Gateway methods + myFunc.AddPermission(jsii.String("ApiGatewayInvoke"), &awslambda.Permission{ + Principal: awsiam.NewServicePrincipal(jsii.String("apigateway.amazonaws.com"), nil), + Action: jsii.String("lambda:InvokeFunction"), + SourceArn: jsii.String(fmt.Sprintf("arn:aws:execute-api:%s:%s:%s/*/*", + *stack.Region(), + *stack.Account(), + *api.RestApiId())), + }) + + // Define API routes + registerResource := api.Root().AddResource(jsii.String("register"), nil) + registerResource.AddMethod(jsii.String("POST"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + + loginResource := api.Root().AddResource(jsii.String("login"), nil) + loginResource.AddMethod(jsii.String("POST"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + + protectedResource := api.Root().AddResource(jsii.String("protected"), nil) + protectedResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + + userResource := api.Root().AddResource(jsii.String("user"), nil) + userResource.AddMethod(jsii.String("PUT"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + + usersResource := api.Root().AddResource(jsii.String("users"), nil) + usersResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + + meResource := api.Root().AddResource(jsii.String("me"), nil) + meResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + + // Skill Management Endpoints + usersSkillsResource := usersResource.AddResource(jsii.String("{username}"), nil) + skillsResource := usersSkillsResource.AddResource(jsii.String("skills"), nil) + skillsResource.AddMethod(jsii.String("POST"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + skillsResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + + skillResource := skillsResource.AddResource(jsii.String("{skillName}"), nil) + skillResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + skillResource.AddMethod(jsii.String("PUT"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + skillResource.AddMethod(jsii.String("DELETE"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + + // Global skill query endpoint + skillsGlobalResource := api.Root().AddResource(jsii.String("skills"), nil) + skillNameResource := skillsGlobalResource.AddResource(jsii.String("{skillName}"), nil) + usersWithSkillResource := skillNameResource.AddResource(jsii.String("users"), nil) + usersWithSkillResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + + // Master Skills Management Endpoints + masterSkillsResource := api.Root().AddResource(jsii.String("master-skills"), nil) + masterSkillsResource.AddMethod(jsii.String("POST"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + masterSkillsResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + + masterSkillResource := masterSkillsResource.AddResource(jsii.String("{skillID}"), nil) + masterSkillResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + masterSkillResource.AddMethod(jsii.String("PUT"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + masterSkillResource.AddMethod(jsii.String("DELETE"), integration, &awsapigateway.MethodOptions{ + AuthorizationType: awsapigateway.AuthorizationType_NONE, + }) + + // Create deployment + deployment := awsapigateway.NewDeployment(stack, jsii.String(id+"-api-deployment"), &awsapigateway.DeploymentProps{ + Api: api, + Description: jsii.String("Deployment triggered by Lambda changes"), + }) + deployment.Node().AddDependency(myFunc) + + // Create stage + timestamp := time.Now().Unix() + stage := awsapigateway.NewStage(stack, jsii.String(fmt.Sprintf("%s-api-stage-%d", id, timestamp)), &awsapigateway.StageProps{ + Deployment: deployment, + StageName: jsii.String("prod"), + ThrottlingBurstLimit: jsii.Number(200), + ThrottlingRateLimit: jsii.Number(100), + LoggingLevel: awsapigateway.MethodLoggingLevel_INFO, + DataTraceEnabled: jsii.Bool(true), + MetricsEnabled: jsii.Bool(true), + }) + + // Create UsagePlan + awsapigateway.NewUsagePlan(stack, jsii.String(id+"-api-gateway-usage-plan"), &awsapigateway.UsagePlanProps{ + Name: jsii.String(id + "-api-gateway-usage-plan"), + Description: jsii.String("Usage plan with rate limiting"), + Throttle: &awsapigateway.ThrottleSettings{ + RateLimit: jsii.Number(100), + BurstLimit: jsii.Number(200), + }, + ApiStages: &[]*awsapigateway.UsagePlanPerApiStage{ + { + Api: api, + Stage: stage, + }, + }, + }) + + // Output the API URL + awscdk.NewCfnOutput(stack, jsii.String("ApiUrl"), &awscdk.CfnOutputProps{ + Value: jsii.String(fmt.Sprintf("https://%s.execute-api.%s.amazonaws.com/%s", *api.RestApiId(), *stack.Region(), *stage.StageName())), + Description: jsii.String("API Gateway endpoint URL"), + ExportName: jsii.String("GladApiUrl"), + }) + + return stack +} diff --git a/deployments/app/database_stack.go b/deployments/app/database_stack.go new file mode 100644 index 0000000..265a515 --- /dev/null +++ b/deployments/app/database_stack.go @@ -0,0 +1,91 @@ +package main + +import ( + "github.com/aws/aws-cdk-go/awscdk/v2" + "github.com/aws/aws-cdk-go/awscdk/v2/awsdynamodb" + "github.com/aws/constructs-go/constructs/v10" + "github.com/aws/jsii-runtime-go" +) + +type DatabaseStackProps struct { + awscdk.StackProps +} + +func NewDatabaseStack(scope constructs.Construct, id string, props *DatabaseStackProps) awscdk.Stack { + var sprops awscdk.StackProps + if props != nil { + sprops = props.StackProps + } + stack := awscdk.NewStack(scope, &id, &sprops) + + ENVIRONMENT := "production" + awscdk.Tags_Of(stack).Add(jsii.String("Environment"), jsii.String(ENVIRONMENT), nil) + + // Create DynamoDB table + entitiesTable := awsdynamodb.NewTableV2(stack, jsii.String(id+"-entities-table"), &awsdynamodb.TablePropsV2{ + TableName: jsii.String("glad-entities"), + PartitionKey: &awsdynamodb.Attribute{ + Name: jsii.String("EntityType"), + Type: awsdynamodb.AttributeType_STRING, + }, + SortKey: &awsdynamodb.Attribute{ + Name: jsii.String("entity_id"), + Type: awsdynamodb.AttributeType_STRING, + }, + GlobalSecondaryIndexes: &[]*awsdynamodb.GlobalSecondaryIndexPropsV2{ + { + IndexName: jsii.String("BySkill"), + PartitionKey: &awsdynamodb.Attribute{ + Name: jsii.String("Category"), + Type: awsdynamodb.AttributeType_STRING, + }, + SortKeys: &[]*awsdynamodb.Attribute{ + { + Name: jsii.String("SkillName"), + Type: awsdynamodb.AttributeType_STRING, + }, + { + Name: jsii.String("ProficiencyLevel"), + Type: awsdynamodb.AttributeType_STRING, + }, + { + Name: jsii.String("YearsOfExperience"), + Type: awsdynamodb.AttributeType_NUMBER, + }, + { + Name: jsii.String("Username"), + Type: awsdynamodb.AttributeType_STRING, + }, + }, + }, + }, + PointInTimeRecovery: jsii.Bool(false), + DynamoStream: awsdynamodb.StreamViewType_NEW_AND_OLD_IMAGES, + RemovalPolicy: awscdk.RemovalPolicy_RETAIN, // Keep table on stack deletion + Tags: &[]*awscdk.CfnTag{ + { + Key: jsii.String("Purpose"), + Value: jsii.String("Single-Table-Design"), + }, + { + Key: jsii.String("DataModel"), + Value: jsii.String("Multi-Entity"), + }, + }, + }) + + // Export table name and ARN for other stacks + awscdk.NewCfnOutput(stack, jsii.String("TableName"), &awscdk.CfnOutputProps{ + Value: entitiesTable.TableName(), + Description: jsii.String("DynamoDB table name"), + ExportName: jsii.String("GladTableName"), + }) + + awscdk.NewCfnOutput(stack, jsii.String("TableArn"), &awscdk.CfnOutputProps{ + Value: entitiesTable.TableArn(), + Description: jsii.String("DynamoDB table ARN"), + ExportName: jsii.String("GladTableArn"), + }) + + return stack +} diff --git a/deployments/app/main.go b/deployments/app/main.go new file mode 100644 index 0000000..138d475 --- /dev/null +++ b/deployments/app/main.go @@ -0,0 +1,39 @@ +package main + +import ( + "os" + + "github.com/aws/aws-cdk-go/awscdk/v2" + "github.com/aws/jsii-runtime-go" +) + +func main() { + defer jsii.Close() + + app := awscdk.NewApp(nil) + + // Create database stack first + NewDatabaseStack(app, "glad-database-stack", &DatabaseStackProps{ + awscdk.StackProps{ + Env: env(), + }, + }) + + // Create application stack (depends on database stack) + NewAppStack(app, "glad-app-stack", &AppStackProps{ + awscdk.StackProps{ + Env: env(), + }, + }) + + app.Synth(nil) +} + +// env determines the AWS environment (account+region) in which our stack is to +// be deployed. For more information see: https://docs.aws.amazon.com/cdk/latest/guide/environments.html +func env() *awscdk.Environment { + return &awscdk.Environment{ + Account: jsii.String(os.Getenv("CDK_DEFAULT_ACCOUNT")), + Region: jsii.String(os.Getenv("CDK_DEFAULT_REGION")), + } +} From e54c17dedf6ed18aa7d32a22376cc4995db24af6 Mon Sep 17 00:00:00 2001 From: Alex Ilies Date: Sun, 21 Dec 2025 16:21:21 +0200 Subject: [PATCH 6/6] infra(cdk): split cdk in different stages --- .gitignore | 2 +- Taskfile.yml | 29 ++- deployments/app/app_stack.go | 12 +- deployments/app/cdk.go | 358 ----------------------------------- deployments/app/cdk.json | 2 +- 5 files changed, 34 insertions(+), 369 deletions(-) delete mode 100644 deployments/app/cdk.go diff --git a/.gitignore b/.gitignore index af40310..7f4ea26 100644 --- a/.gitignore +++ b/.gitignore @@ -87,4 +87,4 @@ yarn-error.log* .terraform.lock.hcl # Go module downloads (keep go.sum) -vendor/ \ No newline at end of file +vendor/cmd/app/testdata/api-testing/http-client.env.json diff --git a/Taskfile.yml b/Taskfile.yml index 2b9738d..ece020d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -203,15 +203,40 @@ tasks: cmds: - echo 'Deploying to AWS with profile {{.profile | default "default"}}' - echo 'CDK will automatically build the Docker image during deployment...' - - cdk deploy --require-approval never --profile {{.profile | default "default"}} + - cdk deploy --all --require-approval never --profile {{.profile | default "default"}} - echo 'Deployment completed!' + cdk:deploy:database: + desc: 'Deploy only database stack' + dir: 'deployments/app' + cmds: + - cdk deploy glad-database-stack --require-approval never --profile {{.profile | default "default"}} + + cdk:deploy:app: + desc: 'Deploy only application stack' + dir: 'deployments/app' + cmds: + - cdk deploy glad-app-stack --require-approval never --profile {{.profile | default "default"}} + cdk:destroy: desc: 'Destroy AWS infrastructure (use with caution!)' dir: 'deployments/app' cmds: - echo 'WARNING This will destroy all AWS resources!' - - cdk destroy --force + - cdk destroy --all --force + + cdk:destroy:app: + desc: 'Destroy only application stack (keeps database)' + dir: 'deployments/app' + cmds: + - cdk destroy glad-app-stack --force + + cdk:destroy:database: + desc: 'Destroy database stack (table will be retained)' + dir: 'deployments/app' + cmds: + - echo 'WARNING Database table will be retained but stack will be removed' + - cdk destroy glad-database-stack --force cdk:bootstrap: desc: 'Bootstrap CDK in your AWS account (run once)' diff --git a/deployments/app/app_stack.go b/deployments/app/app_stack.go index f5bcf95..c3bc503 100644 --- a/deployments/app/app_stack.go +++ b/deployments/app/app_stack.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "time" "github.com/aws/aws-cdk-go/awscdk/v2" "github.com/aws/aws-cdk-go/awscdk/v2/awsapigateway" @@ -62,9 +61,9 @@ func NewAppStack(scope constructs.Construct, id string, props *AppStackProps) aw })) api := awsapigateway.NewRestApi(stack, jsii.String(id+"-api-gateway"), &awsapigateway.RestApiProps{ - RestApiName: jsii.String("glad-api gateway"), - Description: jsii.String("GLAD Stack API"), - Deploy: jsii.Bool(false), + RestApiName: jsii.String("glad-api gateway"), + Description: jsii.String("GLAD Stack API"), + Deploy: jsii.Bool(false), CloudWatchRole: jsii.Bool(true), DefaultCorsPreflightOptions: &awsapigateway.CorsOptions{ AllowOrigins: jsii.Strings("*"), @@ -175,9 +174,8 @@ func NewAppStack(scope constructs.Construct, id string, props *AppStackProps) aw }) deployment.Node().AddDependency(myFunc) - // Create stage - timestamp := time.Now().Unix() - stage := awsapigateway.NewStage(stack, jsii.String(fmt.Sprintf("%s-api-stage-%d", id, timestamp)), &awsapigateway.StageProps{ + // Create stage with fixed logical ID + stage := awsapigateway.NewStage(stack, jsii.String(id+"-api-stage"), &awsapigateway.StageProps{ Deployment: deployment, StageName: jsii.String("prod"), ThrottlingBurstLimit: jsii.Number(200), diff --git a/deployments/app/cdk.go b/deployments/app/cdk.go deleted file mode 100644 index 474c039..0000000 --- a/deployments/app/cdk.go +++ /dev/null @@ -1,358 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "github.com/aws/aws-cdk-go/awscdk/v2" - "github.com/aws/aws-cdk-go/awscdk/v2/awsapigateway" - "github.com/aws/aws-cdk-go/awscdk/v2/awsdynamodb" - "github.com/aws/aws-cdk-go/awscdk/v2/awsiam" - "github.com/aws/aws-cdk-go/awscdk/v2/awslambda" - - // "github.com/aws/aws-cdk-go/awscdk/v2/awssqs" - "github.com/aws/constructs-go/constructs/v10" - "github.com/aws/jsii-runtime-go" -) - -type CdkStackProps struct { - awscdk.StackProps -} - -func createEntitiesTable(stack awscdk.Stack, id *string, environment string) awsdynamodb.TableV2 { - entitiesTable := awsdynamodb.NewTableV2(stack, id, &awsdynamodb.TablePropsV2{ - TableName: jsii.String("glad-entities"), - // Partition Key: EntityType - PartitionKey: &awsdynamodb.Attribute{ - Name: jsii.String("EntityType"), - Type: awsdynamodb.AttributeType_STRING, - }, - SortKey: &awsdynamodb.Attribute{ - Name: jsii.String("entity_id"), - Type: awsdynamodb.AttributeType_STRING, - }, - - GlobalSecondaryIndexes: &[]*awsdynamodb.GlobalSecondaryIndexPropsV2{ - // GSI for flexible category/skill/proficiency queries - // Single PK: Category (allows broad queries) - // Composite SK: SkillName + ProficiencyLevel + YearsOfExperience + Username - // This design provides maximum query flexibility: - // - Query by Category alone - // - Query by Category + SkillName - // - Query by Category + SkillName + ProficiencyLevel - // - Query by Category + SkillName + ProficiencyLevel + YearsOfExperience (with range) - { - IndexName: jsii.String("BySkill"), - PartitionKey: &awsdynamodb.Attribute{ - Name: jsii.String("Category"), - Type: awsdynamodb.AttributeType_STRING, - }, - SortKeys: &[]*awsdynamodb.Attribute{ - { - Name: jsii.String("SkillName"), - Type: awsdynamodb.AttributeType_STRING, - }, - { - Name: jsii.String("ProficiencyLevel"), - Type: awsdynamodb.AttributeType_STRING, - }, - { - Name: jsii.String("YearsOfExperience"), - Type: awsdynamodb.AttributeType_NUMBER, - }, - { - Name: jsii.String("Username"), - Type: awsdynamodb.AttributeType_STRING, - }, - }, - }, - }, - - PointInTimeRecovery: jsii.Bool(false), - DynamoStream: awsdynamodb.StreamViewType_NEW_AND_OLD_IMAGES, - RemovalPolicy: awscdk.RemovalPolicy_DESTROY, - - Tags: &[]*awscdk.CfnTag{ - { - Key: jsii.String("Purpose"), - Value: jsii.String("Single-Table-Design"), - }, - { - Key: jsii.String("DataModel"), - Value: jsii.String("Multi-Entity"), - }, - }, - }) - - return entitiesTable -} - -func NewCdkStack(scope constructs.Construct, id string, props *CdkStackProps) awscdk.Stack { - var sprops awscdk.StackProps - if props != nil { - sprops = props.StackProps - } - stack := awscdk.NewStack(scope, &id, &sprops) - - ENVIRONMENT := "production" // todo: will be parametrised - - // Add environment tag - awscdk.Tags_Of(stack).Add(jsii.String("Environment"), jsii.String(ENVIRONMENT), nil) - - // Create Lambda using Docker image - myFunc := awslambda.NewDockerImageFunction(stack, jsii.String(id+"-go-func"), &awslambda.DockerImageFunctionProps{ - Code: awslambda.DockerImageCode_FromImageAsset(jsii.String("../../"), &awslambda.AssetImageCodeProps{ - File: jsii.String("Dockerfile"), - }), - Timeout: awscdk.Duration_Seconds(jsii.Number(30)), - MemorySize: jsii.Number(512), - Description: jsii.String("GLAD Lambda function using Docker image"), - Architecture: awslambda.Architecture_X86_64(), - }) - - myFunc.AddEnvironment(jsii.String("ENVIRONMENT"), jsii.String(ENVIRONMENT), nil) - - //// Create table | Grant Lambda read/write access to DynamoDB table - entitiesTable := createEntitiesTable(stack, jsii.String(id+"-entities-table-"+ENVIRONMENT), ENVIRONMENT) - // Grant access to table and all GSIs with wildcard to avoid policy size issues - myFunc.AddToRolePolicy(awsiam.NewPolicyStatement(&awsiam.PolicyStatementProps{ - Effect: awsiam.Effect_ALLOW, - Actions: jsii.Strings( - "dynamodb:PutItem", - "dynamodb:GetItem", - "dynamodb:UpdateItem", - "dynamodb:DeleteItem", - "dynamodb:Query", - "dynamodb:Scan", - ), - Resources: jsii.Strings( - *entitiesTable.TableArn(), - *entitiesTable.TableArn()+"/index/*", - ), - })) - - api := awsapigateway.NewRestApi(stack, jsii.String(id+"-api-gateway"), &awsapigateway.RestApiProps{ - RestApiName: jsii.String("glad-api gateway"), - Description: jsii.String("GLAD Stack API"), - // Don't use DeployOptions - we manage deployment explicitly below - Deploy: jsii.Bool(false), // Disable automatic deployment - CloudWatchRole: jsii.Bool(true), // Auto-create IAM role for CloudWatch Logs - DefaultCorsPreflightOptions: &awsapigateway.CorsOptions{ - AllowOrigins: jsii.Strings("*"), - AllowCredentials: jsii.Bool(true), - AllowHeaders: jsii.Strings("Content-Type", "Authorization"), - AllowMethods: jsii.Strings("GET", "POST", "DELETE", "PUT", "OPTIONS"), - }, - }) - - // Create Lambda integration with explicit proxy configuration - // AWS_PROXY mode passes the entire request to Lambda and expects Lambda to return proper API Gateway response - integration := awsapigateway.NewLambdaIntegration(myFunc, &awsapigateway.LambdaIntegrationOptions{ - Proxy: jsii.Bool(true), // Explicitly enable AWS_PROXY mode - }) - - // Add single wildcard permission for all API Gateway methods to avoid policy size limit - myFunc.AddPermission(jsii.String("ApiGatewayInvoke"), &awslambda.Permission{ - Principal: awsiam.NewServicePrincipal(jsii.String("apigateway.amazonaws.com"), nil), - Action: jsii.String("lambda:InvokeFunction"), - SourceArn: jsii.String(fmt.Sprintf("arn:aws:execute-api:%s:%s:%s/*/*", - *stack.Region(), - *stack.Account(), - *api.RestApiId())), - }) - - registerResource := api.Root().AddResource(jsii.String("register"), nil) - registerResource.AddMethod(jsii.String("POST"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - loginResource := api.Root().AddResource(jsii.String("login"), nil) - loginResource.AddMethod(jsii.String("POST"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - protectedResource := api.Root().AddResource(jsii.String("protected"), nil) - protectedResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - userResource := api.Root().AddResource(jsii.String("user"), nil) - userResource.AddMethod(jsii.String("PUT"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // Add missing /users GET endpoint - usersResource := api.Root().AddResource(jsii.String("users"), nil) - usersResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // Add /me GET endpoint for current user - meResource := api.Root().AddResource(jsii.String("me"), nil) - meResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // Skill Management Endpoints - // Pattern: /users/{username}/skills - usersSkillsResource := usersResource.AddResource(jsii.String("{username}"), nil) - skillsResource := usersSkillsResource.AddResource(jsii.String("skills"), nil) - - // POST /users/{username}/skills - Add a skill - skillsResource.AddMethod(jsii.String("POST"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // GET /users/{username}/skills - List all skills for user - skillsResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // Specific skill endpoints - // Pattern: /users/{username}/skills/{skillName} - skillResource := skillsResource.AddResource(jsii.String("{skillName}"), nil) - - // GET /users/{username}/skills/{skillName} - Get specific skill - skillResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // PUT /users/{username}/skills/{skillName} - Update skill - skillResource.AddMethod(jsii.String("PUT"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // DELETE /users/{username}/skills/{skillName} - Delete skill - skillResource.AddMethod(jsii.String("DELETE"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // Global skill query endpoint - // GET /skills/{skillName}/users - Find all users with a skill - skillsGlobalResource := api.Root().AddResource(jsii.String("skills"), nil) - skillNameResource := skillsGlobalResource.AddResource(jsii.String("{skillName}"), nil) - usersWithSkillResource := skillNameResource.AddResource(jsii.String("users"), nil) - usersWithSkillResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // Master Skills Management Endpoints - // Pattern: /master-skills - masterSkillsResource := api.Root().AddResource(jsii.String("master-skills"), nil) - - // POST /master-skills - Create a master skill - masterSkillsResource.AddMethod(jsii.String("POST"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // GET /master-skills - List all master skills - masterSkillsResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // Specific master skill endpoints - // Pattern: /master-skills/{skillID} - masterSkillResource := masterSkillsResource.AddResource(jsii.String("{skillID}"), nil) - - // GET /master-skills/{skillID} - Get specific master skill - masterSkillResource.AddMethod(jsii.String("GET"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // PUT /master-skills/{skillID} - Update master skill - masterSkillResource.AddMethod(jsii.String("PUT"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // DELETE /master-skills/{skillID} - Delete master skill - masterSkillResource.AddMethod(jsii.String("DELETE"), integration, &awsapigateway.MethodOptions{ - AuthorizationType: awsapigateway.AuthorizationType_NONE, - }) - - // Force API Gateway to create new deployment when Lambda changes - // This prevents deployment drift issues when switching between ZIP and Docker images - deployment := awsapigateway.NewDeployment(stack, jsii.String(id+"-api-deployment"), &awsapigateway.DeploymentProps{ - Api: api, - Description: jsii.String("Deployment triggered by Lambda changes"), - }) - - // Add dependency on Lambda function to trigger redeployment when Lambda changes - deployment.Node().AddDependency(myFunc) - - // Update stage to use the explicit deployment - // Use fixed logical ID for stable updates - stage := awsapigateway.NewStage(stack, jsii.String(id+"-api-stage"), &awsapigateway.StageProps{ - Deployment: deployment, - StageName: jsii.String("prod"), - ThrottlingBurstLimit: jsii.Number(200), - ThrottlingRateLimit: jsii.Number(100), - LoggingLevel: awsapigateway.MethodLoggingLevel_INFO, - DataTraceEnabled: jsii.Bool(true), - MetricsEnabled: jsii.Bool(true), - }) - - // Create UsagePlan AFTER all methods are defined - awsapigateway.NewUsagePlan(stack, jsii.String(id+"-api-gateway-usage-plan"), &awsapigateway.UsagePlanProps{ - Name: jsii.String(id + "-api-gateway-usage-plan"), - Description: jsii.String("Usage plan with rate limiting"), - Throttle: &awsapigateway.ThrottleSettings{ - RateLimit: jsii.Number(100), - BurstLimit: jsii.Number(200), - }, - ApiStages: &[]*awsapigateway.UsagePlanPerApiStage{ - { - Api: api, - Stage: stage, // Use explicit stage instead of api.DeploymentStage() - }, - }, - }) - - // Output the API URL - awscdk.NewCfnOutput(stack, jsii.String("ApiUrl"), &awscdk.CfnOutputProps{ - Value: jsii.String(fmt.Sprintf("https://%s.execute-api.%s.amazonaws.com/%s", *api.RestApiId(), *stack.Region(), *stage.StageName())), - Description: jsii.String("API Gateway endpoint URL"), - ExportName: jsii.String("GladApiUrl"), - }) - - return stack -} - -func main() { - defer jsii.Close() - - app := awscdk.NewApp(nil) - - NewCdkStack(app, "glad-stack", &CdkStackProps{ - awscdk.StackProps{ - Env: env(), - }, - }) - - app.Synth(nil) -} - -// env determines the AWS environment (account+region) in which our stack is to -// be deployed. For more information see: https://docs.aws.amazon.com/cdk/latest/guide/environments.html -func env() *awscdk.Environment { - // If unspecified, this stack will be "environment-agnostic". - // Account/Region-dependent features and context lookups will not work, but a - // single synthesized template can be deployed anywhere. - //--------------------------------------------------------------------------- - - // Uncomment if you know exactly what account and region you want to deploy - // the stack to. This is the recommendation for production stacks. - //--------------------------------------------------------------------------- - // return &awscdk.Environment{ - // Account: jsii.String("123456789012"), - // Region: jsii.String("us-east-1"), - // } - - // Uncomment to specialize this stack for the AWS Account and Region that are - // implied by the current CLI configuration. This is recommended for dev - // stacks. - //--------------------------------------------------------------------------- - return &awscdk.Environment{ - Account: jsii.String(os.Getenv("CDK_DEFAULT_ACCOUNT")), - Region: jsii.String(os.Getenv("CDK_DEFAULT_REGION")), - } -} diff --git a/deployments/app/cdk.json b/deployments/app/cdk.json index 2ad95c0..a5e5dee 100644 --- a/deployments/app/cdk.json +++ b/deployments/app/cdk.json @@ -1,5 +1,5 @@ { - "app": "go mod download && go run cdk.go", + "app": "go mod download && go run .", "watch": { "include": [ "**"