diff --git a/README.md b/README.md index 8ab8c98..9bbb695 100644 --- a/README.md +++ b/README.md @@ -153,31 +153,32 @@ The unified `Repository` interface composes all entity repositories, allowing bo - If `ENVIRONMENT=development` or `DB_MOCK=true` → Mock - Default: DynamoDB -## Data Model - Single Table Design with Multi-Key GSI +## Data Model - Optimized Single Table Design -The Single Table Design is modeled using DynamoDB's Multi-Key (composite keys) for GSI feature. -Read more: https://aws.amazon.com/blogs/database/multi-key-support-for-global-secondary-index-in-amazon-dynamodb/ +Optimized single-table design using `EntityType` as partition key to minimize GSI usage and reduce costs while maintaining optimal query performance. ### Table: `glad-entities` -- **Partition Key**: `entity_id` (STRING) +- **Partition Key**: `EntityType` (STRING) - "User", "Skill", "UserSkill" +- **Sort Key**: `entity_id` (STRING) - Unique entity identifier ### Entity ID Format (using `#` delimiter): - **Users**: `USER#` (e.g., `USER#john`) - **User Skills**: `USERSKILL##` (e.g., `USERSKILL#john#python`) - **Master Skills**: `SKILL#` (e.g., `SKILL#python`) -### Global Secondary Indexes (5 GSIs): +### Global Secondary Index (1 GSI): -1. **SkillsByLevel** - Query users by skill and proficiency level - - PK: `SkillName`, SK: `ProficiencyLevel` -2. **ByUser** - Get all entities for a user - - PK: `Username`, SK: `EntityType` -3. **SkillsByCategory** - Find skills by category - - PK: `EntityType`, SK: `Category` -4. **ByEntityType** - Query all entities of a type - - PK: `EntityType`, SK: `SkillName` -5. **BySkillID** - Find all users with a specific skill - - PK: `skill_id`, SK: `Username` +1. **BySkill** - Consolidated skill queries with composite sort keys + - PK: `SkillName` + - SK: `ProficiencyLevel` + `Username` (multi-key sort) + - Supports both skill-only and skill+level queries + +### Query Patterns: +- **List all users**: `EntityType = "User"` (main table) +- **List all skills**: `EntityType = "Skill"` (main table) +- **User's skills**: `EntityType = "UserSkill" AND begins_with(entity_id, "USERSKILL#john#")` (main table) +- **Users with skill**: `SkillName = "Python"` (BySkill GSI) +- **Users with skill at level**: `SkillName = "Python" AND ProficiencyLevel = "Expert"` (BySkill GSI) ## API Endpoints @@ -471,7 +472,8 @@ Deployed resources (via AWS CDK in Go): ### DynamoDB Table - **Name**: `glad-entities` -- **Single table** with 5 Global Secondary Indexes +- **Optimized single table** with 1 Global Secondary Index +- **Table Keys**: `EntityType` (PK) + `entity_id` (SK) - **DynamoDB Streams**: Enabled (NEW_AND_OLD_IMAGES) - **Capacity**: On-demand billing mode - **Point-in-time recovery**: Disabled (dev-friendly) diff --git a/cmd/app/internal/database/constants.go b/cmd/app/internal/database/constants.go index f9fbb59..5548ea5 100644 --- a/cmd/app/internal/database/constants.go +++ b/cmd/app/internal/database/constants.go @@ -4,8 +4,5 @@ const ( // TableName is the single table for all entities TableName = "glad-entities" - GSIByUser = "ByUser" - GSISkillsByLevel = "SkillsByLevel" - GSIBySkillID = "BySkillID" - GSIByEntityType = "ByEntityType" + GSIBySkill = "BySkill" ) diff --git a/cmd/app/internal/database/master_skill_repository_dynamodb.go b/cmd/app/internal/database/master_skill_repository_dynamodb.go index 99397c1..df9b69b 100644 --- a/cmd/app/internal/database/master_skill_repository_dynamodb.go +++ b/cmd/app/internal/database/master_skill_repository_dynamodb.go @@ -55,7 +55,8 @@ func (r *DynamoDBRepository) GetMasterSkill(skillID string) (*models.Skill, erro input := &dynamodb.GetItemInput{ TableName: aws.String(TableName), Key: map[string]*dynamodb.AttributeValue{ - "entity_id": {S: aws.String(entityID)}, + "EntityType": {S: aws.String("Skill")}, + "entity_id": {S: aws.String(entityID)}, }, } @@ -125,7 +126,8 @@ func (r *DynamoDBRepository) DeleteMasterSkill(skillID string) error { input := &dynamodb.DeleteItemInput{ TableName: aws.String(TableName), Key: map[string]*dynamodb.AttributeValue{ - "entity_id": {S: aws.String(entityID)}, + "EntityType": {S: aws.String("Skill")}, + "entity_id": {S: aws.String(entityID)}, }, ConditionExpression: aws.String("attribute_exists(entity_id)"), } @@ -147,10 +149,8 @@ func (r *DynamoDBRepository) ListMasterSkills() ([]*models.Skill, error) { log.Debug("Starting master skills list retrieval") - // Use GSI for better performance instead of Scan input := &dynamodb.QueryInput{ TableName: aws.String(TableName), - IndexName: aws.String(GSIByEntityType), KeyConditionExpression: aws.String("EntityType = :entityType"), ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ ":entityType": {S: aws.String("Skill")}, diff --git a/cmd/app/internal/database/user_repository_dynamodb.go b/cmd/app/internal/database/user_repository_dynamodb.go index 6410dfb..b667224 100644 --- a/cmd/app/internal/database/user_repository_dynamodb.go +++ b/cmd/app/internal/database/user_repository_dynamodb.go @@ -57,7 +57,8 @@ func (r *DynamoDBRepository) GetUser(username string) (*models.User, error) { input := &dynamodb.GetItemInput{ TableName: aws.String(TableName), Key: map[string]*dynamodb.AttributeValue{ - "entity_id": {S: aws.String(entityID)}, + "EntityType": {S: aws.String("User")}, + "entity_id": {S: aws.String(entityID)}, }, } @@ -95,7 +96,8 @@ func (r *DynamoDBRepository) UserExists(username string) (bool, error) { input := &dynamodb.GetItemInput{ TableName: aws.String(TableName), Key: map[string]*dynamodb.AttributeValue{ - "entity_id": {S: aws.String(entityID)}, + "EntityType": {S: aws.String("User")}, + "entity_id": {S: aws.String(entityID)}, }, ProjectionExpression: aws.String("entity_id"), } @@ -144,17 +146,15 @@ func (r *DynamoDBRepository) UpdateUser(user *models.User) error { return nil } -// ListUsers retrieves all users from DynamoDB using Query on EntityType +// ListUsers retrieves all users from DynamoDB using Query on ByEntityType GSI func (r *DynamoDBRepository) ListUsers() ([]*models.User, error) { log := logger.WithComponent("database").With("operation", "ListUsers") start := time.Now() log.Debug("Starting users list retrieval") - // Use Query on GSI for EntityType = "User" input := &dynamodb.QueryInput{ TableName: aws.String(TableName), - IndexName: aws.String(GSIByEntityType), KeyConditionExpression: aws.String("EntityType = :entityType"), ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ ":entityType": {S: aws.String("User")}, @@ -177,6 +177,6 @@ func (r *DynamoDBRepository) ListUsers() ([]*models.User, error) { users = append(users, &user) } - log.Info("Users retrieved successfully", "count", len(users), "scanned_count", *result.ScannedCount, "duration", time.Since(start)) + log.Info("Users retrieved successfully", "count", len(users), "duration", time.Since(start)) return users, nil } diff --git a/cmd/app/internal/database/user_skill_repository.go b/cmd/app/internal/database/user_skill_repository.go index d22e79a..4a95740 100644 --- a/cmd/app/internal/database/user_skill_repository.go +++ b/cmd/app/internal/database/user_skill_repository.go @@ -11,5 +11,4 @@ type SkillRepository interface { ListSkillsForUser(username string) ([]*models.UserSkill, error) ListUsersBySkill(skillName string) ([]*models.UserSkill, error) ListUsersBySkillAndLevel(skillName string, proficiencyLevel models.ProficiencyLevel) ([]*models.UserSkill, error) - QueryUserSkillsBySkillID(skillID string) ([]*models.UserSkill, error) } diff --git a/cmd/app/internal/database/user_skill_repository_dynamodb.go b/cmd/app/internal/database/user_skill_repository_dynamodb.go index 4dc25ec..1e192b5 100644 --- a/cmd/app/internal/database/user_skill_repository_dynamodb.go +++ b/cmd/app/internal/database/user_skill_repository_dynamodb.go @@ -57,7 +57,8 @@ func (r *DynamoDBRepository) GetSkill(username, skillID string) (*models.UserSki input := &dynamodb.GetItemInput{ TableName: aws.String(TableName), Key: map[string]*dynamodb.AttributeValue{ - "entity_id": {S: aws.String(entityID)}, + "EntityType": {S: aws.String("UserSkill")}, + "entity_id": {S: aws.String(entityID)}, }, } @@ -128,7 +129,8 @@ func (r *DynamoDBRepository) DeleteSkill(username, skillID string) error { input := &dynamodb.DeleteItemInput{ TableName: aws.String(TableName), Key: map[string]*dynamodb.AttributeValue{ - "entity_id": {S: aws.String(entityID)}, + "EntityType": {S: aws.String("UserSkill")}, + "entity_id": {S: aws.String(entityID)}, }, ConditionExpression: aws.String("attribute_exists(entity_id)"), } @@ -152,11 +154,10 @@ func (r *DynamoDBRepository) ListSkillsForUser(username string) ([]*models.UserS input := &dynamodb.QueryInput{ TableName: aws.String(TableName), - IndexName: aws.String(GSIByUser), - KeyConditionExpression: aws.String("Username = :username AND EntityType = :entityType"), + KeyConditionExpression: aws.String("EntityType = :entityType AND begins_with(entity_id, :userPrefix)"), ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ - ":username": {S: aws.String(username)}, ":entityType": {S: aws.String("UserSkill")}, + ":userPrefix": {S: aws.String("USERSKILL#" + username + "#")}, }, } @@ -180,7 +181,7 @@ func (r *DynamoDBRepository) ListSkillsForUser(username string) ([]*models.UserS return skills, nil } -// ListUsersBySkill retrieves all users who have a specific skill using GSI SkillsByLevel +// ListUsersBySkill retrieves all users who have a specific skill using GSI BySkill func (r *DynamoDBRepository) ListUsersBySkill(skillName string) ([]*models.UserSkill, error) { log := logger.WithComponent("database").With("operation", "ListUsersBySkill", "skill", skillName) start := time.Now() @@ -189,7 +190,7 @@ func (r *DynamoDBRepository) ListUsersBySkill(skillName string) ([]*models.UserS input := &dynamodb.QueryInput{ TableName: aws.String(TableName), - IndexName: aws.String(GSISkillsByLevel), + IndexName: aws.String(GSIBySkill), KeyConditionExpression: aws.String("SkillName = :skillName"), ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ ":skillName": {S: aws.String(skillName)}, @@ -217,7 +218,6 @@ func (r *DynamoDBRepository) ListUsersBySkill(skillName string) ([]*models.UserS } // ListUsersBySkillAndLevel retrieves users with a specific skill at a specific proficiency level -// Uses composite partition key on SkillName + ProficiencyLevel func (r *DynamoDBRepository) ListUsersBySkillAndLevel(skillName string, proficiencyLevel models.ProficiencyLevel) ([]*models.UserSkill, error) { log := logger.WithComponent("database").With("operation", "ListUsersBySkillAndLevel", "skill", skillName, "level", proficiencyLevel) start := time.Now() @@ -226,7 +226,7 @@ func (r *DynamoDBRepository) ListUsersBySkillAndLevel(skillName string, proficie input := &dynamodb.QueryInput{ TableName: aws.String(TableName), - IndexName: aws.String(GSISkillsByLevel), + IndexName: aws.String(GSIBySkill), KeyConditionExpression: aws.String("SkillName = :skillName AND ProficiencyLevel = :level"), ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ ":skillName": {S: aws.String(skillName)}, @@ -253,40 +253,3 @@ func (r *DynamoDBRepository) ListUsersBySkillAndLevel(skillName string, proficie log.Info("Users with skill and level retrieved successfully", "skill", skillName, "level", proficiencyLevel, "count", len(skills), "duration", time.Since(start)) return skills, nil } - -// QueryUserSkillsBySkillID retrieves all UserSkills that reference a specific skill_id -// Used when syncing denormalized data after master skill updates -func (r *DynamoDBRepository) QueryUserSkillsBySkillID(skillID string) ([]*models.UserSkill, error) { - log := logger.WithComponent("database").With("operation", "QueryUserSkillsBySkillID", "skill_id", skillID) - start := time.Now() - - log.Debug("Starting UserSkills retrieval by skill_id") - - input := &dynamodb.QueryInput{ - TableName: aws.String(TableName), - IndexName: aws.String(GSIBySkillID), - KeyConditionExpression: aws.String("skill_id = :skillID"), - ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ - ":skillID": {S: aws.String(skillID)}, - }, - } - - result, err := r.client.Query(input) - if err != nil { - log.Error("Failed to query UserSkills by skill_id", "error", err.Error(), "duration", time.Since(start)) - return nil, err - } - - var skills []*models.UserSkill - for i, item := range result.Items { - var skill models.UserSkill - if err := dynamodbattribute.UnmarshalMap(item, &skill); err != nil { - log.Error("Failed to unmarshal skill data", "error", err.Error(), "item_index", i, "duration", time.Since(start)) - continue - } - skills = append(skills, &skill) - } - - log.Info("UserSkills by skill_id retrieved successfully", "skill_id", skillID, "count", len(skills), "duration", time.Since(start)) - return skills, nil -} diff --git a/cmd/app/internal/database/user_skill_repository_mock.go b/cmd/app/internal/database/user_skill_repository_mock.go index 97b920b..f57da15 100644 --- a/cmd/app/internal/database/user_skill_repository_mock.go +++ b/cmd/app/internal/database/user_skill_repository_mock.go @@ -134,27 +134,6 @@ func (m *MockRepository) ListUsersBySkill(skillName string) ([]*models.UserSkill return skills, nil } -// QueryUserSkillsBySkillID retrieves all users with a specific skill from memory -func (m *MockRepository) QueryUserSkillsBySkillID(skillName string) ([]*models.UserSkill, error) { - log := logger.WithComponent("database").With("operation", "ListUsersBySkill", "skill", skillName, "repository", "mock") - start := time.Now() - - log.Debug("Starting users list retrieval by skill from mock repository") - - m.mutex.RLock() - defer m.mutex.RUnlock() - - var skills []*models.UserSkill - for _, skill := range m.skills { - if skill.SkillName == skillName { - skills = append(skills, skill) - } - } - - log.Info("Users retrieved successfully by skill from mock repository", "count", len(skills), "duration", time.Since(start)) - return skills, nil -} - // ListUsersBySkillAndLevel retrieves all users with a specific skill and proficiency level from memory func (m *MockRepository) ListUsersBySkillAndLevel(skillName string, proficiencyLevel models.ProficiencyLevel) ([]*models.UserSkill, error) { log := logger.WithComponent("database").With("operation", "ListUsersBySkillAndLevel", "skill", skillName, "level", proficiencyLevel, "repository", "mock") diff --git a/cmd/app/internal/models/user_skill.go b/cmd/app/internal/models/user_skill.go index 8137afc..bf3f99f 100644 --- a/cmd/app/internal/models/user_skill.go +++ b/cmd/app/internal/models/user_skill.go @@ -49,8 +49,9 @@ type UserSkill struct { UpdatedAt time.Time `json:"updated_at" dynamodbav:"UpdatedAt"` // DynamoDB attributes - EntityID string `json:"-" dynamodbav:"entity_id"` - EntityType string `json:"entity_type" dynamodbav:"EntityType"` + EntityID string `json:"-" dynamodbav:"entity_id"` + EntityType string `json:"entity_type" dynamodbav:"EntityType"` + SkillCompositeSort string `json:"-" dynamodbav:"SkillCompositeSort"` } // NewUserSkill creates a new UserSkill with proper validation @@ -99,7 +100,8 @@ func (s *UserSkill) SetKeys() { // Base table key: Unique identifier s.EntityID = BuildUserSkillEntityID(s.Username, s.SkillID) s.EntityType = "UserSkill" - // No GSI concatenation needed - composite keys use actual attribute values! + // Composite sort key: ProficiencyLevel#Username + s.SkillCompositeSort = string(s.ProficiencyLevel) + "#" + s.Username } // UpdateProficiency updates the skill proficiency level diff --git a/deployments/app/cdk.go b/deployments/app/cdk.go index a945786..8b77cb5 100644 --- a/deployments/app/cdk.go +++ b/deployments/app/cdk.go @@ -20,82 +20,33 @@ type CdkStackProps struct { func createEntitiesTable(stack awscdk.Stack, id *string, environment string) awsdynamodb.TableV2 { entitiesTable := awsdynamodb.NewTableV2(stack, id, &awsdynamodb.TablePropsV2{ TableName: jsii.String("glad-entities"), - // Partition Key: PK (stores entity identifier) + // Partition Key: EntityType PartitionKey: &awsdynamodb.Attribute{ + Name: jsii.String("EntityType"), + Type: awsdynamodb.AttributeType_STRING, + }, + SortKey: &awsdynamodb.Attribute{ Name: jsii.String("entity_id"), Type: awsdynamodb.AttributeType_STRING, }, GlobalSecondaryIndexes: &[]*awsdynamodb.GlobalSecondaryIndexPropsV2{ - // GSI1: Query skills by name, optionally filter by level + // GSI for skill-only queries { - IndexName: jsii.String("SkillsByLevel"), + IndexName: jsii.String("BySkill"), PartitionKey: &awsdynamodb.Attribute{ Name: jsii.String("SkillName"), Type: awsdynamodb.AttributeType_STRING, }, - SortKey: &awsdynamodb.Attribute{ - Name: jsii.String("ProficiencyLevel"), - Type: awsdynamodb.AttributeType_STRING, - }, - ProjectionType: awsdynamodb.ProjectionType_INCLUDE, - NonKeyAttributes: jsii.Strings( - "EntityType", - "Notes", - "Email", - "skill_id", - "Category", - "YearsOfExperience", - "Username", - ), - }, - // GSI2: Query all items for a user (profile + skills) - { - IndexName: jsii.String("ByUser"), - PartitionKey: &awsdynamodb.Attribute{ - Name: jsii.String("Username"), - Type: awsdynamodb.AttributeType_STRING, - }, - SortKey: &awsdynamodb.Attribute{ - Name: jsii.String("EntityType"), - Type: awsdynamodb.AttributeType_STRING, - }, - }, - // GSI for querying skills: - { - IndexName: jsii.String("SkillsByCategory"), - PartitionKey: &awsdynamodb.Attribute{ - Name: jsii.String("EntityType"), - Type: awsdynamodb.AttributeType_STRING, - }, - SortKey: &awsdynamodb.Attribute{ - Name: jsii.String("Category"), - Type: awsdynamodb.AttributeType_STRING, - }, - }, - - // GSI for query on EntityType - { - IndexName: jsii.String("ByEntityType"), - PartitionKey: &awsdynamodb.Attribute{ - Name: jsii.String("EntityType"), - Type: awsdynamodb.AttributeType_STRING, - }, - SortKey: &awsdynamodb.Attribute{ - Name: jsii.String("SkillName"), - Type: awsdynamodb.AttributeType_STRING, - }, - }, - // GSI: BySkillID - Query UserSkills by skill_id (for syncing denormalized data) - { - IndexName: jsii.String("BySkillID"), - PartitionKey: &awsdynamodb.Attribute{ - Name: jsii.String("skill_id"), - Type: awsdynamodb.AttributeType_STRING, - }, - SortKey: &awsdynamodb.Attribute{ - Name: jsii.String("Username"), - Type: awsdynamodb.AttributeType_STRING, + SortKeys: &[]*awsdynamodb.Attribute{ + { + Name: jsii.String("ProficiencyLevel"), + Type: awsdynamodb.AttributeType_STRING, + }, + { + Name: jsii.String("Username"), + Type: awsdynamodb.AttributeType_STRING, + }, }, }, }, @@ -131,7 +82,6 @@ func NewCdkStack(scope constructs.Construct, id string, props *CdkStackProps) aw // Add environment tag awscdk.Tags_Of(stack).Add(jsii.String("Environment"), jsii.String(ENVIRONMENT), nil) - entitiesTable := createEntitiesTable(stack, jsii.String(id+"-entities-table-"+ENVIRONMENT), ENVIRONMENT) // Create Lambda myFunc := awslambda.NewFunction(stack, jsii.String(id+"-go-func"), &awslambda.FunctionProps{ Runtime: awslambda.Runtime_PROVIDED_AL2023(), @@ -141,7 +91,8 @@ func NewCdkStack(scope constructs.Construct, id string, props *CdkStackProps) aw myFunc.AddEnvironment(jsii.String("ENVIRONMENT"), jsii.String(ENVIRONMENT), nil) - // Grant Lambda read/write access to DynamoDB table + //// Create table | Grant Lambda read/write access to DynamoDB table + entitiesTable := createEntitiesTable(stack, jsii.String(id+"-entities-table-"+ENVIRONMENT), ENVIRONMENT) entitiesTable.GrantReadWriteData(myFunc) api := awsapigateway.NewRestApi(stack, jsii.String(id+"-api-gateway"), &awsapigateway.RestApiProps{ diff --git a/docs/api-testing/api-test.http b/docs/api-testing/api-test.http index 499d846..e91426d 100644 --- a/docs/api-testing/api-test.http +++ b/docs/api-testing/api-test.http @@ -296,7 +296,7 @@ Authorization: Bearer {{token}} Content-Type: application/json { - "skill_name": "Amazon Web Services", + "skill_name": "aws", "proficiency_level": "Intermediate", "years_of_experience": 3, "notes": "Lambda, DynamoDB, API Gateway" @@ -407,7 +407,7 @@ Content-Type: application/json } ### 4.16 Update User Skill - John Doe - AWS (only notes) -PUT {{API_URL}}/users/john_doe/skills/Amazon Web Services +PUT {{API_URL}}/users/john_doe/skills/aws Authorization: Bearer {{token}} Content-Type: application/json @@ -738,10 +738,10 @@ Authorization: Bearer {{token}} Content-Type: application/json { - "skill_id": "test_skill_1", - "skill_name": "Test Skill 1", + "skill_id": "test-skill-1", + "skill_name": "Test Skill", "description": "Performance test skill 1", - "category": "Testing" + "category": "Programming" } ### 9.2 Create Multiple Master Skills - Skill 2 @@ -750,10 +750,10 @@ Authorization: Bearer {{token}} Content-Type: application/json { - "skill_id": "test_skill_2", + "skill_id": "test-skill-2", "skill_name": "Test Skill 2", "description": "Performance test skill 2", - "category": "Testing" + "category": "Programming" } ### 9.3 Create Multiple Master Skills - Skill 3 @@ -762,10 +762,10 @@ Authorization: Bearer {{token}} Content-Type: application/json { - "skill_id": "test_skill_3", + "skill_id": "test-skill", "skill_name": "Test Skill 3", "description": "Performance test skill 3", - "category": "Testing" + "category": "Programming" } ###############################################################################