create-files.sh

#!/bin/bash
# ============================================================================
# RUN THIS SCRIPT AFTER THE SETUP COMMANDS
# This will create all the TypeScript files for the project
# ============================================================================

# Make sure you're in the project root directory (vector-embedding-server)

# ============================================================================
# EMBEDDINGS MODULE FILES
# ============================================================================

# Create DTO files
cat > src/embeddings/dto/upload-embeddings.dto.ts << 'EOF'
import { ApiProperty } from '@nestjs/swagger';
import { IsString, IsUrl, IsEnum, IsOptional, IsInt, Min } from 'class-validator';

export enum VectorProvider {
  UPSTASH = 'upstash',
  PINECONE = 'pinecone',
}

export enum EmbeddingModel {
  BERT_BASE = 'bert-base-uncased',
  BERT_LARGE = 'bert-large-uncased',
  DISTILBERT = 'distilbert-base-uncased',
}

export class UploadEmbeddingsDto {
  @ApiProperty({
    description: 'URL of the file to process',
    example: 'https://example.com/document.pdf',
  })
  @IsUrl()
  fileUrl: string;

  @ApiProperty({
    description: 'Index name or URL for the vector provider',
    example: 'my-index',
  })
  @IsString()
  indexName: string;

  @ApiProperty({
    description: 'Vector database provider',
    enum: VectorProvider,
    example: VectorProvider.UPSTASH,
  })
  @IsEnum(VectorProvider)
  provider: VectorProvider;

  @ApiProperty({
    description: 'Embedding model to use',
    enum: EmbeddingModel,
    default: EmbeddingModel.BERT_BASE,
    required: false,
  })
  @IsEnum(EmbeddingModel)
  @IsOptional()
  model?: EmbeddingModel = EmbeddingModel.BERT_BASE;

  @ApiProperty({
    description: 'Chunk size in characters',
    default: 500,
    required: false,
  })
  @IsInt()
  @Min(100)
  @IsOptional()
  chunkSize?: number = 500;

  @ApiProperty({
    description: 'Overlap between chunks in characters',
    default: 50,
    required: false,
  })
  @IsInt()
  @Min(0)
  @IsOptional()
  chunkOverlap?: number = 50;
}

export class UploadResponseDto {
  @ApiProperty()
  success: boolean;

  @ApiProperty()
  message: string;

  @ApiProperty()
  chunksProcessed: number;

  @ApiProperty()
  tokensUsed: number;

  @ApiProperty()
  usageId: string;
}
EOF

# Update embeddings controller
cat > src/embeddings/embeddings.controller.ts << 'EOF'
import { Controller, Post, Body, UseGuards } from '@nestjs/common';
import { ApiTags, ApiOperation, ApiResponse, ApiSecurity } from '@nestjs/swagger';
import { EmbeddingsService } from './embeddings.service';
import { UploadEmbeddingsDto, UploadResponseDto } from './dto/upload-embeddings.dto';
import { ApiKeyGuard } from '../common/guards/api-key.guard';

@ApiTags('embeddings')
@Controller('embeddings')
@UseGuards(ApiKeyGuard)
@ApiSecurity('api-key')
export class EmbeddingsController {
  constructor(private readonly embeddingsService: EmbeddingsService) {}

  @Post('upload')
  @ApiOperation({
    summary: 'Upload file embeddings to vector database',
    description: 'Downloads a file, chunks it, creates embeddings, and uploads to the specified vector provider',
  })
  @ApiResponse({
    status: 201,
    description: 'Embeddings successfully uploaded',
    type: UploadResponseDto,
  })
  @ApiResponse({ status: 400, description: 'Invalid input' })
  @ApiResponse({ status: 401, description: 'Unauthorized - Invalid API key' })
  @ApiResponse({ status: 500, description: 'Internal server error' })
  async uploadEmbeddings(@Body() dto: UploadEmbeddingsDto): Promise<UploadResponseDto> {
    return this.embeddingsService.processAndUpload(dto);
  }
}
EOF

# Update embeddings service
cat > src/embeddings/embeddings.service.ts << 'EOF'
import { Injectable, Logger } from '@nestjs/common';
import { PrismaService } from '../prisma/prisma.service';
import { FileService } from './services/file.service';
import { EmbeddingModelService } from './services/embedding-model.service';
import { VectorProviderFactory } from './providers/vector-provider.factory';
import { UploadEmbeddingsDto, UploadResponseDto } from './dto/upload-embeddings.dto';

@Injectable()
export class EmbeddingsService {
  private readonly logger = new Logger(EmbeddingsService.name);

  constructor(
    private readonly prisma: PrismaService,
    private readonly fileService: FileService,
    private readonly embeddingModel: EmbeddingModelService,
    private readonly providerFactory: VectorProviderFactory,
  ) {}

  async processAndUpload(dto: UploadEmbeddingsDto): Promise<UploadResponseDto> {
    const startTime = Date.now();
    let usageId: string;

    try {
      this.logger.log(`Processing file: ${dto.fileUrl}`);

      const text = await this.fileService.downloadAndExtractText(dto.fileUrl);
      this.logger.log(`Extracted ${text.length} characters from file`);

      const chunks = this.fileService.chunkText(text, dto.chunkSize, dto.chunkOverlap);
      this.logger.log(`Created ${chunks.length} chunks`);

      await this.embeddingModel.loadModel(dto.model);

      const embeddings = await this.embeddingModel.createEmbeddings(chunks);
      this.logger.log(`Generated ${embeddings.length} embeddings`);

      const provider = this.providerFactory.getProvider(dto.provider);

      await provider.uploadVectors(dto.indexName, embeddings, chunks);
      this.logger.log(`Uploaded vectors to ${dto.provider}`);

      const tokensUsed = chunks.reduce((sum, chunk) => sum + Math.ceil(chunk.length / 4), 0);

      const usage = await this.prisma.usage.create({
        data: {
          apiKeyId: 'system',
          endpoint: '/embeddings/upload',
          tokensUsed,
          chunksCount: chunks.length,
          provider: dto.provider,
          model: dto.model,
          fileUrl: dto.fileUrl,
          status: 'success',
        },
      });

      usageId = usage.id;

      const duration = Date.now() - startTime;
      this.logger.log(`Processing completed in ${duration}ms`);

      return {
        success: true,
        message: 'Embeddings uploaded successfully',
        chunksProcessed: chunks.length,
        tokensUsed,
        usageId,
      };
    } catch (error) {
      this.logger.error(`Error processing embeddings: ${error.message}`, error.stack);

      await this.prisma.usage.create({
        data: {
          apiKeyId: 'system',
          endpoint: '/embeddings/upload',
          tokensUsed: 0,
          chunksCount: 0,
          provider: dto.provider,
          model: dto.model,
          fileUrl: dto.fileUrl,
          status: 'error',
          errorMessage: error.message,
        },
      });

      throw error;
    }
  }
}
EOF

# Update embeddings module
cat > src/embeddings/embeddings.module.ts << 'EOF'
import { Module } from '@nestjs/common';
import { EmbeddingsController } from './embeddings.controller';
import { EmbeddingsService } from './embeddings.service';
import { FileService } from './services/file.service';
import { EmbeddingModelService } from './services/embedding-model.service';
import { VectorProviderFactory } from './providers/vector-provider.factory';
import { UpstashProvider } from './providers/upstash.provider';
import { PineconeProvider } from './providers/pinecone.provider';

@Module({
  controllers: [EmbeddingsController],
  providers: [
    EmbeddingsService,
    FileService,
    EmbeddingModelService,
    VectorProviderFactory,
    UpstashProvider,
    PineconeProvider,
  ],
})
export class EmbeddingsModule {}
EOF

# Create services
cat > src/embeddings/services/file.service.ts << 'EOF'
import { Injectable, BadRequestException } from '@nestjs/common';
import axios from 'axios';
import * as pdfParse from 'pdf-parse';

@Injectable()
export class FileService {
  async downloadAndExtractText(fileUrl: string): Promise<string> {
    try {
      const response = await axios.get(fileUrl, {
        responseType: 'arraybuffer',
        timeout: 30000,
      });

      const contentType = response.headers['content-type'];
      const buffer = Buffer.from(response.data);

      if (contentType?.includes('pdf')) {
        return await this.extractPdfText(buffer);
      } else if (contentType?.includes('text')) {
        return buffer.toString('utf-8');
      } else {
        return buffer.toString('utf-8');
      }
    } catch (error) {
      throw new BadRequestException(`Failed to download file: ${error.message}`);
    }
  }

  private async extractPdfText(buffer: Buffer): Promise<string> {
    try {
      const data = await pdfParse(buffer);
      return data.text;
    } catch (error) {
      throw new BadRequestException(`Failed to parse PDF: ${error.message}`);
    }
  }

  chunkText(text: string, chunkSize: number = 500, overlap: number = 50): string[] {
    const chunks: string[] = [];
    let start = 0;

    while (start < text.length) {
      const end = Math.min(start + chunkSize, text.length);
      let chunk = text.slice(start, end);

      if (end < text.length) {
        const lastPeriod = chunk.lastIndexOf('.');
        const lastNewline = chunk.lastIndexOf('\n');
        const breakPoint = Math.max(lastPeriod, lastNewline);

        if (breakPoint > chunkSize * 0.5) {
          chunk = chunk.slice(0, breakPoint + 1);
          start += breakPoint + 1;
        } else {
          start = end;
        }
      } else {
        start = end;
      }

      chunks.push(chunk.trim());

      if (start < text.length && overlap > 0) {
        start = Math.max(start - overlap, 0);
      }
    }

    return chunks.filter(chunk => chunk.length > 0);
  }
}
EOF

cat > src/embeddings/services/embedding-model.service.ts << 'EOF'
import { Injectable, Logger } from '@nestjs/common';
import { pipeline } from '@xenova/transformers';
import { EmbeddingModel } from '../dto/upload-embeddings.dto';

@Injectable()
export class EmbeddingModelService {
  private readonly logger = new Logger(EmbeddingModelService.name);
  private model: any;
  private currentModel: string;

  async loadModel(modelName: EmbeddingModel = EmbeddingModel.BERT_BASE): Promise<void> {
    if (this.model && this.currentModel === modelName) {
      return;
    }

    this.logger.log(`Loading model: ${modelName}`);
    this.model = await pipeline('feature-extraction', `Xenova/${modelName}`);
    this.currentModel = modelName;
    this.logger.log(`Model loaded: ${modelName}`);
  }

  async createEmbeddings(texts: string[]): Promise<number[][]> {
    if (!this.model) {
      await this.loadModel();
    }

    const embeddings: number[][] = [];

    for (const text of texts) {
      const output = await this.model(text, { pooling: 'mean', normalize: true });
      embeddings.push(Array.from(output.data));
    }

    return embeddings;
  }
}
EOF

# Create provider interface
cat > src/embeddings/providers/vector-provider.interface.ts << 'EOF'
export interface IVectorProvider {
  uploadVectors(indexName: string, embeddings: number[][], texts: string[]): Promise<void>;
}
EOF

# Create provider factory
cat > src/embeddings/providers/vector-provider.factory.ts << 'EOF'
import { Injectable } from '@nestjs/common';
import { VectorProvider } from '../dto/upload-embeddings.dto';
import { IVectorProvider } from './vector-provider.interface';
import { UpstashProvider } from './upstash.provider';
import { PineconeProvider } from './pinecone.provider';

@Injectable()
export class VectorProviderFactory {
  constructor(
    private readonly upstashProvider: UpstashProvider,
    private readonly pineconeProvider: PineconeProvider,
  ) {}

  getProvider(provider: VectorProvider): IVectorProvider {
    switch (provider) {
      case VectorProvider.UPSTASH:
        return this.upstashProvider;
      case VectorProvider.PINECONE:
        return this.pineconeProvider;
      default:
        throw new Error(`Unsupported provider: ${provider}`);
    }
  }
}
EOF

# Create Upstash provider
cat > src/embeddings/providers/upstash.provider.ts << 'EOF'
import { Injectable, Logger } from '@nestjs/common';
import { Index } from '@upstash/vector';
import { IVectorProvider } from './vector-provider.interface';

@Injectable()
export class UpstashProvider implements IVectorProvider {
  private readonly logger = new Logger(UpstashProvider.name);

  async uploadVectors(indexName: string, embeddings: number[][], texts: string[]): Promise<void> {
    const index = new Index({
      url: process.env.UPSTASH_VECTOR_URL,
      token: process.env.UPSTASH_VECTOR_TOKEN,
    });

    const vectors = embeddings.map((embedding, i) => ({
      id: `${indexName}-${Date.now()}-${i}`,
      vector: embedding,
      metadata: {
        text: texts[i],
        index: indexName,
        timestamp: Date.now(),
      },
    }));

    const batchSize = 100;
    for (let i = 0; i < vectors.length; i += batchSize) {
      const batch = vectors.slice(i, i + batchSize);
      await index.upsert(batch);
      this.logger.log(`Uploaded batch ${i / batchSize + 1} (${batch.length} vectors)`);
    }
  }
}
EOF

# Create Pinecone provider
cat > src/embeddings/providers/pinecone.provider.ts << 'EOF'
import { Injectable, Logger, NotImplementedException } from '@nestjs/common';
import { IVectorProvider } from './vector-provider.interface';

@Injectable()
export class PineconeProvider implements IVectorProvider {
  private readonly logger = new Logger(PineconeProvider.name);

  async uploadVectors(indexName: string, embeddings: number[][], texts: string[]): Promise<void> {
    this.logger.warn('Pinecone provider not fully implemented yet');
    throw new NotImplementedException('Pinecone provider is not yet implemented');
  }
}
EOF

# ============================================================================
# API KEYS MODULE FILES
# ============================================================================

cat > src/api-keys/dto/create-api-key.dto.ts << 'EOF'
import { ApiProperty } from '@nestjs/swagger';
import { IsString } from 'class-validator';

export class CreateApiKeyDto {
  @ApiProperty({
    description: 'Name for the API key',
    example: 'My Application Key',
  })
  @IsString()
  name: string;
}

export class ApiKeyResponseDto {
  @ApiProperty()
  id: string;

  @ApiProperty()
  key: string;

  @ApiProperty()
  name: string;

  @ApiProperty()
  enabled: boolean;

  @ApiProperty()
  createdAt: Date;
}
EOF

cat > src/api-keys/api-keys.controller.ts << 'EOF'
import { Controller, Get, Post, Patch, Delete, Body, Param } from '@nestjs/common';
import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger';
import { ApiKeysService } from './api-keys.service';
import { CreateApiKeyDto, ApiKeyResponseDto } from './dto/create-api-key.dto';

@ApiTags('api-keys')
@Controller('api-keys')
export class ApiKeysController {
  constructor(private readonly apiKeysService: ApiKeysService) {}

  @Post()
  @ApiOperation({ summary: 'Create a new API key' })
  @ApiResponse({ status: 201, description: 'API key created', type: ApiKeyResponseDto })
  create(@Body() dto: CreateApiKeyDto): Promise<ApiKeyResponseDto> {
    return this.apiKeysService.create(dto);
  }

  @Get()
  @ApiOperation({ summary: 'List all API keys' })
  @ApiResponse({ status: 200, description: 'List of API keys', type: [ApiKeyResponseDto] })
  findAll(): Promise<ApiKeyResponseDto[]> {
    return this.apiKeysService.findAll();
  }

  @Get(':id')
  @ApiOperation({ summary: 'Get API key by ID' })
  @ApiResponse({ status: 200, description: 'API key details', type: ApiKeyResponseDto })
  findOne(@Param('id') id: string): Promise<ApiKeyResponseDto> {
    return this.apiKeysService.findOne(id);
  }

  @Patch(':id/toggle')
  @ApiOperation({ summary: 'Enable or disable an API key' })
  @ApiResponse({ status: 200, description: 'API key updated', type: ApiKeyResponseDto })
  toggle(@Param('id') id: string): Promise<ApiKeyResponseDto> {
    return this.apiKeysService.toggle(id);
  }

  @Delete(':id')
  @ApiOperation({ summary: 'Delete an API key' })
  @ApiResponse({ status: 200, description: 'API key deleted' })
  remove(@Param('id') id: string): Promise<void> {
    return this.apiKeysService.remove(id);
  }
}
EOF

cat > src/api-keys/api-keys.service.ts << 'EOF'
import { Injectable, NotFoundException } from '@nestjs/common';
import { PrismaService } from '../prisma/prisma.service';
import { CreateApiKeyDto, ApiKeyResponseDto } from './dto/create-api-key.dto';
import { randomBytes } from 'crypto';

@Injectable()
export class ApiKeysService {
  constructor(private readonly prisma: PrismaService) {}

  async create(dto: CreateApiKeyDto): Promise<ApiKeyResponseDto> {
    const key = `vek_${randomBytes(32).toString('hex')}`;

    return this.prisma.apiKey.create({
      data: {
        key,
        name: dto.name,
      },
    });
  }

  async findAll(): Promise<ApiKeyResponseDto[]> {
    return this.prisma.apiKey.findMany({
      orderBy: { createdAt: 'desc' },
    });
  }

  async findOne(id: string): Promise<ApiKeyResponseDto> {
    const apiKey = await this.prisma.apiKey.findUnique({
      where: { id },
    });

    if (!apiKey) {
      throw new NotFoundException('API key not found');
    }

    return apiKey;
  }

  async toggle(id: string): Promise<ApiKeyResponseDto> {
    const apiKey = await this.findOne(id);

    return this.prisma.apiKey.update({
      where: { id },
      data: { enabled: !apiKey.enabled },
    });
  }

  async remove(id: string): Promise<void> {
    await this.findOne(id);
    await this.prisma.apiKey.delete({ where: { id } });
  }
}
EOF

cat > src/api-keys/api-keys.module.ts << 'EOF'
import { Module } from '@nestjs/common';
import { ApiKeysController } from './api-keys.controller';
import { ApiKeysService } from './api-keys.service';

@Module({
  controllers: [ApiKeysController],
  providers: [ApiKeysService],
})
export class ApiKeysModule {}
EOF

# ============================================================================
# USAGE MODULE FILES
# ============================================================================

cat > src/usage/dto/usage-stats.dto.ts << 'EOF'
import { ApiProperty } from '@nestjs/swagger';

export class UsageStatsDto {
  @ApiProperty()
  totalRequests: number;

  @ApiProperty()
  totalTokens: number;

  @ApiProperty()
  totalChunks: number;

  @ApiProperty()
  successRate: number;

  @ApiProperty()
  byProvider: Record<string, number>;

  @ApiProperty()
  byModel: Record<string, number>;
}

export class UsageRecordDto {
  @ApiProperty()
  id: string;

  @ApiProperty()
  endpoint: string;

  @ApiProperty()
  tokensUsed: number;

  @ApiProperty()
  chunksCount: number;

  @ApiProperty()
  provider: string;

  @ApiProperty()
  model: string;

  @ApiProperty()
  fileUrl: string;

  @ApiProperty()
  status: string;

  @ApiProperty()
  errorMessage?: string;

  @ApiProperty()
  createdAt: Date;
}
EOF

cat > src/usage/usage.controller.ts << 'EOF'
import { Controller, Get, Query } from '@nestjs/common';
import { ApiTags, ApiOperation, ApiResponse, ApiQuery } from '@nestjs/swagger';
import { UsageService } from './usage.service';
import { UsageStatsDto, UsageRecordDto } from './dto/usage-stats.dto';

@ApiTags('usage')
@Controller('usage')
export class UsageController {
  constructor(private readonly usageService: UsageService) {}

  @Get('stats')
  @ApiOperation({ summary: 'Get usage statistics' })
  @ApiResponse({ status: 200, description: 'Usage statistics', type: UsageStatsDto })
  @ApiQuery({ name: 'days', required: false, description: 'Number of days to analyze (default: 30)' })
  getStats(@Query('days') days?: string): Promise<UsageStatsDto> {
    const daysNumber = days ? parseInt(days, 10) : 30;
    return this.usageService.getStats(daysNumber);
  }

  @Get('records')
  @ApiOperation({ summary: 'Get usage records' })
  @ApiResponse({ status: 200, description: 'List of usage records', type: [UsageRecordDto] })
  @ApiQuery({ name: 'limit', required: false, description: 'Number of records to return (default: 100)' })
  getRecords(@Query('limit') limit?: string): Promise<UsageRecordDto[]> {
    const limitNumber = limit ? parseInt(limit, 10) : 100;
    return this.usageService.getRecords(limitNumber);
  }
}
EOF

cat > src/usage/usage.service.ts << 'EOF'
import { Injectable } from '@nestjs/common';
import { PrismaService } from '../prisma/prisma.service';
import { UsageStatsDto, UsageRecordDto } from './dto/usage-stats.dto';

@Injectable()
export class UsageService {
  constructor(private readonly prisma: PrismaService) {}

  async getStats(days: number = 30): Promise<UsageStatsDto> {
    const startDate = new Date();
    startDate.setDate(startDate.getDate() - days);

    const records = await this.prisma.usage.findMany({
      where: {
        createdAt: {
          gte: startDate,
        },
      },
    });

    const totalRequests = records.length;
    const successfulRequests = records.filter(r => r.status === 'success').length;
    const totalTokens = records.reduce((sum, r) => sum + r.tokensUsed, 0);
    const totalChunks = records.reduce((sum, r) => sum + r.chunksCount, 0);

    const byProvider: Record<string, number> = {};
    const byModel: Record<string, number> = {};

    records.forEach(record => {
      byProvider[record.provider] = (byProvider[record.provider] || 0) + 1;
      byModel[record.model] = (byModel[record.model] || 0) + 1;
    });

    return {
      totalRequests,
      totalTokens,
      totalChunks,
      successRate: totalRequests > 0 ? (successfulRequests / totalRequests) * 100 : 0,
      byProvider,
      byModel,
    };
  }

  async getRecords(limit: number = 100): Promise<UsageRecordDto[]> {
    return this.prisma.usage.findMany({
      take: limit,
      orderBy: { createdAt: 'desc' },
    });
  }
}
EOF

cat > src/usage/usage.module.ts << 'EOF'
import { Module } from '@nestjs/common';
import { UsageController } from './usage.controller';
import { UsageService } from './usage.service';

@Module({
  controllers: [UsageController],
  providers: [UsageService],
})
export class UsageModule {}
EOF

# ============================================================================
# COMMON GUARD
# ============================================================================

cat > src/common/guards/api-key.guard.ts << 'EOF'
import { Injectable, CanActivate, ExecutionContext, UnauthorizedException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { PrismaService } from '../../prisma/prisma.service';

@Injectable()
export class ApiKeyGuard implements CanActivate {
  constructor(
    private readonly configService: ConfigService,
    private readonly prisma: PrismaService,
  ) {}

  async canActivate(context: ExecutionContext): Promise<boolean> {
    const enabled = this.configService.get<string>('API_KEY_VALIDATION_ENABLED') === 'true';

    if (!enabled) {
      return true;
    }

    const request = context.switchToHttp().getRequest();
    const apiKey = request.headers['x-api-key'];

    if (!apiKey) {
      throw new UnauthorizedException('API key is required');
    }

    const key = await this.prisma.apiKey.findUnique({
      where: { key: apiKey },
    });

    if (!key || !key.enabled) {
      throw new UnauthorizedException('Invalid or disabled API key');
    }

    request.apiKey = key;
    return true;
  }
}
EOF

echo "✅ All files created successfully!"
echo "📦 Now run: npm run build"
echo "🚀 Then run: npm run start:dev"