feat: add rag-embedding and text-embedding functions

.github/workflows/test-k8s-deployment.yaml

@@ -156,15 +156,17 @@ jobs:
           kubectl rollout restart deploy/knative-job-service -n constructive-functions
           kubectl rollout status deploy/knative-job-service -n constructive-functions --timeout=120s
 
-      - name: Port-forward postgres and run e2e tests
+      - name: Port-forward services and run e2e tests
         env:
           PGHOST: localhost
           PGPORT: "5432"
           PGUSER: postgres
           PGPASSWORD: ${{ env.PG_PASSWORD }}
           PGDATABASE: constructive
+          SERVER_PORT: "3002"
         run: |
           kubectl port-forward -n constructive-functions svc/postgres 5432:5432 &
+          kubectl port-forward -n constructive-functions svc/constructive-server 3002:3000 &
           sleep 3
           pnpm jest tests/e2e/__tests__/job-queue.test.ts tests/e2e/__tests__/${{ matrix.name }}.e2e.test.ts
 

Dockerfile.dev

@@ -28,6 +28,6 @@ RUN node --experimental-strip-types scripts/generate.ts --packages-only && pnpm
 
 # Copy source and build
 COPY . .
-RUN pnpm generate && pnpm build
+RUN pnpm generate && pnpm install --frozen-lockfile && pnpm build
 
 ENV NODE_ENV=development

functions/rag-embedding/__tests__/handler.test.ts

@@ -0,0 +1,172 @@
+const mockRequest = jest.fn();
+
+jest.mock('graphql-request', () => ({
+  gql: jest.fn((strings: TemplateStringsArray) => strings.join('')),
+}));
+
+jest.mock('@agentic-kit/ollama', () => {
+  return jest.fn().mockImplementation(() => ({
+    generateEmbedding: jest.fn().mockResolvedValue(Array(768).fill(0.1)),
+  }));
+});
+
+jest.mock('@constructive-io/graphql-query', () => ({
+  SCHEMA_INTROSPECTION_QUERY: 'query { __schema { types { name } } }',
+  inferTablesFromIntrospection: jest.fn().mockReturnValue([
+    {
+      name: 'article',
+      query: { all: 'articles', create: 'createArticle' },
+      inflection: { allRows: 'articles', tableFieldName: 'article' },
+      primaryKey: 'id',
+      relations: { belongsTo: [], hasMany: [] },
+    },
+    {
+      name: 'article_chunk',
+      query: { all: 'articleChunks', create: 'createArticleChunk' },
+      inflection: { allRows: 'articleChunks', tableFieldName: 'articleChunk', createField: 'createArticleChunk' },
+      primaryKey: 'id',
+      relations: {
+        belongsTo: [{ referencesTable: 'article', fieldName: 'article' }],
+        hasMany: [],
+      },
+    },
+  ]),
+  buildSelect: jest.fn().mockReturnValue({ toString: () => 'query { articles { nodes { id content } } }' }),
+  buildPostGraphileCreate: jest.fn().mockReturnValue({ toString: () => 'mutation { createArticleChunk { articleChunk { id } } }' }),
+  buildPostGraphileDelete: jest.fn().mockReturnValue({ toString: () => 'mutation { deleteArticleChunks { deletedCount } }' }),
+}));
+
+const createMockContext = () => ({
+  job: {
+    jobId: 'test-job-id',
+    workerId: 'test-worker',
+    databaseId: 'test-db',
+  },
+  client: {
+    request: mockRequest,
+  },
+  meta: {
+    request: jest.fn().mockResolvedValue({
+      schemas: { nodes: [{ schemaName: 'test-db-app-public' }] },
+    }),
+  },
+  log: {
+    info: jest.fn(),
+    error: jest.fn(),
+    warn: jest.fn(),
+  },
+  env: {
+    RAG_EMBEDDING_DRY_RUN: 'true',
+    GRAPHQL_URL: 'http://localhost:3000/graphql',
+    GRAPHQL_API_NAME: 'private',
+  },
+});
+
+const loadHandler = () => {
+  const mod = require('../handler');
+  return mod.default ?? mod;
+};
+
+describe('rag-embedding handler', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    mockRequest.mockReset();
+  });
+
+  it('should return early when content is empty', async () => {
+    const handler = loadHandler();
+    const context = createMockContext();
+
+    mockRequest
+      .mockResolvedValueOnce({ __schema: { types: [] } }) // introspection
+      .mockResolvedValueOnce({
+        articles: { nodes: [{ id: 'test-id', content: '' }] },
+      });
+
+    const result = await handler(
+      {
+        table: 'article',
+        schema: 'test-db-app-public',
+        id: 'test-id',
+        chunks_table: 'article_chunk',
+      },
+      context
+    );
+
+    expect(result.complete).toBe(true);
+    expect(result.chunks).toBe(0);
+  });
+
+  it('should chunk content and create embeddings', async () => {
+    const handler = loadHandler();
+    const context = createMockContext();
+
+    mockRequest
+      .mockResolvedValueOnce({ __schema: { types: [] } }) // introspection
+      .mockResolvedValueOnce({
+        articles: { nodes: [{ id: 'test-id', content: 'This is test content for chunking.' }] },
+      })
+      .mockResolvedValueOnce({ deleteArticleChunks: { deletedCount: 0 } })
+      .mockResolvedValueOnce({
+        createArticleChunk: { articleChunk: { id: 'chunk-1' } },
+      });
+
+    const result = await handler(
+      {
+        table: 'article',
+        schema: 'test-db-app-public',
+        id: 'test-id',
+        chunks_table: 'article_chunk',
+        chunk_size: '1000',
+      },
+      context
+    );
+
+    expect(result.complete).toBe(true);
+    expect(result.chunks).toBe(1);
+    expect(result.chunk_ids).toHaveLength(1);
+  });
+
+  it('should throw error when databaseId is missing', async () => {
+    const handler = loadHandler();
+    const context = createMockContext();
+    context.job.databaseId = undefined;
+
+    await expect(
+      handler({ table: 'article', schema: 'test-db-app-public', id: 'test-id', chunks_table: 'article_chunk' }, context)
+    ).rejects.toThrow('Missing X-Database-Id');
+  });
+
+  it('should throw error when required params are missing', async () => {
+    const handler = loadHandler();
+    const context = createMockContext();
+
+    await expect(
+      handler({ table: '', schema: 'test-db-app-public', id: 'test-id', chunks_table: 'article_chunk' }, context)
+    ).rejects.toThrow('Missing required params');
+  });
+
+  it('should return early when content is empty with trigger format', async () => {
+    const handler = loadHandler();
+    const context = createMockContext();
+
+    mockRequest
+      .mockResolvedValueOnce({ __schema: { types: [] } }) // introspection
+      .mockResolvedValueOnce({
+        articles: { nodes: [{ id: 'test-id', content: '' }] },
+      });
+
+    const result = await handler(
+      {
+        table: 'article',
+        schema: 'test-db-app-public',
+        id: 'test-id',
+        chunks_table: 'article_chunk',
+      },
+      context
+    );
+
+    expect(result.complete).toBe(true);
+    expect(result.chunks).toBe(0);
+  });
+});

functions/rag-embedding/handler.json

@@ -0,0 +1,15 @@
+{
+  "name": "rag-embedding",
+  "version": "1.0.0",
+  "type": "node-graphql",
+  "port": 8085,
+  "taskIdentifier": "generate_chunks",
+  "description": "RAG embedding function - chunks text and generates embeddings using Ollama nomic-embed-text",
+  "dependencies": {
+    "@agentic-kit/ollama": "^1.0.3",
+    "@constructive-io/graphql-query": "^3.12.14",
+    "@pgpmjs/env": "^2.11.0",
+    "@pgpmjs/logger": "^2.1.0",
+    "graphql-request": "^7.1.2"
+  }
+}