Add optional compression for response

Georg Traar · Georg Traar · commit 9e0fe2b130f8 · 2025-02-26T21:18:18.000+01:00
diff --git a/README.md b/README.md
@@ -3,10 +3,10 @@
 
 # CrateDB HTTP API Client for Node.js
 
-This library is a lightweight Node.js client derived from `node-crate` for interacting with CrateDB via its **HTTP endpoint**. Unlike libraries such as `node-postgres`, which use the PostgreSQL wire protocol, this client communicates with CrateDB's native HTTP API.
+This library is a Node.js client derived from `node-crate` for interacting with CrateDB via its **HTTP endpoint**. Unlike libraries such as `node-postgres`, which use the PostgreSQL wire protocol, this client communicates with CrateDB's native HTTP API.
 
 > [!CAUTION]
-> While it provides basic functionality to interact with CrateDB, it is not production-ready and lacks the robustness of established libraries.
+> While it provides basic functionality to interact with CrateDB, it is **not** production-ready and lacks the robustness of established libraries.
 > For production use, consider mature libraries like [`node-postgres`](https://node-postgres.com/) which leverage CrateDB's PostgreSQL compatibility.
 
 ## Installation
@@ -107,12 +107,15 @@ export CRATEDB_DEFAULT_SCHEMA=doc
 
 #### RequestCompression
 
-The client supports automatic GZIP compression for large requests to improve network efficiency. When enabled, any request larger than the compression threshold will be automatically compressed.
+The client supports GZIP compression for requests and response to improve network efficiency.
+Request compression is enabled by default, response compression is disabled.
 
 ```typescript
 const client = new CrateDBClient({
-  enableCompression: true,
-  compressionThreshold: 1024, // Default to 1KB
+  compression: {
+    request: 'gzip',
+    response: 'none',
+  },
 });
 ```
 
diff --git a/src/CrateDBClient.ts b/src/CrateDBClient.ts
@@ -39,8 +39,10 @@ const defaultConfig: CrateDBConfig = {
     date: 'date',
   },
   rowMode: 'array',
-  enableCompression: true,
-  compressionThreshold: 1024, // Default to 1KB
+  compression: {
+    request: 'gzip',
+    response: 'none',
+  },
 };
 
 export class CrateDBClient {
@@ -84,6 +86,15 @@ export class CrateDBClient {
       };
     }
 
+    const additionalHeaders: Record<string, string> = {};
+
+    if (cfg.compression.request == 'gzip') {
+      additionalHeaders['Content-Encoding'] = 'gzip';
+    }
+    if (cfg.compression.response == 'gzip') {
+      additionalHeaders['Accept-Encoding'] = 'gzip';
+    }
+
     this.httpOptions = {
       hostname: cfg.host,
       port: cfg.port,
@@ -94,6 +105,7 @@ export class CrateDBClient {
         'Content-Type': 'application/json',
         Accept: 'application/json',
         ...authHeader,
+        ...additionalHeaders,
         ...(cfg.defaultSchema ? { 'Default-Schema': cfg.defaultSchema } : {}),
       },
       auth: cfg.jwt ? undefined : cfg.user && cfg.password ? `${cfg.user}:${cfg.password}` : undefined,
@@ -367,125 +379,85 @@ export class CrateDBClient {
   }
 
   async _makeRequest(options: http.RequestOptions & { body?: string | Buffer }): Promise<CrateDBBaseResponse> {
-    return new Promise((resolve, reject) => {
+    let requestBody = options.body;
+    const headers = { ...options.headers };
+    const requestBodySize = requestBody ? Buffer.byteLength(requestBody) : 0;
+    let encodedSize = requestBodySize;
+    let encodingDuration = 0;
+    let requestDuration = 0;
+    let deserializationDuration = 0;
+
+    // Compress the request body if needed.
+    if (this.cfg.compression.request === 'gzip' && requestBody) {
+      const startEncodingTime = Date.now();
+      requestBody = await promisify(zlib.gzip)(requestBody);
+      encodingDuration = Date.now() - startEncodingTime;
+      encodedSize = Buffer.byteLength(requestBody);
+    }
+
+    // Wrap the HTTP request in a promise.
+    const { response, data } = await new Promise<{
+      response: http.IncomingMessage;
+      data: Buffer[];
+    }>((resolve, reject) => {
+      const protocolHandler = this.protocol === 'https' ? https : http;
+      const req = protocolHandler.request({ ...options, headers }, (response) => {
+        const data: Buffer[] = [];
+        response.on('data', (chunk: Buffer) => data.push(chunk));
+        response.on('end', () => resolve({ response, data }));
+      });
+      req.on('error', (err) => reject(new Error(`Request failed: ${err.message}\nStack trace: ${err.stack}`)));
+      req.end(requestBody);
+    });
+
+    requestDuration = Date.now() - (response.headers.date ? new Date(response.headers.date).getTime() : Date.now());
+    let rawResponse = Buffer.concat(data);
+    const responseBodySize = rawResponse.length;
+
+    // Decompress the response if gzip encoded.
+    if (response.headers['content-encoding'] === 'gzip') {
       try {
-        let requestBody = options.body;
-        const headers = { ...options.headers };
-        const requestBodySize = requestBody ? Buffer.byteLength(requestBody) : 0;
-        let encodedSize = requestBodySize;
-        let encodingDuration = 0;
-        let requestDuration = 0;
-        let deserializationDuration = 0;
-
-        const startEncodingTime = Date.now();
-
-        if (this.cfg.enableCompression && requestBody && requestBodySize > (this.cfg.compressionThreshold ?? 1024)) {
-          promisify(zlib.gzip)(requestBody)
-            .then((compressed) => {
-              const startRequestTime = Date.now();
-              encodingDuration = startRequestTime - startEncodingTime;
-              requestBody = compressed;
-              encodedSize = Buffer.byteLength(requestBody);
-              headers['Content-Encoding'] = 'gzip';
-
-              const reqStartTime = Date.now();
-              const req = (this.protocol === 'https' ? https : http).request({ ...options, headers }, (response) => {
-                const data: Buffer[] = [];
-                response.on('data', (chunk: Buffer) => data.push(chunk));
-                response.on('end', () => {
-                  const startDeserializationTime = Date.now();
-                  requestDuration = startDeserializationTime - reqStartTime;
-                  const rawResponse = Buffer.concat(data);
-                  const responseBodySize = rawResponse.length;
-
-                  try {
-                    const parsedResponse = Serializer.deserialize(rawResponse.toString(), this.cfg.deserialization);
-                    deserializationDuration = Date.now() - startDeserializationTime;
-                    if (response.statusCode === 200) {
-                      resolve({
-                        ...parsedResponse,
-                        sizes: {
-                          response: responseBodySize,
-                          request: encodedSize,
-                          requestUncompressed: requestBodySize,
-                        },
-                        durations: {
-                          encoding: encodingDuration,
-                          request: requestDuration,
-                          deserialization: deserializationDuration,
-                        },
-                      });
-                    } else {
-                      reject(CrateDBError.fromResponse(parsedResponse as CrateDBErrorResponse, response.statusCode));
-                    }
-                  } catch (parseErr: unknown) {
-                    if (parseErr instanceof Error) {
-                      reject(
-                        new Error(
-                          `Failed to parse response: ${parseErr.message}. Raw response: ${rawResponse.toString()}`
-                        )
-                      );
-                    } else {
-                      reject(new Error(`Failed to parse response. Raw response: ${rawResponse.toString()}`));
-                    }
-                  }
-                });
-              });
-
-              req.on('error', (err) => reject(new Error(`Request failed: ${err.message}`)));
-              req.end(requestBody);
-            })
-            .catch((error) => reject(error));
+        rawResponse = await promisify(zlib.gunzip)(rawResponse);
+      } catch (error) {
+        if (error instanceof Error) {
+          throw new Error(`Failed to decompress response: ${error.message}`);
         } else {
-          const startRequestTime = Date.now();
-          const req = (this.protocol === 'https' ? https : http).request({ ...options, headers }, (response) => {
-            const data: Buffer[] = [];
-            response.on('data', (chunk: Buffer) => data.push(chunk));
-            response.on('end', () => {
-              const startDeserializationTime = Date.now();
-              const requestDuration = startDeserializationTime - startRequestTime;
-              const rawResponse = Buffer.concat(data);
-              const responseBodySize = rawResponse.length;
-
-              try {
-                const parsedResponse = Serializer.deserialize(rawResponse.toString(), this.cfg.deserialization);
-                const deserializationDuration = Date.now() - startDeserializationTime;
-                if (response.statusCode === 200) {
-                  resolve({
-                    ...parsedResponse,
-                    sizes: {
-                      response: responseBodySize,
-                      request: encodedSize,
-                      requestUncompressed: requestBodySize,
-                    },
-                    durations: {
-                      encoding: 0,
-                      request: requestDuration,
-                      deserialization: deserializationDuration,
-                    },
-                  });
-                } else {
-                  reject(CrateDBError.fromResponse(parsedResponse as CrateDBErrorResponse, response.statusCode));
-                }
-              } catch (parseErr: unknown) {
-                if (parseErr instanceof Error) {
-                  reject(
-                    new Error(`Failed to parse response: ${parseErr.message}. Raw response: ${rawResponse.toString()}`)
-                  );
-                } else {
-                  reject(new Error(`Failed to parse response. Raw response: ${rawResponse.toString()}`));
-                }
-              }
-            });
-          });
-
-          req.on('error', (err) => reject(new Error(`Request failed: ${err.message}`)));
-          req.end(requestBody);
+          throw new Error(`Failed to decompress response: ${String(error)}`);
         }
-      } catch (error) {
-        reject(error);
       }
-    });
+    }
+
+    // Deserialize the response.
+    const startDeserializationTime = Date.now();
+    let parsedResponse;
+    try {
+      parsedResponse = Serializer.deserialize(rawResponse.toString(), this.cfg.deserialization);
+    } catch (err) {
+      if (err instanceof Error) {
+        throw new Error(`Failed to parse response: ${err.message}. Raw response: ${rawResponse.toString()}`);
+      } else {
+        throw new Error(`Failed to parse response: ${String(err)}. Raw response: ${rawResponse.toString()}`);
+      }
+    }
+    deserializationDuration = Date.now() - startDeserializationTime;
+
+    if (response.statusCode === 200) {
+      return {
+        ...parsedResponse,
+        sizes: {
+          response: responseBodySize,
+          request: encodedSize,
+          requestUncompressed: requestBodySize,
+        },
+        durations: {
+          encoding: encodingDuration,
+          request: requestDuration,
+          deserialization: deserializationDuration,
+        },
+      };
+    } else {
+      throw CrateDBError.fromResponse(parsedResponse as CrateDBErrorResponse, response.statusCode!);
+    }
   }
 
   protected _transformResponse(
diff --git a/src/interfaces.ts b/src/interfaces.ts
@@ -13,10 +13,14 @@ export interface CrateDBConfig {
   maxConnections: number;
   deserialization: DeserializationConfig;
   rowMode?: 'array' | 'object';
-  enableCompression?: boolean; // If true, enables gzip compression for large requests
-  compressionThreshold?: number; // Size in bytes before compression is applied
+  compression: CompressionOptions;
 }
 
+type CompressionOptions = {
+  request: 'gzip' | 'none';
+  response: 'gzip' | 'none';
+};
+
 export type DeserializationConfig = {
   long: 'bigint' | 'number';
   timestamp: 'date' | 'number';
diff --git a/tests/CrateDBClient.test.ts b/tests/CrateDBClient.test.ts
@@ -548,20 +548,18 @@ describe('CrateDBClient Integration Tests', () => {
   });
 
   describe('Compression', () => {
-    it('should compress large requests when enabled', async () => {
-      // Create a large payload that will trigger compression
-      const largeData = Array(100).fill('test data').join(' ');
+    it('should compress by default', async () => {
+      const records = Array(20000)
+        .fill({})
+        .map((_, i) => ({ id: i, data: 'Alice' }));
       const tableName = 'compression_test';
 
       await client.createTable(tableName, {
         id: { type: 'INTEGER', primaryKey: true },
         data: { type: 'TEXT' },
       });
 
-      const response = await client.insert(tableName, {
-        id: 1,
-        data: largeData,
-      });
+      const response = await client.insertMany(tableName, records);
 
       // Verify compression metrics
       expect(response.sizes.requestUncompressed).toBeGreaterThan(response.sizes.request);
@@ -573,34 +571,11 @@ describe('CrateDBClient Integration Tests', () => {
       await client.drop(tableName);
     });
 
-    it('should not compress small requests', async () => {
-      const smallData = 'small test data';
-      const tableName = 'small_data_test';
-
-      await client.createTable(tableName, {
-        id: { type: 'INTEGER', primaryKey: true },
-        data: { type: 'TEXT' },
-      });
-
-      const response = await client.insert(tableName, {
-        id: 1,
-        data: smallData,
-      });
-
-      // Verify no compression was applied
-      expect(response.sizes.request).toBe(response.sizes.requestUncompressed);
-      expect(response.sizes.request).toBeLessThan(1024);
-      expect(response.durations.encoding).toBe(0);
-      expect(response.durations.total).toBeGreaterThan(0);
-
-      await client.drop(tableName);
-    });
-
     it('should respect compression setting when disabled', async () => {
       const clientWithoutCompression = new CrateDBClient({
         host: container.getHost(),
         port: container.getMappedPort(4200),
-        enableCompression: false,
+        compression: { request: 'none' },
       });
 
       const largeData = Array(1000).fill('test data').join(' ');
diff --git a/tests/StatementGenerator.test.ts b/tests/StatementGenerator.test.ts
@@ -1,5 +1,6 @@
 import { describe, it, expect } from 'vitest';
 import { StatementGenerator } from '../src/StatementGenerator';
+import { ColumnDefinition } from '../src/interfaces';
 
 describe('StatementGenerator SQL Generation', () => {
   // CREATE TABLE tests
@@ -34,7 +35,7 @@ describe('StatementGenerator SQL Generation', () => {
 
   it('should generate a CREATE TABLE query with default values and NOT NULL constraints', () => {
     const tableName = 'default_table';
-    const columns = {
+    const columns: Record<string, ColumnDefinition> = {
       id: { type: 'integer', primaryKey: true, notNull: true, defaultValue: 1 },
       name: { type: 'string', primaryKey: false, notNull: false, defaultValue: "'unknown'" },
     };
