sdk-core: overhaul of database and storage [INT-355]

Author: perf2711

packages/sdk-core/src/BacktraceCoreClient.ts

@@ -9,9 +9,12 @@ import {
     BacktraceReportSubmissionResult,
     BacktraceRequestHandler,
     BacktraceSessionProvider,
+    BacktraceStorageModule,
     BacktraceSubmissionResponse,
     DebugIdProvider,
-    FileSystem,
+    DefaultReportBacktraceDatabaseRecordFactory,
+    ReportBacktraceDatabaseRecordSender,
+    ReportBacktraceDatabaseRecordSerializer,
     SdkOptions,
     SessionFiles,
 } from './index.js';
@@ -117,7 +120,7 @@ export abstract class BacktraceCoreClient<
 
     protected readonly attributeManager: AttributeManager;
     protected readonly attachmentManager: AttachmentManager;
-    protected readonly fileSystem?: FileSystem;
+    protected readonly databaseStorage?: BacktraceStorageModule;
 
     private readonly _modules: BacktraceModules = new Map();
     private readonly _dataBuilder: BacktraceDataBuilder;
@@ -133,7 +136,6 @@ export abstract class BacktraceCoreClient<
         super();
 
         this.options = setup.options;
-        this.fileSystem = setup.fileSystem;
         this._sdkOptions = setup.sdkOptions;
         this._sessionProvider = setup.sessionProvider ?? new SingleSessionProvider();
         this._reportSubmission =
@@ -167,27 +169,35 @@ export abstract class BacktraceCoreClient<
             new DebugIdProvider(stackTraceConverter, setup.debugIdMapProvider),
         );
 
-        if (this.options?.database?.enable === true && setup.fileSystem) {
+        if (this.options?.database?.enable === true && setup.database?.storage) {
+            this.databaseStorage = setup.database.storage;
+            this.addModule(this.databaseStorage);
+
             const provider = BacktraceDatabaseFileStorageProvider.createIfValid(
-                setup.fileSystem,
+                {
+                    report: new ReportBacktraceDatabaseRecordSerializer(),
+                    ...setup.database?.recordSerializers,
+                },
+                this.databaseStorage,
                 this.options.database,
             );
 
-            if (this.fileSystem) {
-                const sessionFiles = new SessionFiles(
-                    this.fileSystem,
-                    this.options.database.path,
-                    this.sessionId,
-                    this.options.database.maximumOldSessions ?? 1,
-                );
-                this._modules.set(SessionFiles, sessionFiles);
-            }
+            const sessionFiles = new SessionFiles(
+                this.databaseStorage,
+                this.sessionId,
+                this.options.database.maximumOldSessions ?? 1,
+            );
+            this._modules.set(SessionFiles, sessionFiles);
 
             if (provider) {
                 const database = new BacktraceDatabase(
                     this.options.database,
                     provider,
-                    this._reportSubmission,
+                    {
+                        report: new ReportBacktraceDatabaseRecordSender(this._reportSubmission),
+                        ...setup.database?.recordSenders?.(this._reportSubmission),
+                    },
+                    setup.database?.reportRecordFactory ?? DefaultReportBacktraceDatabaseRecordFactory.default(),
                     this.sessionFiles,
                 );
                 this._modules.set(BacktraceDatabase, database);
@@ -220,16 +230,6 @@ export abstract class BacktraceCoreClient<
     public initialize() {
         this.validateAttributes();
 
-        if (this.fileSystem && this.options.database?.createDatabaseDirectory) {
-            if (!this.options.database.path) {
-                throw new Error(
-                    'Missing mandatory path to the database. Please define the database.path option in the configuration.',
-                );
-            }
-
-            this.fileSystem.createDirSync(this.options.database?.path);
-        }
-
         for (const module of this._modules.values()) {
             if (module.bind) {
                 module.bind(this.getModuleBindData());
@@ -403,7 +403,7 @@ export abstract class BacktraceCoreClient<
         return data instanceof BacktraceReport;
     }
 
-    private getModuleBindData(): BacktraceModuleBindData {
+    private getModuleBindData(): BacktraceModuleBindData<O> {
         return {
             client: this,
             options: this.options,
@@ -413,7 +413,7 @@ export abstract class BacktraceCoreClient<
             requestHandler: this._requestHandler,
             database: this.database,
             sessionFiles: this.sessionFiles,
-            fileSystem: this.fileSystem,
+            storage: this.databaseStorage,
         };
     }
 

packages/sdk-core/src/builder/BacktraceCoreClientBuilder.ts

@@ -12,7 +12,7 @@ import { BacktraceSessionProvider } from '../modules/metrics/BacktraceSessionPro
 import { MetricsQueue } from '../modules/metrics/MetricsQueue.js';
 import { SummedEvent } from '../modules/metrics/model/SummedEvent.js';
 import { UniqueEvent } from '../modules/metrics/model/UniqueEvent.js';
-import { FileSystem } from '../modules/storage/index.js';
+import { BacktraceStorageModule } from '../modules/storage/index.js';
 import { CoreClientSetup } from './CoreClientSetup.js';
 
 type Writeable<T> = { -readonly [P in keyof T]: T[P] };
@@ -77,8 +77,11 @@ export abstract class BacktraceCoreClientBuilder<S extends Partial<CoreClientSet
         return this;
     }
 
-    public useFileSystem(fileSystem: FileSystem): this {
-        this.clientSetup.fileSystem = fileSystem;
+    public useDatabaseStorage(storage: BacktraceStorageModule): this {
+        this.clientSetup.database = {
+            ...this.clientSetup.database,
+            storage,
+        };
         return this;
     }
 

packages/sdk-core/src/builder/CoreClientSetup.ts

@@ -5,11 +5,14 @@ import { BacktraceModule } from '../modules/BacktraceModule.js';
 import { BacktraceAttributeProvider } from '../modules/attribute/BacktraceAttributeProvider.js';
 import { BreadcrumbsSetup } from '../modules/breadcrumbs/index.js';
 import { BacktraceStackTraceConverter } from '../modules/converter/index.js';
+import { BacktraceDatabaseRecordSenders } from '../modules/database/BacktraceDatabaseRecordSender.js';
+import { BacktraceDatabaseRecordSerializers } from '../modules/database/BacktraceDatabaseRecordSerializer.js';
+import { ReportBacktraceDatabaseRecordFactory } from '../modules/database/ReportBacktraceDatabaseRecordFactory.js';
 import { BacktraceSessionProvider } from '../modules/metrics/BacktraceSessionProvider.js';
 import { MetricsQueue } from '../modules/metrics/MetricsQueue.js';
 import { SummedEvent } from '../modules/metrics/model/SummedEvent.js';
 import { UniqueEvent } from '../modules/metrics/model/UniqueEvent.js';
-import { FileSystem } from '../modules/storage/index.js';
+import { BacktraceStorageModule } from '../modules/storage/BacktraceStorage.js';
 import { DebugIdMapProvider } from '../sourcemaps/index.js';
 import { SdkOptions } from './SdkOptions.js';
 
@@ -28,8 +31,13 @@ export interface CoreClientSetup<O extends BacktraceConfiguration = BacktraceCon
     readonly debugIdMapProvider?: DebugIdMapProvider;
     readonly breadcrumbsSetup?: BreadcrumbsSetup;
     readonly reportSubmission?: BacktraceReportSubmission;
-    readonly fileSystem?: FileSystem;
     readonly modules?: BacktraceModule[];
     readonly summedMetricsQueue?: MetricsQueue<SummedEvent>;
     readonly uniqueMetricsQueue?: MetricsQueue<UniqueEvent>;
+    readonly database?: {
+        readonly storage?: BacktraceStorageModule;
+        readonly recordSerializers?: BacktraceDatabaseRecordSerializers;
+        readonly recordSenders?: (submission: BacktraceReportSubmission) => BacktraceDatabaseRecordSenders;
+        readonly reportRecordFactory?: ReportBacktraceDatabaseRecordFactory;
+    };
 }

packages/sdk-core/src/common/asyncGenerator.ts

@@ -0,0 +1,7 @@
+export async function toArray<T>(generator: AsyncGenerator<T>): Promise<T[]> {
+    const result: T[] = [];
+    for await (const element of generator) {
+        result.push(element);
+    }
+    return result;
+}

packages/sdk-core/src/model/configuration/BacktraceDatabaseConfiguration.ts

@@ -3,15 +3,6 @@ export interface EnabledBacktraceDatabaseConfiguration {
      * Determine if the Database is enabled
      */
     enable: true;
-    /**
-     * Path where the SDK can store data.
-     */
-    path: string;
-    /**
-     * Determine if the directory should be auto created by the SDK.
-     * @default true
-     */
-    createDatabaseDirectory?: boolean;
 
     /**
      * Sends reports to the server based on the retry settings.

packages/sdk-core/src/modules/BacktraceModule.ts

@@ -2,28 +2,28 @@ import {
     BacktraceConfiguration,
     BacktraceCoreClient,
     BacktraceRequestHandler,
-    FileSystem,
+    BacktraceStorageModule,
     SessionFiles,
 } from '../index.js';
 import { BacktraceReportSubmission } from '../model/http/BacktraceReportSubmission.js';
 import { AttachmentManager } from './attachments/AttachmentManager.js';
 import { AttributeManager } from './attribute/AttributeManager.js';
 import { BacktraceDatabase } from './database/BacktraceDatabase.js';
 
-export interface BacktraceModuleBindData {
+export interface BacktraceModuleBindData<O extends BacktraceConfiguration = BacktraceConfiguration> {
     readonly client: BacktraceCoreClient;
-    readonly options: BacktraceConfiguration;
+    readonly options: O;
     readonly attributeManager: AttributeManager;
     readonly attachmentManager: AttachmentManager;
     readonly reportSubmission: BacktraceReportSubmission;
     readonly requestHandler: BacktraceRequestHandler;
     readonly database?: BacktraceDatabase;
     readonly sessionFiles?: SessionFiles;
-    readonly fileSystem?: FileSystem;
+    readonly storage?: BacktraceStorageModule;
 }
 
-export interface BacktraceModule {
-    bind?(client: BacktraceModuleBindData): void;
+export interface BacktraceModule<O extends BacktraceConfiguration = BacktraceConfiguration> {
+    bind?(client: BacktraceModuleBindData<O>): void;
     initialize?(): void;
     dispose?(): void;
 }

packages/sdk-core/src/modules/attribute/FileAttributeManager.ts

@@ -1,7 +1,8 @@
 import { jsonEscaper } from '../../common/jsonEscaper.js';
 import { AttributeType } from '../../model/data/index.js';
 import { BacktraceModule, BacktraceModuleBindData } from '../BacktraceModule.js';
-import { FileSystem, SessionFiles } from '../storage/index.js';
+import { BacktraceStorage, BacktraceSyncStorage } from '../storage/BacktraceStorage.js';
+import { SessionFiles } from '../storage/index.js';
 import { AttributeManager } from './AttributeManager.js';
 
 const ATTRIBUTE_FILE_NAME = 'bt-attributes';
@@ -10,17 +11,17 @@ export class FileAttributeManager implements BacktraceModule {
     private _attributeManager?: AttributeManager;
 
     constructor(
-        private readonly _fileSystem: FileSystem,
+        private readonly _storage: BacktraceStorage,
         private _fileName?: string,
     ) {}
 
-    public static create(fileSystem: FileSystem) {
-        return new FileAttributeManager(fileSystem);
+    public static create(storage: BacktraceStorage) {
+        return new FileAttributeManager(storage);
     }
 
-    public static createFromSession(sessionFiles: SessionFiles, fileSystem: FileSystem) {
+    public static createFromSession(sessionFiles: SessionFiles, storage: BacktraceStorage & BacktraceSyncStorage) {
         const fileName = sessionFiles.getFileName(ATTRIBUTE_FILE_NAME);
-        return new FileAttributeManager(fileSystem, fileName);
+        return new FileAttributeManager(storage, fileName);
     }
 
     public initialize(): void {
@@ -51,7 +52,10 @@ export class FileAttributeManager implements BacktraceModule {
         }
 
         try {
-            const content = await this._fileSystem.readFile(this._fileName);
+            const content = await this._storage.get(this._fileName);
+            if (!content) {
+                return {};
+            }
             return JSON.parse(content);
         } catch {
             return {};
@@ -64,6 +68,6 @@ export class FileAttributeManager implements BacktraceModule {
         }
 
         const reportData = this._attributeManager.get('scoped');
-        await this._fileSystem.writeFile(this._fileName, JSON.stringify(reportData.attributes, jsonEscaper()));
+        await this._storage.set(this._fileName, JSON.stringify(reportData.attributes, jsonEscaper()));
     }
 }