Merge pull request #27 from Libertech-FR/backend-feature

tacxou · web-flow · commit f52b95603788 · 2024-03-20T09:28:30.000+01:00
implements auth + backends execution
diff --git a/.env.example b/.env.example
@@ -1,8 +1,6 @@
-LOG_LEVEL="info"
-NAME_QUEUE="backend"
-SECRET="secret"
+# SESAME_LOG_LEVEL="info"
+# SESAME_NAME_QUEUE="sesame"
+SESAME_JWT_SECRET="zeaezazeaezazaeeazrftrqezfqfqszewfsqddfqsqsqsdqdsqdsqdzsdqzsdqzs"
 
-REDIS_URI="redis://sesame-redis:6379"
-MONGO_URI="mongodb://sesame-mongodb:27017/sesame"
-SWAGGER_PATH="/swagger"
-SWAGGER_API="swagger.json"
+SESAME_REDIS_URI="redis://sesame-redis:6379"
+SESAME_MONGO_URI="mongodb://sesame-mongodb:27017/sesame"
diff --git a/Makefile b/Makefile
@@ -3,6 +3,7 @@ APP_PORT = 4002
 IMG_NAME = "ghcr.io/libertech-fr/sesame-orchestrator"
 BASE_NAME = "sesame"
 APP_NAME = "sesame-orchestrator"
+PLATFORM = "linux/amd64"
 
 .DEFAULT_GOAL := help
 help:
diff --git a/src/_common/abstracts/abstract.queue.processor.ts b/src/_common/abstracts/abstract.queue.processor.ts
@@ -1,19 +1,38 @@
-import { ConfigService } from '@nestjs/config';
 import { Queue, QueueEvents } from 'bullmq';
-import Redis from 'ioredis';
+import { AbstractService, AbstractServiceContext } from './abstract.service';
+import { Redis, getRedisConnectionToken } from '@nestjs-modules/ioredis';
+import { OnModuleInit } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+
+export abstract class AbstractQueueProcessor extends AbstractService implements OnModuleInit {
+  protected config: ConfigService;
+
+  private redis: Redis;
+  protected _queue: Queue;
+  public queueEvents: QueueEvents;
+
+  public get queue(): Queue {
+    return this._queue;
+  }
 
-export abstract class AbstractQueueProcessor {
-  protected readonly queue: Queue;
-  protected readonly queueEvents: QueueEvents;
-  public constructor(
-    protected readonly config: ConfigService,
-    protected readonly redis: Redis,
-  ) {
-    this.queue = new Queue(this.config.get('nameQueue'), {
+  public constructor(context?: AbstractServiceContext) {
+    super(context);
+    if (!this.moduleRef) throw new Error('ModuleRef is not defined in ' + this.constructor.name);
+  }
+
+  public async onModuleInit() {
+    this.config = this.moduleRef.get<ConfigService>(ConfigService, { strict: false });
+    this.redis = this.moduleRef.get<Redis>(getRedisConnectionToken(), { strict: false });
+
+    this._queue = new Queue(this.config.get<string>('application.nameQueue'), {
       connection: this.redis,
     });
-    this.queueEvents = new QueueEvents(this.config.get('nameQueue'), {
+    this.queueEvents = new QueueEvents(this.config.get<string>('application.nameQueue'), {
       connection: this.redis,
     });
+
+    this.queueEvents.on('failed', (errors) => {
+      this.logger.warn('Queue failed', errors);
+    });
   }
 }
diff --git a/src/_common/pipes/dto-validation.pipe.ts b/src/_common/pipes/dto-validation.pipe.ts
@@ -7,6 +7,7 @@ import {
   Injectable,
   Scope,
   Inject,
+  ArgumentMetadata,
 } from '@nestjs/common';
 import { REQUEST } from '@nestjs/core';
 import { Request } from 'express';
@@ -17,6 +18,10 @@ interface ValidationRecursive {
 
 @Injectable({ scope: Scope.REQUEST })
 export class DtoValidationPipe extends ValidationPipe {
+  public async transform(value: any, metadata: ArgumentMetadata): Promise<any> {
+    return (await super.transform(value, metadata)) || value;
+  }
+
   public constructor(@Inject(REQUEST) protected readonly request: Request) {
     super({
       transform: true,
diff --git a/src/app.controller.ts b/src/app.controller.ts
@@ -2,7 +2,9 @@ import { Controller, Get, Res } from '@nestjs/common';
 import { AppService } from './app.service';
 import { Response } from 'express';
 import { AbstractController } from '~/_common/abstracts/abstract.controller';
+import { Public } from './_common/decorators/public.decorator';
 
+@Public()
 @Controller()
 export class AppController extends AbstractController {
   constructor(private readonly appService: AppService) {
diff --git a/src/app.module.ts b/src/app.module.ts
@@ -11,6 +11,10 @@ import { RedisOptions } from 'ioredis';
 import { AppController } from './app.controller';
 import { AppService } from './app.service';
 import { RequestContextModule } from 'nestjs-request-context';
+import { APP_FILTER, APP_GUARD, APP_PIPE } from '@nestjs/core';
+import { AuthGuard } from './_common/guards/auth.guard';
+import { MongooseValidationFilter } from './_common/filters/mongoose-validation.filter';
+import { DtoValidationPipe } from './_common/pipes/dto-validation.pipe';
 
 @Module({
   imports: [
@@ -58,6 +62,24 @@ import { RequestContextModule } from 'nestjs-request-context';
     ManagementModule.register(),
   ],
   controllers: [AppController],
-  providers: [AppService],
+  providers: [
+    AppService,
+    {
+      provide: APP_GUARD,
+      useClass: AuthGuard,
+    },
+    // {
+    //   provide: APP_FILTER,
+    //   useClass: AllExceptionFilter,
+    // },
+    {
+      provide: APP_FILTER,
+      useClass: MongooseValidationFilter,
+    },
+    {
+      provide: APP_PIPE,
+      useClass: DtoValidationPipe,
+    },
+  ],
 })
 export class AppModule {}
diff --git a/src/config.ts b/src/config.ts
@@ -13,6 +13,7 @@ export interface MongoosePlugin {
 export interface ConfigInstance {
   application: {
     logLevel: string;
+    nameQueue: string;
     bodyParser: {
       limit: string;
     };
@@ -38,13 +39,12 @@ export interface ConfigInstance {
     api: string;
     options: SwaggerCustomOptions;
   };
-  nameQueue: string;
-  secret: string;
 }
 
 export default (): ConfigInstance => ({
   application: {
-    logLevel: process.env['LOG_LEVEL'] || 'info',
+    logLevel: process.env['SESAME_LOG_LEVEL'] || 'info',
+    nameQueue: process.env['SESAME_NAME_QUEUE'] || 'sesame',
     bodyParser: {
       limit: '500mb',
     },
@@ -63,14 +63,14 @@ export default (): ConfigInstance => ({
     },
   },
   ioredis: {
-    uri: process.env['REDIS_URI'] || 'redis://localhost:6379/0',
+    uri: process.env['SESAME_REDIS_URI'] || 'redis://localhost:6379/0',
     options: {
       showFriendlyErrorStack: true,
       maxRetriesPerRequest: null,
     },
   },
   mongoose: {
-    uri: process.env['MONGO_URI'] || 'mongodb://localhost:27017/backend',
+    uri: process.env['SESAME_MONGO_URI'] || 'mongodb://localhost:27017/backend',
     options: {
       directConnection: true,
     },
@@ -88,15 +88,13 @@ export default (): ConfigInstance => ({
       /**
        * @see https://randomkeygen.com/
        */
-      secret: process.env['JWT_SECRET'],
+      secret: process.env['SESAME_JWT_SECRET'],
       // jwksUri: 'http://127.0.0.1:2000/jwks',
     },
   },
-  nameQueue: process.env['NAME_QUEUE'] || 'backend',
-  secret: process.env['SECRET'] || 'mySecret',
   swagger: {
-    path: process.env['SWAGGER_PATH'] || '/swagger',
-    api: process.env['SWAGGER_API'] || '/swagger/json',
+    path: '/swagger',
+    api: '/swagger/json',
     options: {
       swaggerOptions: {
         persistAuthorization: true,
diff --git a/src/core/auth/auth.service.ts b/src/core/auth/auth.service.ts
@@ -100,7 +100,7 @@ export class AuthService extends AbstractService implements OnModuleInit {
     access_token: string;
     refresh_token?: string;
   }> {
-    const scopes = ['teaket'];
+    const scopes = ['sesame'];
     if (refresh_token === false) scopes.push('offline');
     const jwtid = `${identity._id}_${randomBytes(16).toString('hex')}`;
     const access_token = this.jwtService.sign(
diff --git a/src/core/backends/_dto/execute-job.dto.ts b/src/core/backends/_dto/execute-job.dto.ts
@@ -0,0 +1,15 @@
+import { IsEnum, IsNotEmpty, IsObject, IsOptional } from 'class-validator';
+import { ActionType } from '../_enum/action-type.enum';
+import { ApiProperty } from '@nestjs/swagger';
+
+export class ExecuteJobDto {
+  @IsNotEmpty()
+  @IsEnum(ActionType)
+  @ApiProperty({ type: String })
+  public action: ActionType;
+
+  @IsOptional()
+  @IsObject()
+  @ApiProperty({ type: Object })
+  public payload: Record<string | number, any>;
+}
diff --git a/src/core/backends/_enum/action-type.enum.ts b/src/core/backends/_enum/action-type.enum.ts
@@ -0,0 +1,9 @@
+export enum ActionType {
+  LIST_BACKENDS = 'LIST_BACKENDS',
+  PING_TARGET = 'PING_TARGET',
+  IDENTITY_CREATE = 'IDENTITY_CREATE',
+  IDENTITY_UPDATE = 'IDENTITY_UPDATE',
+  IDENTITY_DELETE = 'IDENTITY_DELETE',
+  IDENTITY_PASSWORD_RESET = 'IDENTITY_PASSWORD_RESET',
+  IDENTITY_PASSWORD_CHANGE = 'IDENTITY_PASSWORD_CHANGE',
+}
diff --git a/src/core/backends/_interfaces/backend-codes.enum.ts b/src/core/backends/_interfaces/backend-codes.enum.ts
@@ -0,0 +1,17 @@
+/**
+ * BackendCodesEnum
+ *
+ * @description Enum for backend codes
+ * @see https://tldp.org/LDP/abs/html/exitcodes.html
+ * @see https://www.linuxdoc.org/LDP/abs/html/exitcodes.html
+ * @see https://www.gnu.org/software/libc/manual/html_node/Exit-Status.html
+ */
+export enum BackendCodesEnum {
+  OK = 0,
+  GENERIC_ERROR = 1 << 0,
+  GENERIC_STOPPED = 1 << 1,
+  CONNECTION_ERROR = 1 << 2,
+  INVALID_LOGIN = 1 << 3,
+  INVALID_CREDENTIALS = 1 << 4,
+  INVALID_JSON_RESPONSE = 1 << 5,
+}
diff --git a/src/core/backends/_interfaces/backend-result.interface.ts b/src/core/backends/_interfaces/backend-result.interface.ts
@@ -0,0 +1,12 @@
+export interface BackendResultInterface {
+  backend: string;
+  status: number;
+  output?: BackendResultInfoInterface;
+  error?: BackendResultInfoInterface;
+}
+
+export interface BackendResultInfoInterface {
+  status: number;
+  message?: string;
+  data?: object;
+}
diff --git a/src/core/backends/_interfaces/execute-job-options.interface.ts b/src/core/backends/_interfaces/execute-job-options.interface.ts
@@ -0,0 +1,8 @@
+import { JobsOptions } from 'bullmq';
+
+export interface ExecuteJobOptions {
+  job?: JobsOptions;
+  async?: boolean;
+  syncTimeout?: number;
+  timeoutDiscard?: boolean;
+}
diff --git a/src/core/backends/_interfaces/worker-result.interface.ts b/src/core/backends/_interfaces/worker-result.interface.ts
@@ -0,0 +1,11 @@
+import { BackendResultInterface } from './backend-result.interface';
+
+export interface WorkerResultInterface {
+  jobId: string;
+  status: number;
+  data: WorkerResultInfoInterface;
+}
+
+export interface WorkerResultInfoInterface {
+  [backendName: string]: Partial<BackendResultInterface>;
+}
diff --git a/src/core/backends/backends.controller.ts b/src/core/backends/backends.controller.ts
@@ -1,32 +1,84 @@
-import { Controller, Logger, Get, Res, UseGuards } from '@nestjs/common';
-import { BackendsService } from './backends.service';
-import { ApiBearerAuth, ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger';
-import { AuthGuard } from '@nestjs/passport';
+import { InjectRedis } from '@nestjs-modules/ioredis';
+import {
+  Body,
+  Controller,
+  HttpStatus,
+  Logger,
+  ParseIntPipe,
+  Post,
+  Query,
+  Res,
+  Sse,
+  UnauthorizedException,
+} from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
 import { Response } from 'express';
+import Redis from 'ioredis';
+import { Observable, Subscriber } from 'rxjs';
+import { Public } from '~/_common/decorators/public.decorator';
+import { ExecuteJobDto } from './_dto/execute-job.dto';
+import { BackendsService } from './backends.service';
+
+function fireMessage(observer: Subscriber<MessageEvent>, channel: string, message: any, loggername: string) {
+  try {
+    observer.next({
+      data: { channel, payload: message },
+    } as MessageEvent);
+    Logger.debug(`Observer send to <${channel}> with data <${message}>`, loggername);
+  } catch (err) {
+    Logger.error(`Observer error from <${channel}> with data <${message}>. Error: ${err}`, loggername);
+  }
+}
 
 @Controller('backends')
 @ApiTags('core')
 export class BackendsController {
   private readonly logger = new Logger(BackendsController.name);
 
-  constructor(private backendsService: BackendsService) {}
+  constructor(
+    private backendsService: BackendsService,
+    @InjectRedis() protected readonly redis: Redis,
+  ) {}
 
-  @Get('list')
-  @ApiOperation({ summary: 'List backends from daemon' })
-  @ApiResponse({ status: 200, description: 'List ok ' })
-  @ApiBearerAuth()
-  @UseGuards(AuthGuard('api-key'))
-  async list(@Res() res: Response): Promise<Response> {
-    const backend = await this.backendsService.list();
-    return res.status(200).json(backend);
+  @Post('execute')
+  public async executeJob(
+    @Res() res: Response,
+    @Body() body: ExecuteJobDto,
+    @Query('async') asyncQuery: string,
+    @Query('timeoutDiscard') timeoutDiscardQuery: string,
+    @Query('syncTimeout', new ParseIntPipe({ optional: true, errorHttpStatusCode: 406 })) syncTimeout?: number,
+  ): Promise<Response> {
+    const async = /true|on|yes|1/i.test(asyncQuery);
+    const timeoutDiscard = /true|on|yes|1/i.test(timeoutDiscardQuery);
+    const [job, response] = await this.backendsService.executeJob(body.action, body.payload, {
+      async,
+      syncTimeout,
+      timeoutDiscard,
+    });
+    return res.status(HttpStatus.ACCEPTED).json({ async, job, response });
   }
-  @Get('alive')
-  @ApiOperation({ summary: 'test backends ' })
-  @ApiResponse({ status: 200, description: 'command executed ' })
-  @ApiBearerAuth()
-  @UseGuards(AuthGuard('api-key'))
-  async alive(@Res() res: Response): Promise<Response> {
-    const backend = await this.backendsService.alive();
-    return res.status(200).json(backend);
+
+  @Public()
+  @Sse('sse')
+  public async sse(@Res() res: Response, @Query('key') key: string): Promise<Observable<MessageEvent>> {
+    if (key !== 'hZcdVqHScVDsDFdHOdcjmufEKFJVKaS8') throw new UnauthorizedException();
+
+    res.socket.on('close', () => {
+      Logger.debug(`Observer close connection`, this.constructor.name);
+    });
+
+    return new Observable((observer) => {
+      this.backendsService.queueEvents.on('added', (added) =>
+        fireMessage(observer, 'job:added', added, this.constructor.name),
+      );
+
+      this.backendsService.queueEvents.on('completed', (completed) =>
+        fireMessage(observer, 'job:completed', completed, this.constructor.name),
+      );
+
+      this.backendsService.queueEvents.on('failed', (failed) =>
+        fireMessage(observer, 'job:failed', failed, this.constructor.name),
+      );
+    });
   }
 }
diff --git a/src/core/backends/backends.service.ts b/src/core/backends/backends.service.ts
diff --git a/src/core/logger/internal.logger.ts b/src/core/logger/internal.logger.ts
diff --git a/src/main.ts b/src/main.ts
diff --git a/src/management/identities/identities.module.ts b/src/management/identities/identities.module.ts
