REF WebServices separated request and response

Common/AbstractAPISchemaPrototype.php

@@ -2,8 +2,6 @@
 
 namespace axenox\ETL\Common;
 
-use axenox\ETL\Common\AbstractETLPrototype;
-use axenox\ETL\Common\OpenAPI\OpenAPI3;
 use axenox\ETL\Interfaces\APISchema\APISchemaInterface;
 use axenox\ETL\Interfaces\ETLStepDataInterface;
 use exface\Core\CommonLogic\UxonObject;
@@ -12,7 +10,6 @@
 use exface\Core\Exceptions\InvalidArgumentException;
 use exface\Core\Factories\DataSheetFactory;
 use exface\Core\Interfaces\DataSheets\DataSheetInterface;
-use exface\Core\Widgets\DebugMessage;
 use exface\Core\Interfaces\Tasks\TaskInterface;
 use exface\Core\Interfaces\Tasks\HttpTaskInterface;
 use axenox\ETL\Interfaces\OpenApiFacadeInterface;
@@ -63,27 +60,6 @@ protected function getAPISchema(ETLStepDataInterface $stepData) : APISchemaInter
         return $model;
     }
 
-
-    /**
-     * @param ETLStepDataInterface $stepData
-     * @param array $requestedColumns
-     * @return DataSheetInterface
-     */
-    protected function loadRequestData(ETLStepDataInterface $stepData, array $requestedColumns): DataSheetInterface
-    {
-        $requestLogData = DataSheetFactory::createFromObjectIdOrAlias($this->getWorkbench(), 'axenox.ETL.webservice_request');
-        $requestLogData->getColumns()->addFromSystemAttributes();
-        $requestLogData->getColumns()->addMultiple($requestedColumns);
-        $requestLogData->getFilters()->addConditionFromString('flow_run', $stepData->getFlowRunUid());
-        $requestLogData->dataRead();
-
-        if ($requestLogData->countRows() > 1) {
-            throw new InvalidArgumentException('Ambiguous web requests!');
-        }
-
-        return $requestLogData;
-    }
-
     /**
      * Datasheets from OpenApi JSON data cannot have relations.
      * There are only used for dynamic formulars like =Lookup()

Common/AbstractETLPrototype.php

@@ -6,6 +6,8 @@
 use exface\Core\CommonLogic\DataSheets\CrudCounter;
 use exface\Core\CommonLogic\Debugger\LogBooks\FlowStepLogBook;
 use exface\Core\Exceptions\DataSheets\DataCheckFailedErrorMultiple;
+use exface\Core\Exceptions\InvalidArgumentException;
+use exface\Core\Factories\DataSheetFactory;
 use exface\Core\Interfaces\DataSheets\DataSheetInterface;
 use exface\Core\Interfaces\WorkbenchInterface;
 use exface\Core\CommonLogic\UxonObject;
@@ -441,4 +443,62 @@ public function createDebugWidget(DebugMessage $debug_widget, ?ETLStepDataInterf
         }
         return $this->getLogBook($stepData)->createDebugWidget($debug_widget);
     }
+
+    /**
+     * @param ETLStepDataInterface $stepData
+     * @param array $requestedColumns
+     * @return DataSheetInterface
+     */
+    protected function loadRequestData(ETLStepDataInterface $stepData, array $requestedColumns = []): DataSheetInterface
+    {
+        $requestData = DataSheetFactory::createFromObjectIdOrAlias($this->getWorkbench(), 'axenox.ETL.webservice_request');
+        $requestData->getColumns()->addFromSystemAttributes();
+        $requestData->getColumns()->addMultiple($requestedColumns);
+        $requestData->getFilters()->addConditionFromString('flow_run', $stepData->getFlowRunUid());
+        $requestData->dataRead();
+
+        if ($requestData->countRows() > 1) {
+            throw new InvalidArgumentException('Ambiguous web requests!');
+        }
+
+        return $requestData;
+    }
+
+    /**
+     * @param string $requestUid
+     * @param array  $requestedColumns
+     * @param bool   $createIfNotFound
+     * @return DataSheetInterface
+     */
+    protected function loadResponseData(
+        string $requestUid, 
+        array $requestedColumns = [], 
+        bool $createIfNotFound = true): DataSheetInterface
+    {
+        if(empty($requestUid)) {
+            throw new InvalidArgumentException('Cannot load response: Missing request UID!');
+        }
+
+        $responseData = DataSheetFactory::createFromObjectIdOrAlias($this->getWorkbench(), 'axenox.ETL.webservice_response');
+        $responseData->getColumns()->addFromSystemAttributes();
+        $responseData->getColumns()->addMultiple($requestedColumns);
+        $responseData->getColumns()->addFromExpression('webservice_request');
+        $responseData->getFilters()->addConditionFromString('webservice_request', $requestUid);
+        $responseData->dataRead();
+
+        if ($responseData->countRows() > 1) {
+            throw new InvalidArgumentException('Cannot load response: Ambiguous web requests!');
+        }
+
+        if($createIfNotFound && $responseData->countRows() === 0) {
+            $responseData->addRow([
+                'webservice_request' => $requestUid,
+                'http_response_code' => 200,
+            ]);
+            $responseData->dataCreate();
+            $responseData->dataRead();
+        }
+
+        return $responseData;
+    }
 }

Common/AbstractOpenApiPrototype.php

@@ -2,23 +2,16 @@
 
 namespace axenox\ETL\Common;
 
-use axenox\ETL\Common\AbstractETLPrototype;
-use axenox\ETL\Interfaces\ETLStepDataInterface;
-use axenox\ETL\Interfaces\ETLStepResultInterface;
-use exface\Core\CommonLogic\DataSheets\DataSheet;
 use exface\Core\DataTypes\StringDataType;
 use exface\Core\Exceptions\InvalidArgumentException;
 use exface\Core\Facades\AbstractHttpFacade\Middleware\RouteConfigLoader;
-use exface\Core\Factories\DataSheetFactory;
 use exface\Core\Factories\FormulaFactory;
 use exface\Core\Interfaces\DataSheets\DataSheetInterface;
-use exface\Core\Widgets\DebugMessage;
 use Flow\JSONPath\JSONPath;
 use Flow\JSONPath\JSONPathException;
 use Psr\Http\Message\ServerRequestInterface;
 use exface\Core\Interfaces\Tasks\TaskInterface;
 use exface\Core\Interfaces\Tasks\HttpTaskInterface;
-use exface\Core\Exceptions\RuntimeException;
 use axenox\ETL\Interfaces\OpenApiFacadeInterface;
 
 abstract class AbstractOpenApiPrototype extends AbstractETLPrototype
@@ -129,29 +122,9 @@ function getSchema(ServerRequestInterface $request, string $openApiJson, string
         return json_decode(json_encode($data), true);
     }
 
-
-    /**
-     * @param ETLStepDataInterface $stepData
-     * @param array $requestedColumns
-     * @return DataSheet|DataSheetInterface
-     */
-    protected function loadRequestData(ETLStepDataInterface $stepData, array $requestedColumns): DataSheet|DataSheetInterface
-    {
-        $requestLogData = DataSheetFactory::createFromObjectIdOrAlias($this->getWorkbench(), 'axenox.ETL.webservice_request');
-        $requestLogData->getColumns()->addFromSystemAttributes();
-        $requestLogData->getColumns()->addMultiple($requestedColumns);
-        $requestLogData->getFilters()->addConditionFromString('flow_run', $stepData->getFlowRunUid());
-        $requestLogData->dataRead();
-
-        if ($requestLogData->countRows() > 1) {
-            throw new InvalidArgumentException('Ambiguous web requests!');
-        }
-
-        return $requestLogData;
-    }
-
     /**
-     * Adds additional data provided by the ´additional_rows´ config within the step to given row into the given datasheet.
+     * Adds additional data provided by the ´additional_rows´ config within the step to given row into the given
+     * datasheet.
      *
      * @param DataSheetInterface $dataSheet
      * @param array $placeholder

ETLPrototypes/DataSheetJsonToOpenApi.php

@@ -199,9 +199,13 @@ public function run(ETLStepDataInterface $stepData) : \Generator
             $index = 0;
         }
 
-        $requestLogData = $this->loadRequestData($stepData, ['response_body', 'response_header']);
+        $responseData = $this->loadResponseData(
+            $this->loadRequestData($stepData)->getRow()['UID'], 
+            ['body_file__CONTENTS', 'response_header']
+        );
+
         $request = $stepTask->getHttpRequest();
-        $this->updateRequestData($requestLogData, $request, $openApiJson, $content, $fromObjectSchema[self::OPEN_API_ATTRIBUTE_TO_OBJECT_ALIAS], $placeholders);
+        $this->updateRequestData($responseData, $request, $openApiJson, $content, $fromObjectSchema[self::OPEN_API_ATTRIBUTE_TO_OBJECT_ALIAS], $placeholders);
 
         return $result->setProcessedRowsCounter(count($content));
     }
@@ -310,24 +314,24 @@ protected function createBodyFromSchema(array $responseSchema, array $newContent
     }
 
     /**
-     * @param DataSheetInterface $requestLogData
+     * @param DataSheetInterface     $responseData
      * @param ServerRequestInterface $request
-     * @param string|null $openApiJson
-     * @param array $rows
-     * @param string $objectAlias
-     * @param array $placeholders
+     * @param string|null            $openApiJson
+     * @param array                  $rows
+     * @param string                 $objectAlias
+     * @param array                  $placeholders
      * @return void
      * @throws JSONPathException
      */
     protected function updateRequestData(
-        DataSheetInterface $requestLogData,
+        DataSheetInterface     $responseData,
         ServerRequestInterface $request,
-        ?string $openApiJson,
-        array $rows,
-        string $objectAlias,
-        array $placeholders): void
+        ?string                $openApiJson,
+        array                  $rows,
+        string                 $objectAlias,
+        array                  $placeholders): void
     {
-        $currentBody = json_decode($requestLogData->getCellValue('response_body', 0), true);
+        $currentBody = json_decode($responseData->getCellValue('body_file__CONTENTS', 0), true);
         $jsonPath = '$.paths.[#routePath#].[#methodType#].responses.200.content.[#ContentType#].schema';
         $responseSchema = $this->getSchema($request, $openApiJson, $jsonPath);
 
@@ -339,9 +343,9 @@ protected function updateRequestData(
 
         $newBody = $this->createBodyFromSchema($responseSchema, $rows, $objectAlias, $placeholders);
         $newBody = $currentBody === null ? $newBody : $this->deepMerge($currentBody, $newBody);
-        $requestLogData->setCellValue('response_header', 0, 'application/json');
-        $requestLogData->setCellValue('response_body', 0, json_encode($newBody));
-        $requestLogData->dataUpdate();
+        $responseData->setCellValue('response_header', 0, json_encode(['Content-Type' => 'application/json']));
+        $responseData->setCellValue('body_file__CONTENTS', 0, json_encode($newBody));
+        $responseData->dataUpdate(true);
     }
 
     /**

ETLPrototypes/DataSheetToJsonApi.php

@@ -184,8 +184,12 @@ public function run(ETLStepDataInterface $stepData) : \Generator
             }
         }
 
-        $requestLogData = $this->loadRequestData($stepData, ['response_body', 'response_header']);
-        $this->updateRequestData($requestLogData, $fromObjectSchema, $content, $placeholders);
+        $responseData = $this->loadResponseData(
+            $this->loadRequestData($stepData)->getRow()['UID'], 
+            ['body_file__CONTENTS', 'response_header']
+        );
+
+        $this->updateResponseData($responseData, $fromObjectSchema, $content, $placeholders);
 
         return $result->setProcessedRowsCounter(count($content));
     }
@@ -277,28 +281,25 @@ protected function createBodyFromSchema(array $jsonSchema, array $newContent, st
     }
 
     /**
-     * @param DataSheetInterface $requestLogData
-     * @param ServerRequestInterface $request
-     * @param string|null $openApiJson
-     * @param array $rows
-     * @param string $objectAlias
-     * @param array $placeholders
+     * @param DataSheetInterface       $responseData
+     * @param APIObjectSchemaInterface $objectSchema
+     * @param array                    $rows
+     * @param array                    $placeholders
      * @return void
-     * @throws JSONPathException
      */
-    protected function updateRequestData(
-        DataSheetInterface $requestLogData,
+    protected function updateResponseData(
+        DataSheetInterface       $responseData,
         APIObjectSchemaInterface $objectSchema,
-        array $rows,
-        array $placeholders): void
+        array                    $rows,
+        array                    $placeholders): void
     {
         $responseSchema = $objectSchema->getJsonSchema();
-        $currentBody = json_decode($requestLogData->getCellValue('response_body', 0), true);
+        $currentBody = json_decode($responseData->getCellValue('body_file__CONTENTS', 0), true);
         $newBody = $this->createBodyFromSchema($responseSchema, $rows, $objectSchema->getMetaObject()->getAliasWithNamespace(), $placeholders);
         $newBody = $currentBody === null ? $newBody : $this->deepMerge($currentBody, $newBody);
-        $requestLogData->setCellValue('response_header', 0, 'application/json');
-        $requestLogData->setCellValue('response_body', 0, json_encode($newBody));
-        $requestLogData->dataUpdate();
+        $responseData->setCellValue('response_header', 0, json_encode(['Content-Type' => 'application/json']));
+        $responseData->setCellValue('body_file__CONTENTS', 0, json_encode($newBody));
+        $responseData->dataUpdate();
     }
 
     /**

ETLPrototypes/JsonApiToDataSheet.php

@@ -210,8 +210,8 @@ public function run(ETLStepDataInterface $stepData) : \Generator
 
         $this->getWorkbench()->eventManager()->dispatch(new OnBeforeETLStepRun($this, $logBook));
 
-        $requestLogData = $this->loadRequestData($stepData, ['http_body', 'http_content_type'])->getRow(0);
-        $requestBody = $requestLogData['http_body'];
+        $requestLogData = $this->loadRequestData($stepData, ['body_file__CONTENTS', 'http_content_type'])->getRow(0);
+        $requestBody = $requestLogData['body_file__CONTENTS'];
 
         if ($requestLogData['http_content_type'] !== 'application/json' || $requestBody === null) {
             $logBook->addLine($msg = 'No HTTP content found to process.');

ETLPrototypes/OpenApiJsonToDataSheet.php

@@ -111,8 +111,8 @@ public function run(ETLStepDataInterface $stepData) : \Generator
         $this->baseSheet = $baseSheet;
         $this->getWorkbench()->eventManager()->dispatch(new OnBeforeETLStepRun($this));
 
-        $requestLogData = $this->loadRequestData($stepData, ['http_body', 'http_content_type'])->getRow();
-        $requestBody = json_decode($requestLogData['http_body'], true);
+        $requestLogData = $this->loadRequestData($stepData, ['body_file__CONTENTS', 'http_content_type'])->getRow();
+        $requestBody = json_decode($requestLogData['body_file__CONTENTS'], true);
 
         if ($requestLogData['http_content_type'] !== 'application/json' || $requestBody === null) {
             yield 'No HTTP content found to process' . PHP_EOL;