Feat: 💫New paradigm to configure the Etl chain.

.github/workflows/php.yml

@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        php-versions: ['8.2', '8.3']
+        php-versions: ['8.3', '8.4']
 
     steps:
       - name: Checkout

.gitignore

@@ -5,3 +5,5 @@ build
 ./composer.lock
 var
 /docs/_site/
+/examples/**.csv
+/examples/00-SimpleCases/*.csv

CHANGELOG.md

@@ -1,5 +1,6 @@
 # 🌟 2.0.0 🌟
 
+- :star2: - **NEW PARADIGM** - use php for configurations instead of yaml (**Breaking Change**)
 - :star2: - Complete rewrite of the ChainProcessor
 - :star2: - Optimized chain processor not to send as many stop items when multiple chains are involved
 - :star2: - Allows ChainProcessor to output through generators the items at the end. This is great to remove all limitations of the current sub chains.

composer.json

@@ -4,20 +4,20 @@
     "license": "MIT",
     "description": "A small etl coded in php, to extract/transform/load Edit with ease.",
     "require": {
-        "php": ">=8.2",
+        "php": ">=8.3",
         "oliverde8/associative-array-simplified": "^1.0",
         "psr/log": "^2.0|^3.0",
-        "symfony/expression-language": "^6.0|^7.0",
-        "symfony/filesystem": "^6.0|^7.0",
-        "symfony/validator": "^6.0|^7.0",
-        "symfony/yaml": "^6.0|^7.0"
+        "symfony/expression-language": "^6.4|^7.4|^8.0",
+        "symfony/filesystem": "^6.4|^7.4|^8.0",
+        "symfony/validator": "^6.4|^7.4|^8.0",
+        "symfony/yaml": "^6.4|^7.4|^8.0"
     },
     "require-dev": {
         "phpunit/php-code-coverage": "^9.2",
         "phpunit/phpunit": "^9.6",
         "rector/rector": "^1.2",
-        "symfony/console": "^7.1",
-        "symfony/http-client": "^7.2"
+        "symfony/console": "^7.0|^8.0",
+        "symfony/http-client": "^6.4|^7.4|^8.0"
     },
     "suggest": {
         "league/flysystem": "For transfaring files from external file systems",

docs/_includes/doc/getting-started/usage-symfony.md

@@ -1,39 +1,260 @@
 #### Creating an ETL chain
 
-Each chain is declared in a single file. The name of the chain is the name of the file created in `/config/etl/`.
+To create an ETL chain in Symfony, you need to create a service that implements `ChainDefinitionInterface`.
+The chain is built using typed PHP configuration objects.
+
+**1. Create a chain definition service:**
+
+```php
+<?php
+
+namespace App\Etl\ChainDefinition;
+
+use Oliverde8\Component\PhpEtl\ChainConfig;
+use Oliverde8\PhpEtlBundle\Etl\ChainDefinitionInterface\ChainDefinitionInterface;
+use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig;
+use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig;
+use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig;
+
+class CustomerImportDefinition implements ChainDefinitionInterface
+{
+    public function getKey(): string
+    {
+        return 'customer-import';
+    }
+
+    public function build(): ChainConfig
+    {
+        return (new ChainConfig())
+            ->addLink(new CsvExtractConfig())
+            ->addLink((new RuleTransformConfig(false))
+                ->addColumn('customer_id', [['get' => ['field' => 'ID']]])
+                ->addColumn('full_name', [
+                    ['implode' => [
+                        'values' => [
+                            [['get' => ['field' => 'FirstName']]],
+                            [['get' => ['field' => 'LastName']]],
+                        ],
+                        'with' => ' ',
+                    ]]
+                ])
+                ->addColumn('email', [['get' => ['field' => 'Email']]])
+            )
+            ->addLink(new CsvFileWriterConfig('output/customers.csv'));
+    }
+}
+```
+
+**2. Register the service** (if not using autoconfigure):
+
+The interface has the `#[AutoconfigureTag('etl.chain_definition')]` attribute, so services implementing it are automatically tagged when autoconfigure is enabled (default in Symfony).
+
+```yaml
+services:
+    App\Etl\ChainDefinition\CustomerImportDefinition:
+        tags: ['etl.chain_definition']
+```
+
+**3. Configure maxAsynchronousItems and other chain settings:**
+
+```php
+class HighVolumeImportDefinition implements ChainDefinitionInterface
+{
+    public function getKey(): string
+    {
+        return 'high-volume-import';
+    }
+
+    public function build(): ChainConfig
+    {
+        $chainConfig = new ChainConfig();
+        $chainConfig->setMaxAsynchronousItems(100); // Process up to 100 items in parallel
+
+        return $chainConfig
+            ->addLink(new CsvExtractConfig())
+            ->addLink((new RuleTransformConfig(false))
+                ->addColumn('id', [['get' => ['field' => 'ID']]])
+            )
+            ->addLink(new CsvFileWriterConfig('output/processed.csv'));
+    }
+}
+```
+
+**4. You can also inject dependencies** into your chain definition:
+
+```php
+class ApiImportDefinition implements ChainDefinitionInterface
+{
+    public function __construct(
+        private string $apiUrl,
+    ) {}
+
+    public function getKey(): string
+    {
+        return 'api-import';
+    }
+
+    public function build(): ChainConfig
+    {
+        return (new ChainConfig())
+            ->addLink(new SimpleHttpConfig(
+                url: $this->apiUrl,
+                method: 'GET',
+                responseIsJson: true
+            ))
+            ->addLink(new LogConfig(
+                message: 'Imported record',
+                level: 'info'
+            ))
+            ->addLink(new CsvFileWriterConfig('output/api-data.csv'));
+    }
+}
+```
+
+#### Creating custom operations
+
+Custom operations are automatically registered when they implement `ConfigurableChainOperationInterface`. The bundle's compiler pass discovers them and sets up dependency injection automatically.
+
+**1. Create a config class:**
+
+```php
+<?php
+
+namespace App\Etl\Config;
+
+use Oliverde8\Component\PhpEtl\OperationConfig\OperationConfigInterface;
+
+class CustomTransformConfig implements OperationConfigInterface
+{
+    public function __construct(
+        public readonly string $targetField,
+        public readonly string $transformation,
+    ) {}
+}
+```
+
+**2. Create the operation:**
+
+```php
+<?php
+
+namespace App\Etl\Operation;
+
+use App\Etl\Config\CustomTransformConfig;
+use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface;
+use Oliverde8\Component\PhpEtl\ChainBuilderV2;
+use Psr\Log\LoggerInterface;
+
+class CustomTransformOperation implements ConfigurableChainOperationInterface
+{
+    public function __construct(
+        private CustomTransformConfig $config,
+        private ChainBuilderV2 $chainBuilder,
+        private string $flavor,
+        private LoggerInterface $logger, // Auto-injected by Symfony
+    ) {}
+
+    public function process(mixed $item, ?array &$output = null, mixed $context = null): void
+    {
+        $this->logger->info('Processing item', ['field' => $this->config->targetField]);
+
+        // Your transformation logic here
+        $item[$this->config->targetField] = strtoupper($item[$this->config->targetField] ?? '');
+
+        $output[] = $item;
+    }
+}
+```
+
+The operation is automatically registered and all dependencies (except `$config`, `$chainBuilder`, and `$flavor`) are auto-injected using Symfony's autowiring.
+
+**3. Use it in your chain:**
+
+```php
+class MyChainDefinition implements ChainDefinitionInterface
+{
+    public function getKey(): string
+    {
+        return 'my-custom-chain';
+    }
+
+    public function build(): ChainConfig
+    {
+        return (new ChainConfig())
+            ->addLink(new CsvExtractConfig())
+            ->addLink(new CustomTransformConfig('email', 'uppercase'))
+            ->addLink(new CsvFileWriterConfig('output/transformed.csv'));
+    }
+}
+```
+
+**How automatic dependency injection works:**
+
+The `ChainBuilderV2Compiler` compiler pass:
+- Discovers all services implementing `ConfigurableChainOperationInterface`
+- Identifies the config class from the constructor (must implement `OperationConfigInterface`)
+- Resolves all constructor dependencies at compile time using Symfony's dependency injection
+- Creates a `GenericChainFactory` for each operation with resolved dependencies
+- Automatically skips injection for `$config`, `$chainBuilder`, and `$flavor` (handled by the factory)
+
+**Manual dependency injection:**
+
+If you need more control over dependency injection, you can configure arguments manually:
 
-Example:
 ```yaml
-chain:
-  "Dummy Step":
-    operation: rule-engine-transformer
-    options:
-      add: true
-      columns:
-        test:
-          rules:
-            - get : {field: [0, 'uid']}
+services:
+    App\Etl\Operation\CustomTransformOperation:
+        arguments:
+            $logger: '@monolog.logger.etl'
 ```
 
+Or use the `#[Autowire]` attribute in PHP 8.1+:
+
+```php
+use Symfony\Component\DependencyInjection\Attribute\Autowire;
+
+class CustomTransformOperation implements ConfigurableChainOperationInterface
+{
+    public function __construct(
+        private CustomTransformConfig $config,
+        private ChainBuilderV2 $chainBuilder,
+        private string $flavor,
+        #[Autowire(service: 'monolog.logger.etl')]
+        private LoggerInterface $logger,
+        #[Autowire('%app.etl.batch_size%')]
+        private int $batchSize,
+    ) {}
+}
+```
+
+The compiler pass respects:
+- Manually configured service arguments
+- `#[Autowire]` attributes on constructor parameters
+- Default parameter values
+- Nullable parameters
+
 #### Executing a chain
 
 ```sh
-./bin/console etl:execute demo '[["test1"],["test2"]]' '{"opt1": "val1"}'
+./bin/console etl:execute customer-import '[["test1"],["test2"]]' '{"opt1": "val1"}'
 ```
 
-The first argument is the input, depending on your chain it can be empty.
-The second are parameters that will be available in the context of each link in the chain.
+The first argument is the chain key (returned by `getKey()`).
+The second argument is the input data, depending on your chain it can be empty or a JSON array.
+The third argument contains parameters that will be available in the execution context.
 
 #### Get a definition
 
 ```sh
-./bin/console etl:get-definition demo
+./bin/console etl:get-definition customer-import
 ```
 
+This displays the chain configuration and all its operations.
+
 #### Get definition graph
 
 ```sh
-./bin/console etl:definition:graph demo
+./bin/console etl:definition:graph customer-import
 ```
 
-This will return a mermaid graph. Adding a `-u` will return the url to the mermaid graph image.
+This returns a Mermaid graph visualization of your ETL chain. Adding `-u` will return the URL to the Mermaid graph image.

docs/_includes/menu.html

@@ -111,6 +111,9 @@
                         <a class="item" href="/doc/10-examples/150-api-csv.html">Api to CSV N°1</a>
                         <a class="item" href="/doc/10-examples/160-api-csv.html">Api to CSV N°2</a>
                         <a class="item" href="/doc/10-examples/170-sub-chains.html">Sub chains</a>
+                        <a class="item" href="/doc/10-examples/180-api-pagination.html">Api with Pagination</a>
+                        <a class="item" href="/doc/10-examples/190-error-handling.md">Error handling</a>
+                        <a class="item" href="/doc/10-examples/200-chain-merge.md">Merge multiple source of data</a>
                     </div>
 
                     <div class="header">