diff --git a/.github/workflows/php.yml b/.github/workflows/php.yml index 6e5b032..2eda52d 100644 --- a/.github/workflows/php.yml +++ b/.github/workflows/php.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - php-versions: ['8.2', '8.3'] + php-versions: ['8.3', '8.4'] steps: - name: Checkout diff --git a/.gitignore b/.gitignore index fdbb5a5..68e5859 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ build ./composer.lock var /docs/_site/ +/examples/**.csv +/examples/00-SimpleCases/*.csv diff --git a/.phpunit.result.cache b/.phpunit.result.cache new file mode 100644 index 0000000..9fd033b --- /dev/null +++ b/.phpunit.result.cache @@ -0,0 +1 @@ +{"version":1,"defects":{"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainRepeatOperationTest::testRepeatDataItem":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainRepeatOperationTest::testRepeatGroupedItem":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainRepeatOperationTest::testAsyncDisabled":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainRepeatOperationTest::testAsyncEnabled":3,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainSplitOperationTest::testDataProcessing":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainSplitOperationTest::testStopProcess":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\FailSafeOperationTest::testSingleFail":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\FailSafeOperationTest::testToManyFail":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Loader\\FileWriterOperationTest::testWrite":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Loader\\FileWriterOperationTest::testTmpFileDeleted":4,"ChainOperation\\SimpleHttpOperationTest::testTimeout":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\RuleTransformOperationTest::testMultiColumnRulesAddingData":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\RuleTransformOperationTest::testMultiColumnRulesReplacingData":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\RuleTransformOperationTest::testDynamicColumn":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\RuleTransformOperationTest::testMultipleDynamic":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\RuleTransformOperationTest::testSimpleDynamic":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testSimpleChain":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testChainBreak":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testSingleGrouping":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testDoubleGrouping":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testMixItems":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testAsyncItems":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testReturnOfAsyncItems":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testMaxAsyncItems":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testAsyncItemsWithMixItem":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testAsyncDisabledWithSplit":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testException":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testCreateChainWithNoFactories":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testCreateChainWithMaxAsynchronousItems":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testSupportsReturnsTrueForMatchingConfigAndFlavor":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testSupportsReturnsFalseForDifferentConfigClass":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testSupportsReturnsFalseForDifferentFlavor":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testSupportsReturnsTrueForMatchingSubclass":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildCreatesOperationWithConfig":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildInjectsChainBuilder":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildInjectsCustomDependencies":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildInjectsFlavor":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildUsesDefaultParameterValues":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildThrowsExceptionForMissingParameter":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildInjectsMultipleParameters":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildWithNoConstructor":4,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testSupportsChecksFlavorMatch":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testGroupBySingleKey":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testGroupByNestedKey":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testGroupWithIdentifierKey":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testGroupWithNestedIdentifierKey":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testMultipleGroupsWithoutIdentifier":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testDataClearedAfterStop":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testMultipleStopCycles":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testGroupingPreservesAllDataFields":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testOverwriteWithSameIdentifier":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testNumericGroupKeys":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithDefaultDelimiter":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithCustomDelimiter":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithCustomEnclosure":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvFromArrayWithFileKey":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithNestedFileKey":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractEmptyCsv":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithQuotedFields":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testFileExtractedItemContainsCorrectFilename":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractMultipleRows":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithEscapeCharacter":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractFromArrayWithFileKey":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractWithNestedFileKey":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testFileExtractedItemContainsOriginalData":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testFileExtractedItemWithArrayInput":4,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testProcessStopCallsAllChains":3},"times":{"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderTest::testSimpleBuild":0.005,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderTest::testBuildWihMultipleSameOperation":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderTest::testWithVariableOptions":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderTest::testUnknownOperation":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainRepeatOperationTest::testRepeatDataItem":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainRepeatOperationTest::testRepeatGroupedItem":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainRepeatOperationTest::testAsyncDisabled":3.584,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainRepeatOperationTest::testAsyncEnabled":6,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainSplitOperationTest::testDataProcessing":0.006,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainSplitOperationTest::testStopProcess":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\FailSafeOperationTest::testSingleFail":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\FailSafeOperationTest::testToManyFail":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Loader\\FileWriterOperationTest::testWrite":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Loader\\FileWriterOperationTest::testTmpFileDeleted":0.001,"ChainOperation\\SimpleHttpOperationTest::testTimeout":76.017,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\RuleTransformOperationTest::testMultiColumnRulesAddingData":0.003,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\RuleTransformOperationTest::testMultiColumnRulesReplacingData":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\RuleTransformOperationTest::testDynamicColumn":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\RuleTransformOperationTest::testMultipleDynamic":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\RuleTransformOperationTest::testSimpleDynamic":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testSimpleChain":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testChainBreak":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testSingleGrouping":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testDoubleGrouping":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testMixItems":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testAsyncItems":1.931,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testReturnOfAsyncItems":2,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testMaxAsyncItems":4.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testAsyncItemsWithMixItem":1.999,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testAsyncDisabledWithSplit":3.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainProcessorTest::testException":0.002,"Oliverde8\\Component\\RuleEngine\\Tests\\RuleApplierTest::testApplyingSimpleRule":0.002,"Oliverde8\\Component\\RuleEngine\\Tests\\RuleApplierTest::testApplyConstantValue":0,"Oliverde8\\Component\\RuleEngine\\Tests\\RuleApplierTest::testApplyConsecutiveRules":0,"Oliverde8\\Component\\RuleEngine\\Tests\\RuleApplierTest::testApplyConsecutiveRulesWithConstant":0,"Oliverde8\\Component\\RuleEngine\\Tests\\RuleApplierTest::testNoResults":0,"Oliverde8\\Component\\RuleEngine\\Tests\\RuleApplierTest::testUnknownRule":0.001,"Oliverde8\\Component\\RuleEngine\\Tests\\RuleApplierTest::testValidationError":0.001,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\ConstantTest::testConstants":0.001,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\ConstantTest::testRuleCode":0,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\GetTest::testWithoutLocale":0,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\GetTest::testUnfound":0,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\GetTest::testRuleCode":0,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\ImplodeTest::testImplode":0,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\ImplodeTest::testTwoLevelImplode":0,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\ImplodeTest::testFourLevelImplode":0,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\ImplodeTest::testRuleCode":0,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\StrToLowerTest::testStrToLower":0,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\StrToLowerTest::testRuleCode":0,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\StrToUpperTest::testStrToUpper":0,"Oliverde8\\Component\\RuleEngine\\Tests\\Rules\\StrToUpperTest::testRuleCode":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testCreateChainWithNoFactories":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testCreateChainWithSingleOperation":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testCreateChainWithMultipleOperations":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testCreateChainWithMultipleFactories":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testCreateChainWithFactoryNotSupporting":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testCreateChainWithNoMatchingFactory":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testCreateChainWithMaxAsynchronousItems":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testFactoryReceivesChainBuilderInstance":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testCreateChainWithIterableFactories":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainBuilderV2Test::testCreateChainPreservesOperationOrder":0,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testConstructorThrowsExceptionForNonConfigurableOperation":0.002,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testSupportsReturnsTrueForMatchingConfigAndFlavor":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testSupportsReturnsFalseForDifferentConfigClass":0,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testSupportsReturnsFalseForDifferentFlavor":0,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testSupportsReturnsTrueForMatchingSubclass":0,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildCreatesOperationWithConfig":0,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildInjectsChainBuilder":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildInjectsCustomDependencies":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildInjectsFlavor":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildUsesDefaultParameterValues":0,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildThrowsExceptionForMissingParameter":0,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildInjectsMultipleParameters":0,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testBuildWithNoConstructor":0,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testSupportsChecksFlavorMatch":0,"Oliverde8\\Component\\PhpEtl\\Tests\\GenericChainFactoryTest::testConstructorValidatesOperationClass":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testGroupBySingleKey":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testGroupByNestedKey":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testGroupWithIdentifierKey":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testGroupWithNestedIdentifierKey":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testMultipleGroupsWithoutIdentifier":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testEmptyDataReturnsStopItem":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testDataClearedAfterStop":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testMultipleStopCycles":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testGroupingPreservesAllDataFields":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testOverwriteWithSameIdentifier":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Grouping\\SimpleGroupingOperationTest::testNumericGroupKeys":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithDefaultDelimiter":0.005,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithCustomDelimiter":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithCustomEnclosure":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvFromArrayWithFileKey":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithNestedFileKey":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractEmptyCsv":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithQuotedFields":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testFileExtractedItemContainsCorrectFilename":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractMultipleRows":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testExtractCsvWithEscapeCharacter":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\CsvExtractOperationTest::testGetConfigurationClass":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractSimpleJsonArray":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractJsonObject":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractFromArrayWithFileKey":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractWithNestedFileKey":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractEmptyJsonArray":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractNestedJsonStructure":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractJsonWithSpecialCharacters":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testFileExtractedItemContainsOriginalData":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testFileExtractedItemWithArrayInput":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractJsonWithNumericKeys":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractJsonWithBooleanAndNull":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\JsonExtractOperationTest::testExtractLargeJsonArray":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testFindFilesWithSimplePattern":0.001,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testFindFilesWithNoMatches":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testFindFilesWithComplexPattern":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testFindFilesInEmptyDirectory":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testFindAllFilesWithMatchAllPattern":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testFileSystemIsPassedToExternalFileItem":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testExternalFileItemInitialState":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testFindFilesWithCaseInsensitivePattern":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testFindFilesWithNumericPattern":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testDirectoryWithExpression":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testDirectoryExpressionWithComplexPath":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testFindFilesWithSpecialCharactersInName":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Extract\\ExternalFileFinderOperationTest::testMultipleExtensionsPattern":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testProcessNewFileMovesToProcessing":0.005,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testProcessProcessingFileMovesToProcessed":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testProcessNewFileCreatesLocalCopy":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testProcessFileWithNestedDirectory":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testProcessFilePreservesFilename":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testProcessMultipleFilesInSequence":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testNewFileResultContainsDataItemAndExternalFileItem":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testProcessingFileOnlyReturnsExternalFileItem":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testProcessFileCreatesProcessingDirectory":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testProcessFileCreatesProcessedDirectory":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testStateTransitionFromNewToProcessing":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testStateTransitionFromProcessingToProcessed":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\Transformer\\ExternalFileProcessorOperationTest::testFilenameExtraction":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testMergeTwoChains":0.002,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testMergeThreeChains":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testMergeWithMultipleItemsPerChain":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testMergePreservesOrder":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testProcessStopCallsAllChains":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testMergeWithEmptyChain":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testGetChainProcessors":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testMergeWithDifferentDataTypes":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testMergeWithChainTransformations":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testGetConfigurationClass":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testMergeDoesNotStopOnFirstChain":0,"Oliverde8\\Component\\PhpEtl\\Tests\\ChainOperation\\ChainMergeOperationTest::testMergeWithComplexChainOperations":0}} \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1977fba..0000000 --- a/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: php -php: - - '8.1' - - '8.2' - - '8.3'' - -env: - - SYMFONY_DEPRECATIONS_HELPER=disabled - -before_script: - - composer self-update - - composer install - -script: - - ./vendor/bin/phpunit -c phpunit.xml.dist --coverage-clover=coverage.clover - -notifications: - email: false - -after_script: - - wget https://scrutinizer-ci.com/ocular.phar && php ocular.phar code-coverage:upload --format=php-clover coverage.clover diff --git a/CHANGELOG.md b/CHANGELOG.md index cd71985..b31a13c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # šŸŒŸ 2.0.0 šŸŒŸ +- :star2: - **NEW PARADIGM** - use php for configurations instead of yaml (**Breaking Change**) - :star2: - Complete rewrite of the ChainProcessor - :star2: - Optimized chain processor not to send as many stop items when multiple chains are involved - :star2: - Allows ChainProcessor to output through generators the items at the end. This is great to remove all limitations of the current sub chains. diff --git a/composer.json b/composer.json index d7b01ae..42d855d 100644 --- a/composer.json +++ b/composer.json @@ -4,20 +4,20 @@ "license": "MIT", "description": "A small etl coded in php, to extract/transform/load Edit with ease.", "require": { - "php": ">=8.2", + "php": ">=8.3", "oliverde8/associative-array-simplified": "^1.0", "psr/log": "^2.0|^3.0", - "symfony/expression-language": "^6.0|^7.0", - "symfony/filesystem": "^6.0|^7.0", - "symfony/validator": "^6.0|^7.0", - "symfony/yaml": "^6.0|^7.0" + "symfony/expression-language": "^6.4|^7.4|^8.0", + "symfony/filesystem": "^6.4|^7.4|^8.0", + "symfony/validator": "^6.4|^7.4|^8.0", + "symfony/yaml": "^6.4|^7.4|^8.0" }, "require-dev": { "phpunit/php-code-coverage": "^9.2", "phpunit/phpunit": "^9.6", "rector/rector": "^1.2", - "symfony/console": "^7.1", - "symfony/http-client": "^7.2" + "symfony/console": "^7.0|^8.0", + "symfony/http-client": "^6.4|^7.4|^8.0" }, "suggest": { "league/flysystem": "For transfaring files from external file systems", diff --git a/docs/_includes/doc/getting-started/usage-symfony.md b/docs/_includes/doc/getting-started/usage-symfony.md index 2c30ab6..c5a4804 100644 --- a/docs/_includes/doc/getting-started/usage-symfony.md +++ b/docs/_includes/doc/getting-started/usage-symfony.md @@ -1,39 +1,260 @@ #### Creating an ETL chain -Each chain is declared in a single file. The name of the chain is the name of the file created in `/config/etl/`. +To create an ETL chain in Symfony, you need to create a service that implements `ChainDefinitionInterface`. +The chain is built using typed PHP configuration objects. + +**1. Create a chain definition service:** + +```php +addLink(new CsvExtractConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('full_name', [ + ['implode' => [ + 'values' => [ + [['get' => ['field' => 'FirstName']]], + [['get' => ['field' => 'LastName']]], + ], + 'with' => ' ', + ]] + ]) + ->addColumn('email', [['get' => ['field' => 'Email']]]) + ) + ->addLink(new CsvFileWriterConfig('output/customers.csv')); + } +} +``` + +**2. Register the service** (if not using autoconfigure): + +The interface has the `#[AutoconfigureTag('etl.chain_definition')]` attribute, so services implementing it are automatically tagged when autoconfigure is enabled (default in Symfony). + +```yaml +services: + App\Etl\ChainDefinition\CustomerImportDefinition: + tags: ['etl.chain_definition'] +``` + +**3. Configure maxAsynchronousItems and other chain settings:** + +```php +class HighVolumeImportDefinition implements ChainDefinitionInterface +{ + public function getKey(): string + { + return 'high-volume-import'; + } + + public function build(): ChainConfig + { + $chainConfig = new ChainConfig(); + $chainConfig->setMaxAsynchronousItems(100); // Process up to 100 items in parallel + + return $chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'ID']]]) + ) + ->addLink(new CsvFileWriterConfig('output/processed.csv')); + } +} +``` + +**4. You can also inject dependencies** into your chain definition: + +```php +class ApiImportDefinition implements ChainDefinitionInterface +{ + public function __construct( + private string $apiUrl, + ) {} + + public function getKey(): string + { + return 'api-import'; + } + + public function build(): ChainConfig + { + return (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: $this->apiUrl, + method: 'GET', + responseIsJson: true + )) + ->addLink(new LogConfig( + message: 'Imported record', + level: 'info' + )) + ->addLink(new CsvFileWriterConfig('output/api-data.csv')); + } +} +``` + +#### Creating custom operations + +Custom operations are automatically registered when they implement `ConfigurableChainOperationInterface`. The bundle's compiler pass discovers them and sets up dependency injection automatically. + +**1. Create a config class:** + +```php +logger->info('Processing item', ['field' => $this->config->targetField]); + + // Your transformation logic here + $item[$this->config->targetField] = strtoupper($item[$this->config->targetField] ?? ''); + + $output[] = $item; + } +} +``` + +The operation is automatically registered and all dependencies (except `$config`, `$chainBuilder`, and `$flavor`) are auto-injected using Symfony's autowiring. + +**3. Use it in your chain:** + +```php +class MyChainDefinition implements ChainDefinitionInterface +{ + public function getKey(): string + { + return 'my-custom-chain'; + } + + public function build(): ChainConfig + { + return (new ChainConfig()) + ->addLink(new CsvExtractConfig()) + ->addLink(new CustomTransformConfig('email', 'uppercase')) + ->addLink(new CsvFileWriterConfig('output/transformed.csv')); + } +} +``` + +**How automatic dependency injection works:** + +The `ChainBuilderV2Compiler` compiler pass: +- Discovers all services implementing `ConfigurableChainOperationInterface` +- Identifies the config class from the constructor (must implement `OperationConfigInterface`) +- Resolves all constructor dependencies at compile time using Symfony's dependency injection +- Creates a `GenericChainFactory` for each operation with resolved dependencies +- Automatically skips injection for `$config`, `$chainBuilder`, and `$flavor` (handled by the factory) + +**Manual dependency injection:** + +If you need more control over dependency injection, you can configure arguments manually: -Example: ```yaml -chain: - "Dummy Step": - operation: rule-engine-transformer - options: - add: true - columns: - test: - rules: - - get : {field: [0, 'uid']} +services: + App\Etl\Operation\CustomTransformOperation: + arguments: + $logger: '@monolog.logger.etl' ``` +Or use the `#[Autowire]` attribute in PHP 8.1+: + +```php +use Symfony\Component\DependencyInjection\Attribute\Autowire; + +class CustomTransformOperation implements ConfigurableChainOperationInterface +{ + public function __construct( + private CustomTransformConfig $config, + private ChainBuilderV2 $chainBuilder, + private string $flavor, + #[Autowire(service: 'monolog.logger.etl')] + private LoggerInterface $logger, + #[Autowire('%app.etl.batch_size%')] + private int $batchSize, + ) {} +} +``` + +The compiler pass respects: +- Manually configured service arguments +- `#[Autowire]` attributes on constructor parameters +- Default parameter values +- Nullable parameters + #### Executing a chain ```sh -./bin/console etl:execute demo '[["test1"],["test2"]]' '{"opt1": "val1"}' +./bin/console etl:execute customer-import '[["test1"],["test2"]]' '{"opt1": "val1"}' ``` -The first argument is the input, depending on your chain it can be empty. -The second are parameters that will be available in the context of each link in the chain. +The first argument is the chain key (returned by `getKey()`). +The second argument is the input data, depending on your chain it can be empty or a JSON array. +The third argument contains parameters that will be available in the execution context. #### Get a definition ```sh -./bin/console etl:get-definition demo +./bin/console etl:get-definition customer-import ``` +This displays the chain configuration and all its operations. + #### Get definition graph ```sh -./bin/console etl:definition:graph demo +./bin/console etl:definition:graph customer-import ``` -This will return a mermaid graph. Adding a `-u` will return the url to the mermaid graph image. \ No newline at end of file +This returns a Mermaid graph visualization of your ETL chain. Adding `-u` will return the URL to the Mermaid graph image. diff --git a/docs/_includes/menu.html b/docs/_includes/menu.html index e72357b..7a18fbb 100644 --- a/docs/_includes/menu.html +++ b/docs/_includes/menu.html @@ -111,6 +111,9 @@ Api to CSV NĀ°1 Api to CSV NĀ°2 Sub chains + Api with Pagination + Error handling + Merge multiple source of data
diff --git a/docs/doc/01-understand-the-etl/custom-operation.md b/docs/doc/01-understand-the-etl/custom-operation.md index 72ca7a3..03a3b56 100644 --- a/docs/doc/01-understand-the-etl/custom-operation.md +++ b/docs/doc/01-understand-the-etl/custom-operation.md @@ -5,54 +5,201 @@ subTitle: Creating custom operations width: large --- -To create your own operation you need to extend `Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation` +Creating custom operations involves three main components: -In your class you will need to create a method taking in parameter an `ItemInterface` and a `ExecutionContext` +1. A configuration class that holds the operation's parameters +2. An operation class that implements the actual logic +3. Registering the operation with the ChainBuilder + +## Step 1: Create a Configuration Class + +First, create a configuration class that extends `AbstractOperationConfig`. This class holds all the parameters needed for your operation. -**Example** ```php -class MyOperation extends Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation +getData(); + + // Your custom logic here + if ($this->config->myFlag) { + $data[$this->config->myParameter] = 'modified'; + } + + return new DataItem($data); + } } ``` -If you wish your operation to only process certain item types, for example data items, you can change the signature -of your `processItem` method. +### Processing Specific Item Types + +If your operation only processes data items, implement `DataChainOperationInterface` and use the `processData` method as shown above. + +For operations that need to process all item types, you can use the generic `processItem` method: ```php -class MyOperation extends Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation +class MyGenericOperation extends AbstractChainOperation implements ConfigurableChainOperationInterface { - protected function processItem(\Oliverde8\Component\PhpEtl\Item\DataItemInterface $item, \Oliverde8\Component\PhpEtl\Model\ExecutionContext $executionContext): \Oliverde8\Component\PhpEtl\Item\ItemInterface + public function __construct(private readonly MyCustomConfig $config) { - // TODO - return $item; } + protected function processItem(ItemInterface $item, ExecutionContext $context): ItemInterface + { + // Process any item type + return $item; + } } ``` -The name of the method is not important, only the type of the first argument is important. If you wish to process -multiple types of items you can create multiple methods. +### Processing Multiple Item Types + +If you need to handle different item types differently, you can create multiple methods with different type hints: ```php -class MyOperation extends Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation +use Oliverde8\Component\PhpEtl\Item\StopItem; + +class MyMultiTypeOperation extends AbstractChainOperation implements ConfigurableChainOperationInterface { - protected function processDataItem(\Oliverde8\Component\PhpEtl\Item\DataItemInterface $item, \Oliverde8\Component\PhpEtl\Model\ExecutionContext $executionContext): \Oliverde8\Component\PhpEtl\Item\ItemInterface + public function __construct(private readonly MyCustomConfig $config) + { + } + + protected function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface { - // TODO + // Handle data items return $item; } - protected function processStopItem(\Oliverde8\Component\PhpEtl\Item\StopItem $item, \Oliverde8\Component\PhpEtl\Model\ExecutionContext $executionContext): \Oliverde8\Component\PhpEtl\Item\ItemInterface + protected function processStopItem(StopItem $item, ExecutionContext $context): ItemInterface { - // TODO + // Handle stop items return $item; } } ``` + +## Step 3: Register the Operation + +Register your custom operation with the `ChainBuilderV2` using `GenericChainFactory`: + +```php + $myLogger] + ), + ] +); +``` + +## Step 4: Use Your Custom Operation + +Now you can use your custom operation in a chain configuration: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use App\Etl\OperationConfig\MyCustomConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new MyCustomConfig( + myParameter: 'example', + myFlag: true + )) + ->addLink(/* other operations */); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +``` diff --git a/docs/doc/01-understand-the-etl/execution-context.md b/docs/doc/01-understand-the-etl/execution-context.md index aa5e2d1..65045a1 100644 --- a/docs/doc/01-understand-the-etl/execution-context.md +++ b/docs/doc/01-understand-the-etl/execution-context.md @@ -37,6 +37,11 @@ handled for you. {% capture code %} ```php addLink(new SimpleHttpConfig( + method: 'GET', + url: 'https://63b687951907f863aaf90ab1.mockapi.io/test', + responseIsJson: true + )) + ->addLink(new SplitItemConfig( + keys: ['content'], + singleElement: true + )) + ->addLink(new CsvFileWriterConfig('output.csv')); +``` +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +{% capture description %} +Finally, we create the chain processor from our configuration and execute it with the execution context. +The execution is identified with objects of type ExecutionInterface set in the options array: +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\ChainBuilderV2; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\Model\PockExecution; + +// Assuming $chainBuilder is a configured ChainBuilderV2 instance +$chainProcessor = $chainBuilder->createChain($chainConfig); + $options = [ 'etl' => [ 'execution' => new PockExecution(new DateTime()) @@ -63,7 +104,7 @@ $options = [ ]; $chainProcessor->process( - new ArrayIterator([[]]), + new ArrayIterator([new DataItem([])]), $options ); ``` diff --git a/docs/doc/01-understand-the-etl/the-concept.md b/docs/doc/01-understand-the-etl/the-concept.md index 2ebc24d..aa0e7c2 100644 --- a/docs/doc/01-understand-the-etl/the-concept.md +++ b/docs/doc/01-understand-the-etl/the-concept.md @@ -7,43 +7,41 @@ width: large ## Basic Concept of PHP-ETL -PHP-ETL is structured around a series of operations that process items. -Each item represents either a unit of data that can be passed from one operation to the next in a defined chain or the +PHP-ETL is structured around a series of operations that process items. +Each item represents either a unit of data that can be passed from one operation to the next in a defined chain or the result of an action inside the ETL. Each operation in the can define specific item types it will handle (each operation can handle any number of different types of items) and any item that doesnā€™t match an operationā€™s defined input simply skips that step. - ### Item Types Different item types serve distinct purposes in the ETL chain. Here are the basic ones you will need to understand how the etl works. -- **DataItem:** This is the most common item type and is typically used to represent rows of data. +- **DataItem:** This is the most common item type and is typically used to represent rows of data. It can be both input and output in most operations, serving as the primary "payload" of data transformations. -- **ChainBreakItem:** This item type, if returned by an operation, signals that the current chain should stop processing +- **ChainBreakItem:** This item type, if returned by an operation, signals that the current chain should stop processing the current item and move to the next one. Itā€™s mostly used to filter data inside the chain. -- **StopItem:** This item type is automatically created when all items in the chain are processed. -It signals the end of processing, allowing operations to finish any remaining tasks, such as persisting data -or releasing resources. +- **StopItem:** This item type is automatically created when all items in the chain are processed. +It signals the end of processing, allowing operations to finish any remaining tasks, such as persisting data +or releasing resources. StopItems should never be manually created, only utilized when they naturally occur at the end of processing. -- **GroupedItem:** This item encapsulates multiple row of data within an iterator. -When a GroupedItem is encountered, the items it contains are processed as individual DataItems downstream, +- **GroupedItem:** This item encapsulates multiple row of data within an iterator. +When a GroupedItem is encountered, the items it contains are processed as individual DataItems downstream, so a GroupedItem can not be in the input of an operation, they can only be the output. You can find the list of all native item types [here](/doc/01-understand-the-etl/item-types.html). - ### How does it works We will have more detailed real use cases with sample data a bit further in the document. {% capture column1 %} -In the simplest case the chains receive an iterator containing 2 items in input, both items are processed by each chain operation. +In the simplest case the chains receive an iterator containing 2 items in input, both items are processed by each chain operation. This could be for example a list of customer. Each operation changes the items. images/concept-flows {% endcapture %} @@ -55,7 +53,7 @@ images/concept-flows {% include block/divider.html %} {% capture column1 %} -In the following example the iterator sends a single item. The first operation will then send GroupedItems containing 2 items. +In the following example the iterator sends a single item. The first operation will then send GroupedItems containing 2 items. The first item could be a customer, and then we fetch each order of the customer in the operation1. Operation 2 will then receive each order as a DataItem. It will therefore receive 2 GroupedItems, each containing a single order. @@ -69,11 +67,12 @@ Group Items uses iterators, therefore all the orders are not necessarily in memo {% include block/divider.html %} {% capture column1 %} -We can also group items, to make aggregations. The chain receives an iterator containg 2 items, the first operation processes both items. -It breaks the chain for the first item, and returns an aggregation of item1 & item 2. +We can also group items, to make aggregations. The chain receives an iterator containg 2 items, the first operation processes both items. +It breaks the chain for the first item, and returns an aggregation of item1 & item 2. -This can be used to: -- count the number of customers. +This can be used to: + +- count the number of customers. - to get customers grouped by country. - to create chunks and improving performance while inserting items in a database. - ... @@ -91,6 +90,7 @@ This can be used to: Chains can also be split, this would allow 2 different operations to be executed on the same item. This can be usefull to : + - store data in 2 different places. {% endcapture %} @@ -104,30 +104,31 @@ This can be usefull to : The split operations is among the building blocks of complex executions. There are additional operations to merge multiple branches or to repeat a part of the chain. - ## Example: Simple CSV Transformation -To demonstrate PHP-ETLā€™s capabilities, letā€™s walk through a basic example where we read multiple CSV files, +To demonstrate PHP-ETLā€™s capabilities, letā€™s walk through a basic example where we read multiple CSV files, modify each line, and output a new CSV file with selected columns containing the rows of all the files. ### Step 1: Read the Input CSV File {% capture code %} -```yaml -chain: - read-file: - operation: csv-read - options: [] + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; + +$chainConfig->addLink(new CsvExtractConfig()); ``` + {% endcapture %} {% capture description %} -The first step is reading the input CSV. PHP-ETL can handle large volumes of data efficiently by processing each row individually, +The first step is reading the input CSV. PHP-ETL can handle large volumes of data efficiently by processing each row individually, allowing memory-efficient transformations. -This operation reads the input file line by line, it does this by: -- Creating an iterator that will iterate on the csv file. -- Returning a GroupedItem containing this iterator. +This operation reads the input file line by line, it does this by: + +- Creating an iterator that will iterate on the csv file. +- Returning a GroupedItem containing this iterator. The internals in the ETL will then transform each line returned by the iterator into a DataItem and send it to the next step. While this step will return a single `GroupedItem` the next step will receive as many `DataItem`'s as the @@ -139,39 +140,47 @@ lines of the csv file. ### Step 2: Transform the Data {% capture code %} -```yaml -chain: - keep-only-name-and-subscription: - operation: rule-engine-transformer - options: - # Replace existing columns with transformed columns. - add: false - columns: - Name: - rules: - - implode: # Concatenate FirstName and LastName - values: - - [{get : {field: 'FirstName'}}] - - [{get : {field: 'LastName'}}] - with: " " - SubscriptionStatus: - rules: - - get: {field: 'IsSubscribed'} + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; + +$chainConfig->addLink(new RuleTransformConfig( + columns: [ + 'Name' => [ + 'rules' => [ + ['implode' => [ + 'values' => [ + [['get' => ['field' => 'FirstName']]], + [['get' => ['field' => 'LastName']]] + ], + 'with' => ' ' + ]] + ] + ], + 'SubscriptionStatus' => [ + 'rules' => [ + ['get' => ['field' => 'IsSubscribed']] + ] + ] + ], + add: false // Replace existing columns with transformed columns +)); ``` + {% endcapture %} {% capture description %} -Weā€™ll use a **rule-engine-transformer** operation to transform the data. This operation allows us to define custom +We'll use a **RuleTransformConfig** operation to transform the data. This operation allows us to define custom transformations for each column. - In this example: + - **Name** is created by concatenating FirstName and LastName with a space. - **SubscriptionStatus** copies the IsSubscribed field without any transformation. {% endcapture %} Let's also note that `add: false` is used to replace existing columns with transformed columns. Without -this option, the operation would add new columns while keeping the original ones. This mean FirstName, LastName +this option, the operation would add new columns while keeping the original ones. This mean FirstName, LastName and IsSubscribed would remain in the output. {% include block/etl-step.html code=code description=description %} @@ -179,18 +188,18 @@ and IsSubscribed would remain in the output. ### Step 3: Write the Output CSV File {% capture code %} -```yaml -chain: - write-new-file: - operation: csv-write - options: - file: "output.csv" + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig->addLink(new CsvFileWriterConfig('output.csv')); ``` + {% endcapture %} {% capture description %} -The transformed data is then written to a new CSV file using the csv-write operation. This operation -will write each line one by one but close the file only when the etl finishes. +The transformed data is then written to a new CSV file using the CsvFileWriterConfig operation. This operation +will write each line one by one but close the file only when the etl finishes. > ##### TIP > We will dive later into how this operations should be used in a context of exporting data. Indeed when exporting @@ -203,26 +212,47 @@ will write each line one by one but close the file only when the etl finishes. ### Step 4: Run the Chain Processor -The chain processor is then initialized with the input file(s). +The chain processor is created from the configuration and then initialized with the input file(s). {% capture column1 %} + #### šŸ˜ Standalone -For instance, the following code will process two input files and merge their output: + +For instance, the following code will create the chain and process two input files, merging their output: ```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +// Create the chain configuration +$chainConfig = new ChainConfig(); +// ... add operations as shown above + +// Create the chain processor from the configuration +$chainProcessor = $chainBuilder->createChain($chainConfig); + +// Process the files $chainProcessor->process( - new ArrayIterator([__DIR__ . "/customers.csv", __DIR__ . "/customers2.csv"]), + new ArrayIterator([ + new DataItem(['file' => __DIR__ . "/customers.csv"]), + new DataItem(['file' => __DIR__ . "/customers2.csv"]) + ]), [] ); ``` + {% endcapture %} {% capture column2 %} + #### šŸŽµ Symfony + For instance, the following command will process two input files and merge their output: + ```bash ./bin/console etl:execute myetl "['./customers1.csv', './customers2.csv']" ``` + {% endcapture %} {% include block/2column.html column1=column1 column2=column2 %} @@ -230,4 +260,3 @@ For instance, the following command will process two input files and merge their ### Output The output of this ETL will be a output.csv file containing all customer data from files customers1.csv and customers2.csv. - diff --git a/docs/doc/10-examples/100-grouping.md b/docs/doc/10-examples/100-grouping.md index c988ad7..8109b08 100644 --- a/docs/doc/10-examples/100-grouping.md +++ b/docs/doc/10-examples/100-grouping.md @@ -12,16 +12,18 @@ We will write this in json as its more suited to understand what we are doing. Let's start by reading our csv file {% endcapture %} {% capture code %} -```yaml - read-file: - operation: csv-read - options: [] # The default delimeter + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; + +$chainConfig->addLink(new CsvExtractConfig()); ``` + {% endcapture %} {% include block/etl-step.html code=code description=description %} {% capture description %} -We will use the `simple-grouping` operation for this. **This operation needs to put all the data in memory +We will use the `SimpleGroupingConfig` operation for this. **This operation needs to put all the data in memory and should therefore be used with caution.** We have a single **grouping-key**, we can make more complex grouping operations, by grouping by subscription status and @@ -31,48 +33,62 @@ Grouping identifier allows us to remove duplicates, if we had customer emails we that information for example. {% endcapture %} {% capture code %} -```yaml -group-per-subscription: - operation: simple-grouping - options: - grouping-key: ['IsSubscribed'] - group-identifier: [] + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Grouping\SimpleGroupingConfig; + +$chainConfig->addLink(new SimpleGroupingConfig( + groupingKey: ['IsSubscribed'], + groupIdentifier: [] +)); ``` + {% endcapture %} {% include block/etl-step.html code=code description=description %} {% capture description %} -We will also use json write operation. +We will also use the JSON file writer operation. This works like the csv file, but is more suited for complex multi level datas as we have after the grouping. {% endcapture %} {% capture code %} -```yaml -write-new-file: - operation: json-write - options: - file: "output.json" + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig->addLink(new CsvFileWriterConfig( + file: 'output.json', + fileFormat: 'json' +)); ``` + {% endcapture %} {% include block/etl-step.html code=code description=description %} -### Complete yaml +## Complete Configuration -```yaml -chain: - read-file: - operation: csv-read - options: [] # The default delimeter +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Grouping\SimpleGroupingConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; - group-per-subscription: - operation: simple-grouping - options: - grouping-key: ['IsSubscribed'] - group-identifier: [] +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new SimpleGroupingConfig( + groupingKey: ['IsSubscribed'], + groupIdentifier: [] + )) + ->addLink(new CsvFileWriterConfig( + file: 'output.json', + fileFormat: 'json' + )); - write-new-file: - operation: json-write - options: - file: "output.json" - -``` \ No newline at end of file +// Create and execute the chain +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'customers.csv'])]), + [] +); +``` diff --git a/docs/doc/10-examples/110-filtering.md b/docs/doc/10-examples/110-filtering.md index 4e4b461..5611f98 100644 --- a/docs/doc/10-examples/110-filtering.md +++ b/docs/doc/10-examples/110-filtering.md @@ -7,42 +7,49 @@ width: large {% capture description %} We can also filter data preventing some of it from being propagated through all the chain, in our example -it will prevent unsubscribed customers from being written in our final csv file. So we can add this opertion to our +it will prevent unsubscribed customers from being written in our final csv file. So we can add this operation to our chain. -The rule engine is used for the filtering, If the rule returns false, 0, empty string or null then the item **will not -be propagated**. We can also inverse this rule, but changing `negate: true`, in this case the rule needs to return +The rule engine is used for the filtering. If the rule returns false, 0, empty string or null then the item **will not +be propagated**. We can also inverse this rule by setting `negate: true`, in this case the rule needs to return false for the item **to be propagated**. This might seem limiting but the rule engine does support SymfonyExpressions which opens a whole lot of flexibility. {% endcapture %} {% capture code %} -```yaml -filter-unsubscribed: - operation: filter - options: - rule: [{get : {field: 'IsSubscribed'}}] + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; + +$chainConfig->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], negate: false +)); ``` + {% endcapture %} {% include block/etl-step.html code=code description=description %} -### Complete yaml - -```yaml -chain: - read-file: - operation: csv-read - options: [] # The default delimeter,& - - filter-unsubscribed: - operation: filter - options: - rule: [{get : {field: 'IsSubscribed'}}] - negate: false - - write-new-file: - operation: csv-write - options: - file: "output.csv" - -``` \ No newline at end of file +## Complete Configuration + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], + negate: false + )) + ->addLink(new CsvFileWriterConfig('output.csv')); + +// Create and execute the chain +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'customers.csv'])]), + [] +); +``` diff --git a/docs/doc/10-examples/120-split-the-chain.md b/docs/doc/10-examples/120-split-the-chain.md index d36820a..647990b 100644 --- a/docs/doc/10-examples/120-split-the-chain.md +++ b/docs/doc/10-examples/120-split-the-chain.md @@ -5,7 +5,8 @@ subTitle: Split/Fork the chain width: large --- -In our next example +In our next example + - One file containing all the customers - A second file containing unsubscribed customers - A third file with subscribed customers. @@ -20,80 +21,90 @@ first example. But before writing the files we will add a split operation to cre filter to get subscribed customers and write them. The second branch will filter to get un subscribed customers and write them. {% endcapture %} + {% capture code %} -```yaml - branch-out: - operation: split - options: - branches: - - - filter-unsubscribed: - operation: filter - options: - rule: [{get : {field: 'IsSubscribed'}}] - negate: false - - write-new-file: - operation: csv-write - options: - file: "subscribed.csv" - - - filter-subscribed: - operation: filter - options: - rule: [{get : {field: 'IsSubscribed'}}] - negate: true - - write-new-file: - operation: csv-write - options: - file: "unsubscribed.csv" + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainSplitConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +// First branch: subscribed customers +$subscribedBranch = new ChainConfig(); +$subscribedBranch + ->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], + negate: false + )) + ->addLink(new CsvFileWriterConfig('subscribed.csv')); + +// Second branch: unsubscribed customers +$unsubscribedBranch = new ChainConfig(); +$unsubscribedBranch + ->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], + negate: true + )) + ->addLink(new CsvFileWriterConfig('unsubscribed.csv')); + +// Add split operation with both branches +$chainConfig->addLink(new ChainSplitConfig( + branches: [$subscribedBranch, $unsubscribedBranch] +)); ``` + {% endcapture %} {% include block/etl-step.html code=code description=description %} ### Complete Code {% capture column1 %} -```yaml -chain: - read-file: - operation: csv-read - options: [] # The default delimeter,& - - branch-out: - operation: split - options: - branches: - - - filter-subscribed: - operation: filter - options: - rule: [{get : {field: 'IsSubscribed'}}] - negate: false - - write-new-file: - operation: csv-write - options: - file: "subscribed.csv" - - - filter-unsubscribed: - operation: filter - options: - rule: [{get : {field: 'IsSubscribed'}}] - negate: true - - write-new-file: - operation: csv-write - options: - file: "unsubscribed.csv" - - write-new-file: - operation: csv-write - options: - file: "output.csv" +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainSplitConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig->addLink(new CsvExtractConfig()); + +// First branch: subscribed customers +$subscribedBranch = new ChainConfig(); +$subscribedBranch + ->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], + negate: false + )) + ->addLink(new CsvFileWriterConfig('subscribed.csv')); + +// Second branch: unsubscribed customers +$unsubscribedBranch = new ChainConfig(); +$unsubscribedBranch + ->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], + negate: true + )) + ->addLink(new CsvFileWriterConfig('unsubscribed.csv')); + +// Add split operation +$chainConfig->addLink(new ChainSplitConfig( + branches: [$subscribedBranch, $unsubscribedBranch] +)); + +// Write all customers to main output +$chainConfig->addLink(new CsvFileWriterConfig('output.csv')); + +// Create and execute the chain +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'customers.csv'])]), + [] +); ``` + {% endcapture %} {% capture mermaid %} flowchart TB @@ -106,7 +117,6 @@ S2 -->|UnSubscribed| S2A1 S2 -->|Susbscribed| S2B1(Filter UnSubscribed) S2 -->|UnSubscribed| S2B1 - subgraph SubFlow S2A1 -->|Susbscribed| S2A2(Write Subscribed) @@ -116,8 +126,9 @@ end S2 --->|Susbscribed| S3(Write Both) S2 --->|UnSubscribed| S3 {% endcapture %} + {% capture column2 %} {% include block/mermaid.html mermaid=mermaid %} {% endcapture %} -{% include block/2column.html column1=column1 column2=column2 %} \ No newline at end of file +{% include block/2column.html column1=column1 column2=column2 %} diff --git a/docs/doc/10-examples/130-configuration.md b/docs/doc/10-examples/130-configuration.md index 5b7dcbd..263e09a 100644 --- a/docs/doc/10-examples/130-configuration.md +++ b/docs/doc/10-examples/130-configuration.md @@ -9,95 +9,138 @@ width: large You are able to configure through the input the names of the files that are being read. {% endcapture %} {% capture code %} + ```php +use Oliverde8\Component\PhpEtl\Item\DataItem; + $chainProcessor->process( - new ArrayIterator([__DIR__ . "/customers.csv"]), + new ArrayIterator([new DataItem(['file' => __DIR__ . "/customers.csv"])]), [] ); ``` + {% endcapture %} {% include block/etl-step.html code=code description=description %} {% capture description %} But we might need to configure some operations independently from the input. For example the name of the csv output file. {% endcapture %} + {% capture code %} -```yaml -write-new-file: - operation: csv-write - options: - file: "output.csv" + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig->addLink(new CsvFileWriterConfig('output.csv')); ``` + {% endcapture %} {% include block/etl-step.html code=code description=description %} {% capture description %} -The name "output.csv" is hardcoded here. But we can make this dynamic with symfony expression language. We will need -to start our line with the `!` character. +The name "output.csv" is hardcoded here. But we can make this dynamic with PHP variables. Simply define your +configuration parameters and use them when creating the operation configs. {% endcapture %} + {% capture code %} -```yaml -write-new-file: - operation: csv-write - options: - file: "!filewriter['outputfile']['name']" + +```php +// Define configuration +$outputFileName = 'configured-output.csv'; + +// Use in configuration +$chainConfig->addLink(new CsvFileWriterConfig($outputFileName)); ``` + {% endcapture %} {% include block/etl-step.html code=code description=description %} -We will also need to give this informaton when the chain is being created: +You can pass configuration values from external sources (environment variables, config files, etc.) and use them when building your chain: {% capture column1 %} -#### šŸ˜ Standalone + +## šŸ˜ Standalone + ```php -$inputOptions = ['filewriter' => - ['outputfile' => - ['name' => 'configured-output.csv'] - ] +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +// Get configuration from your source +$config = [ + 'outputFileName' => 'configured-output.csv' ]; -$chainProcessor = $builder->buildChainProcessor( - Yaml::parse(file_get_contents($fileName))['chain'], - $inputOptions +// Build the chain with configuration +$chainConfig = new ChainConfig(); +// ... add operations using $config values +$chainConfig->addLink( + new CsvFileWriterConfig($config['outputFileName']) +); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => './customers.csv'])]), + [] ); ``` + {% endcapture %} + {% capture column2 %} -#### šŸŽµ Symfony + +## šŸŽµ Symfony + ```sh ./bin/console etl:execute myetl "['./customers.csv']" "{'outputfile': {'name': 'configured-output.csv'}}" ``` + {% endcapture %} {% include block/2column.html column1=column1 column2=column2 %} ### Complete Code -```yaml -chain: - read-file: - operation: csv-read - options: [] # The default delimeter,& - - keep-only-name-and-subscription: - operation: rule-engine-transformer - options: - add: false # We want to replace all existing columns with our new columns. - columns: - Name: - rules: - - implode: # Concat both firstname & lastname - values: - - [{get : {field: 'FirstName'}}] - - [{get : {field: "LastName"}}] - with: " " - SubscriptionStatus: - rules: - - get: {field: 'IsSubscribed'} - - write-new-file: - operation: csv-write - options: - file: "!filewriter['outputfile']['name']" - -``` \ No newline at end of file +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +// Configuration from your source (env, config file, etc.) +$config = [ + 'outputFileName' => 'configured-output.csv' +]; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new RuleTransformConfig( + columns: [ + 'Name' => [ + 'rules' => [ + ['implode' => [ + 'values' => [ + [['get' => ['field' => 'FirstName']]], + [['get' => ['field' => 'LastName']]] + ], + 'with' => ' ' + ]] + ] + ], + 'SubscriptionStatus' => [ + 'rules' => [ + ['get' => ['field' => 'IsSubscribed']] + ] + ] + ], + add: false + )) + ->addLink(new CsvFileWriterConfig($config['outputFileName'])); + +// Create and execute the chain +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'customers.csv'])]), + [] +); +``` diff --git a/docs/doc/10-examples/140-complex-data.md b/docs/doc/10-examples/140-complex-data.md index 0cad06e..29093d7 100644 --- a/docs/doc/10-examples/140-complex-data.md +++ b/docs/doc/10-examples/140-complex-data.md @@ -6,15 +6,16 @@ width: large --- {% capture description %} -Complex `json` files/api responses can be flattened and have multiple columns using the rule engine. -In our example we have a list of products with their name, their skus etc. +Complex `json` files/api responses can be flattened and have multiple columns using the rule engine. +In our example we have a list of products with their name, their skus etc. The name of the product is different for each locale. We could manually create a list of columns for each locale using the rule engine, but this will not be very generic, and if we have a lot of locales & a lot of translatable fields on our products this will be complicated to maintain. {% endcapture %} {% capture code %} -#### Example products file + +## Example products file ```json [ @@ -44,6 +45,7 @@ and if we have a lot of locales & a lot of translatable fields on our products t } ] ``` + {% endcapture %} {% include block/etl-step.html code=code description=description %} @@ -51,14 +53,18 @@ and if we have a lot of locales & a lot of translatable fields on our products t We can use dynamic columns for this purpose. To use this we will need to list locales when starting the process: {% endcapture %} {% capture code %} + ```php +use Oliverde8\Component\PhpEtl\Item\DataItem; + $chainProcessor->process( - new ArrayIterator([__DIR__ . "/products.json"]), + new ArrayIterator([new DataItem(['file' => __DIR__ . "/products.json"])]), [ 'locales' => ['fr_FR', 'en_US'] ] ); ``` + {% endcapture %} {% include block/etl-step.html code=code description=description %} @@ -70,41 +76,64 @@ data from that product. We could also have used symfony expression language but given locale is missing. `get` will simply return an empty column, symfony expression language rule will fail. {% endcapture %} {% capture code %} -```yaml - 'name-{@context/locales}': - rules: - - get : {field: ['name', '@context/locales']} + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; + +$chainConfig->addLink(new RuleTransformConfig( + columns: [ + 'name-{@context/locales}' => [ + 'rules' => [ + ['get' => ['field' => ['name', '@context/locales']]] + ] + ] + ] +)); ``` + {% endcapture %} {% include block/etl-step.html code=code description=description %} ### Complete Code -```yaml -chain: - read-file: - operation: json-read - options: [] - - flatten: - operation: rule-engine-transformer - options: - add: false # We want to replace all existing columns with our new columns. - columns: - productId: - rules: - - get: {field: 'productId'} - sku: - rules: - - get: {field: 'sku'} - 'name-{@context/locales}': - rules: - - get : {field: ['name', '@context/locales']} - - write-new-file: - operation: csv-write - options: - file: "output.csv" +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\JsonExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; -``` +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new JsonExtractConfig()) + ->addLink(new RuleTransformConfig( + columns: [ + 'productId' => [ + 'rules' => [ + ['get' => ['field' => 'productId']] + ] + ], + 'sku' => [ + 'rules' => [ + ['get' => ['field' => 'sku']] + ] + ], + 'name-{@context/locales}' => [ + 'rules' => [ + ['get' => ['field' => ['name', '@context/locales']]] + ] + ] + ], + add: false + )) + ->addLink(new CsvFileWriterConfig('output.csv')); +// Create and execute the chain +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => __DIR__ . '/products.json'])]), + [ + 'locales' => ['fr_FR', 'en_US'] + ] +); +``` diff --git a/docs/doc/10-examples/150-api-csv.md b/docs/doc/10-examples/150-api-csv.md index e7e77aa..692dd7f 100644 --- a/docs/doc/10-examples/150-api-csv.md +++ b/docs/doc/10-examples/150-api-csv.md @@ -5,30 +5,30 @@ subTitle: Api to CSV width: large --- -### Simple API to CSV - The php etl also provides a basic http client operation, this operation will allow us to get or push data using rest api's. {% capture description %} Let's call a mock api returning a list of users. -Using `response_is_json` allow us to decode the json returned by the api automatically. `option_key` will allow us to +Using `responseIsJson` allows us to decode the json returned by the api automatically. `optionKey` will allow us to pass additional options to the query. This can be used to add dynamic headers, or data that needs to be posted. -If `response_key` is set that the response data will be added to the original data object. If not the response will +If `responseKey` is set then the response data will be added to the original data object. If not, the response will replace the input data. {% endcapture %} {% capture code %} -```yaml - get-from-api: - operation: http - options: - url: https://63b687951907f863aaf90ab1.mockapi.io/test - method: GET - response_is_json: true - option_key: ~ - response_key: ~ - options: - headers: {'Accept': 'application/json'} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; + +$chainConfig->addLink(new SimpleHttpConfig( + url: 'https://63b687951907f863aaf90ab1.mockapi.io/test', + method: 'GET', + responseIsJson: true, + optionKey: null, + responseKey: null, + options: [ + 'headers' => ['Accept' => 'application/json'] + ] +)); ``` {% endcapture %} {% include block/etl-step.html code=code description=description %} @@ -38,12 +38,13 @@ This will return a single DataItem with all the users of the api. We will need t each users individually. {% endcapture %} {% capture code %} -```yaml - split-item: - operation: split-item - options: - keys: ['content'] - singleElement: true +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; + +$chainConfig->addLink(new SplitItemConfig( + keys: ['content'], + singleElement: true +)); ``` {% endcapture %} {% include block/etl-step.html code=code description=description %} @@ -54,29 +55,37 @@ Now we can write the users into the csv file, as we have done so in our previous #### Complete Code -```yaml -chain: - get-from-api: - operation: http - options: - url: https://63b687951907f863aaf90ab1.mockapi.io/test - method: GET - response_is_json: true - option_key: ~ - response_key: ~ - options: - headers: {'Accept': 'application/json'} - - split-item: - operation: split-item - options: - keys: ['content'] - singleElement: true - - write-new-file: - operation: csv-write - options: - file: "output.csv" +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new SimpleHttpConfig( + url: 'https://63b687951907f863aaf90ab1.mockapi.io/test', + method: 'GET', + responseIsJson: true, + optionKey: null, + responseKey: null, + options: [ + 'headers' => ['Accept' => 'application/json'] + ] + )) + ->addLink(new SplitItemConfig( + keys: ['content'], + singleElement: true + )) + ->addLink(new CsvFileWriterConfig('output.csv')); + +// Create and execute the chain +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem([])]), + [] +); ``` @@ -89,27 +98,29 @@ user id. {% capture description %} We can achieve this by using symfony expressions in the url key. To tell the operation that a symfony expression is being used just prefix it with a `@`. (`!` is for using values from the input, @ is for using data from the current data. -All fields do not support `@` as it's handled by each operation. but all fields support `!` as it's generated before the etl starts +All fields do not support `@` as it's handled by each operation, but all fields support `!` as it's generated before the etl starts processing). -We will also change the `option_key`, if not our data (id = 1), will be sent into the options of the HttpClient, which +We will also change the `optionKey`, if not our data (id = 1), will be sent into the options of the HttpClient, which will cause an error. Having an invalid key here will allow us not to have any options. Let us note that this operation runs multiple queries with concurrency. A single Symfony HttpClient is created for this operation. And using the AsyncItems functionality of the ETL, we can run all the http requests in parallel. {% endcapture %} {% capture code %} -```yaml - get-from-api: - operation: http - options: - url: '@"https://63b687951907f863aaf90ab1.mockapi.io/test/"~data["id"]' - method: GET - response_is_json: true - option_key: "-placeholder-" - response_key: ~ - options: - headers: {'Accept': 'application/json'} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; + +$chainConfig->addLink(new SimpleHttpConfig( + url: '@"https://63b687951907f863aaf90ab1.mockapi.io/test/"~data["id"]', + method: 'GET', + responseIsJson: true, + optionKey: '-placeholder-', + responseKey: null, + options: [ + 'headers' => ['Accept' => 'application/json'] + ] +)); ``` {% endcapture %} {% include block/etl-step.html code=code description=description %} @@ -120,40 +131,56 @@ Now we can write the users into the csv file, as we have done so in our previous #### Complete Code -```yaml -chain: - get-from-api: - operation: http - options: - url: '@"https://63b687951907f863aaf90ab1.mockapi.io/test/"~data["id"]' - method: GET - response_is_json: true - option_key: "-placeholder-" - response_key: ~ - options: - headers: {'Accept': 'application/json'} - - content-only: - operation: rule-engine-transformer - options: - add: false # We want to replace all existing columns with our new columns. - columns: - createdAt: - rules: - - get: {field: ['content','createdAt']} - name: - rules: - - get: {field: ['content','name']} - avatar: - rules: - - get: {field: ['content','avatar']} - id: - rules: - - get: {field: ['content','id']} - - write-new-file: - operation: csv-write - options: - file: "output.csv" +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new SimpleHttpConfig( + url: '@"https://63b687951907f863aaf90ab1.mockapi.io/test/"~data["id"]', + method: 'GET', + responseIsJson: true, + optionKey: '-placeholder-', + responseKey: null, + options: [ + 'headers' => ['Accept' => 'application/json'] + ] + )) + ->addLink(new RuleTransformConfig( + columns: [ + 'createdAt' => [ + 'rules' => [ + ['get' => ['field' => ['content', 'createdAt']]] + ] + ], + 'name' => [ + 'rules' => [ + ['get' => ['field' => ['content', 'name']]] + ] + ], + 'avatar' => [ + 'rules' => [ + ['get' => ['field' => ['content', 'avatar']]] + ] + ], + 'id' => [ + 'rules' => [ + ['get' => ['field' => ['content', 'id']]] + ] + ] + ], + add: false + )) + ->addLink(new CsvFileWriterConfig('output.csv')); + +// Create and execute the chain +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['id' => 1])]), + [] +); ``` - diff --git a/docs/doc/10-examples/160-api-csv.md b/docs/doc/10-examples/160-api-csv.md index 88dec37..684ef4c 100644 --- a/docs/doc/10-examples/160-api-csv.md +++ b/docs/doc/10-examples/160-api-csv.md @@ -14,27 +14,29 @@ user id. {% capture description %} We can achieve this by using symfony expressions in the url key. To tell the operation that a symfony expression is being used just prefix it with a `@`. (`!` is for using values from the input, @ is for using data from the current data. -All fields do not support `@` as it's handled by each operation. but all fields support `!` as it's generated before the etl starts +All fields do not support `@` as it's handled by each operation, but all fields support `!` as it's generated before the etl starts processing). -We will also change the `option_key`, if not our data (id = 1), will be sent into the options of the HttpClient, which +We will also change the `optionKey`, if not our data (id = 1), will be sent into the options of the HttpClient, which will cause an error. Having an invalid key here will allow us not to have any options. Let us note that this operation runs multiple queries with concurrency. A single Symfony HttpClient is created for this operation. And using the AsyncItems functionality of the ETL, we can run all the http requests in parallel. {% endcapture %} {% capture code %} -```yaml - get-from-api: - operation: http - options: - url: '@"https://63b687951907f863aaf90ab1.mockapi.io/test/"~data["id"]' - method: GET - response_is_json: true - option_key: "-placeholder-" - response_key: ~ - options: - headers: {'Accept': 'application/json'} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; + +$chainConfig->addLink(new SimpleHttpConfig( + url: '@"https://63b687951907f863aaf90ab1.mockapi.io/test/"~data["id"]', + method: 'GET', + responseIsJson: true, + optionKey: '-placeholder-', + responseKey: null, + options: [ + 'headers' => ['Accept' => 'application/json'] + ] +)); ``` {% endcapture %} {% include block/etl-step.html code=code description=description %} @@ -45,39 +47,56 @@ Now we can write the users into the csv file, as we have done so in our previous #### Complete Code -```yaml -chain: - get-from-api: - operation: http - options: - url: '@"https://63b687951907f863aaf90ab1.mockapi.io/test/"~data["id"]' - method: GET - response_is_json: true - option_key: "-placeholder-" - response_key: ~ - options: - headers: {'Accept': 'application/json'} +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; - content-only: - operation: rule-engine-transformer - options: - add: false # We want to replace all existing columns with our new columns. - columns: - createdAt: - rules: - - get: {field: ['content','createdAt']} - name: - rules: - - get: {field: ['content','name']} - avatar: - rules: - - get: {field: ['content','avatar']} - id: - rules: - - get: {field: ['content','id']} +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new SimpleHttpConfig( + url: '@"https://63b687951907f863aaf90ab1.mockapi.io/test/"~data["id"]', + method: 'GET', + responseIsJson: true, + optionKey: '-placeholder-', + responseKey: null, + options: [ + 'headers' => ['Accept' => 'application/json'] + ] + )) + ->addLink(new RuleTransformConfig( + columns: [ + 'createdAt' => [ + 'rules' => [ + ['get' => ['field' => ['content', 'createdAt']]] + ] + ], + 'name' => [ + 'rules' => [ + ['get' => ['field' => ['content', 'name']]] + ] + ], + 'avatar' => [ + 'rules' => [ + ['get' => ['field' => ['content', 'avatar']]] + ] + ], + 'id' => [ + 'rules' => [ + ['get' => ['field' => ['content', 'id']]] + ] + ] + ], + add: false + )) + ->addLink(new CsvFileWriterConfig('output.csv')); - write-new-file: - operation: csv-write - options: - file: "output.csv" +// Create and execute the chain +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['id' => 1])]), + [] +); ``` diff --git a/docs/doc/10-examples/170-sub-chains.md b/docs/doc/10-examples/170-sub-chains.md index 95c144e..fa2af3e 100644 --- a/docs/doc/10-examples/170-sub-chains.md +++ b/docs/doc/10-examples/170-sub-chains.md @@ -16,145 +16,184 @@ and one with those not subscribed. We do not do any additional process to change Let's now imagine we would like to extract only the firstName and Lastname from the csv file for the subscribed customers. The resulting chain would look like: -```yaml - branch-out: - operation: split - options: - branches: - - - filter-unsubscribed: - operation: filter - options: - rule: [{get : {field: 'IsSubscribed'}}] - negate: false - transform: - operation: rule-engine-transformer - options: - add: false # We want to replace all existing columns with our new columns. - columns: - FirstName: - rules: - - get : {field: 'FirstName'} - LastName: - rules: - - get : {field: "LastName"} - write-new-file: - operation: csv-write - options: - file: "subscribed.csv" - - - filter-subscribed: - operation: filter - options: - rule: [{get : {field: 'IsSubscribed'}}] - negate: true - - write-new-file: - operation: csv-write - options: - file: "unsubscribed.csv" +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainSplitConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +// First branch: subscribed customers with transformation +$subscribedBranch = new ChainConfig(); +$subscribedBranch + ->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], + negate: false + )) + ->addLink(new RuleTransformConfig( + columns: [ + 'FirstName' => [ + 'rules' => [['get' => ['field' => 'FirstName']]] + ], + 'LastName' => [ + 'rules' => [['get' => ['field' => 'LastName']]] + ] + ], + add: false + )) + ->addLink(new CsvFileWriterConfig('subscribed.csv')); + +// Second branch: unsubscribed customers +$unsubscribedBranch = new ChainConfig(); +$unsubscribedBranch + ->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], + negate: true + )) + ->addLink(new CsvFileWriterConfig('unsubscribed.csv')); + +$chainConfig->addLink(new ChainSplitConfig( + branches: [$subscribedBranch, $unsubscribedBranch] +)); ``` -In order to do the same for both subscribed & unsubscribed customer we would need to duplicate the whole `transform` -operation. That would be quite inefficient. Also this is a very simple case, if we wanted to add grouping and more +In order to do the same for both subscribed & unsubscribed customer we would need to duplicate the whole transformation +config. That would be quite inefficient. Also this is a very simple case, if we wanted to add grouping and more transforms it makes the amount of duplications even more important. -The subChain can be used in such cases: +With the new v2 paradigm, we can use reusable ChainConfig objects (functions or factory methods) to avoid duplication: {% capture description %} -We can create such a subchain that will make the necessary transformations. +We can create a function that returns a configured ChainConfig for the necessary transformations. {% endcapture %} {% capture code %} -```yaml -subChains: - customTransform: - chain: - - - operation: rule-engine-transformer - options: - add: false # We want to replace all existing columns with our new columns. - columns: - FirstName: - rules: - - get : {field: 'FirstName'} - LastName: - rules: - - get : {field: "LastName"} +```php +function createCustomTransform(): ChainConfig +{ + $config = new ChainConfig(); + $config->addLink(new RuleTransformConfig( + columns: [ + 'FirstName' => [ + 'rules' => [['get' => ['field' => 'FirstName']]] + ], + 'LastName' => [ + 'rules' => [['get' => ['field' => 'LastName']]] + ] + ], + add: false + )); + return $config; +} ``` {% endcapture %} {% include block/etl-step.html code=code description=description %} {% capture description %} -We can use this operation anywhere within our chain +We can then use this function to create transformation chains for both branches, avoiding duplication. {% endcapture %} {% capture code %} -```yaml - transform: - operation: subchain - options: - name: customTransform +```php +// First branch: subscribed customers +$subscribedBranch = new ChainConfig(); +$subscribedBranch + ->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], + negate: false + )); +// Add the reusable transformation +foreach (createCustomTransform()->getLinks() as $link) { + $subscribedBranch->addLink($link); +} +$subscribedBranch->addLink(new CsvFileWriterConfig('subscribed.csv')); + +// Second branch: unsubscribed customers +$unsubscribedBranch = new ChainConfig(); +$unsubscribedBranch + ->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], + negate: true + )); +// Add the same reusable transformation +foreach (createCustomTransform()->getLinks() as $link) { + $unsubscribedBranch->addLink($link); +} +$unsubscribedBranch->addLink(new CsvFileWriterConfig('unsubscribed.csv')); ``` {% endcapture %} {% include block/etl-step.html code=code description=description %} -**The following rules applies for subchains:** -- Sub chains can have multiple operations as with a normal chain. -- Operation for subchains are cloned, so a grouping operation will not share memory. Unless option; shared is true. -- subchains can use subchains, so it's possible to have multiple levels of subchains. +**The following benefits apply for reusable ChainConfig objects:** +- Reusable chains can have multiple operations as with a normal chain. +- You can use PHP functions, factory methods, or classes to create reusable configurations. +- Reusable chains can themselves use other reusable chains, enabling multiple levels of composition. +- Full IDE support with autocompletion and type checking. +- Easy to test and refactor using standard PHP patterns. #### Complete Code -```yaml -subChains: - customTransform: - chain: - generic-subchain-transformation: - operation: rule-engine-transformer - options: - add: false # We want to replace all existing columns with our new columns. - columns: - FirstName: - rules: - - get : {field: 'FirstName'} - LastName: - rules: - - get : {field: "LastName"} - -chain: - read-file: - operation: csv-read - options: [] # The default delimeter,& - branch-out: - operation: split - options: - branches: - - - filter-unsubscribed: - operation: filter - options: - rule: [{get : {field: 'IsSubscribed'}}] - negate: false - transform: - operation: subchain - options: - name: customTransform - write-new-file: - operation: csv-write - options: - file: "subscribed.csv" - - - filter-subscribed: - operation: filter - options: - rule: [{get : {field: 'IsSubscribed'}}] - negate: true - transform: - operation: subchain - options: - name: customTransform - write-new-file: - operation: csv-write - options: - file: "unsubscribed.csv" +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainSplitConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +// Reusable transformation function +function createCustomTransform(): ChainConfig +{ + $config = new ChainConfig(); + $config->addLink(new RuleTransformConfig( + columns: [ + 'FirstName' => [ + 'rules' => [['get' => ['field' => 'FirstName']]] + ], + 'LastName' => [ + 'rules' => [['get' => ['field' => 'LastName']]] + ] + ], + add: false + )); + return $config; +} + +// Main chain configuration +$chainConfig = new ChainConfig(); +$chainConfig->addLink(new CsvExtractConfig()); + +// First branch: subscribed customers +$subscribedBranch = new ChainConfig(); +$subscribedBranch->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], + negate: false +)); +foreach (createCustomTransform()->getLinks() as $link) { + $subscribedBranch->addLink($link); +} +$subscribedBranch->addLink(new CsvFileWriterConfig('subscribed.csv')); + +// Second branch: unsubscribed customers +$unsubscribedBranch = new ChainConfig(); +$unsubscribedBranch->addLink(new FilterDataConfig( + rules: [['get' => ['field' => 'IsSubscribed']]], + negate: true +)); +foreach (createCustomTransform()->getLinks() as $link) { + $unsubscribedBranch->addLink($link); +} +$unsubscribedBranch->addLink(new CsvFileWriterConfig('unsubscribed.csv')); + +// Add split operation with both branches +$chainConfig->addLink(new ChainSplitConfig( + branches: [$subscribedBranch, $unsubscribedBranch] +)); + +// Create and execute the chain +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'customers.csv'])]), + [] +); ``` \ No newline at end of file diff --git a/docs/doc/10-examples/180-api-pagination.md b/docs/doc/10-examples/180-api-pagination.md new file mode 100644 index 0000000..b5e57af --- /dev/null +++ b/docs/doc/10-examples/180-api-pagination.md @@ -0,0 +1,222 @@ +--- +layout: base +title: PHP-ETL - Cook Books +subTitle: API Pagination with ChainRepeat +width: large +--- + +### API Pagination with ChainRepeat + +A very common use case when working with APIs is pagination. Most APIs return data in pages, requiring multiple +requests to fetch all data. The `ChainRepeatConfig` operation is designed specifically for this scenario. + +## How ChainRepeat Works + +The `ChainRepeat` operation executes a sub-chain repeatedly until a validation condition returns false. This makes it +perfect for: +- Fetching all pages from a paginated API +- Retrying failed operations +- Processing data in iterative batches + +### Basic Pagination Example + +{% capture description %} +Let's fetch data from a paginated API. The key is to use a validation expression that checks if there's a next page. +The repeat chain will execute until `hasNextPage` becomes false. + +We use a `CallBackTransformerConfig` to simulate API calls, but in a real scenario, you would use `SimpleHttpConfig`. +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\ChainRepeatConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; +use Oliverde8\Component\PhpEtl\Item\DataItemInterface; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +$page = 1; + +$repeatConfig = new ChainRepeatConfig( + chainConfig: (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItemInterface $dataItem) use (&$page) { + echo "Fetching page {$page}...\n"; + + // Simulate API response + $response = [ + 'items' => [...], // Your data items + 'page' => $page, + 'hasNextPage' => $page < 5, // Continue until page 5 + ]; + + $page++; + return new DataItem($response); + })), + validationExpression: 'data["hasNextPage"] == true' +); + +$chainConfig->addLink($repeatConfig); +``` +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Real API Pagination Example + +{% capture description %} +Here's a more realistic example using actual HTTP requests with cursor-based pagination (common with modern APIs). +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; + +// Prepare initial data with API URL +$initialData = new DataItem([ + 'cursor' => null, + 'hasMore' => true, +]); + +$repeatConfig = new ChainRepeatConfig( + chainConfig: (new ChainConfig()) + // Make API call with cursor + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/items?cursor="~(data["cursor"] ?? "")', + method: 'GET', + responseIsJson: true, + responseKey: 'apiResponse' + )) + // Extract pagination info from response + ->addLink(new CallBackTransformerConfig(function(DataItemInterface $item) { + $data = $item->getData(); + $response = $data['apiResponse']; + + return new DataItem([ + 'items' => $response['data'], + 'cursor' => $response['pagination']['next_cursor'] ?? null, + 'hasMore' => !empty($response['pagination']['next_cursor']), + ]); + })), + validationExpression: 'data["hasMore"] == true' +); +``` +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Processing Paginated Results + +{% capture description %} +After fetching all pages, we need to process the accumulated data. Use `SplitItemConfig` to split the items array +into individual DataItems for downstream processing. +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig + ->addLink($repeatConfig) + // Split the items array into individual items + ->addLink(new SplitItemConfig( + keys: ['items'], + singleElement: false + )) + // Now each item is processed individually + ->addLink(new CsvFileWriterConfig('all-items.csv')); +``` +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Page Number Based Pagination + +{% capture description %} +Some APIs use traditional page numbers instead of cursors. Here's how to handle that pattern: +{% endcapture %} +{% capture code %} +```php +$page = 1; +$pageSize = 100; + +$repeatConfig = new ChainRepeatConfig( + chainConfig: (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/items?page="~data["page"]~"&limit="~data["pageSize"]', + method: 'GET', + responseIsJson: true, + responseKey: 'apiResponse' + )) + ->addLink(new CallBackTransformerConfig(function(DataItemInterface $item) use (&$page, $pageSize) { + $data = $item->getData(); + $response = $data['apiResponse']; + $totalItems = $response['total']; + + $currentPage = $page++; + $hasMore = ($currentPage * $pageSize) < $totalItems; + + return new DataItem([ + 'items' => $response['items'], + 'page' => $page, + 'pageSize' => $pageSize, + 'hasMore' => $hasMore, + ]); + })), + validationExpression: 'data["hasMore"] == true' +); +``` +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Complete Example + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainRepeatConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\Item\DataItemInterface; + +$chainConfig = new ChainConfig(); + +$page = 1; +$totalPages = 5; + +// Create a repeat configuration for pagination +$repeatConfig = new ChainRepeatConfig( + chainConfig: (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItemInterface $dataItem) use (&$page, $totalPages) { + echo "Fetching page {$page}/{$totalPages}...\n"; + + // Simulate API response with paginated data + $items = []; + for ($i = 1; $i <= 10; $i++) { + $itemId = (($page - 1) * 10) + $i; + $items[] = [ + 'id' => $itemId, + 'name' => "Item {$itemId}", + 'page' => $page, + ]; + } + + $hasNextPage = $page < $totalPages; + $page++; + + return new DataItem([ + 'items' => $items, + 'hasNextPage' => $hasNextPage, + ]); + })), + validationExpression: 'data["hasNextPage"] == true', + allowAsynchronous: false +); + +$chainConfig + ->addLink($repeatConfig) + ->addLink(new SplitItemConfig(keys: ['items'])) + ->addLink(new CsvFileWriterConfig('paginated-results.csv')); + +// Create and execute the chain +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem([])]), + [] +); +``` diff --git a/docs/doc/10-examples/190-error-handling.md b/docs/doc/10-examples/190-error-handling.md new file mode 100644 index 0000000..486da37 --- /dev/null +++ b/docs/doc/10-examples/190-error-handling.md @@ -0,0 +1,319 @@ +--- +layout: base +title: PHP-ETL - Cook Books +subTitle: Error Handling with FailSafe +width: large +--- + +### Error Handling with FailSafe + +In production ETL pipelines, errors are inevitable. Network issues, temporary API failures, database locks, or +invalid data can all cause operations to fail. The `FailSafeConfig` operation provides error handling +and automatic retry logic to make your ETL chains resilient. + +## How FailSafe Works + +The `FailSafe` operation wraps a chain of operations and: +- **Catches specified exceptions** and retries the operation +- **Limits retry attempts** to prevent infinite loops +- **Continues processing** other items even if one fails +- **Logs failures** for monitoring and debugging + +This is critical for production use where you want graceful degradation instead of complete failure. + +### Basic Error Handling + +{% capture description %} +Let's start with a simple example that retries a failing operation. This simulates a transient error (like a +temporary network issue) that succeeds after a few retries. +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +// Chain that might fail +$failingChain = (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + static $attempt = 0; + $attempt++; + + if ($attempt < 3) { + throw new RuntimeException('Transient error, please retry'); + } + + // Success on 3rd attempt + return $item; + })); + +// Wrap it in FailSafe +$failSafeConfig = new FailSafeConfig( + chainConfig: $failingChain, + exceptionsToCatch: [RuntimeException::class], + nbAttempts: 5 +); + +$chainConfig->addLink($failSafeConfig); +``` +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### API Retry Pattern + +{% capture description %} +A common use case is retrying failed API calls. This handles temporary network issues, rate limits, or server errors. +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; + +// API call that might fail +$apiChain = (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: 'https://api.example.com/data', + method: 'GET', + responseIsJson: true + )); + +// Retry up to 3 times on any exception +$failSafeConfig = new FailSafeConfig( + chainConfig: $apiChain, + exceptionsToCatch: [\Exception::class], // Catch all exceptions + nbAttempts: 3 +); + +$chainConfig->addLink($failSafeConfig); +``` +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Catching Specific Exceptions + +{% capture description %} +You can specify which exceptions to catch and retry. This allows different handling for different types of errors. +For example, retry network errors but not validation errors. +{% endcapture %} +{% capture code %} +```php +use Symfony\Component\HttpClient\Exception\TransportException; + +// Only retry on network/transport errors +$failSafeConfig = new FailSafeConfig( + chainConfig: $apiChain, + exceptionsToCatch: [ + TransportException::class, // Network errors + \RuntimeException::class, // Runtime issues + ], + nbAttempts: 3 +); + +// ValidationException would NOT be caught and would fail immediately +``` +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Continue Processing on Failure + +{% capture description %} +One of the most powerful features of FailSafe is that it allows the chain to continue processing other items +even if one fails. This is critical for batch processing where you don't want a single bad record to stop +the entire job. +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$processingChain = (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + $data = $item->getData(); + + // Some items might fail validation or processing + if (empty($data['email'])) { + throw new \InvalidArgumentException('Email is required'); + } + + return $item; + })) + ->addLink(new CsvFileWriterConfig('valid-records.csv')); + +// Failed items are skipped, but processing continues +$failSafeConfig = new FailSafeConfig( + chainConfig: $processingChain, + exceptionsToCatch: [\InvalidArgumentException::class], + nbAttempts: 1 // Don't retry validation errors +); + +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink($failSafeConfig); +``` +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Logging Failed Items + +{% capture description %} +For production systems, you'll want to log which items failed so you can investigate or reprocess them later. +{% endcapture %} +{% capture code %} +```php +$failedItems = []; + +$processingChain = (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) use (&$failedItems) { + $data = $item->getData(); + + try { + // Your processing logic + processData($data); + return $item; + } catch (\Exception $e) { + // Log the failure + $failedItems[] = [ + 'data' => $data, + 'error' => $e->getMessage(), + 'timestamp' => date('Y-m-d H:i:s') + ]; + throw $e; // Re-throw to trigger FailSafe + } + })); + +$failSafeConfig = new FailSafeConfig( + chainConfig: $processingChain, + exceptionsToCatch: [\Exception::class], + nbAttempts: 3 +); + +// After processing, write failed items to a separate file +file_put_contents('failed-items.json', json_encode($failedItems, JSON_PRETTY_PRINT)); +``` +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Exponential Backoff for Retries + +{% capture description %} +For API calls, you often want to wait between retries with increasing delays (exponential backoff). +This is especially important when dealing with rate limits. +{% endcapture %} +{% capture code %} +```php +$apiChain = (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + static $attempt = 0; + $attempt++; + + // Exponential backoff: 1s, 2s, 4s, 8s + if ($attempt > 1) { + $delay = pow(2, $attempt - 2); + echo "Waiting {$delay} seconds before retry...\n"; + sleep($delay); + } + + // Make API call + // ... + + return $item; + })) + ->addLink(new SimpleHttpConfig( + url: 'https://api.example.com/data', + method: 'POST', + responseIsJson: true + )); + +$failSafeConfig = new FailSafeConfig( + chainConfig: $apiChain, + exceptionsToCatch: [\Exception::class], + nbAttempts: 5 +); +``` +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Complete Example + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +$chainConfig = new ChainConfig(); + +// Read customer data +$chainConfig->addLink(new CsvExtractConfig()); + +// Enrich with API data (with retry logic) +$apiEnrichmentChain = (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + static $attempt = 0; + $attempt++; + + $data = $item->getData(); + echo "Fetching data for customer {$data['id']} (attempt {$attempt})...\n"; + + // Simulate occasional failures + if (rand(1, 10) > 7 && $attempt < 3) { + throw new RuntimeException('API timeout'); + } + + return $item; + })) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/customer/"~data["id"]', + method: 'GET', + responseIsJson: true, + responseKey: 'apiData' + )); + +$chainConfig->addLink(new FailSafeConfig( + chainConfig: $apiEnrichmentChain, + exceptionsToCatch: [RuntimeException::class], + nbAttempts: 3 +)); + +// Write enriched data +$chainConfig->addLink(new CsvFileWriterConfig('enriched-customers.csv')); + +// Create and execute the chain +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'customers.csv'])]), + [] +); +``` + +### Best Practices + +**1. Choose Appropriate Retry Counts** + +- Transient errors (network): 3-5 attempts +- Rate limits: 5-10 attempts with backoff +- Validation errors: 1 attempt (don't retry) + +**2. Catch Specific Exceptions** + +- Only catch exceptions you know can be retried +- Let validation errors fail fast +- Different retry strategies for different error types + + +**3. Add Delays Between Retries** + +- Use exponential backoff for APIs +- Respect rate limits +- Don't overwhelm failing services + +### Common Use Cases + +- **API Integration**: Retry failed HTTP requests due to network issues +- **Database Operations**: Retry on deadlocks or connection issues +- **File Processing**: Handle temporary file system errors +- **Batch Processing**: Skip invalid records while continuing with valid ones +- **External Services**: Handle service unavailability gracefully diff --git a/docs/doc/10-examples/200-chain-merge.md b/docs/doc/10-examples/200-chain-merge.md new file mode 100644 index 0000000..4bd3ad8 --- /dev/null +++ b/docs/doc/10-examples/200-chain-merge.md @@ -0,0 +1,435 @@ +--- +layout: base +title: PHP-ETL - Cook Books +subTitle: ChainMerge - Combining Multiple Data Sources +width: large +--- + +### ChainMerge - Combining Multiple Data Sources + +The `ChainMergeConfig` operation allows you to process a single data item through multiple parallel chains and +combine all the results. This is powerful for enriching data from multiple sources, creating different views +of the same data, or denormalizing complex datasets. + +## How ChainMerge Works + +ChainMerge takes one input item and: +- **Processes it through multiple chains in parallel** +- **Each chain can transform the data differently** +- **All results are returned to the next step** in the main chain +- **Order is preserved** - items from each branch are returned in order + +āš ļø **Important**: If branches don't filter items, the same item will appear multiple times in the output. +This is by design and allows for powerful data splitting and denormalization patterns. + +### Basic Data Enrichment + +{% capture description %} +The simplest use case is to create different views of the same data. For example, extracting different +fields from a customer record into separate output items. +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainMergeConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink((new ChainMergeConfig()) + // Branch 1: Extract customer contact info + ->addMerge((new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('full_name', [ + ['implode' => [ + 'values' => [ + [['get' => ['field' => 'FirstName']]], + [['get' => ['field' => 'LastName']]], + ], + 'with' => ' ', + ]] + ]) + ->addColumn('email', [['get' => ['field' => 'Email']]]) + ) + ) + // Branch 2: Extract customer subscription status + ->addMerge((new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('status', [['get' => ['field' => 'subscribed']]]) + ->addColumn('subscription_date', [['get' => ['field' => 'created_at']]]) + ) + ) + ) + ->addLink(new CsvFileWriterConfig('enriched-data.csv')); +``` + +**Result**: Each customer record becomes 2 items - one with contact info, one with subscription info. +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Combining Data from Multiple APIs + +{% capture description %} +A common pattern is enriching data by calling multiple APIs in parallel. Each branch fetches different +information about the same entity. +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) // Read customer IDs + ->addLink((new ChainMergeConfig()) + // Branch 1: Get customer profile from API + ->addMerge((new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/customers/"~data["customer_id"]', + method: 'GET', + responseIsJson: true, + responseKey: 'profile' + )) + ) + // Branch 2: Get customer orders from API + ->addMerge((new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/orders?customer="~data["customer_id"]', + method: 'GET', + responseIsJson: true, + responseKey: 'orders' + )) + ) + // Branch 3: Get customer preferences from API + ->addMerge((new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/preferences/"~data["customer_id"]', + method: 'GET', + responseIsJson: true, + responseKey: 'preferences' + )) + ) + ) + ->addLink(new CsvFileWriterConfig('customer-full-data.csv')); +``` + +**Result**: Each customer becomes 3 items containing profile, orders, and preferences data. +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Conditional Data Routing + +{% capture description %} +You can use filters in branches to route data to different outputs based on conditions. Combined with +ChainMerge, this allows sophisticated data splitting. +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink((new ChainMergeConfig()) + // Branch 1: High-value customers (> $1000) + ->addMerge((new ChainConfig()) + ->addLink(new FilterDataConfig('@data["total_spent"] > 1000')) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('segment', [['const' => 'premium']]) + ->addColumn('total_spent', [['get' => ['field' => 'total_spent']]]) + ) + ->addLink(new CsvFileWriterConfig('premium-customers.csv')) + ) + // Branch 2: Regular customers + ->addMerge((new ChainConfig()) + ->addLink(new FilterDataConfig('@data["total_spent"] <= 1000')) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('segment', [['const' => 'regular']]) + ->addColumn('total_spent', [['get' => ['field' => 'total_spent']]]) + ) + ->addLink(new CsvFileWriterConfig('regular-customers.csv')) + ) + // Branch 3: All customers to main file + ->addMerge((new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('name', [['get' => ['field' => 'name']]]) + ) + ) + ) + ->addLink(new CsvFileWriterConfig('all-customers.csv')); +``` + +**Result**: Premium and regular customers go to separate files, and all customers go to a main file. +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Denormalizing Hierarchical Data + +{% capture description %} +ChainMerge is excellent for denormalizing complex JSON or hierarchical data into flat structures. +For example, converting a product with multiple variants into separate rows. +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\JsonExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; + +// Input: {"product_id": 123, "name": "T-Shirt", "variants": ["Small", "Medium", "Large"]} + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new JsonExtractConfig()) + ->addLink((new ChainMergeConfig()) + // Branch 1: Create row for product base info + ->addMerge((new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('product_id', [['get' => ['field' => 'product_id']]]) + ->addColumn('name', [['get' => ['field' => 'name']]]) + ->addColumn('type', [['const' => 'base']]) + ) + ) + // Branch 2: Create row for each variant + ->addMerge((new ChainConfig()) + ->addLink(new SplitItemConfig(path: 'variants')) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('product_id', [['get' => ['field' => 'product_id']]]) + ->addColumn('name', [['get' => ['field' => 'name']]]) + ->addColumn('variant', [['get' => ['field' => 'data']]]) // Split item data + ->addColumn('type', [['const' => 'variant']]) + ) + ) + ) + ->addLink(new CsvFileWriterConfig('products-denormalized.csv')); +``` + +**Result**: One base product row + one row for each variant (4 rows total). +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Database and API Combination + +{% capture description %} +Combine data from a database with real-time API enrichment. This is useful when you have master data +in a database but need current information from external services. +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) // Customer IDs from database export + ->addLink((new ChainMergeConfig()) + // Branch 1: Local customer data (fast) + ->addMerge((new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) use ($pdo) { + $id = $item->getData()['customer_id']; + $stmt = $pdo->prepare('SELECT * FROM customers WHERE id = ?'); + $stmt->execute([$id]); + $data = $stmt->fetch(PDO::FETCH_ASSOC); + + return new DataItem(array_merge(['source' => 'database'], $data)); + })) + ) + // Branch 2: Real-time credit score from external API (slower) + ->addMerge((new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://credit-api.example.com/score/"~data["customer_id"]', + method: 'GET', + responseIsJson: true + )) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'customer_id']]]) + ->addColumn('credit_score', [['get' => ['field' => 'score']]]) + ->addColumn('source', [['const' => 'credit_api']]) + ) + ) + ) + ->addLink(new CsvFileWriterConfig('customer-enriched.csv')); +``` + +**Result**: Each customer gets 2 rows - one with database data, one with API credit score. +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Error Isolation per Branch + +{% capture description %} +Wrap each branch in a FailSafe to handle errors independently. If one data source fails, others can +still succeed. +{% endcapture %} +{% capture code %} +```php +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink((new ChainMergeConfig()) + // Branch 1: Critical API (must succeed) + ->addMerge((new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api1.example.com/data/"~data["id"]', + method: 'GET', + responseIsJson: true, + responseKey: 'api1_data' + )) + ) + // Branch 2: Optional API (can fail gracefully) + ->addMerge((new FailSafeConfig( + chainConfig: (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api2.example.com/data/"~data["id"]', + method: 'GET', + responseIsJson: true, + responseKey: 'api2_data' + )), + exceptionsToCatch: [\Exception::class], + nbAttempts: 2 + ))) + // Branch 3: Another optional API + ->addMerge((new FailSafeConfig( + chainConfig: (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api3.example.com/data/"~data["id"]', + method: 'GET', + responseIsJson: true, + responseKey: 'api3_data' + )), + exceptionsToCatch: [\Exception::class], + nbAttempts: 2 + ))) + ) + ->addLink(new CsvFileWriterConfig('merged-data.csv')); +``` + +**Result**: If API2 or API3 fail, you still get data from API1 and the working APIs. +{% endcapture %} +{% include block/etl-step.html code=code description=description %} + +### Complete Example: Multi-Source Customer Enrichment + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\ChainBuilderV2; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainMergeConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); + +// Extract customer IDs +$chainConfig->addLink(new CsvExtractConfig()); + +// Merge data from multiple sources +$chainConfig->addLink((new ChainMergeConfig()) + + // Branch 1: Basic customer info (always include) + ->addMerge((new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('full_name', [ + ['implode' => [ + 'values' => [ + [['get' => ['field' => 'FirstName']]], + [['get' => ['field' => 'LastName']]], + ], + 'with' => ' ', + ]] + ]) + ->addColumn('email', [['get' => ['field' => 'Email']]]) + ->addColumn('data_source', [['const' => 'basic_info']]) + ) + ) + + // Branch 2: Subscription status (with retry) + ->addMerge(new FailSafeConfig( + chainConfig: (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/subscriptions/"~data["ID"]', + method: 'GET', + responseIsJson: true + )) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('subscription_tier', [['get' => ['field' => 'tier']]]) + ->addColumn('subscription_status', [['get' => ['field' => 'status']]]) + ->addColumn('data_source', [['const' => 'subscription_api']]) + ), + exceptionsToCatch: [\Exception::class], + nbAttempts: 3 + )) + + // Branch 3: Purchase history (with retry) + ->addMerge(new FailSafeConfig( + chainConfig: (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/purchases/"~data["ID"]', + method: 'GET', + responseIsJson: true + )) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('total_purchases', [['get' => ['field' => 'total']]]) + ->addColumn('last_purchase_date', [['get' => ['field' => 'last_date']]]) + ->addColumn('data_source', [['const' => 'purchase_api']]) + ), + exceptionsToCatch: [\Exception::class], + nbAttempts: 3 + )) + + // Branch 4: Calculated metrics + ->addMerge((new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + $data = $item->getData(); + + // Calculate customer lifetime value + $lifetimeValue = calculateLifetimeValue($data); + $segment = $lifetimeValue > 1000 ? 'premium' : 'regular'; + + return new DataItem([ + 'customer_id' => $data['ID'], + 'lifetime_value' => $lifetimeValue, + 'segment' => $segment, + 'calculated_at' => date('Y-m-d H:i:s'), + 'data_source' => 'calculated' + ]); + })) + ) +); + +// Write all enriched data +$chainConfig->addLink(new CsvFileWriterConfig('customer-360-view.csv')); + +// Execute +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'customers.csv'])]), + [] +); +``` + +## Common Use Cases + +- **Customer 360 View**: Combine CRM, orders, support tickets, analytics +- **Product Enrichment**: Add images, reviews, inventory from different systems +- **Multi-API Aggregation**: Fetch related data from multiple APIs diff --git a/docs/doc/10-examples/500-api-csv.md b/docs/doc/10-examples/500-api-csv.md index 236dfb2..801f04a 100644 --- a/docs/doc/10-examples/500-api-csv.md +++ b/docs/doc/10-examples/500-api-csv.md @@ -7,35 +7,50 @@ width: large ### With Context - Api to CSV +This example uses **execution contexts**, which isolate each ETL execution in its own directory. +If you're not familiar with execution contexts, please read [Execution Context - Why to have an execution context & what it does](/doc/01-understand-the-etl/execution-context.html). + {% capture description %} The chain definition is identical to our previous [definition](/doc/10-examples/150-api-csv.html) one without a context. -It's the end results that changes, as now our file is created within the unique context. +It's the end results that changes, as now our file is created within the unique context directory. {% endcapture %} {% capture code %} -```yaml -chain: - get-from-api: - operation: http - options: - url: https://63b687951907f863aaf90ab1.mockapi.io/test - method: GET - response_is_json: true - option_key: ~ - response_key: ~ - options: - headers: {'Accept': 'application/json'} - - split-item: - operation: split-item - options: - keys: ['content'] - singleElement: true +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\Model\PockExecution; - write-new-file: - operation: csv-write - options: - file: "output.csv" +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new SimpleHttpConfig( + url: 'https://63b687951907f863aaf90ab1.mockapi.io/test', + method: 'GET', + responseIsJson: true, + optionKey: null, + responseKey: null, + options: [ + 'headers' => ['Accept' => 'application/json'] + ] + )) + ->addLink(new SplitItemConfig( + keys: ['content'], + singleElement: true + )) + ->addLink(new CsvFileWriterConfig('output.csv')); +// Create and execute the chain with execution context +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem([])]), + [ + 'etl' => [ + 'execution' => new PockExecution(new DateTime()) + ] + ] +); ``` {% endcapture %} {% include block/etl-step.html code=code description=description %} diff --git a/docs/doc/10-examples/510-import-file.md b/docs/doc/10-examples/510-import-file.md index a9072a7..a433488 100644 --- a/docs/doc/10-examples/510-import-file.md +++ b/docs/doc/10-examples/510-import-file.md @@ -13,7 +13,7 @@ The ETL can be used to fetch files from another filesystem and process them usin {% capture description %} -We will use the `ExternalFileFinderOperation` to find the files. And use the `ExternalFileProcessorOperation` to copy +We will use the `ExternalFileFinderConfig` to find the files. And use the `ExternalFileProcessorConfig` to copy the files to the context of the ETL execution. The file finder will use a directory that we will add to the context at the beginning of the execution. @@ -21,16 +21,15 @@ The file finder will use a directory that we will add to the context at the begi We will also need to provide the finder with a regex as chain input so that it can find all the files. {% endcapture %} {% capture code %} -```yaml - find-file1: - operation: external-file-finder-local - options: - directory: "@context['dir']" - - process-new-file1: - operation: external-file-processor - options: [] - +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; + +$chainConfig + ->addLink(new ExternalFileFinderConfig( + directory: "@context['dir']" + )) + ->addLink(new ExternalFileProcessorConfig()); ``` {% endcapture %} {% include block/etl-step.html code=code description=description %} @@ -38,13 +37,18 @@ We will also need to provide the finder with a regex as chain input so that it c {% capture column1 %} #### šŸ˜ Standalone ```php +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\Model\PockExecution; + $options = [ - // ... + 'etl' => [ + 'execution' => new PockExecution(new DateTime()) + ], 'dir' => $dir, ]; $chainProcessor->process( - new ArrayIterator(['/^file[0-9]\.csv$/']), + new ArrayIterator([new DataItem(['pattern' => '/^file[0-9]\.csv$/'])]), $options ); ``` @@ -59,27 +63,34 @@ $chainProcessor->process( #### Complete Code -```yaml -chain: - find-file1: - operation: external-file-finder-local - options: - directory: "@context['dir']" - - process-new-file1: - operation: external-file-processor - options: [] - - read-file: - operation: csv-read - options: [] - - write-new-file: - operation: csv-write - options: - file: "output.csv" - - process-finalized-file1: - operation: external-file-processor - options: [] +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\Model\PockExecution; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new ExternalFileFinderConfig( + directory: "@context['dir']" + )) + ->addLink(new ExternalFileProcessorConfig()) + ->addLink(new CsvExtractConfig()) + ->addLink(new CsvFileWriterConfig('output.csv')) + ->addLink(new ExternalFileProcessorConfig()); + +// Create and execute the chain with context +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['pattern' => '/^file[0-9]\.csv$/'])]), + [ + 'etl' => [ + 'execution' => new PockExecution(new DateTime()) + ], + 'dir' => '/var/import' + ] +); ``` diff --git a/docs/doc/20-operations/10-building/010-split.md b/docs/doc/20-operations/10-building/010-split.md index 2634637..4d9e021 100644 --- a/docs/doc/20-operations/10-building/010-split.md +++ b/docs/doc/20-operations/10-building/010-split.md @@ -1,38 +1,168 @@ --- layout: base title: PHP-ETL - Operations -subTitle: Building Blocks - Chain Split(split) +subTitle: Building Blocks - Chain Split --- -The `split` operation executes multiple, independent chains of operations with the same input data. +The `ChainSplitConfig` operation executes multiple, independent chains of operations with the same input data. Each chain, or "branch," processes data in parallel without affecting other branches or the main chain. This is useful for performing distinct tasks simultaneously, such as logging, sending to an API, and saving to a database. -## Options +**Key characteristics:** +- Each branch receives the **same input data** +- Branches execute **independently** - one branch's modifications don't affect others +- The **original item** continues to the next step in the main chain +- Useful for parallel processing, routing data to multiple destinations, or creating filtered outputs -- **branches:** An array of chains of operations. +## Configuration -## Example +Use `ChainSplitConfig` and add branches with the `addSplit()` method: -Here's an example of how to use the `split` operation to process a CSV file in two different ways simultaneously: +```php +use Oliverde8\Component\PhpEtl\OperationConfig\ChainSplitConfig; +use Oliverde8\Component\PhpEtl\ChainConfig; -```yaml -chain: - - operation: extract-csv - options: - path: /path/to/input.csv +$splitConfig = new ChainSplitConfig(); +$splitConfig + ->addSplit($branch1ChainConfig) + ->addSplit($branch2ChainConfig) + ->addSplit($branch3ChainConfig); +``` + +Each branch is a `ChainConfig` that can contain any sequence of operations. + +## Example: Splitting Data into Multiple Files + +Here's an example that reads a CSV file and splits data into different files based on subscription status: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainSplitConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; - - operation: split - options: - branches: - - - operation: rule-transformer - options: - # ... rules for transformation - - operation: load-csv - options: - path: /path/to/output.csv +$chainConfig = new ChainConfig(); - - - operation: log - options: - message: "Processing item {{ item.id }}" +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink( + (new ChainSplitConfig()) + // Branch 1: Subscribed customers only + ->addSplit( + (new ChainConfig()) + ->addLink(new FilterDataConfig('@data["IsSubscribed"]')) + ->addLink(new CsvFileWriterConfig('customers-subscribed.csv')) + ) + // Branch 2: Non-subscribed customers only + ->addSplit( + (new ChainConfig()) + ->addLink(new FilterDataConfig('@!data["IsSubscribed"]')) + ->addLink(new CsvFileWriterConfig('customers-not-subscribed.csv')) + ) + ) + // After the split, all data continues to the main chain + ->addLink(new CsvFileWriterConfig('customers-all.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'data/customers.csv'])]), + [] +); ``` + +**Result**: +- `customers-subscribed.csv` contains only subscribed customers +- `customers-not-subscribed.csv` contains only non-subscribed customers +- `customers-all.csv` contains all customers (unfiltered) + +## Example: Logging and Processing Simultaneously + +Split is useful for logging or monitoring without affecting the main processing flow: + +```php +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink( + (new ChainSplitConfig()) + // Branch 1: Main processing + ->addSplit( + (new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'ID']]]) + ->addColumn('name', [['get' => ['field' => 'Name']]]) + ) + ->addLink(new CsvFileWriterConfig('output/processed.csv')) + ) + // Branch 2: Logging only + ->addSplit( + (new ChainConfig()) + ->addLink(new LogConfig( + message: 'Processing customer ID: @data["ID"]', + level: 'info' + )) + ) + // Branch 3: Send to API + ->addSplit( + (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: 'https://api.example.com/track', + method: 'POST', + responseIsJson: false + )) + ) + ); +``` + +## Example: Multi-Destination Export + +Send the same data to different systems with different transformations: + +```php +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink( + (new ChainSplitConfig()) + // Branch 1: Export to CRM (full data) + ->addSplit( + (new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('email', [['get' => ['field' => 'Email']]]) + ->addColumn('name', [['get' => ['field' => 'Name']]]) + ->addColumn('phone', [['get' => ['field' => 'Phone']]]) + ) + ->addLink(new CsvFileWriterConfig('export/crm-full.csv')) + ) + // Branch 2: Export to Marketing (email only) + ->addSplit( + (new ChainConfig()) + ->addLink(new FilterDataConfig('@data["IsSubscribed"]')) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('email', [['get' => ['field' => 'Email']]]) + ->addColumn('first_name', [['get' => ['field' => 'FirstName']]]) + ) + ->addLink(new CsvFileWriterConfig('export/marketing-emails.csv')) + ) + // Branch 3: Export to Analytics (aggregated) + ->addSplit( + (new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('country', [['get' => ['field' => 'Country']]]) + ->addColumn('subscription_date', [['get' => ['field' => 'CreatedAt']]]) + ) + ->addLink(new CsvFileWriterConfig('export/analytics-summary.csv')) + ) + ); +``` + +## Common Use Cases + +- **Data routing**: Send different subsets to different destinations +- **Parallel exports**: Export same data to multiple formats/systems +- **Multi-format output**: Generate CSV, JSON, and XML from same data diff --git a/docs/doc/20-operations/10-building/020-merge.md b/docs/doc/20-operations/10-building/020-merge.md index 5b82218..2071ad8 100644 --- a/docs/doc/20-operations/10-building/020-merge.md +++ b/docs/doc/20-operations/10-building/020-merge.md @@ -1,48 +1,233 @@ --- layout: base title: PHP-ETL - Operations -subTitle: Building Blocks - Chain Merge(merge) +subTitle: Building Blocks - Chain Merge --- -The `merge` operation executes multiple chains of operations (branches) with the same input data, then combines their results back into the main chain. -This is useful for performing different transformations on the same data and combining the outcomes. +The `ChainMergeConfig` operation executes multiple chains of operations (branches) with the same input data, then combines their results back into the main chain. Each branch can transform the data differently, and **all outputs from all branches** are passed to the next step. + +**Key characteristics:** +- Each branch receives the **same input data** +- Each branch can **transform data differently** +- **All branch outputs are combined** and sent to the next step +- One input item can become **multiple output items** +- Useful for creating multiple views, enriching from multiple sources, or denormalizing data **Warning:** If branches don't filter or modify items, subsequent steps will receive duplicate data, as `merge` doesn't handle duplicates automatically. This behavior can be leveraged to create multiple versions of an item. -## Options +## Configuration + +Use `ChainMergeConfig` and add branches with the `addMerge()` method: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\ChainMergeConfig; +use Oliverde8\Component\PhpEtl\ChainConfig; + +$mergeConfig = new ChainMergeConfig(); +$mergeConfig + ->addMerge($branch1ChainConfig) + ->addMerge($branch2ChainConfig) + ->addMerge($branch3ChainConfig); +``` -- **branches:** An array of chains of operations. +Each branch is a `ChainConfig` that can contain any sequence of operations. -## Example +## Example: Creating Multiple Product Variants -Here's an example of how to use the `merge` operation to create two different versions of a product from a single input item: +Here's an example that creates both simple and configurable product records from a single input: -```yaml -chain: - - operation: extract-csv - options: - path: /path/to/products.csv +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainMergeConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; - - operation: merge - options: - branches: - - - operation: rule-transformer - options: - # Rules to create a simple product. - rules: - sku: "{{ item.sku }}-simple" - type: simple +$chainConfig = new ChainConfig(); - - - operation: rule-transformer - options: - # Rules to create a configurable product. - rules: - sku: "{{ item.sku }}-configurable" - type: configurable +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink( + (new ChainMergeConfig()) + // Branch 1: Create simple product + ->addMerge( + (new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('sku', [['get' => ['field' => 'sku']], ['append' => '-simple']]) + ->addColumn('type', [['const' => 'simple']]) + ->addColumn('name', [['get' => ['field' => 'name']]]) + ) + ) + // Branch 2: Create configurable product + ->addMerge( + (new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('sku', [['get' => ['field' => 'sku']], ['append' => '-configurable']]) + ->addColumn('type', [['const' => 'configurable']]) + ->addColumn('name', [['get' => ['field' => 'name']]]) + ) + ) + ) + // All branch outputs (2 products per input) go to the CSV file + ->addLink(new CsvFileWriterConfig('merged-products.csv')); - - operation: load-csv - options: - path: /path/to/merged-products.csv +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'products.csv'])]), + [] +); ``` + +**Result**: Each input product becomes **2 output rows** - one simple, one configurable. + +## Example: Customer Data with Multiple Views + +Create different data views for different systems from the same customer data: + +```php +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink( + (new ChainMergeConfig()) + // Branch 1: Contact information view + ->addMerge( + (new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('full_name', [ + ['implode' => [ + 'values' => [ + [['get' => ['field' => 'FirstName']]], + [['get' => ['field' => 'LastName']]], + ], + 'with' => ' ', + ]] + ]) + ->addColumn('email', [['get' => ['field' => 'Email']]]) + ->addColumn('data_type', [['const' => 'contact']]) + ) + ) + // Branch 2: Subscription status view + ->addMerge( + (new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('is_subscribed', [['get' => ['field' => 'IsSubscribed']]]) + ->addColumn('subscription_date', [['get' => ['field' => 'CreatedAt']]]) + ->addColumn('data_type', [['const' => 'subscription']]) + ) + ) + ) + ->addLink(new CsvFileWriterConfig('customer-views.csv')); +``` + +**Result**: Each customer becomes **2 rows** - one with contact info, one with subscription data. + +## Example: Multi-Source API Enrichment + +Enrich data by calling multiple APIs and merging all responses: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink( + (new ChainMergeConfig()) + // Branch 1: Get customer profile + ->addMerge( + (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/customers/"~data["customer_id"]', + method: 'GET', + responseIsJson: true, + responseKey: 'profile' + )) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'customer_id']]]) + ->addColumn('name', [['get' => ['field' => 'profile', 'name']]]) + ->addColumn('source', [['const' => 'profile_api']]) + ) + ) + // Branch 2: Get order history (with error handling) + ->addMerge( + new FailSafeConfig( + chainConfig: (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/orders?customer="~data["customer_id"]', + method: 'GET', + responseIsJson: true, + responseKey: 'orders' + )) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'customer_id']]]) + ->addColumn('total_orders', [['get' => ['field' => 'orders', 'total']]]) + ->addColumn('source', [['const' => 'orders_api']]) + ), + exceptionsToCatch: [\Exception::class], + nbAttempts: 3 + ) + ) + ) + ->addLink(new CsvFileWriterConfig('customer-enriched.csv')); +``` + +**Result**: Each customer becomes **2 rows** (or 1 if orders API fails) with data from different APIs. + +## Example: Denormalizing Hierarchical Data + +Split hierarchical JSON data into flat records: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\JsonExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; + +// Input: {"product_id": 123, "name": "T-Shirt", "variants": ["Small", "Medium", "Large"]} + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new JsonExtractConfig()) + ->addLink( + (new ChainMergeConfig()) + // Branch 1: Base product info + ->addMerge( + (new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('product_id', [['get' => ['field' => 'product_id']]]) + ->addColumn('name', [['get' => ['field' => 'name']]]) + ->addColumn('type', [['const' => 'base']]) + ->addColumn('variant', [['const' => null]]) + ) + ) + // Branch 2: Each variant becomes a separate row + ->addMerge( + (new ChainConfig()) + ->addLink(new SplitItemConfig(path: 'variants')) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('product_id', [['get' => ['field' => 'product_id']]]) + ->addColumn('name', [['get' => ['field' => 'name']]]) + ->addColumn('type', [['const' => 'variant']]) + ->addColumn('variant', [['get' => ['field' => 'data']]]) + ) + ) + ) + ->addLink(new CsvFileWriterConfig('products-flat.csv')); +``` + +**Result**: 1 base row + 3 variant rows = **4 total rows** per product. + +## Common Use Cases + +- **Multi-view data export**: Create different data views for different systems +- **Product variants**: Generate simple and configurable products from base data +- **Data enrichment**: Combine information from multiple APIs or databases +- **Customer 360**: Merge profile, orders, support tickets into comprehensive view diff --git a/docs/doc/20-operations/10-building/030-repeat.md b/docs/doc/20-operations/10-building/030-repeat.md index 29a2e3a..55fbe15 100644 --- a/docs/doc/20-operations/10-building/030-repeat.md +++ b/docs/doc/20-operations/10-building/030-repeat.md @@ -1,43 +1,305 @@ --- layout: base title: PHP-ETL - Operations -subTitle: Building Blocks - Chain Repeat(repeat) +subTitle: Building Blocks - Chain Repeat --- -The `repeat` operation executes a chain of operations repeatedly until a specified condition is met. -This is useful for processing data in a loop, such as paginating through API results or processing a file until its end. +The `ChainRepeatConfig` operation executes a chain of operations repeatedly until a specified condition is met. +This is the perfect solution for paginated APIs, iterative data processing, or any scenario where you need to loop until a condition becomes false. -## Options +**Key characteristics:** +- Executes a sub-chain **repeatedly in a loop** +- Continues while validation expression evaluates to **true** +- Stops when validation expression returns **false** +- Each iteration can modify data for the next iteration +- Supports asynchronous processing within the loop -- **chain:** The chain of operations to be executed in each iteration. -- **while:** An expression that determines whether the loop should continue. The expression is evaluated before each iteration. The loop continues as long as the expression evaluates to `true`. -- **allow_async:** (Optional) A boolean that specifies whether the operations within the loop can be executed asynchronously. Defaults to `false`. +## Configuration -## Example +Use `ChainRepeatConfig` with these parameters: -Here's an example of how to use the `repeat` operation to fetch and process data from a paginated API: +```php +use Oliverde8\Component\PhpEtl\OperationConfig\ChainRepeatConfig; +use Oliverde8\Component\PhpEtl\ChainConfig; -```yaml -chain: - - operation: repeat - options: - while: "context.page <= context.totalPages" - allow_async: true - chain: - - operation: http-request - options: - url: "https://api.example.com/data?page={{ context.page }}" - method: GET +$repeatConfig = new ChainRepeatConfig( + chainConfig: $chainToRepeat, // ChainConfig to execute repeatedly + validationExpression: 'expression', // Symfony expression - loop while true + allowAsynchronous: false // Optional: allow async operations (default: false) +); +``` - - operation: rule-transformer - options: - # Rules to transform the API response. +**Parameters:** +- `chainConfig`: A `ChainConfig` containing operations to execute in each iteration +- `validationExpression`: A Symfony Expression Language expression that must return boolean. Loop continues while `true` +- `allowAsynchronous`: Whether operations within the loop can execute asynchronously - - operation: load-to-database - options: - # Options to load the data into a database. +## Example: Basic API Pagination - - operation: expression - options: - expression: "context.page = context.page + 1" -``` \ No newline at end of file +Fetch all pages from a paginated API: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainRepeatConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\Item\DataItemInterface; + +$chainConfig = new ChainConfig(); + +$page = 1; + +$repeatConfig = new ChainRepeatConfig( + chainConfig: (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItemInterface $dataItem) use (&$page) { + echo "Fetching page {$page}...\n"; + + $totalPages = 5; + + // Simulate API call returning paginated data + $items = []; + for ($i = 1; $i <= 10; $i++) { + $itemId = (($page - 1) * 10) + $i; + $items[] = [ + 'id' => $itemId, + 'name' => "Item {$itemId}", + 'page' => $page, + ]; + } + + $hasNextPage = $page < $totalPages; + $page++; + + return new DataItem([ + 'items' => $items, + 'hasNextPage' => $hasNextPage, + ]); + })), + validationExpression: 'data["hasNextPage"] == true', + allowAsynchronous: false +); + +$chainConfig + ->addLink($repeatConfig) + ->addLink(new SplitItemConfig(keys: ['items'])) // Extract items array + ->addLink(new CsvFileWriterConfig('all-items.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem([])]), + [] +); +``` + +**Result**: Fetches pages 1-5, extracts all items, writes to CSV. + +## Example: Cursor-Based API Pagination + +Modern APIs often use cursor-based pagination instead of page numbers: + +```php +$repeatConfig = new ChainRepeatConfig( + chainConfig: (new ChainConfig()) + // Make API request with cursor + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/items?cursor="~(data["cursor"] ?? "")', + method: 'GET', + responseIsJson: true, + responseKey: 'response' + )) + // Process response and extract next cursor + ->addLink(new CallBackTransformerConfig(function(DataItemInterface $item) { + $data = $item->getData(); + $response = $data['response']; + + return new DataItem([ + 'items' => $response['data'], + 'cursor' => $response['pagination']['next_cursor'] ?? null, + 'hasMore' => !empty($response['pagination']['next_cursor']), + ]); + })) + ->addLink(new SplitItemConfig(keys: ['items'])), + validationExpression: 'data["hasMore"] == true' +); + +$chainConfig + ->addLink($repeatConfig) + ->addLink(new CsvFileWriterConfig('api-data.csv')); +``` + +## Example: Page Number Pagination + +APIs that use page numbers with a total page count: + +```php +$repeatConfig = new ChainRepeatConfig( + chainConfig: (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/users?page="~data["currentPage"]', + method: 'GET', + responseIsJson: true, + responseKey: 'response' + )) + ->addLink(new CallBackTransformerConfig(function(DataItemInterface $item) { + $data = $item->getData(); + $response = $data['response']; + + return new DataItem([ + 'items' => $response['results'], + 'currentPage' => $data['currentPage'] + 1, + 'totalPages' => $response['total_pages'], + ]); + })) + ->addLink(new SplitItemConfig(keys: ['items'])), + validationExpression: 'data["currentPage"] <= data["totalPages"]' +); + +// Start with page 1 +$chainProcessor->process( + new ArrayIterator([new DataItem(['currentPage' => 1, 'totalPages' => 1])]), + [] +); +``` + +## Example: Offset-Based Pagination + +APIs using offset/limit pattern: + +```php +$repeatConfig = new ChainRepeatConfig( + chainConfig: (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/records?offset="~data["offset"]~"&limit=100"', + method: 'GET', + responseIsJson: true, + responseKey: 'response' + )) + ->addLink(new CallBackTransformerConfig(function(DataItemInterface $item) { + $data = $item->getData(); + $response = $data['response']; + $fetchedCount = count($response['records']); + + return new DataItem([ + 'items' => $response['records'], + 'offset' => $data['offset'] + $fetchedCount, + 'total' => $response['total'], + 'hasMore' => $data['offset'] + $fetchedCount < $response['total'], + ]); + })) + ->addLink(new SplitItemConfig(keys: ['items'])), + validationExpression: 'data["hasMore"] == true' +); + +// Start at offset 0 +$chainProcessor->process( + new ArrayIterator([new DataItem(['offset' => 0])]), + [] +); +``` + +## Example: Retry Until Success + +Use ChainRepeat for retry logic (though FailSafeConfig is usually better for this): + +```php +$attempts = 0; +$maxAttempts = 3; + +$repeatConfig = new ChainRepeatConfig( + chainConfig: (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItemInterface $item) use (&$attempts, $maxAttempts) { + $attempts++; + echo "Attempt {$attempts}/{$maxAttempts}\n"; + + // Simulate operation that might fail + $success = rand(1, 10) > 7; // 30% success rate + + if ($success) { + echo "Success!\n"; + } + + return new DataItem([ + 'success' => $success, + 'shouldRetry' => !$success && $attempts < $maxAttempts, + ]); + })), + validationExpression: 'data["shouldRetry"] == true' +); +``` + +## Example: With Error Handling + +Combine ChainRepeat with FailSafe for robust pagination: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; + +$repeatConfig = new ChainRepeatConfig( + chainConfig: (new ChainConfig()) + ->addLink(new FailSafeConfig( + chainConfig: (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: '@"https://api.example.com/data?page="~data["page"]', + method: 'GET', + responseIsJson: true + )), + exceptionsToCatch: [\Exception::class], + nbAttempts: 3 + )) + ->addLink(new CallBackTransformerConfig(function(DataItemInterface $item) { + $data = $item->getData(); + + return new DataItem([ + 'items' => $data['results'] ?? [], + 'page' => $data['page'] + 1, + 'hasMore' => !empty($data['next']), + ]); + })) + ->addLink(new SplitItemConfig(keys: ['items'])), + validationExpression: 'data["hasMore"] == true' +); +``` + +## Understanding the Validation Expression + +The validation expression is evaluated **before each iteration**. The loop continues while the expression returns `true`: + +```php +// Loop while there are more pages +validationExpression: 'data["hasMore"] == true' + +// Loop while page number is less than total +validationExpression: 'data["page"] < data["totalPages"]' + +// Loop while cursor exists +validationExpression: 'data["cursor"] != null' + +// Complex condition +validationExpression: 'data["hasMore"] == true and data["errorCount"] < 5' + +// Using context +validationExpression: 'context["shouldContinue"] == true' +``` + +## Asynchronous Processing + +Set `allowAsynchronous: true` to allow operations within the repeat chain to execute asynchronously: + +```php +$repeatConfig = new ChainRepeatConfig( + chainConfig: $paginationChain, + validationExpression: 'data["hasMore"] == true', + allowAsynchronous: true // Enable async processing +); +``` + +## Common Use Cases + +- **API Pagination**: Fetch all pages from paginated REST APIs +- **Cursor Navigation**: Process cursor-based API responses +- **Retry Logic**: Retry operations until success (use FailSafe instead when possible) +- **Polling**: Poll an endpoint until data is available +- **Queue Processing**: Process messages from a queue until empty diff --git a/docs/doc/20-operations/10-building/040-safe.md b/docs/doc/20-operations/10-building/040-safe.md index bb4ba06..b434d32 100644 --- a/docs/doc/20-operations/10-building/040-safe.md +++ b/docs/doc/20-operations/10-building/040-safe.md @@ -1,39 +1,336 @@ --- layout: base title: PHP-ETL - Operations -subTitle: Building Blocks - Fail Safe(safe) +subTitle: Building Blocks - Fail Safe --- -The `safe` operation (`FailSafeOperation`) handles exceptions within an ETL chain. It wraps a chain in a "safe" block, catching specified exceptions and retrying the chain a set number of times. If an exception is not caught or retries are exhausted, it's re-thrown. This enhances ETL process robustness against transient errors like network issues. +The `FailSafeConfig` operation handles exceptions within an ETL chain, making your pipelines resilient to transient errors. It wraps a chain in a "safe" block, catching specified exceptions and automatically retrying the operation. This is critical for production ETL processes that need to handle network issues, temporary API failures, or other recoverable errors gracefully. -## Options +**Key characteristics:** +- **Catches specified exceptions** and retries automatically +- **Limits retry attempts** to prevent infinite loops +- **Continues processing** other items if one fails +- **Re-throws unhandled exceptions** that aren't in the catch list +- Essential for production reliability and error resilience -- **chain:** The chain of operations to be executed within the safe block. -- **exceptions_to_catch:** A list of exception classes to catch. If an exception of a different type is thrown, it will not be caught. -- **max_retries:** The maximum number of times to retry the chain if a caught exception is thrown. +## Configuration -## Example +Use `FailSafeConfig` with these parameters: -Here's an example of how to use the `safe` operation to handle `ApiConnectionException` when making an HTTP request: +```php +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; +use Oliverde8\Component\PhpEtl\ChainConfig; -```yaml -chain: - - operation: safe - options: - exceptions_to_catch: - - App\Exception\ApiConnectionException - max_retries: 3 - chain: - - operation: http-request - options: - url: "https://api.example.com/data" - method: GET +$failSafeConfig = new FailSafeConfig( + chainConfig: $chainToProtect, // ChainConfig to wrap + exceptionsToCatch: [\Exception::class], // Array of exception classes to catch + nbAttempts: 3 // Number of attempts (default: 3) +); +``` - - operation: rule-transformer - options: - # Rules to transform the API response. +**Parameters:** +- `chainConfig`: The `ChainConfig` to execute with error protection +- `exceptionsToCatch`: Array of exception class names to catch and retry +- `nbAttempts`: Total number of attempts (including first try). Default is 3 - - operation: load-to-database - options: - # Options to load the data into a database. -``` \ No newline at end of file +## Example: Basic API Error Handling + +Retry failed API calls automatically: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); + +// API call that might fail due to network issues +$apiChain = (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: 'https://api.example.com/data', + method: 'GET', + responseIsJson: true + )); + +// Wrap in FailSafe - retry up to 3 times on any exception +$chainConfig + ->addLink(new FailSafeConfig( + chainConfig: $apiChain, + exceptionsToCatch: [\Exception::class], + nbAttempts: 3 + )) + ->addLink(new CsvFileWriterConfig('output.csv')); +``` + +## Example: Catching Specific Exceptions + +Only retry certain types of errors: + +```php +use Symfony\Component\HttpClient\Exception\TransportException; +use Symfony\Component\HttpClient\Exception\ServerException; + +$apiChain = (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: 'https://api.example.com/users', + method: 'GET', + responseIsJson: true + )); + +// Only retry network and server errors, not client errors (4xx) +$failSafeConfig = new FailSafeConfig( + chainConfig: $apiChain, + exceptionsToCatch: [ + TransportException::class, // Network errors + ServerException::class, // 5xx server errors + \RuntimeException::class, // Runtime issues + ], + nbAttempts: 3 +); + +$chainConfig->addLink($failSafeConfig); +``` + +**Note**: Validation errors (like 400 Bad Request) won't be caught, causing immediate failure - which is correct for non-retryable errors. + +## Example: Transient Error Simulation + +Handle transient errors that succeed after a few attempts: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +$attempt = 0; + +$failingChain = (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) use (&$attempt) { + $attempt++; + echo "Attempt {$attempt}\n"; + + // Fail on first 2 attempts, succeed on 3rd + if ($attempt < 3) { + throw new RuntimeException('Transient error, please retry'); + } + + echo "Success!\n"; + return $item; + })); + +$chainConfig->addLink(new FailSafeConfig( + chainConfig: $failingChain, + exceptionsToCatch: [RuntimeException::class], + nbAttempts: 5 // Allow up to 5 attempts +)); +``` + +**Result**: Fails twice, succeeds on 3rd attempt, processing continues. + +## Example: Continue Processing on Failure + +Process a batch of items where some might fail, but others should continue: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; + +$processingChain = (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + $data = $item->getData(); + + // Validate item + if (empty($data['email'])) { + throw new \InvalidArgumentException('Email is required'); + } + + // Process valid item + processItem($data); + return $item; + })); + +// Failed items are skipped, but processing continues with next items +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new FailSafeConfig( + chainConfig: $processingChain, + exceptionsToCatch: [\InvalidArgumentException::class], + nbAttempts: 1 // Don't retry validation errors + )) + ->addLink(new CsvFileWriterConfig('valid-items.csv')); +``` + +**Result**: Invalid items are skipped, valid items are processed and saved. + +## Example: API Retry with Error Logging + +Log failures while retrying: + +```php +$failedItems = []; + +$apiChain = (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) use (&$failedItems) { + static $attempt = 0; + $attempt++; + + try { + // Make API call + $result = makeApiCall($item->getData()); + return new DataItem($result); + } catch (\Exception $e) { + // Log the failure + $failedItems[] = [ + 'data' => $item->getData(), + 'error' => $e->getMessage(), + 'attempt' => $attempt, + 'timestamp' => date('Y-m-d H:i:s') + ]; + throw $e; // Re-throw to trigger retry + } + })); + +$chainConfig->addLink(new FailSafeConfig( + chainConfig: $apiChain, + exceptionsToCatch: [\Exception::class], + nbAttempts: 3 +)); + +// After processing, save failed items for review +file_put_contents('failed-items.json', json_encode($failedItems, JSON_PRETTY_PRINT)); +``` + +## Example: Exponential Backoff Pattern + +Add delays between retries for rate-limited APIs: + +```php +$apiChain = (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + static $attempt = 0; + $attempt++; + + // Exponential backoff: 0s, 1s, 2s, 4s, 8s + if ($attempt > 1) { + $delay = pow(2, $attempt - 2); + echo "Waiting {$delay} seconds before retry...\n"; + sleep($delay); + } + + // Make API call + return makeApiCall($item); + })) + ->addLink(new SimpleHttpConfig( + url: 'https://api.example.com/data', + method: 'POST', + responseIsJson: true + )); + +$failSafeConfig = new FailSafeConfig( + chainConfig: $apiChain, + exceptionsToCatch: [\Exception::class], + nbAttempts: 5 +); +``` + +## Example: Database Operation with Retry + +Handle database deadlocks or connection issues: + +```php +use Doctrine\DBAL\Exception\DeadlockException; +use Doctrine\DBAL\Exception\ConnectionException; + +$dbChain = (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) use ($entityManager) { + $data = $item->getData(); + + // Database operation that might fail + $entity = new Customer(); + $entity->setName($data['name']); + $entity->setEmail($data['email']); + + $entityManager->persist($entity); + $entityManager->flush(); + + return $item; + })); + +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new FailSafeConfig( + chainConfig: $dbChain, + exceptionsToCatch: [ + DeadlockException::class, + ConnectionException::class, + ], + nbAttempts: 5 + )); +``` + +## Example: Multiple FailSafe Layers + +Different retry strategies for different operations: + +```php +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) + + // First FailSafe: API call with 3 retries + ->addLink(new FailSafeConfig( + chainConfig: (new ChainConfig()) + ->addLink(new SimpleHttpConfig( + url: 'https://api.example.com/enrich', + method: 'POST', + responseIsJson: true + )), + exceptionsToCatch: [\Exception::class], + nbAttempts: 3 + )) + + // Second FailSafe: Database save with 5 retries + ->addLink(new FailSafeConfig( + chainConfig: (new ChainConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + saveToDatabase($item->getData()); + return $item; + })), + exceptionsToCatch: [DeadlockException::class], + nbAttempts: 5 + )) + + ->addLink(new CsvFileWriterConfig('completed.csv')); +``` + +## Understanding Retry Behavior + +**Retry Count:** + +- `nbAttempts: 1` = No retries (runs once, fails if exception) +- `nbAttempts: 3` = Up to 2 retries (runs 3 times total) +- `nbAttempts: 5` = Up to 4 retries (runs 5 times total) + +**Exception Handling:** + +```php +exceptionsToCatch: [\Exception::class] // Catches all exceptions +exceptionsToCatch: [RuntimeException::class] // Only RuntimeException +exceptionsToCatch: [ + NetworkException::class, + TimeoutException::class +] // Multiple specific exceptions +``` + +**When Retry Stops:** + +- Operation succeeds (no exception thrown) +- Maximum attempts reached +- Different exception type is thrown (not in catch list) + +## Common Use Cases + +- **API resilience**: Handle temporary API outages or rate limits +- **Network reliability**: Retry on connection timeouts or DNS failures +- **Database operations**: Handle deadlocks and connection pool exhaustion +- **Batch processing**: Skip invalid items while processing valid ones diff --git a/docs/doc/20-operations/20-extract/010-file-finder.md b/docs/doc/20-operations/20-extract/010-file-finder.md index 98e00f8..a5c6fa8 100644 --- a/docs/doc/20-operations/20-extract/010-file-finder.md +++ b/docs/doc/20-operations/20-extract/010-file-finder.md @@ -1,11 +1,11 @@ --- layout: base title: PHP-ETL - Operations -subTitle: Extract- File Finder +subTitle: Extract - File Finder --- -The `ExternalFileFinder` operation is the base operation for importing files from **remote or external file systems**. -It is responsible for **locating files** based on a given pattern and returning them as `ExternalFileItem`s for further processing. +The `ExternalFileFinderConfig` operation is the base operation for importing files from **remote or external file systems**. +It is responsible for **locating files** based on a given pattern and returning them as `FileExtractedItem`s for further processing. This operation works with any file system supported by [Flysystem](https://flysystem.thephpleague.com/), including **SFTP, local files, AWS S3**, and more. @@ -13,54 +13,419 @@ This operation works with any file system supported by [Flysystem](https://flysy ## How It Works -The `ExternalFileFinder` searches a directory for files matching a provided pattern. For each file found, -it returns an `ExternalFileItem`. These items are typically passed down the chain to: +The `ExternalFileFinder` searches a directory for files matching a provided regex pattern. For each file found, +it returns a `FileExtractedItem`. These items are typically passed down the chain to: -- **Copy the file locally** using a dedicated operation. -- **Read/process the file** content using format-specific operations (e.g., `csv-read`, `xml-read`, etc.). +- **Process the file** using `ExternalFileProcessorConfig` to copy it locally +- **Read/process the file** content using format-specific operations (e.g., `CsvExtractConfig`, `JsonExtractConfig`, etc.) > šŸ“˜ Refer to the [Cookbook section](/doc/10-examples/510-import-file.html) for complete examples of end-to-end remote file import flows. --- +## Configuration + +Use `ExternalFileFinderConfig` with these parameters: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; + +$fileFinderConfig = new ExternalFileFinderConfig( + directory: '/path/to/remote/directory' +); +``` + +**Parameters:** +- `directory`: The directory path on the file system to search for files + +**Input Data:** The operation expects a DataItem containing a regex pattern string to match files. + +--- + ### Registering the Operation Because multiple instances of this operation may be needed (e.g., different connections or directories), the -`ExternalFileFinder` must be **manually registered** using a factory. - -Here is how you would register an instance using a local file system: +`ExternalFileFinder` must be **manually registered** using a factory with a Flysystem adapter. {% capture column1 %} #### šŸ˜ Standalone ```php -$builder->registerFactory( - new ExternalFileFinderFactory( - 'external-file-finder-local', - ExternalFileFinderOperation::class, - new LocalFileSystem("/") // or any Flysystem-compatible adapter - ) +use Oliverde8\Component\PhpEtl\GenericChainFactory; +use Oliverde8\Component\PhpEtl\ChainOperation\Extract\ExternalFileFinderOperation; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; +use Oliverde8\Component\PhpEtl\Model\File\LocalFileSystem; + +// Register with local file system +$chainBuilder = new ChainBuilderV2( + $executionContextFactory, + [ + // ... other factories + new GenericChainFactory( + ExternalFileFinderOperation::class, + ExternalFileFinderConfig::class, + injections: ['fileSystem' => new LocalFileSystem("/")] + ), + ] ); ``` -You can use any Flysystem adapter (e.g., SFTP, AWS S3, Azure Blob Storage, etc.) when creating the factory. +**Using SFTP:** +```php +use League\Flysystem\PhpseclibV3\SftpAdapter; +use League\Flysystem\PhpseclibV3\SftpConnectionProvider; + +$adapter = new SftpAdapter( + new SftpConnectionProvider( + host: 'sftp.example.com', + username: 'user', + password: 'password', + port: 22 + ), + '/remote/path' +); + +new GenericChainFactory( + ExternalFileFinderOperation::class, + ExternalFileFinderConfig::class, + injections: ['fileSystem' => $adapter] +) +``` + +**Using AWS S3:** +```php +use Aws\S3\S3Client; +use League\Flysystem\AwsS3V3\AwsS3V3Adapter; + +$client = new S3Client([ + 'credentials' => [ + 'key' => 'your-key', + 'secret' => 'your-secret', + ], + 'region' => 'us-east-1', + 'version' => 'latest', +]); + +$adapter = new AwsS3V3Adapter($client, 'your-bucket-name'); + +new GenericChainFactory( + ExternalFileFinderOperation::class, + ExternalFileFinderConfig::class, + injections: ['fileSystem' => $adapter] +) +``` {% endcapture %} {% capture column2 %} #### šŸŽµ Symfony -In a Symfony application, you should register the operation via Dependency Injection, defining it as a service and tagging it accordingly. +In a Symfony application, you should register the operation via Dependency Injection, defining it as a service with the appropriate filesystem adapter. + +```yaml +services: + # Define your filesystem adapter + app.filesystem.sftp: + class: League\Flysystem\PhpseclibV3\SftpAdapter + arguments: + $connectionProvider: '@app.sftp.connection' + $root: '/remote/path' + + # Register the ETL operation factory + app.etl.file_finder.sftp: + class: Oliverde8\Component\PhpEtl\GenericChainFactory + arguments: + $operationClass: 'Oliverde8\Component\PhpEtl\ChainOperation\Extract\ExternalFileFinderOperation' + $configClass: 'Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig' + $injections: + fileSystem: '@app.filesystem.sftp' + tags: + - { name: etl.operation-factory } +``` +**With multiple file systems:** ```yaml services: - app.etl.operation.external_file_finder.my-local: - class: Oliverde8\Component\PhpEtl\Builder\Factories\Extract\ExternalFileFinderLocal - autowire: true + # SFTP File Finder + app.etl.file_finder.sftp: + class: Oliverde8\Component\PhpEtl\GenericChainFactory + arguments: + $operationClass: 'Oliverde8\Component\PhpEtl\ChainOperation\Extract\ExternalFileFinderOperation' + $configClass: 'Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig' + $injections: + fileSystem: '@app.filesystem.sftp' + tags: + - { name: etl.operation-factory } + + # S3 File Finder + app.etl.file_finder.s3: + class: Oliverde8\Component\PhpEtl\GenericChainFactory arguments: - $operation: 'my_custom_finder' - $class: 'Oliverde8\Component\PhpEtl\ChainOperation\Extract\ExternalFileFinderOperation' - $filesystem: '@your.flysystem.adapter.service' + $operationClass: 'Oliverde8\Component\PhpEtl\ChainOperation\Extract\ExternalFileFinderOperation' + $configClass: 'Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig' + $injections: + fileSystem: '@app.filesystem.s3' tags: - { name: etl.operation-factory } ``` {% endcapture %} -{% include block/2column.html column1=column1 column2=column2 %} \ No newline at end of file +{% include block/2column.html column1=column1 column2=column2 %} + +--- + +## Example: Finding and Processing Files + +Basic example that finds CSV files matching a pattern and processes them: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +$chainConfig = new ChainConfig(); + +$chainConfig + // Find files in directory matching pattern + ->addLink(new ExternalFileFinderConfig( + directory: '/remote/data/imports' + )) + // Copy file locally for processing + ->addLink(new ExternalFileProcessorConfig()) + // Extract CSV data + ->addLink(new CsvExtractConfig()) + // Write processed data + ->addLink(new CsvFileWriterConfig('output.csv')) + // Clean up local file + ->addLink(new ExternalFileProcessorConfig()); + +$chainProcessor = $chainBuilder->createChain($chainConfig); + +// Input: regex pattern to match files +$chainProcessor->process( + new ArrayIterator([ + new DataItem('/^customer_export_[0-9]{8}\.csv$/') // Matches: customer_export_20231215.csv + ]), + [] +); +``` + +## Example: Multiple File Patterns + +Process different file types from the same directory: + +```php +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new ExternalFileFinderConfig(directory: '/data/inbox')) + ->addLink(new ExternalFileProcessorConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + $data = $item->getData(); + $filename = $data['file']; + + echo "Processing: {$filename}\n"; + + // Route to different processors based on file type + if (preg_match('/\.csv$/', $filename)) { + return new DataItem(['file' => $filename, 'type' => 'csv']); + } elseif (preg_match('/\.json$/', $filename)) { + return new DataItem(['file' => $filename, 'type' => 'json']); + } + + return $item; + })) + ->addLink(new CsvFileWriterConfig('processed-files.csv')); + +// Process multiple patterns +$patterns = [ + new DataItem('/^sales_.*\.csv$/'), + new DataItem('/^inventory_.*\.json$/'), + new DataItem('/^orders_[0-9]{4}-[0-9]{2}\.csv$/'), +]; + +$chainProcessor->process(new ArrayIterator($patterns), []); +``` + +## Example: With Date-Based Filtering + +Find files from a specific date range: + +```php +$chainConfig = new ChainConfig(); + +$date = date('Ymd'); // e.g., 20231215 + +$chainConfig + ->addLink(new ExternalFileFinderConfig(directory: '/data/daily')) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) use ($date) { + $data = $item->getData(); + $filename = basename($data['file']); + + // Only process files from today + if (!preg_match("/_{$date}\./", $filename)) { + echo "Skipping old file: {$filename}\n"; + return null; // Skip this file + } + + return $item; + })) + ->addLink(new ExternalFileProcessorConfig()) + ->addLink(new CsvExtractConfig()) + ->addLink(new CsvFileWriterConfig("processed_{$date}.csv")); + +// Find all CSV files, filter by date in callback +$chainProcessor->process( + new ArrayIterator([new DataItem('/^report_.*\.csv$/')]), + [] +); +``` + +## Example: SFTP Import with Error Handling + +Robust file import from SFTP with retry logic: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; + +$importChain = (new ChainConfig()) + ->addLink(new ExternalFileFinderConfig(directory: '/remote/exports')) + ->addLink(new ExternalFileProcessorConfig()) + ->addLink(new CsvExtractConfig()) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + // Transform data + $data = $item->getData(); + // ... processing logic + return $item; + })) + ->addLink(new CsvFileWriterConfig('imported-data.csv')) + ->addLink(new ExternalFileProcessorConfig()); // Clean up + +// Wrap in FailSafe for network reliability +$chainConfig = new ChainConfig(); +$chainConfig->addLink(new FailSafeConfig( + chainConfig: $importChain, + exceptionsToCatch: [\Exception::class], + nbAttempts: 3 +)); + +$chainProcessor->process( + new ArrayIterator([new DataItem('/^export_[0-9]{8}\.csv$/')]), + [] +); +``` + +## Example: S3 File Import + +Import files from AWS S3: + +```php +// Assuming S3 adapter is registered with GenericChainFactory + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new ExternalFileFinderConfig(directory: 'data/incoming')) + ->addLink(new LogConfig( + message: 'Found file: @data["file"]', + level: 'info' + )) + ->addLink(new ExternalFileProcessorConfig()) + ->addLink(new CsvExtractConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'ID']]]) + ->addColumn('name', [['get' => ['field' => 'Name']]]) + ) + ->addLink(new CsvFileWriterConfig('s3-import-results.csv')) + ->addLink(new ExternalFileProcessorConfig()); + +// Find all CSV files in S3 bucket +$chainProcessor->process( + new ArrayIterator([new DataItem('/\.csv$/')]), + [] +); +``` + +## Understanding File Flow + +The typical flow when using `ExternalFileFinderConfig`: + +1. **Find**: `ExternalFileFinderConfig` searches directory and returns `FileExtractedItem` for each match +2. **Copy**: `ExternalFileProcessorConfig` copies the file to local execution context directory +3. **Process**: Format-specific operations (`CsvExtractConfig`, `JsonExtractConfig`) read the local file +4. **Transform**: Apply transformations to the data +5. **Load**: Save results to output +6. **Cleanup**: `ExternalFileProcessorConfig` removes local copy + +```php +$chainConfig + ->addLink(new ExternalFileFinderConfig(...)) // Step 1: Find + ->addLink(new ExternalFileProcessorConfig()) // Step 2: Copy locally + ->addLink(new CsvExtractConfig()) // Step 3: Read + ->addLink(new RuleTransformConfig(...)) // Step 4: Transform + ->addLink(new CsvFileWriterConfig('output.csv')) // Step 5: Load + ->addLink(new ExternalFileProcessorConfig()); // Step 6: Cleanup +``` + +## Best Practices + +**1. Use Specific Patterns** +```php +// Good: Specific pattern +new DataItem('/^customer_export_[0-9]{8}\.csv$/') + +// Too broad: Might match unwanted files +new DataItem('/customer\.csv/') +``` + +**2. Always Clean Up** +```php +// Copy file locally +->addLink(new ExternalFileProcessorConfig()) +// ... process file ... +// Remove local copy +->addLink(new ExternalFileProcessorConfig()) +``` + +**3. Add Error Handling** +```php +// Wrap in FailSafe for network issues +$chainConfig->addLink(new FailSafeConfig( + chainConfig: $fileImportChain, + exceptionsToCatch: [\Exception::class], + nbAttempts: 3 +)); +``` + +**4. Log File Processing** +```php +->addLink(new LogConfig( + message: 'Processing file: @data["file"], size: @data["size"] bytes', + level: 'info' +)) +``` + +**5. Validate Files Before Processing** +```php +->addLink(new CallBackTransformerConfig(function(DataItem $item) { + $data = $item->getData(); + + // Validate file size + if ($data['size'] > 100 * 1024 * 1024) { // 100MB + throw new \RuntimeException('File too large'); + } + + return $item; +})) +``` + +## Common Use Cases + +- **SFTP Import**: Fetch files from SFTP servers (partners, vendors) +- **S3 Import**: Process files uploaded to AWS S3 buckets +- **FTP Import**: Import files from legacy FTP servers +- **Azure Blob**: Process files from Azure Blob Storage +- **Multi-Source**: Import from multiple remote locations +- **Scheduled Imports**: Daily/hourly file imports from remote systems +- **EDI Processing**: Import EDI files from partner systems +- **Data Lake**: Process files from data lake storage diff --git a/docs/doc/20-operations/20-extract/020-csv.md b/docs/doc/20-operations/20-extract/020-csv.md index f098a1d..95eb21c 100644 --- a/docs/doc/20-operations/20-extract/020-csv.md +++ b/docs/doc/20-operations/20-extract/020-csv.md @@ -1,38 +1,410 @@ --- layout: base title: PHP-ETL - Operations -subTitle: Extract- CSV File(csv-read) +subTitle: Extract - CSV File --- -The `csv-read` operation reads a CSV file, outputting a `GroupedItem` with an iterator for row-by-row processing. It typically follows an operation providing a file path, like `file-finder`. +The `CsvExtractConfig` operation reads a CSV file and outputs individual `DataItem`s for each row. It's typically used at the beginning of a chain or after file-finding operations to extract data from CSV files for processing. -The operation receives a `DataItem` that contains the path to the csv file to read. It will return a list DataItem's. -Should e used after a `external file processor` operation. +**Key characteristics:** +- Reads CSV files row by row +- Returns one `DataItem` per CSV row +- Configurable delimiter, enclosure, and escape characters +- Supports nested file paths in data items +- Efficient for large files (streaming, not loading entire file into memory) -## Options +--- + +## Configuration + +Use `CsvExtractConfig` with these optional parameters: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; + +$csvConfig = new CsvExtractConfig( + delimiter: ';', // Field separator (default: ';') + enclosure: '"', // Field enclosure (default: '"') + escape: '\\', // Escape character (default: '\') + fileKey: 'file' // Key containing file path (default: 'file') +); +``` + +**Parameters:** +- `delimiter`: Character separating fields. Common values: `','`, `';'`, `'\t'` +- `enclosure`: Character enclosing fields (for fields containing delimiters). Must be `"` or `'` +- `escape`: Character for escaping special characters within fields +- `fileKey`: The key in the input data that contains the CSV file path (supports nested keys like `'data/file'`) + +--- + +## Example: Basic CSV Reading + +Read a CSV file and process each row: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('name', [['get' => ['field' => 'Name']]]) + ->addColumn('email', [['get' => ['field' => 'Email']]]) + ) + ->addLink(new CsvFileWriterConfig('output.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); + +// Input: DataItem with file path +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'customers.csv'])]), + [] +); +``` + +## Example: Comma-Delimited CSV + +For standard comma-separated CSV files: + +```php +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig( + delimiter: ',', + enclosure: '"', + escape: '\\' + )) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'ID']]]) + ->addColumn('product', [['get' => ['field' => 'Product']]]) + ->addColumn('price', [['get' => ['field' => 'Price']]]) + ) + ->addLink(new CsvFileWriterConfig('products.csv')); + +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'data/products.csv'])]), + [] +); +``` + +## Example: Tab-Delimited File + +For TSV (tab-separated values) files: + +```php +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig( + delimiter: "\t", // Tab delimiter + enclosure: '"', + escape: '\\' + )) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'UserID']]]) + ->addColumn('username', [['get' => ['field' => 'Username']]]) + ) + ->addLink(new CsvFileWriterConfig('users.csv')); + +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'users.tsv'])]), + [] +); +``` + +## Example: Custom File Key + +When the file path is in a different key or nested: + +```php +// Input data: ['data' => ['csv_file' => 'import.csv']] + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig( + fileKey: 'data/csv_file' // Nested path to file + )) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('order_id', [['get' => ['field' => 'OrderID']]]) + ->addColumn('amount', [['get' => ['field' => 'Amount']]]) + ) + ->addLink(new CsvFileWriterConfig('orders.csv')); + +$chainProcessor->process( + new ArrayIterator([ + new DataItem(['data' => ['csv_file' => 'orders.csv']]) + ]), + [] +); +``` + +## Example: Processing Multiple CSV Files + +Read and process multiple CSV files in one chain: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig(delimiter: ',')) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + $data = $item->getData(); + echo "Processing row from {$data['source_file']}: {$data['ID']}\n"; + return $item; + })) + ->addLink(new CsvFileWriterConfig('combined.csv')); + +// Process multiple files +$files = [ + new DataItem(['file' => 'customers1.csv']), + new DataItem(['file' => 'customers2.csv']), + new DataItem(['file' => 'customers3.csv']), +]; + +$chainProcessor->process(new ArrayIterator($files), []); +``` + +## Example: With External File Finder + +Import files from remote location and process CSV content: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + // Find CSV files on remote filesystem + ->addLink(new ExternalFileFinderConfig(directory: '/remote/exports')) + // Copy file locally + ->addLink(new ExternalFileProcessorConfig()) + // Extract CSV data + ->addLink(new CsvExtractConfig(delimiter: ',')) + // Transform data + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'ID']]]) + ->addColumn('status', [['get' => ['field' => 'Status']]]) + ) + // Save results + ->addLink(new CsvFileWriterConfig('imported-data.csv')) + // Clean up local file + ->addLink(new ExternalFileProcessorConfig()); + +$chainProcessor->process( + new ArrayIterator([new DataItem('/^export_[0-9]{8}\.csv$/')]), + [] +); +``` + +## Example: With Filtering + +Extract CSV and filter rows based on conditions: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig(delimiter: ',')) + // Only process active customers + ->addLink(new FilterDataConfig('@data["Status"] == "Active"')) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('name', [['get' => ['field' => 'Name']]]) + ->addColumn('email', [['get' => ['field' => 'Email']]]) + ) + ->addLink(new CsvFileWriterConfig('active-customers.csv')); + +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'all-customers.csv'])]), + [] +); +``` + +## Example: With Grouping + +Extract CSV and group data: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Grouping\SimpleGroupingConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig(delimiter: ',')) + // Group orders by customer + ->addLink(new SimpleGroupingConfig( + groupBy: [['get' => ['field' => 'CustomerID']]], + aggregations: [ + 'total_orders' => [['count' => []]], + 'total_amount' => [['sum' => ['field' => 'Amount']]], + ] + )) + ->addLink(new CsvFileWriterConfig('customer-summary.csv')); + +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'orders.csv'])]), + [] +); +``` + +## Example: Large File Processing with Logging + +Process large CSV files with progress logging: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\LogConfig; + +$chainConfig = new ChainConfig(); + +$rowCount = 0; + +$chainConfig + ->addLink(new CsvExtractConfig(delimiter: ',')) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) use (&$rowCount) { + $rowCount++; + if ($rowCount % 1000 == 0) { + echo "Processed {$rowCount} rows...\n"; + } + return $item; + })) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'ID']]]) + ->addColumn('data', [['get' => ['field' => 'Data']]]) + ) + ->addLink(new CsvFileWriterConfig('output.csv')); + +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'large-dataset.csv'])]), + [] +); + +echo "Completed! Total rows: {$rowCount}\n"; +``` + +## Example: CSV with Error Handling + +Handle malformed CSV files gracefully: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; + +$csvChain = (new ChainConfig()) + ->addLink(new CsvExtractConfig(delimiter: ',')) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'ID']]]) + ->addColumn('name', [['get' => ['field' => 'Name']]]) + ); + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new FailSafeConfig( + chainConfig: $csvChain, + exceptionsToCatch: [\Exception::class], + nbAttempts: 1 // Don't retry malformed files + )) + ->addLink(new CsvFileWriterConfig('valid-records.csv')); + +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'input.csv'])]), + [] +); +``` + +## Understanding CSV Structure + +**Input CSV file:** +```csv +ID,Name,Email,Status +1,John Doe,john@example.com,Active +2,Jane Smith,jane@example.com,Inactive +3,Bob Wilson,bob@example.com,Active +``` + +**After CsvExtractConfig, each row becomes a DataItem:** +```php +DataItem(['ID' => '1', 'Name' => 'John Doe', 'Email' => 'john@example.com', 'Status' => 'Active']) +DataItem(['ID' => '2', 'Name' => 'Jane Smith', 'Email' => 'jane@example.com', 'Status' => 'Inactive']) +DataItem(['ID' => '3', 'Name' => 'Bob Wilson', 'Email' => 'bob@example.com', 'Status' => 'Active']) +``` + +**Key points:** +- First row is used as column headers +- Each subsequent row becomes a separate DataItem +- Column headers become array keys in the DataItem +- All values are strings (type conversion happens in transformers) + +## Best Practices + +**1. Match Delimiter to File Format** +```php +// Standard CSV (comma) +new CsvExtractConfig(delimiter: ',') + +// European CSV (semicolon) +new CsvExtractConfig(delimiter: ';') + +// Tab-delimited +new CsvExtractConfig(delimiter: "\t") +``` -- **delimiter:** (Optional) Character separating fields. Defaults to `;`. -- **enclosure:** (Optional) Character enclosing fields. Defaults to `"`. -- **escape:** (Optional) Character escaping special characters. Defaults to `\`. -- **file_key:** (Optional) If input is an array, the key (e.g., `key/subkey`) containing the CSV file path. +**2. Handle Missing Headers** +```php +// If CSV has no header row, add one first or use CallbackTransformer +->addLink(new CallBackTransformerConfig(function(DataItem $item) { + $data = $item->getData(); + // Map numeric indices to named keys + return new DataItem([ + 'id' => $data[0] ?? null, + 'name' => $data[1] ?? null, + 'email' => $data[2] ?? null, + ]); +})) +``` -## Example +**3. Validate Data Early** +```php +->addLink(new FilterDataConfig('@data["ID"] != null and data["Email"] != ""')) +``` -```yaml -chain: - - operation: file-finder - options: - path: /path/to/input.csv +**4. Log Progress for Large Files** +```php +->addLink(new LogConfig( + message: 'Processing record @data["ID"]', + level: 'debug' +)) +``` - - operation: csv-read - options: - delimiter: "," +**5. Handle Special Characters** +```php +// Ensure enclosure is set correctly for fields with delimiters +new CsvExtractConfig( + delimiter: ',', + enclosure: '"', // Fields like "Smith, John" are handled correctly + escape: '\\' +) +``` - - operation: rule-transformer - options: - # Rules to transform the CSV data. +**6. Use Appropriate File Keys** +```php +// For simple file path +fileKey: 'file' - - operation: load-csv - options: - path: /path/to/output.csv -``` \ No newline at end of file +// For nested data structures +fileKey: 'input/csv_file' +fileKey: 'data/source/path' +``` diff --git a/docs/doc/20-operations/20-extract/030-json.md b/docs/doc/20-operations/20-extract/030-json.md index b3e0847..5308a2d 100644 --- a/docs/doc/20-operations/20-extract/030-json.md +++ b/docs/doc/20-operations/20-extract/030-json.md @@ -1,33 +1,371 @@ --- layout: base title: PHP-ETL - Operations -subTitle: Extract - JSON File(json-read) +subTitle: Extract - JSON File --- -The `json-read` operation reads a JSON file, outputting a `GroupedItem` with an iterator for the JSON data. It typically follows an operation providing a file path, like `file-finder`. +The `JsonExtractConfig` operation reads a JSON file and outputs `DataItem`s for each element in the JSON structure. It's designed to handle both single JSON objects and arrays of objects, making it ideal for API responses, configuration files, and data exports in JSON format. -The operation receives a `DataItem` that contains the path to the csv file to read. It will return a list DataItem's. -Should e used after a `external file processor` operation. +--- + +## Configuration + +Use `JsonExtractConfig` with an optional fileKey parameter: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\JsonExtractConfig; + +$jsonConfig = new JsonExtractConfig( + fileKey: 'file' // Optional: key containing file path (default: null, reads from data directly) +); +``` + +**Parameters:** +- `fileKey`: The key in the input data that contains the JSON file path. If null, the input data itself is treated as the file path (supports nested keys like `'data/file'`) + +**Input Data:** +- When `fileKey` is null: DataItem containing the file path as a string +- When `fileKey` is set: DataItem containing an array with the file path at the specified key + +--- + +## Example: Basic JSON Reading + +Read a JSON file containing an array of objects: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\JsonExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new JsonExtractConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'id']]]) + ->addColumn('name', [['get' => ['field' => 'name']]]) + ->addColumn('email', [['get' => ['field' => 'email']]]) + ) + ->addLink(new CsvFileWriterConfig('customers.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); + +// Input: file path as string +$chainProcessor->process( + new ArrayIterator([new DataItem('data/customers.json')]), + [] +); +``` + +**Input JSON file (data/customers.json):** +```json +[ + {"id": 1, "name": "John Doe", "email": "john@example.com"}, + {"id": 2, "name": "Jane Smith", "email": "jane@example.com"}, + {"id": 3, "name": "Bob Wilson", "email": "bob@example.com"} +] +``` + +## Example: With FileKey + +When the file path is in a specific key: + +```php +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new JsonExtractConfig(fileKey: 'file')) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('product_id', [['get' => ['field' => 'productId']]]) + ->addColumn('sku', [['get' => ['field' => 'sku']]]) + ->addColumn('price', [['get' => ['field' => 'price']]]) + ) + ->addLink(new CsvFileWriterConfig('products.csv')); + +// Input: DataItem with 'file' key +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'data/products.json'])]), + [] +); +``` + +## Example: Nested JSON Structure + +Extract data from nested JSON objects: + +```php +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new JsonExtractConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('order_id', [['get' => ['field' => 'id']]]) + ->addColumn('customer_name', [['get' => ['field' => ['customer', 'name']]]]) + ->addColumn('customer_email', [['get' => ['field' => ['customer', 'email']]]]) + ->addColumn('total', [['get' => ['field' => 'total']]]) + ->addColumn('status', [['get' => ['field' => 'status']]]) + ) + ->addLink(new CsvFileWriterConfig('orders.csv')); + +$chainProcessor->process( + new ArrayIterator([new DataItem('data/orders.json')]), + [] +); +``` + +**Input JSON with nested structure:** +```json +[ + { + "id": 1001, + "customer": { + "name": "John Doe", + "email": "john@example.com" + }, + "total": 150.00, + "status": "completed" + } +] +``` + +## Example: Processing API Response + +Convert API JSON response to CSV: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + // Fetch data from API + ->addLink(new SimpleHttpConfig( + url: 'https://api.example.com/users', + method: 'GET', + responseIsJson: true, + responseKey: 'api_response' + )) + // Save JSON response to file temporarily + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + $data = $item->getData(); + $jsonFile = 'temp/api-response.json'; + file_put_contents($jsonFile, json_encode($data['api_response'])); + return new DataItem($jsonFile); + })) + // Extract JSON + ->addLink(new JsonExtractConfig()) + // Transform + ->addLink((new RuleTransformConfig(false)) + ->addColumn('user_id', [['get' => ['field' => 'id']]]) + ->addColumn('username', [['get' => ['field' => 'username']]]) + ) + ->addLink(new CsvFileWriterConfig('users.csv')); + +$chainProcessor->process( + new ArrayIterator([new DataItem([])]), + [] +); +``` + +## Example: With Dynamic Columns + +Use context to create dynamic column names: + +```php +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new JsonExtractConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('productId', [['get' => ['field' => 'productId']]]) + ->addColumn('sku', [['get' => ['field' => 'sku']]]) + // Dynamic column for each locale + ->addColumn('name-{@context/locales}', [['get' => ['field' => ['name', '@context/locales']]]]) + ) + ->addLink(new CsvFileWriterConfig('products.csv')); + +$chainProcessor->process( + new ArrayIterator([new DataItem('data/products.json')]), + ['locales' => ['fr_FR', 'en_US']] +); +``` + +**Input JSON:** +```json +[ + { + "productId": 1, + "sku": "PROD-001", + "name": { + "fr_FR": "Produit Un", + "en_US": "Product One" + } + } +] +``` + +**Output CSV:** +```csv +productId,sku,name-fr_FR,name-en_US +1,PROD-001,Produit Un,Product One +``` + +## Example: Flattening Complex JSON + +Convert complex nested JSON to flat CSV structure: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new JsonExtractConfig()) + // Split items array into separate DataItems + ->addLink(new SplitItemConfig(keys: ['items'])) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('order_id', [['get' => ['field' => 'orderId']]]) + ->addColumn('item_name', [['get' => ['field' => 'data', 'name']]]) + ->addColumn('item_price', [['get' => ['field' => 'data', 'price']]]) + ->addColumn('quantity', [['get' => ['field' => 'data', 'quantity']]]) + ) + ->addLink(new CsvFileWriterConfig('order-items.csv')); + +$chainProcessor->process( + new ArrayIterator([new DataItem('data/orders.json')]), + [] +); +``` + +**Input JSON:** +```json +[ + { + "orderId": 1, + "items": [ + {"name": "Widget", "price": 10.00, "quantity": 2}, + {"name": "Gadget", "price": 15.00, "quantity": 1} + ] + } +] +``` + +**Output:** One row per item (2 rows from 1 order) + +## Example: With Filtering + +Extract and filter JSON data: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new JsonExtractConfig()) + // Only process active products + ->addLink(new FilterDataConfig('@data["status"] == "active"')) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'id']]]) + ->addColumn('name', [['get' => ['field' => 'name']]]) + ->addColumn('price', [['get' => ['field' => 'price']]]) + ) + ->addLink(new CsvFileWriterConfig('active-products.csv')); + +$chainProcessor->process( + new ArrayIterator([new DataItem('data/products.json')]), + [] +); +``` + +## Example: Multiple JSON Files + +Process multiple JSON files in one chain: + +```php +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new JsonExtractConfig(fileKey: 'file')) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'id']]]) + ->addColumn('data', [['get' => ['field' => 'data']]]) + ->addColumn('source_file', [['get' => ['field' => 'file']]]) + ) + ->addLink(new CsvFileWriterConfig('combined.csv')); + +// Process multiple files +$files = [ + new DataItem(['file' => 'data1.json']), + new DataItem(['file' => 'data2.json']), + new DataItem(['file' => 'data3.json']), +]; + +$chainProcessor->process(new ArrayIterator($files), []); +``` + +## Example: With External File Finder + +Import JSON files from remote location: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; + +$chainConfig = new ChainConfig(); + +$chainConfig + // Find JSON files on remote filesystem + ->addLink(new ExternalFileFinderConfig(directory: '/remote/exports')) + // Copy file locally + ->addLink(new ExternalFileProcessorConfig()) + // Extract JSON data + ->addLink(new JsonExtractConfig()) + // Transform data + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'id']]]) + ->addColumn('status', [['get' => ['field' => 'status']]]) + ) + // Save results + ->addLink(new CsvFileWriterConfig('imported-data.csv')) + // Clean up local file + ->addLink(new ExternalFileProcessorConfig()); + +$chainProcessor->process( + new ArrayIterator([new DataItem('/^export_[0-9]{8}\.json$/')]), + [] +); +``` -## Options +## Example: Error Handling for Malformed JSON -- **file_key:** (Optional) If the input data is an array, the key (e.g., `key/subkey`) containing the JSON file path. +Handle invalid JSON files gracefully: -## Example +```php +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; -```yaml -chain: - - operation: file-finder - options: - path: /path/to/input.json +$jsonChain = (new ChainConfig()) + ->addLink(new JsonExtractConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'id']]]) + ->addColumn('name', [['get' => ['field' => 'name']]]) + ); - - operation: json-read +$chainConfig = new ChainConfig(); - - operation: rule-transformer - options: - # Rules to transform the JSON data. +$chainConfig + ->addLink(new FailSafeConfig( + chainConfig: $jsonChain, + exceptionsToCatch: [\JsonException::class, \Exception::class], + nbAttempts: 1 // Don't retry malformed JSON + )) + ->addLink(new CsvFileWriterConfig('valid-records.csv')); - - operation: load-json - options: - path: /path/to/output.json -``` \ No newline at end of file +$chainProcessor->process( + new ArrayIterator([new DataItem('data.json')]), + [] +); +``` diff --git a/docs/doc/20-operations/40-transform/010-external-file.md b/docs/doc/20-operations/40-transform/010-external-file.md index c21957c..67e385e 100644 --- a/docs/doc/20-operations/40-transform/010-external-file.md +++ b/docs/doc/20-operations/40-transform/010-external-file.md @@ -1,22 +1,18 @@ --- layout: base title: PHP-ETL - Operations -subTitle: Transform - External File(external-file) +subTitle: Transform - External File Processor --- -The `external-file-processor` operation is designed to **move and register external files** -(e.g., from SFTP, local FS, cloud storage) into the ETL execution context. -This operation works hand-in-hand with the `external-file-finder` and is essential for enabling -further processing of remote files. +The **External File Processor** operation moves and registers external files (e.g., from SFTP, local FS, cloud storage) into the ETL execution context. This operation works hand-in-hand with the `ExternalFileFinderConfig` and is essential for enabling further processing of remote files. ## Purpose -When `external-file-finder` locates a remote file, it returns an `ExternalFileItem`. -However, that file is not yet part of the ETL's working context. +When `ExternalFileFinderConfig` locates a remote file, it returns an `FileExtractedItem`. However, that file is not yet part of the ETL's working context. -The `external-file-processor` operation: +The `ExternalFileProcessorConfig` operation: -- **Copies the external file into the ETL context**, making it accessible to extract operations like `csv-read`, `xml-read`, etc. +- **Copies the external file into the ETL context**, making it accessible to extract operations like `CsvExtractConfig`, `JsonExtractConfig`, etc. - **Archives the file** within the ETL execution history, so it can be tracked and audited later. - **Returns a `DataItem`** containing the path of the new local file, making it usable by downstream operations. @@ -25,22 +21,327 @@ The `external-file-processor` operation: The operation follows a structured file management flow across multiple runs: 1. **Initial Processing**: - - The file is moved from its source directory into a `processing/` subdirectory (within the external filesystem). - - It is also copied to the local ETL context (temporary working directory). - - A `DataItem` is emitted with the new local file path. + - The file is moved from its source directory into a `processing/` subdirectory (within the external filesystem). + - It is also copied to the local ETL context (temporary working directory). + - A `DataItem` is emitted with the new local file path. 2. **Post-Processing**: - - If the operation is used a second time in the same chain (e.g., near the end of the flow), it will: - - Move the remote file from `processing/` to `processed/`, effectively archiving it. - - This signals the file has been fully and successfully handled. + - If the operation is used a second time in the same chain (e.g., near the end of the flow), it will: + - Move the remote file from `processing/` to `processed/`, effectively archiving it. + - This signals the file has been fully and successfully handled. > šŸ’” **Best Practice**: -> Use `external-file-processor` **twice** in a chain: -> - Once immediately after the `external-file-finder`. +> Use `ExternalFileProcessorConfig` **twice** in a chain: +> - Once immediately after the `ExternalFileFinderConfig`. > - Once at the end of the chain, to archive the file remotely after successful processing. - ## Filesystem Agnostic -The operation does **not require manual configuration of the filesystem**. -It uses the File system instance already embedded in the `ExternalFileItem` provided by the `externalFileFinder`. +The operation does **not require manual configuration of the filesystem**. It uses the File system instance already embedded in the `FileExtractedItem` provided by the `ExternalFileFinderConfig`. + +--- + +## Configuration + +### Basic Configuration + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; + +$config = new ExternalFileProcessorConfig(); +``` + +### Constructor Parameters + +```php +public function __construct( + string $flavor = 'default' +) +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `flavor` | `string` | `'default'` | Operation flavor for custom implementations | + +--- + +## Input/Output + +- **Input**: Expects `FileExtractedItem` objects from `ExternalFileFinderConfig` +- **Output**: Produces `DataItem` objects containing local file paths + +--- + +## Examples + +### Example 1: Basic File Processing + +Process files from a local directory, move them through processing stages: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new ExternalFileFinderConfig(directory: '/data/incoming')) + ->addLink(new ExternalFileProcessorConfig()) // Copy to local context + ->addLink(new CsvExtractConfig()) + ->addLink(new CsvFileWriterConfig('output.csv')) + ->addLink(new ExternalFileProcessorConfig()); // Archive to processed/ + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('/^orders_.*\.csv$/')]), + ['dir' => '/data/incoming'] +); +``` + +**What happens:** +1. `ExternalFileFinderConfig` finds files matching the pattern in `/data/incoming` +2. First `ExternalFileProcessorConfig` moves files to `processing/` and copies to local context +3. Files are processed (CSV extraction and transformation) +4. Second `ExternalFileProcessorConfig` moves files from `processing/` to `processed/` + +### Example 2: SFTP File Processing + +Process files from an SFTP server with error handling: + +```php +use League\Flysystem\Ftp\FtpAdapter; +use League\Flysystem\Ftp\FtpConnectionOptions; +use League\Flysystem\Filesystem; +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +// Create SFTP filesystem +$adapter = new FtpAdapter( + FtpConnectionOptions::fromArray([ + 'host' => 'sftp.example.com', + 'username' => 'user', + 'password' => 'pass', + 'port' => 22, + ]) +); +$filesystem = new Filesystem($adapter); + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new ExternalFileFinderConfig( + directory: '/uploads', + fileSystem: $filesystem + )) + ->addLink(new ExternalFileProcessorConfig()) + ->addLink(new CsvExtractConfig(delimiter: ';')) + ->addLink(new RuleTransformConfig([ + 'status' => '"processed"', + 'processed_at' => 'now()' + ])) + ->addLink(new CsvFileWriterConfig('processed_orders.csv')) + ->addLink(new ExternalFileProcessorConfig()); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('/^order_.*\.csv$/')]), + [] +); +``` + +### Example 3: Multiple File Types with Conditional Processing + +Process different file types differently after initial file processor: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Building\ChainSplitConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\JsonExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$csvChain = new ChainConfig(); +$csvChain + ->addLink(new CsvExtractConfig()) + ->addLink(new CsvFileWriterConfig('csv_output.csv')); + +$jsonChain = new ChainConfig(); +$jsonChain + ->addLink(new JsonExtractConfig()) + ->addLink(new CsvFileWriterConfig('json_output.csv')); + +$mainChain = new ChainConfig(); +$mainChain + ->addLink(new ExternalFileFinderConfig(directory: '/data/mixed')) + ->addLink(new ExternalFileProcessorConfig()) + ->addLink(new ChainSplitConfig() + ->addSplit('preg_match("/\.csv$/", item.data)', $csvChain) + ->addSplit('preg_match("/\.json$/", item.data)', $jsonChain) + ) + ->addLink(new ExternalFileProcessorConfig()); + +$chainProcessor = $chainBuilder->createChain($mainChain); +$chainProcessor->process( + new ArrayIterator([new DataItem('/^data_.*\.(csv|json)$/')]), + [] +); +``` + +### Example 4: Processing with Validation + +Validate files before processing and only archive valid ones: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new ExternalFileFinderConfig(directory: '/data/incoming')) + ->addLink(new ExternalFileProcessorConfig()) + ->addLink(new CsvExtractConfig()) + // Validate: only keep rows with required fields + ->addLink(new FilterDataConfig( + 'item.id != null and item.name != null and item.email != null' + )) + // Transform: add validation status + ->addLink(new CallBackTransformerConfig( + function($item, $context) { + $item['validated'] = true; + $item['validation_date'] = date('Y-m-d H:i:s'); + return $item; + } + )) + ->addLink(new CsvFileWriterConfig('validated_output.csv')) + ->addLink(new ExternalFileProcessorConfig()); // Only archive if processing succeeded + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('/^customers_.*\.csv$/')]), + [] +); +``` + +### Example 5: Re-processing Failed Files + +Process files that failed in previous runs: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Building\FailSafeConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +// Create chain with error handling +$processingChain = new ChainConfig(); +$processingChain + ->addLink(new CsvExtractConfig()) + ->addLink(new CsvFileWriterConfig('output.csv')); + +$mainChain = new ChainConfig(); +$mainChain + // Look in both incoming and processing directories (for retry) + ->addLink(new ExternalFileFinderConfig( + directory: '/data/incoming', + additionalDirectories: ['/data/processing'] + )) + ->addLink(new ExternalFileProcessorConfig()) + ->addLink(new FailSafeConfig( + chain: $processingChain, + retries: 3 + )) + ->addLink(new ExternalFileProcessorConfig()); + +$chainProcessor = $chainBuilder->createChain($mainChain); +$chainProcessor->process( + new ArrayIterator([new DataItem('/^data_.*\.csv$/')]), + [] +); +``` + +**Note**: Files that failed previously will be in the `processing/` directory. The `ExternalFileFinderConfig` can be configured to look in multiple directories. + +--- + +## File Lifecycle Details + +### Directory Structure + +The operation creates and uses the following directory structure: + +``` +/data/incoming/ # Original files arrive here +ā”œā”€ā”€ file1.csv +ā”œā”€ā”€ file2.csv +ā””ā”€ā”€ processing/ # Files being processed + ā””ā”€ā”€ file1.csv +ā””ā”€ā”€ processed/ # Successfully processed files + ā””ā”€ā”€ file1.csv +``` + +### State Transitions + +1. **Found**: File is discovered by `ExternalFileFinderConfig` in the source directory +2. **Processing**: First `ExternalFileProcessorConfig` moves file to `processing/` subdirectory +3. **Processed**: Second `ExternalFileProcessorConfig` moves file to `processed/` subdirectory + +--- + +## Best Practices + +### 1. Always Use Twice + +Always use `ExternalFileProcessorConfig` twice in your chain: + +```php +$chainConfig + ->addLink(new ExternalFileFinderConfig(...)) + ->addLink(new ExternalFileProcessorConfig()) // Move to processing/ + // ... processing operations ... + ->addLink(new ExternalFileProcessorConfig()); // Archive to processed/ +``` + +This ensures proper file lifecycle management. + +### 2. Log File Processing + +Add logging to track file processing: + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\LogConfig; + +$chainConfig + ->addLink(new ExternalFileFinderConfig(...)) + ->addLink(new LogConfig('Found file: {item.filename}')) + ->addLink(new ExternalFileProcessorConfig()) + ->addLink(new LogConfig('Processing file: {item.data}')) + // ... processing ... + ->addLink(new LogConfig('Completed file: {item.data}')) + ->addLink(new ExternalFileProcessorConfig()); +``` + +## Related Operations + +- [ExternalFileFinderConfig](../20-extract/010-file-finder.html) - Find files in external filesystems +- [CsvExtractConfig](../20-extract/020-csv.html) - Extract data from CSV files +- [JsonExtractConfig](../20-extract/030-json.html) - Extract data from JSON files +- [FailSafeConfig](../10-building/040-safe.html) - Handle errors during processing diff --git a/docs/doc/20-operations/40-transform/020-filter-data.md b/docs/doc/20-operations/40-transform/020-filter-data.md index 5f6ae22..544f66d 100644 --- a/docs/doc/20-operations/40-transform/020-filter-data.md +++ b/docs/doc/20-operations/40-transform/020-filter-data.md @@ -1,43 +1,410 @@ --- layout: base title: PHP-ETL - Operations -subTitle: Transform - Filter Data(filter) +subTitle: Transform - Filter Data --- -The `filter` operation selectively skips items in the chain based on a rule. It uses the [rule engine](/doc/20-operations/40-transform/030-rule-transformer) to evaluate a condition; if the condition is not met, the item is not passed to subsequent operations. +The **Filter Data** operation selectively skips items in the chain based on rules. It uses the [rule engine](030-rule-transformer.html) to evaluate conditions; if the condition is not met, the item is not passed to subsequent operations. -## Options +--- + +## Purpose + +Use `FilterDataConfig` to: +- Remove unwanted items from the data stream +- Keep only items that meet specific criteria +- Filter based on field values or Symfony expressions +- Implement conditional data processing logic + +--- + +## Configuration + +### Basic Configuration + +```php +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; + +// Keep items where field value is truthy +$config = new FilterDataConfig([ + ["get" => ["field" => "IsSubscribed"]] +]); +``` + +### Constructor Parameters + +```php +public function __construct( + public readonly array $rules, + public readonly bool $negate = false, + string $flavor = 'default' +) +``` -- **rule:** The rule to be evaluated. If the rule evaluates to a "truthy" value (not `null` and not `false`), the item is kept. Otherwise, it is filtered out. -- **negate:** (Optional) A boolean that, if set to `true`, inverts the result of the rule. In this case, items that evaluate to a "truthy" value are filtered out, and items that evaluate to a "falsy" value are kept. +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `rules` | `array` | *required* | Rule engine rules to evaluate. Item kept if rule evaluates to truthy value | +| `negate` | `bool` | `false` | If `true`, inverts the logic (keeps items that evaluate to falsy) | +| `flavor` | `string` | `'default'` | Operation flavor for custom implementations | -## Example +--- -Here's an example of how to use the `filter` operation to keep only the items where the `status` field is equal to `"published"`: +## Rule Engine Basics -```yaml -chain: - - operation: filter - options: - rule: - #... +The `rules` parameter uses the rule engine syntax. The rule engine has the following operations: - - operation: load-to-database - options: - # Options to load the filtered data into a database. +### Get Field Value +```php +// Keep items where field exists and is truthy +["get" => ["field" => "fieldName"]] ``` -Here's an example of how to use the `filter` operation with `negate` to filter out items where the `age` is less than `18`: +### Expression Language (for complex conditions) +```php +// Keep items where status equals "published" using Symfony Expression Language +["expression_language" => [ + "expression" => "rowData.status == 'published'" +]] + +// Keep items where age >= 18 +["expression_language" => [ + "expression" => "rowData.age >= 18" +]] -```yaml -chain: - - operation: filter - options: - rule: - #... - negate: true +// Complex condition: status is published AND age >= 18 +["expression_language" => [ + "expression" => "rowData.status == 'published' and rowData.age >= 18" +]] +``` + +### Constant Value +```php +// Always keep all items (returns true constant) +["constant" => ["value" => true]] - - operation: load-to-database - options: - # Options to load the filtered data into a database. +// Never keep items (returns false constant) +["constant" => ["value" => false]] ``` + +For all available rule engine operations, see the [Rule Transformer documentation](030-rule-transformer.html). + +--- + +## Input/Output + +- **Input**: `DataItem` objects with data to filter +- **Output**: `DataItem` objects that match the filter criteria (others are discarded) + +--- + +## Examples + +### Example 1: Simple Field Filter + +Keep only subscribed customers: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new FilterDataConfig([ + ["get" => ["field" => "IsSubscribed"]] + ])) + ->addLink(new CsvFileWriterConfig('subscribed_customers.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'data/customers.csv'])]), + [] +); +``` + +**Result**: Only rows where `IsSubscribed` is truthy (non-null, non-false) are written to output. + +### Example 2: Negated Filter + +Keep only non-subscribed customers using negate: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new FilterDataConfig( + rules: [["get" => ["field" => "IsSubscribed"]]], + negate: true + )) + ->addLink(new CsvFileWriterConfig('not_subscribed_customers.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'data/customers.csv'])]), + [] +); +``` + +**Result**: Only rows where `IsSubscribed` is falsy (null or false) are written to output. + +### Example 3: Equality Check with Expression Language + +Keep only published articles: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\JsonExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new JsonExtractConfig()) + ->addLink(new FilterDataConfig([ + ["expression_language" => [ + "expression" => "rowData.status == 'published'" + ]] + ])) + ->addLink(new CsvFileWriterConfig('published_articles.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('articles.json')]), + [] +); +``` + +### Example 4: Numeric Range Filter + +Keep only adults (age >= 18): + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new FilterDataConfig([ + ["expression_language" => [ + "expression" => "rowData.age >= 18" + ]] + ])) + ->addLink(new CsvFileWriterConfig('adults.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('users.csv')]), + [] +); +``` + +### Example 5: Multiple Conditions (AND) + +Keep only active, verified users: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new FilterDataConfig([ + ["expression_language" => [ + "expression" => "rowData.status == 'active' and rowData.verified == true" + ]] + ])) + ->addLink(new CsvFileWriterConfig('active_verified_users.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('users.csv')]), + [] +); +``` + +### Example 6: Multiple Conditions (OR) + +Keep orders that are either pending or processing: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new FilterDataConfig([ + ["expression_language" => [ + "expression" => "rowData.status in ['pending', 'processing']" + ]] + ])) + ->addLink(new CsvFileWriterConfig('active_orders.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('orders.csv')]), + [] +); +``` + +### Example 7: Pattern Matching with Expression Language + +Keep emails from specific domains: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new FilterDataConfig([ + ["expression_language" => [ + "expression" => "rowData.email matches '/@(company|partner)\\.com$/'" + ]] + ])) + ->addLink(new CsvFileWriterConfig('business_contacts.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('contacts.csv')]), + [] +); +``` + +### Example 8: Split into Multiple Files + +Use `ChainSplitConfig` with filters to split data into multiple output files: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Building\ChainSplitConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +// Chain for subscribed customers +$subscribedChain = new ChainConfig(); +$subscribedChain + ->addLink(new FilterDataConfig([ + ["get" => ["field" => "IsSubscribed"]] + ])) + ->addLink(new CsvFileWriterConfig('customers-subscribed.csv')); + +// Chain for non-subscribed customers +$notSubscribedChain = new ChainConfig(); +$notSubscribedChain + ->addLink(new FilterDataConfig( + rules: [["get" => ["field" => "IsSubscribed"]]], + negate: true + )) + ->addLink(new CsvFileWriterConfig('customers-not-subscribed.csv')); + +// Main chain +$mainChain = new ChainConfig(); +$mainChain + ->addLink(new CsvExtractConfig()) + ->addLink(new ChainSplitConfig() + ->addSplit($subscribedChain) + ->addSplit($notSubscribedChain) + ) + ->addLink(new CsvFileWriterConfig('customers-all.csv')); + +$chainProcessor = $chainBuilder->createChain($mainChain); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'data/customers.csv'])]), + [] +); +``` + +**Result**: Three files are created: +- `customers-subscribed.csv` - Only subscribed customers +- `customers-not-subscribed.csv` - Only non-subscribed customers +- `customers-all.csv` - All customers + +### Example 9: Date Range Filter + +Keep records from the last 30 days: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new FilterDataConfig([ + ["expression_language" => [ + "expression" => "rowData.created_at >= date('-30 days')", + "values" => [] + ]] + ])) + ->addLink(new CsvFileWriterConfig('recent_orders.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('orders.csv')]), + [] +); +``` + +### Example 10: Null/Empty Check + +Keep only records with non-empty email addresses: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink(new FilterDataConfig([ + ["expression_language" => [ + "expression" => "rowData.email is not null and rowData.email != ''" + ]] + ])) + ->addLink(new CsvFileWriterConfig('contacts_with_email.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('contacts.csv')]), + [] +); +``` + +--- + +## Available Rule Engine Operations +For complex conditions (comparisons, AND/OR logic, etc.), use `expression_language` with [Symfony Expression Language syntax](https://symfony.com/doc/current/components/expression_language/syntax.html). diff --git a/docs/doc/20-operations/40-transform/030-rule-transformer.md b/docs/doc/20-operations/40-transform/030-rule-transformer.md index 9b48ec3..46149a7 100644 --- a/docs/doc/20-operations/40-transform/030-rule-transformer.md +++ b/docs/doc/20-operations/40-transform/030-rule-transformer.md @@ -1,12 +1,12 @@ --- layout: base title: PHP-ETL - Operations -subTitle: Transform - Rule Engine ā€“ Data Transformation(rule-engine-transformer) +subTitle: Transform - Rule Engine ā€“ Data Transformation --- -The Rule Engine is a lightweight transformation component that converts an associative array into another associative array using a flexible set of rules. +The **Rule Engine** is a lightweight transformation component that converts an associative array into another associative array using a flexible set of rules. -It is designed to be used within PHP-ETL through the `RuleTransformOperation`. +It is designed to be used within PHP-ETL through the `RuleTransformConfig`. ## Available Rules @@ -108,7 +108,9 @@ options: expression: "rowData['IsSubscribed'] == 'yes'" ``` -## Adding your own rules +--- + +## Adding Your Own Rules While PHP-ETL provides a powerful set of built-in rules, you may encounter situations where you need to implement your own custom logic. You can extend the `RuleApplier` class to add your own rules. @@ -135,9 +137,9 @@ class CustomRuleApplier extends RuleApplier } ``` -**2. Use your custom `RuleApplier` in the `ChainProcessor`:** +**2. Register your custom `RuleApplier` with the `ChainBuilderV2`:** -When you create your `ChainProcessor`, you need to tell it to use your custom `RuleApplier`. +When creating your `ChainBuilderV2`, pass your custom `RuleApplier` to the `GenericChainFactory` for `RuleTransformConfig`. {% capture column1 %} #### šŸ˜ Standalone @@ -145,31 +147,35 @@ When you create your `ChainProcessor`, you need to tell it to use your custom `R add( - new RuleTransformOperation( - $customRuleApplier, - [ - 'my_custom_field' => [ - 'rules' => [ - 'myCustomRule' => [ - 'field1' => ['get' => ['field' => 'FirstName']], - 'field2' => ['get' => ['field' => 'LastName']], - ], - ], - ], - ], - false - ) +$chainBuilder = new ChainBuilderV2([ + new GenericChainFactory( + RuleTransformOperation::class, + RuleTransformConfig::class, + injections: ['ruleApplier' => $customRuleApplier] + ), + // ... other factories +]); + +// Use the custom rule in your chain +$chainConfig = new ChainConfig(); +$chainConfig->addLink((new RuleTransformConfig(add: false)) + ->addColumn('MyCustomField', [ + ['myCustomRule' => [ + 'field1' => ['get' => ['field' => 'FirstName']], + 'field2' => ['get' => ['field' => 'LastName']], + ]] + ]) ); -$processor = new ChainProcessor($chainBuilder); +$processor = $chainBuilder->createChain($chainConfig); ``` {% endcapture %} {% capture column2 %} @@ -182,21 +188,25 @@ services: autowire: true tags: - { name: etl.rule } + + # The GenericChainFactory will automatically use your custom RuleApplier + # when it's injected via dependency injection ``` {% endcapture %} {% include block/2column.html column1=column1 column2=column2 %} -**3. Use your custom rule in your YAML configuration:** +**3. Use your custom rule in your chain configuration:** -Once you have configured your `ChainProcessor` to use your custom `RuleApplier`, you can use your custom rule in your YAML files. +Once you have configured your `ChainBuilderV2` to use your custom `RuleApplier`, you can use your custom rule in your chain. -```yaml -operation: rule-engine-transformer -options: - columns: - MyCustomField: - rules: - - myCustomRule: - field1: { get: { field: "FirstName" } } - field2: { get: { field: "LastName" } } +```php +$chainConfig = new ChainConfig(); +$chainConfig->addLink((new RuleTransformConfig(add: false)) + ->addColumn('MyCustomField', [ + ['myCustomRule' => [ + 'field1' => ['get' => ['field' => 'FirstName']], + 'field2' => ['get' => ['field' => 'LastName']], + ]] + ]) +); ``` \ No newline at end of file diff --git a/docs/doc/20-operations/40-transform/040-split-item.md b/docs/doc/20-operations/40-transform/040-split-item.md index 06d4029..e7d94b4 100644 --- a/docs/doc/20-operations/40-transform/040-split-item.md +++ b/docs/doc/20-operations/40-transform/040-split-item.md @@ -32,16 +32,18 @@ This example shows how to split an array of addresses into separate items. } ``` -**YAML Configuration:** - -```yaml -chain: - - operation: item-split - options: - keys: ["addresses"] - single_element: true - duplicate_keys: - customer_id: customer_id +**Configuration:** + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; + +$chainConfig = new ChainConfig(); +$chainConfig->addLink(new SplitItemConfig( + keys: ['addresses'], + singleElement: true, + duplicateKeys: ['customer_id' => 'customer_id'] +)); ``` **Output:** @@ -78,15 +80,18 @@ This example shows how to create separate items for the `billing_address` and `s } ``` -**YAML Configuration:** +**Configuration:** -```yaml -chain: - - operation: item-split - options: - keys: ["billing_address", "shipping_address"] - keep_keys: true - key_name: "address" +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; + +$chainConfig = new ChainConfig(); +$chainConfig->addLink(new SplitItemConfig( + keys: ['billing_address', 'shipping_address'], + keepKeys: true, + keyName: 'address' +)); ``` **Output:** diff --git a/docs/doc/20-operations/40-transform/050-http.md b/docs/doc/20-operations/40-transform/050-http.md index 573ac1f..e8d2857 100644 --- a/docs/doc/20-operations/40-transform/050-http.md +++ b/docs/doc/20-operations/40-transform/050-http.md @@ -17,18 +17,17 @@ The `http` operation makes HTTP requests to external services using the [Symfony ## Example -Here's an example of how to use the `http` operation to fetch data from a JSON API and store the response in the `api_response` key: +Here's an example of how to use the HTTP operation to fetch data from a JSON API and store the response in the `api_response` key: -```yaml -chain: - - operation: http - options: - method: GET - url: "@'https://api.example.com/users/' ~ data.user_id" - response_is_json: true - response_key: api_response +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; - - operation: rule-transformer - options: - # Rules to transform the data, including the api_response. -``` \ No newline at end of file +$chainConfig = new ChainConfig(); +$chainConfig->addLink(new SimpleHttpConfig( + method: 'GET', + url: '@"https://api.example.com/users/"~data["user_id"]', + responseIsJson: true, + responseKey: 'api_response' +)); +``` diff --git a/docs/doc/20-operations/40-transform/060-log.md b/docs/doc/20-operations/40-transform/060-log.md index eab90f8..d0f4b96 100644 --- a/docs/doc/20-operations/40-transform/060-log.md +++ b/docs/doc/20-operations/40-transform/060-log.md @@ -14,15 +14,19 @@ The `log` operation writes messages to the logger, useful for debugging and trac ## Example -Here's an example of how to use the `log` operation to log a message with the user's ID and email address: +Here's an example of how to use the log operation to log a message with the user's ID and email address: -```yaml -chain: - - operation: log - options: - level: info - message: "@'Processing user ' ~ data.id" - context: - user_id: id - email: email +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\LogConfig; + +$chainConfig = new ChainConfig(); +$chainConfig->addLink(new LogConfig( + message: '@"Processing user " ~ data["id"]', + level: 'info', + context: [ + 'user_id' => 'id', + 'email' => 'email' + ] +)); ``` diff --git a/docs/doc/20-operations/40-transform/070-callback.md b/docs/doc/20-operations/40-transform/070-callback.md index 1ec0ce3..ec59dc4 100644 --- a/docs/doc/20-operations/40-transform/070-callback.md +++ b/docs/doc/20-operations/40-transform/070-callback.md @@ -4,9 +4,7 @@ title: PHP-ETL - Operations subTitle: Transform - Callback --- -The `callback` operation executes a custom PHP function within your ETL chain, useful for complex transformations not covered by built-in operations. - -**Note:** This operation is for programmatic use in PHP; it cannot be configured via YAML due to callback serialization limitations. +The callback operation executes a custom PHP function within your ETL chain, useful for complex transformations not covered by built-in operations. ## Options @@ -14,28 +12,23 @@ The `callback` operation executes a custom PHP function within your ETL chain, u ## Example -Here's an example of how to use the `callback` operation to transform a data item with a custom function: +Here's an example of how to use the callback operation to transform a data item with a custom function: ```php add( - new CallbackTransformerOperation( - function (ItemInterface $item, ExecutionContext $context) { - $data = $item->getData(); - $data['full_name'] = $data['first_name'] . ' ' . $data['last_name']; - return new DataItem($data); - } - ) -); +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; + +$chainConfig = new ChainConfig(); +$chainConfig->addLink(new CallBackTransformerConfig( + function (DataItem $dataItem) { + $data = $dataItem->getData(); + $data['full_name'] = $data['first_name'] . ' ' . $data['last_name']; + return new DataItem($data); + } +)); // ... ``` diff --git a/docs/doc/20-operations/60-aggregation/010-grouping.md b/docs/doc/20-operations/60-aggregation/010-grouping.md index 04b5f89..95fc098 100644 --- a/docs/doc/20-operations/60-aggregation/010-grouping.md +++ b/docs/doc/20-operations/60-aggregation/010-grouping.md @@ -13,7 +13,7 @@ The `simple-grouping` operation groups items based on a common key, useful for d ## Example -Here's an example of how to use the `simple-grouping` operation to group a list of customers by their city. +Here's an example of how to use the grouping operation to group a list of customers by their city. **Input Data (a sequence of items):** @@ -25,24 +25,22 @@ Here's an example of how to use the `simple-grouping` operation to group a list ] ``` -**YAML Configuration:** +**Configuration:** -```yaml -chain: - - operation: simple-grouping - options: - grouping_key: ["city"] +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Grouping\SimpleGroupingConfig; - - operation: rule-transformer - options: - # Rules to process the grouped data. - # The input to this operation will be an iterator of groups. - # Each group will be an array of customers. +$chainConfig = new ChainConfig(); +$chainConfig->addLink(new SimpleGroupingConfig(['city'])); + +// The next operation will receive grouped data +// Each group will be an array of customers with the same city ``` **Output:** -The `rule-transformer` will receive an iterator with two groups: +The next operation in the chain will receive an iterator with two groups: - **Group 1 (New York):** ```json diff --git a/docs/doc/20-operations/70-load/010-csv.md b/docs/doc/20-operations/70-load/010-csv.md index b0eb63a..dbf1c32 100644 --- a/docs/doc/20-operations/70-load/010-csv.md +++ b/docs/doc/20-operations/70-load/010-csv.md @@ -16,29 +16,31 @@ The `csv-write` operation writes data to a CSV file. It expects `DataItem` objec ## Example -Here's an example of how to use the `csv-write` operation to save transformed data to a new CSV file: - -```yaml -chain: - - operation: extract-csv - options: - path: /path/to/input.csv - - - operation: rule-transformer - options: - # Rules to transform the CSV data. - columns: - FullName: - rules: - - implode: - values: - - { get: { field: "FirstName" } } - - { get: { field: "LastName" } } - with: " " - - - operation: csv-write - options: - file: /path/to/output.csv - delimiter: "," - has_headers: true +Here's an example of how to use the CSV write operation to save transformed data to a new CSV file: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink((new RuleTransformConfig(add: false)) + ->addColumn('FullName', [ + ['implode' => [ + 'values' => [ + [['get' => ['field' => 'FirstName']]], + [['get' => ['field' => 'LastName']]], + ], + 'with' => ' ', + ]], + ]) + ) + ->addLink(new CsvFileWriterConfig( + file: '/path/to/output.csv', + delimiter: ',', + hasHeaders: true + )); ``` \ No newline at end of file diff --git a/docs/doc/20-operations/70-load/020-json.md b/docs/doc/20-operations/70-load/020-json.md index 2c76acc..640271d 100644 --- a/docs/doc/20-operations/70-load/020-json.md +++ b/docs/doc/20-operations/70-load/020-json.md @@ -12,27 +12,27 @@ The `json-write` operation writes `DataItem` objects (associative arrays) to a J ## Example -Here's an example of how to use the `json-write` operation to save transformed data to a new JSON file: - -```yaml -chain: - - operation: extract-json - options: - path: /path/to/input.json - - - operation: rule-transformer - options: - # Rules to transform the JSON data. - columns: - full_name: - rules: - - implode: - values: - - { get: { field: "first_name" } } - - { get: { field: "last_name" } } - with: " " - - - operation: json-write - options: - file: /path/to/output.json +Here's an example of how to use JSON file writing to save transformed data to a new JSON file: + +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\JsonExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; + +$chainConfig = new ChainConfig(); +$chainConfig + ->addLink(new JsonExtractConfig()) + ->addLink((new RuleTransformConfig(add: false)) + ->addColumn('full_name', [ + ['implode' => [ + 'values' => [ + [['get' => ['field' => 'first_name']]], + [['get' => ['field' => 'last_name']]], + ], + 'with' => ' ', + ]], + ]) + ); + // Note: JsonFileWriterConfig is not yet available in v2. + // Use the old YAML-based approach or contribute the implementation. ``` \ No newline at end of file diff --git a/docs/doc/getting-started/standalone.md b/docs/doc/getting-started/standalone.md index 6faecaa..9905807 100644 --- a/docs/doc/getting-started/standalone.md +++ b/docs/doc/getting-started/standalone.md @@ -8,90 +8,259 @@ subTitle: šŸ˜ Standalone You probably will never run the ETL in a stand-alone mode, but if you do intend to create an adapter for a different Framework or CMS you need to understand how to start the ETL from "nothing". So this document -focused on that aspect. +focuses on that aspect. -That said that does not mean the ETL can't be run standalone, it's just that the "init" part is not ideal. +That said, this does not mean the ETL can't be run standalone, it's just that the "init" part is not ideal. ## Writing some code - Start by installing the necessary dependencies ```sh - composer require oliverde8/php-etl +composer require oliverde8/php-etl ``` -- We will also need a factory to create execution context's, you can read more about what an execution context is +- We will also need a factory to create execution contexts, you can read more about what an execution context is [here](/doc/01-understand-the-etl/execution-context.html). ```php -$executionContextFaxtory = new ExecutionContextFactory(); +use Oliverde8\Component\PhpEtl\ExecutionContextFactory; + +$executionContextFactory = new ExecutionContextFactory(); ``` > ##### TIP -> The default provider we use here is very simple and basically bypasses the context's. +> The default provider we use here is very simple and basically bypasses the contexts. {: .block-tip } -- You will need to initialize all your operations. +- You will need to initialize the rule applier and all your operation factories. ```php - // We need to create the rule applier operaiton with all the rules we have. Additional rules can be added. - $ruleApplier = new \Oliverde8\Component\RuleEngine\RuleApplier( - new \Psr\Log\NullLogger(), - [ - new \Oliverde8\Component\RuleEngine\Rules\Get(new \Psr\Log\NullLogger()), - new \Oliverde8\Component\RuleEngine\Rules\Implode(new \Psr\Log\NullLogger()), - new \Oliverde8\Component\RuleEngine\Rules\StrToLower(new \Psr\Log\NullLogger()), - new \Oliverde8\Component\RuleEngine\Rules\StrToUpper(new \Psr\Log\NullLogger()), - new \Oliverde8\Component\RuleEngine\Rules\ExpressionLanguage(new \Psr\Log\NullLogger()), - ] - ); - - $builder = new ChainBuilder(getExecutionContextFactory()); - $builder->registerFactory(new RuleTransformFactory('rule-engine-transformer', RuleTransformOperation::class, $ruleApplier)); - $builder->registerFactory(new FilterDataFactory('filter', FilterDataOperation::class, $ruleApplier)); - $builder->registerFactory(new SimpleGroupingFactory('simple-grouping', SimpleGroupingOperation::class)); - $builder->registerFactory(new ChainSplitFactory('split', ChainSplitOperation::class, $builder)); - $builder->registerFactory(new CsvFileWriterFactory('csv-write', FileWriterOperation::class)); - $builder->registerFactory(new JsonFileWriterFactory('json-write', FileWriterOperation::class)); - $builder->registerFactory(new CsvExtractFactory('csv-read', CsvExtractOperation::class)); - $builder->registerFactory(new JsonExtractFactory('json-read', JsonExtractOperation::class)); - $builder->registerFactory(new SplitItemFactory('split-item', SplitItemOperation::class)); - $builder->registerFactory(new SimpleHttpOperationFactory('http', SimpleHttpOperation::class)); - $builder->registerFactory(new ExternalFileFinderFactory('external-file-finder-local', ExternalFileFinderOperation::class, new LocalFileSystem("/"))); - $builder->registerFactory(new ExternalFileProcessorFactory("external-file-processor", ExternalFileProcessorOperation::class)); +use Oliverde8\Component\PhpEtl\ChainBuilderV2; +use Oliverde8\Component\PhpEtl\GenericChainFactory; +use Oliverde8\Component\RuleEngine\RuleApplier; +use Oliverde8\Component\RuleEngine\Rules\Get; +use Oliverde8\Component\RuleEngine\Rules\Implode; +use Oliverde8\Component\RuleEngine\Rules\StrToLower; +use Oliverde8\Component\RuleEngine\Rules\StrToUpper; +use Oliverde8\Component\RuleEngine\Rules\ExpressionLanguage; +use Psr\Log\NullLogger; +use Symfony\Component\HttpClient\HttpClient; + +// Create the rule applier with all available rules +$ruleApplier = new RuleApplier( + new NullLogger(), + [ + new Get(new NullLogger()), + new Implode(new NullLogger()), + new StrToLower(new NullLogger()), + new StrToUpper(new NullLogger()), + new ExpressionLanguage(new NullLogger()), + ] +); + +// Create HTTP client for API operations +$client = HttpClient::create(['headers' => ['Accept' => 'application/json']]); + +// Initialize the chain builder with all operation factories +$chainBuilder = new ChainBuilderV2( + $executionContextFactory, + [ + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Extract\CsvExtractOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Extract\JsonExtractOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Extract\JsonExtractConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Transformer\RuleTransformOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig::class, + injections: ['ruleApplier' => $ruleApplier] + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Transformer\FilterDataOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig::class, + injections: ['ruleApplier' => $ruleApplier] + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Transformer\CallbackTransformerOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Transformer\SimpleHttpOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig::class, + injections: ['client' => $client] + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Transformer\SplitItemOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Loader\FileWriterOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Grouping\SimpleGroupingOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Grouping\SimpleGroupingConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\ChainSplitOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\ChainSplitConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\ChainMergeOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\ChainMergeConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\ChainRepeatOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\ChainRepeatConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\FailSafeOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Transformer\LogOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Transformer\LogConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Extract\ExternalFileFinderOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig::class, + injections: ['fileSystem' => new \Oliverde8\Component\PhpEtl\Model\File\LocalFileSystem("/")] + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Transformer\ExternalFileProcessorOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig::class + ), + ], +); ``` > ##### TIP -> We are here initializing all possible operations, most of the documentation -> will assume you are using symfony as framework, you will need to register your factories manually -> if you are not. +> We are here initializing all possible operations. Most of the documentation +> will assume you are using Symfony as a framework. You will need to register your factories manually +> if you are not using Symfony. {: .block-tip } -- We can start describing our etl in a `Yaml` file. We will create here a single step chain that just dumps the data. -```yaml -chain: - dump-data: - operation: dump - options: [] +- Now we can describe our ETL using typed PHP configuration objects instead of YAML. Let's create a simple chain that transforms and outputs data. +```php +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\CsvExtractConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig; +use Oliverde8\Component\PhpEtl\Item\DataItem; + +// Create the chain configuration +$chainConfig = new ChainConfig(); + +$chainConfig + ->addLink(new CsvExtractConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('id', [['get' => ['field' => 'ID']]]) + ->addColumn('name', [['get' => ['field' => 'Name']]]) + ->addColumn('email', [['get' => ['field' => 'Email']]]) + ) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + // Dump the data to console + var_dump($item->getData()); + return $item; + })) + ->addLink(new CsvFileWriterConfig('output.csv')); ``` -- We can now build our etl +- Build the chain processor from the configuration ```php -$chainProcessor = $builder->buildChainProcessor(Yaml::parse(file_get_contents($fileName)),[]); +$chainProcessor = $chainBuilder->createChain($chainConfig); ``` -- Before starting the etl we need our input data +- Prepare your input data. For a CSV file, wrap it in a DataItem: ```php -$inputData = [['myKey' => "value1"], ['myKey' => "value1"]] +$inputData = new ArrayIterator([ + new DataItem(['file' => 'customers.csv']) +]); ``` -- We can now start it +- Execute the ETL chain ```php -$chainProcessor->process(new ArrayIterator($inputData)); +$chainProcessor->process($inputData, []); ``` -## Conclusion +## Complete Example -We have created a very simple ETL that only outputs to the console the data we have given it. To move on further you -should read: +Here's a complete standalone ETL script: + +```php + $ruleApplier] + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Transformer\CallbackTransformerOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig::class + ), + new GenericChainFactory( + \Oliverde8\Component\PhpEtl\ChainOperation\Loader\FileWriterOperation::class, + \Oliverde8\Component\PhpEtl\OperationConfig\Loader\CsvFileWriterConfig::class + ), + ], +); + +// Build the ETL chain +$chainConfig = (new ChainConfig()) + ->addLink(new CsvExtractConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('customer_name', [['get' => ['field' => 'Name']]]) + ) + ->addLink(new CallBackTransformerConfig(function(DataItem $item) { + echo "Processing: " . json_encode($item->getData()) . "\n"; + return $item; + })) + ->addLink(new CsvFileWriterConfig('output.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); + +// Execute +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'input.csv'])]), + [] +); + +``` diff --git a/examples/.init.php b/examples/.init.php new file mode 100644 index 0000000..b4f383b --- /dev/null +++ b/examples/.init.php @@ -0,0 +1,103 @@ + ['Accept' => 'application/json']]); + + +$chainBuilder = new ChainBuilderV2( + getEtlExecutionContextFactory(), + [ + new GenericChainFactory(CsvExtractOperation::class, CsvExtractConfig::class), + new GenericChainFactory(CallbackTransformerOperation::class, CallBackTransformerConfig::class), + new GenericChainFactory(RuleTransformOperation::class, RuleTransformConfig::class, injections: ['ruleApplier' => $ruleApplier]), + new GenericChainFactory(FileWriterOperation::class, CsvFileWriterConfig::class), + new GenericChainFactory(SimpleGroupingOperation::class, SimpleGroupingConfig::class), + new GenericChainFactory(FilterDataOperation::class, FilterDataConfig::class, injections: ['ruleApplier' => $ruleApplier]), + new GenericChainFactory(ChainMergeOperation::class, ChainMergeConfig::class), + new GenericChainFactory(ChainRepeatOperation::class, ChainRepeatConfig::class), + new GenericChainFactory(ChainSplitOperation::class, ChainSplitConfig::class), + new GenericChainFactory(JsonExtractOperation::class, JsonExtractConfig::class), + new GenericChainFactory(SimpleHttpOperation::class, SimpleHttpConfig::class, injections: ['client' => $client]), + new GenericChainFactory(SplitItemOperation::class, SplitItemConfig::class), + new GenericChainFactory(LogOperation::class, LogConfig::class), + new GenericChainFactory(FailSafeOperation::class, FailSafeConfig::class), + new GenericChainFactory(ExternalFileFinderOperation::class, ExternalFileFinderConfig::class, injections: ['fileSystem' => new LocalFileSystem("/")]), + new GenericChainFactory(ExternalFileProcessorOperation::class, ExternalFileProcessorConfig::class), + ], +); + +if (!function_exists('getEtlExecutionContextFactory')) { + function getEtlExecutionContextFactory() { + return new ExecutionContextFactory(); + } +} diff --git a/examples/00-SimpleCases/00-CsvRead.php b/examples/00-SimpleCases/00-CsvRead.php new file mode 100644 index 0000000..884d089 --- /dev/null +++ b/examples/00-SimpleCases/00-CsvRead.php @@ -0,0 +1,25 @@ +addLink(New CsvExtractConfig()); +$chainConfig->addLink(new CallBackTransformerConfig(function (DataItem $dataItem) { + var_dump($dataItem->getData()); + return $dataItem; +})); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new DataItem([ + 'file' => 'data/customers.csv', + ]), + [] +); \ No newline at end of file diff --git a/examples/00-SimpleCases/01-CsvTransform.php b/examples/00-SimpleCases/01-CsvTransform.php new file mode 100644 index 0000000..cf2e1d0 --- /dev/null +++ b/examples/00-SimpleCases/01-CsvTransform.php @@ -0,0 +1,43 @@ +addLink(New CsvExtractConfig()) + ->addLink((new RuleTransformConfig(add: false, flavor: 'default')) + ->addColumn('Name', [ + ['implode' => [ + 'values' => [ + [[ 'get' => [ 'field' => 'FirstName' ]]], + [[ 'get' => [ 'field' => 'LastName' ]]], + ], + 'with' => ' ', + ]], + ]) + ->addColumn('SubscriptionStatus', [ + ['get' => [ 'field' => 'IsSubscribed' ]] + ]) + ) + ->addLink(new CsvFileWriterConfig('customers-transformed.csv')) + ->addLink(new CallBackTransformerConfig(function (DataItem $dataItem) { + var_dump($dataItem->getData()); + return $dataItem; + })) +; + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new DataItem([ + 'file' => 'data/customers.csv', + ]), + [] +); diff --git a/examples/00-SimpleCases/02-CsvMerge.php b/examples/00-SimpleCases/02-CsvMerge.php new file mode 100644 index 0000000..c819822 --- /dev/null +++ b/examples/00-SimpleCases/02-CsvMerge.php @@ -0,0 +1,41 @@ +addLink(New CsvExtractConfig()) + ->addLink((new RuleTransformConfig(add: false, flavor: 'default')) + ->addColumn('Name', [ + ['implode' => [ + 'values' => [ + [[ 'get' => [ 'field' => 'FirstName' ]]], + [[ 'get' => [ 'field' => 'LastName' ]]], + ], + 'with' => ' ', + ]], + ]) + ->addColumn('SubscriptionStatus', [ + ['get' => [ 'field' => 'IsSubscribed' ]] + ]) + ) + ->addLink(new CsvFileWriterConfig('customers-transformed.csv')) + ->addLink(new CallBackTransformerConfig(function (DataItem $dataItem) { + var_dump($dataItem->getData()); + return $dataItem; + })) +; + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'data/customers.csv',]), new DataItem(['file' => 'data/customers2.csv',])]), + [] +); diff --git a/examples/00-SimpleCases/03-CsvMergeGrouoIntoJson.php b/examples/00-SimpleCases/03-CsvMergeGrouoIntoJson.php new file mode 100644 index 0000000..70a29a9 --- /dev/null +++ b/examples/00-SimpleCases/03-CsvMergeGrouoIntoJson.php @@ -0,0 +1,25 @@ +addLink(new CsvExtractConfig()) + ->addLink(new SimpleGroupingConfig(['IsSubscribed'])) + ->addLink(new CallBackTransformerConfig(function (DataItem $dataItem) { + var_dump($dataItem->getData()); + return $dataItem; + })); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'data/customers.csv',]), new DataItem(['file' => 'data/customers2.csv',])]), + [] +); diff --git a/examples/00-SimpleCases/04-CsvFileFilter.php b/examples/00-SimpleCases/04-CsvFileFilter.php new file mode 100644 index 0000000..25a0901 --- /dev/null +++ b/examples/00-SimpleCases/04-CsvFileFilter.php @@ -0,0 +1,26 @@ +addLink(new CsvExtractConfig()) + ->addLink(new FilterDataConfig([["get" => ["field" => 'IsSubscribed']]])) + ->addLink(new CallBackTransformerConfig(function (DataItem $dataItem) { + var_dump($dataItem->getData()); + return $dataItem; + })); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'data/customers.csv',]), new DataItem(['file' => 'data/customers2.csv',])]), + [] +); diff --git a/examples/00-SimpleCases/05-SplitDataIn3Files.php b/examples/00-SimpleCases/05-SplitDataIn3Files.php new file mode 100644 index 0000000..95f87b4 --- /dev/null +++ b/examples/00-SimpleCases/05-SplitDataIn3Files.php @@ -0,0 +1,35 @@ +addLink(new CsvExtractConfig()) + ->addLink( + (new ChainSplitConfig()) + ->addSplit( + (new ChainConfig()) + ->addLink(new FilterDataConfig([["get" => ["field" => 'IsSubscribed']]])) + ->addLink(new CsvFileWriterConfig('customers-subscribed.csv')) + ) + ->addSplit( + (new ChainConfig()) + ->addLink(new FilterDataConfig([["get" => ["field" => 'IsSubscribed']]], true)) + ->addLink(new CsvFileWriterConfig('customers-not-subscribed.csv')) + ) + ) + ->addLink(new CsvFileWriterConfig('customers-all.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem(['file' => 'data/customers.csv',]), new DataItem(['file' => 'data/customers2.csv',])]), + [] +); diff --git a/examples/00-SimpleCases/07-JsonToCsv.php b/examples/00-SimpleCases/07-JsonToCsv.php new file mode 100644 index 0000000..a89982e --- /dev/null +++ b/examples/00-SimpleCases/07-JsonToCsv.php @@ -0,0 +1,26 @@ +addLink(new JsonExtractConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('productId', [['get' => ['field' => 'productId']]]) + ->addColumn('sku', [['get' => ['field' => 'sku']]]) + ->addColumn('name-{@context/locales}', [['get' => ['field' => ['name', '@context/locales']]]]) + ) + ->addLink(new CsvFileWriterConfig('products.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('data/products.json')]), + ['locales' => ['fr_FR', 'en_US']] +); diff --git a/examples/00-SimpleCases/08-ApiToCsv.php b/examples/00-SimpleCases/08-ApiToCsv.php new file mode 100644 index 0000000..fadbe07 --- /dev/null +++ b/examples/00-SimpleCases/08-ApiToCsv.php @@ -0,0 +1,31 @@ +addLink(new SimpleHttpConfig( + method: 'GET', + url: 'https://63b687951907f863aaf90ab1.mockapi.io/test', + responseIsJson: true + )) + ->addLink(new SplitItemConfig( + keys: ['content'], + singleElement: true + )) + ->addLink(new CsvFileWriterConfig('output.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem([])]), + [] +); + diff --git a/examples/00-SimpleCases/09-ApiToCsv2.php b/examples/00-SimpleCases/09-ApiToCsv2.php new file mode 100644 index 0000000..577a28e --- /dev/null +++ b/examples/00-SimpleCases/09-ApiToCsv2.php @@ -0,0 +1,52 @@ +addLink(new SimpleHttpConfig( + method: 'GET', + url: '@"https://63b687951907f863aaf90ab1.mockapi.io/test/"~data["id"]', + responseIsJson: true, + optionKey: '-placeholder-', + )) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('createdAt', [['get' => ['field' => ['content', 'createdAt']]]]) + ->addColumn('name', [['get' => ['field' => ['content', 'name']]]]) + ->addColumn('avatar', [['get' => ['field' => ['content', 'avatar']]]]) + ->addColumn('id', [['get' => ['field' => ['content', 'id']]]]) + ) + ->addLink(new CsvFileWriterConfig('output.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([ + ["id" => 1], + ["id" => 2], + ["id" => 3], + ["id" => 4], + ["id" => 5], + ["id" => 6], + ["id" => 7], + ["id" => 8], + ["id" => 9], + ["id" => 10], + ["id" => 11], + ["id" => 12], + ["id" => 13], + ["id" => 14], + ["id" => 15], + ["id" => 16], + ["id" => 17], + ["id" => 18], + ["id" => 19], + ]), + [] +); + diff --git a/examples/00-SimpleCases/10-CsvWithLogging.php b/examples/00-SimpleCases/10-CsvWithLogging.php new file mode 100644 index 0000000..7c21f9b --- /dev/null +++ b/examples/00-SimpleCases/10-CsvWithLogging.php @@ -0,0 +1,29 @@ +addLink(new LogConfig( + message: 'Starting CSV extraction', + level: 'info' + )) + ->addLink(new CsvExtractConfig()) + ->addLink(new LogConfig( + message: '@"Processing customer: " ~ data["FirstName"]', + level: 'debug' + )); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('data/customers.csv')]), + [] +); + diff --git a/examples/00-SimpleCases/11-CsvMergeChains.php b/examples/00-SimpleCases/11-CsvMergeChains.php new file mode 100644 index 0000000..a31e02c --- /dev/null +++ b/examples/00-SimpleCases/11-CsvMergeChains.php @@ -0,0 +1,50 @@ +addLink(new CsvExtractConfig()) + ->addLink((new ChainMergeConfig()) + ->addMerge((new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('full_name', [ + ['implode' => [ + 'values' => [ + [[ 'get' => [ 'field' => 'FirstName' ]]], + [[ 'get' => [ 'field' => 'LastName' ]]], + ], + 'with' => ' ', + ]] + ]) + ) + ) + ->addMerge((new ChainConfig()) + ->addLink((new RuleTransformConfig(false)) + ->addColumn('customer_id', [['get' => ['field' => 'ID']]]) + ->addColumn('status', [['get' => ['field' => 'subscribed']]]) + ) + ) + + ) + ->addLink(new CallBackTransformerConfig(function (DataItem $dataItem) { + var_dump($dataItem->getData()); + return $dataItem; + })); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([new DataItem('data/customers.csv')]), + [] +); diff --git a/examples/00-SimpleCases/12-ApiPagination.php b/examples/00-SimpleCases/12-ApiPagination.php new file mode 100644 index 0000000..b0d0084 --- /dev/null +++ b/examples/00-SimpleCases/12-ApiPagination.php @@ -0,0 +1,66 @@ +addLink(new CallBackTransformerConfig(function(DataItemInterface $dataItem) use (&$page) { + $totalPages = 5; // Simulate 5 pages of data + + echo "Fetching page {$page}/{$totalPages}...\n"; + + // Simulate API response with paginated data + $items = []; + for ($i = 1; $i <= 10; $i++) { + $itemId = (($page - 1) * 10) + $i; + $items[] = [ + 'id' => $itemId, + 'name' => "Item {$itemId}", + 'page' => $page, + ]; + } + + $hasNextPage = $page < $totalPages; + + return new DataItem([ + 'items' => $items, + 'page' => $page++, + 'hasNextPage' => $hasNextPage, + ]); + })), + validationExpression: 'data["hasNextPage"] == true', + allowAsynchronous: false +); + +$chainConfig + ->addLink($repeatConfig) + ->addLink(new SplitItemConfig(keys: ['items'])) + ->addLink(new CsvFileWriterConfig('paginated-results.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process( + new ArrayIterator([ + new DataItem([[]]), + ]), + [] +); + + diff --git a/examples/00-SimpleCases/13-FailSafe.php b/examples/00-SimpleCases/13-FailSafe.php new file mode 100644 index 0000000..7569986 --- /dev/null +++ b/examples/00-SimpleCases/13-FailSafe.php @@ -0,0 +1,46 @@ +addLink(new CallBackTransformerConfig(function(DataItem $dataItem) { + static $attempt = 0; $attempt++; + $data = $dataItem->getData(); + echo "Attempt {$attempt} for id {$data['id']}\n"; + if ($attempt < 3) { + echo " -> Simulating transient error\n"; + throw new RuntimeException('Transient error, please retry'); + } + + echo " -> Success!\n"; + $data['status'] = 'success'; + $data['attempt'] = $attempt; + return new DataItem($data); + })); + +$failSafeConfig = new FailSafeConfig( + chainConfig: $failingChain, + exceptionsToCatch: [RuntimeException::class], + nbAttempts: 5, +); + +$rootChain = new ChainConfig(); +$rootChain->addLink($failSafeConfig); + +$processor = $chainBuilder->createChain($rootChain); +$processor->process( + new ArrayIterator([ + new DataItem(['id' => 1]), + ]), + [] +); + diff --git a/examples/00-SimpleCases/customers-transformed.csv b/examples/00-SimpleCases/customers-transformed.csv new file mode 100644 index 0000000..f81b02e --- /dev/null +++ b/examples/00-SimpleCases/customers-transformed.csv @@ -0,0 +1,11 @@ +Name;SubscriptionStatus +"Fahima Mathews";1 +"Stephan Patel";0 +"Leena Rennie";0 +"Christine Findlay";1 +"Jeffrey Oneal";1 +"Nikhil Amin";0 +"Yannis Bonner";1 +"Jess Doherty";1 +"Tiya Harris";0 +"Amira Merritt";0 diff --git a/examples/00-SimpleCases/data/customers.csv b/examples/00-SimpleCases/data/customers.csv new file mode 100644 index 0000000..e5e2f7b --- /dev/null +++ b/examples/00-SimpleCases/data/customers.csv @@ -0,0 +1,6 @@ +ID;FirstName;LastName;IsSubscribed +1;"Fahima";"Mathews";1 +2;"Stephan";"Patel";0 +3;"Leena";"Rennie";0 +4;"Christine";"Findlay";1 +5;"Jeffrey";"Oneal";1 \ No newline at end of file diff --git a/examples/00-SimpleCases/data/customers2.csv b/examples/00-SimpleCases/data/customers2.csv new file mode 100644 index 0000000..ab24b13 --- /dev/null +++ b/examples/00-SimpleCases/data/customers2.csv @@ -0,0 +1,6 @@ +ID;FirstName;LastName;IsSubscribed +10;"Nikhil";"Amin";0 +20;"Yannis";"Bonner";1 +30;"Jess";"Doherty";1 +40;"Tiya";"Harris";0 +50;"Amira";"Merritt";0 \ No newline at end of file diff --git a/examples/00-SimpleCases/data/products.json b/examples/00-SimpleCases/data/products.json new file mode 100644 index 0000000..11dc8ec --- /dev/null +++ b/examples/00-SimpleCases/data/products.json @@ -0,0 +1,26 @@ +[ + { + "productId": 1, + "sku": "sku1", + "name": { + "fr_FR": "Mon Produit 1", + "en_US": "My Product 1" + } + }, + { + "productId": 2, + "sku": "sku2", + "name": { + "fr_FR": "Mon Produit 2", + "en_US": "My Product 2" + } + }, + { + "productId": 3, + "sku": "sku3", + "name": { + "fr_FR": "Mon Produit 3", + "en_US": "My Product 3" + } + } +] diff --git a/examples/10-contextual/.init.php b/examples/10-contextual/.init.php new file mode 100644 index 0000000..3e055e7 --- /dev/null +++ b/examples/10-contextual/.init.php @@ -0,0 +1,18 @@ + [ + 'execution' => new \Oliverde8\Component\PhpEtl\Model\PockExecution(new DateTime()) + ] +]; \ No newline at end of file diff --git a/examples/10-contextual/01-ApiToCsv.php b/examples/10-contextual/01-ApiToCsv.php new file mode 100644 index 0000000..fa9f0a8 --- /dev/null +++ b/examples/10-contextual/01-ApiToCsv.php @@ -0,0 +1,27 @@ +addLink(new SimpleHttpConfig( + method: 'GET', + url: 'https://63b687951907f863aaf90ab1.mockapi.io/test', + responseIsJson: true + )) + ->addLink(new SplitItemConfig( + keys: ['content'], + singleElement: true + )) + ->addLink(New CsvFileWriterConfig('test.csv')); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process(new ArrayIterator([new DataItem([])]), $options); \ No newline at end of file diff --git a/examples/10-contextual/02-FindFile.php b/examples/10-contextual/02-FindFile.php new file mode 100644 index 0000000..47c7632 --- /dev/null +++ b/examples/10-contextual/02-FindFile.php @@ -0,0 +1,30 @@ +addLink(new ExternalFileFinderConfig(directory: $dir)) + ->addLink(new ExternalFileProcessorConfig()) + ->addLink(new CsvExtractConfig()) + ->addLink(new CsvFileWriterConfig('output.csv')) + ->addLink(new ExternalFileProcessorConfig()); + +$chainProcessor = $chainBuilder->createChain($chainConfig); +$chainProcessor->process(new ArrayIterator([new DataItem('/^file[0-9]\.csv$/')]), $options); + diff --git a/rector.php b/rector.php index 4d23113..d53557c 100644 --- a/rector.php +++ b/rector.php @@ -6,9 +6,9 @@ return RectorConfig::configure() ->withPaths([ - __DIR__ . '/old-docs', + __DIR__ . '/examples', __DIR__ . '/src', ]) // uncomment to reach your current PHP version - // ->withPhpSets() + ->withPhpSets(php83: true) ->withTypeCoverageLevel(0); diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/AbstractFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/AbstractFactory.php index 031b58f..936cbd4 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/AbstractFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/AbstractFactory.php @@ -74,10 +74,9 @@ protected function configureValidator(): Constraint } /** - * @param mixed ...$arguments * @return ChainOperationInterface */ - protected function create(...$arguments): ChainOperationInterface + protected function create(mixed ...$arguments): ChainOperationInterface { $class = $this->class; return new $class(...$arguments); diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/ChainRepeatFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/ChainRepeatFactory.php index 660bd65..6a565a0 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/ChainRepeatFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/ChainRepeatFactory.php @@ -15,6 +15,7 @@ public function __construct(string $operation, string $class, protected readonly parent::__construct($operation, $class); } + #[\Override] public function build(string $operation, array $options): ChainOperationInterface { $maxAsync = $options['maxASynchronousItems'] ?? 0; @@ -25,6 +26,7 @@ public function build(string $operation, array $options): ChainOperationInterfac return $this->create($chainProcessor, $options['validationExpr'], $allowAsync); } + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection([ diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/ChainSplitFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/ChainSplitFactory.php index 3230c20..3aebee3 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/ChainSplitFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/ChainSplitFactory.php @@ -21,6 +21,7 @@ public function __construct(string $operation, string $class, ChainBuilder $buil /** * @inheritdoc */ + #[\Override] public function build(string $operation, array $options): ChainOperationInterface { $chainProcessors = []; @@ -31,6 +32,7 @@ public function build(string $operation, array $options): ChainOperationInterfac return $this->create($chainProcessors); } + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection([ diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/CsvExtractFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/CsvExtractFactory.php index d92103f..105a8d8 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/CsvExtractFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/CsvExtractFactory.php @@ -9,6 +9,7 @@ class CsvExtractFactory extends AbstractFactory { + #[\Override] protected function build(string $operation, array $options): ChainOperationInterface { return $this->create( diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/ExternalFileFinderFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/ExternalFileFinderFactory.php index 1ef7732..3ba5e3f 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/ExternalFileFinderFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/ExternalFileFinderFactory.php @@ -11,20 +11,19 @@ class ExternalFileFinderFactory extends AbstractFactory { - protected FileSystemInterface $fileSystem; - - public function __construct(string $operation, string $class, FileSystemInterface $fileSystem) + public function __construct(string $operation, string $class, protected FileSystemInterface $fileSystem) { parent::__construct($operation, $class); - $this->fileSystem = $fileSystem; } + #[\Override] protected function build(string $operation, array $options): ChainOperationInterface { return $this->create($this->fileSystem, $options['directory']); } + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection([ diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/JsonExtractFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/JsonExtractFactory.php index 9403ce8..c8fde5d 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/JsonExtractFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Extract/JsonExtractFactory.php @@ -7,6 +7,7 @@ class JsonExtractFactory extends AbstractFactory { + #[\Override] protected function build(string $operation, array $options): ChainOperationInterface { return $this->create($options['fileKey'] ?? 'file', $options['scoped'] ?? true); diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Grouping/SimpleGroupingFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Grouping/SimpleGroupingFactory.php index 842c18d..4e0d49e 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Grouping/SimpleGroupingFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Grouping/SimpleGroupingFactory.php @@ -21,6 +21,7 @@ class SimpleGroupingFactory extends AbstractFactory /** * @inheritdoc */ + #[\Override] public function build(string $operation, array $options): ChainOperationInterface { return $this->create($options['grouping-key'], $options['group-identifier']); @@ -29,6 +30,7 @@ public function build(string $operation, array $options): ChainOperationInterfac /** * @inheritdoc */ + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection([ diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Loader/CsvFileWriterFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Loader/CsvFileWriterFactory.php index 5ee4dc5..97990b7 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Loader/CsvFileWriterFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Loader/CsvFileWriterFactory.php @@ -22,6 +22,7 @@ class CsvFileWriterFactory extends AbstractFactory /** * @inheritdoc */ + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection([ @@ -32,6 +33,7 @@ protected function configureValidator(): Constraint /** * @inheritdoc */ + #[\Override] protected function build($operation, $options): ChainOperationInterface { $tmp = tempnam(sys_get_temp_dir(), 'etl'); diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Loader/JsonFileWriterFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Loader/JsonFileWriterFactory.php index 584085a..484c1d3 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Loader/JsonFileWriterFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Loader/JsonFileWriterFactory.php @@ -22,6 +22,7 @@ class JsonFileWriterFactory extends AbstractFactory /** * @inheritdoc */ + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection([ @@ -32,6 +33,7 @@ protected function configureValidator(): Constraint /** * @inheritdoc */ + #[\Override] protected function build($operation, $options): ChainOperationInterface { $tmp = tempnam(sys_get_temp_dir(), 'etl'); diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/ExternalFileProcessorFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/ExternalFileProcessorFactory.php index 073f441..c69c910 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/ExternalFileProcessorFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/ExternalFileProcessorFactory.php @@ -9,6 +9,7 @@ class ExternalFileProcessorFactory extends AbstractFactory { + #[\Override] protected function build(string $operation, array $options): ChainOperationInterface { return $this->create(); diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/FilterDataFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/FilterDataFactory.php index cba7bb1..16637fc 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/FilterDataFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/FilterDataFactory.php @@ -38,6 +38,7 @@ public function __construct(string $operation, string $class, RuleApplier $ruleA /** * @inheritdoc */ + #[\Override] public function build(string $operation, array $options): ChainOperationInterface { return $this->create($this->ruleApplier, $options['rule'], $options['negate']); @@ -46,6 +47,7 @@ public function build(string $operation, array $options): ChainOperationInterfac /** * @inheritdoc */ + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection([ diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/LogFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/LogFactory.php index 9b1f8f1..d4c124b 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/LogFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/LogFactory.php @@ -17,11 +17,13 @@ public function __construct(string $operation, string $class) parent::__construct($operation, $class); } + #[\Override] protected function build($operation, $options): ChainOperationInterface { return $this->create($options['message'], $options['level'], $options['context'] ?? []); } + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection([ diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/RuleTransformFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/RuleTransformFactory.php index dc555ad..4751882 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/RuleTransformFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/RuleTransformFactory.php @@ -38,6 +38,7 @@ public function __construct(string $operation, string $class, RuleApplier $ruleA /** * @inheritdoc */ + #[\Override] public function build(string $operation, array $options): ChainOperationInterface { return $this->create($this->ruleApplier, $options['columns'], $options['add']); @@ -46,6 +47,7 @@ public function build(string $operation, array $options): ChainOperationInterfac /** * @inheritdoc */ + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection([ diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SimpleHttpOperationFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SimpleHttpOperationFactory.php index 04d5cfb..a59f5c8 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SimpleHttpOperationFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SimpleHttpOperationFactory.php @@ -13,6 +13,7 @@ class SimpleHttpOperationFactory extends AbstractFactory { + #[\Override] protected function build(string $operation, array $options): ChainOperationInterface { if (!class_exists(HttpClient::class)) { @@ -31,6 +32,7 @@ protected function build(string $operation, array $options): ChainOperationInter ); } + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection([ diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SimpleHttpOperationWithClientFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SimpleHttpOperationWithClientFactory.php index 10e2064..a61cec9 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SimpleHttpOperationWithClientFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SimpleHttpOperationWithClientFactory.php @@ -12,18 +12,15 @@ class SimpleHttpOperationWithClientFactory extends AbstractFactory { - private HttpClientInterface $httpClient; - /** * @param HttpClientInterface $httpClient */ - public function __construct(string $operation, HttpClientInterface $httpClient) + public function __construct(string $operation, private readonly HttpClientInterface $httpClient) { - $this->httpClient = $httpClient; - parent::__construct($operation, SimpleHttpOperation::class); } + #[\Override] protected function build(string $operation, array $options): ChainOperationInterface { return new SimpleHttpOperation( @@ -36,6 +33,7 @@ protected function build(string $operation, array $options): ChainOperationInter ); } + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection([ diff --git a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SplitItemFactory.php b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SplitItemFactory.php index e007958..fa4b3bf 100644 --- a/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SplitItemFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Builder/Factories/Transformer/SplitItemFactory.php @@ -26,6 +26,7 @@ public function __construct(string $operation, string $class) /** * @inheritdoc */ + #[\Override] public function build(string $operation, array $options): ChainOperationInterface { return $this->create( @@ -40,6 +41,7 @@ public function build(string $operation, array $options): ChainOperationInterfac /** * @inheritdoc */ + #[\Override] protected function configureValidator(): Constraint { return new Assert\Collection( diff --git a/src/Oliverde8/Component/PhpEtl/ChainBuilder.php b/src/Oliverde8/Component/PhpEtl/ChainBuilder.php index 8a292a7..08fcf57 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainBuilder.php +++ b/src/Oliverde8/Component/PhpEtl/ChainBuilder.php @@ -25,17 +25,13 @@ class ChainBuilder /** @var ChainProcessor[] */ protected array $subChainProcessors = []; - protected ExecutionContextFactoryInterface $contextFactory; - protected ExpressionLanguage $expressionLanguage; /** * @param ExecutionContextFactoryInterface $contextFactory */ - public function __construct(ExecutionContextFactoryInterface $contextFactory) + public function __construct(protected ExecutionContextFactoryInterface $contextFactory) { - $this->contextFactory = $contextFactory; - $this->expressionLanguage = new ExpressionLanguage(); } @@ -70,7 +66,7 @@ public function buildChainProcessor(array $configs, array $inputOptions = [], in foreach ($configs['subChains'] ?? [] as $subChainName => $subChainConfigs) { $chainOperations = []; foreach ($subChainConfigs['chain'] as $id => $operation) { - $chainOperations[$id] = $this->getOperationFromConfig($operation, $inputOptions, $subChainProcessors); + $chainOperations[$id] = $this->getOperationFromConfig($operation, $inputOptions); } $this->subChainProcessors[$subChainName] = new ChainProcessor( $chainOperations, @@ -97,13 +93,13 @@ public function buildChainProcessor(array $configs, array $inputOptions = [], in protected function getOperationFromConfig(array $config, array $inputOptions): ChainOperationInterface { foreach ($config['options'] as &$option) { - if (is_string($option) && strpos($option, "!") === 0) { + if (is_string($option) && str_starts_with($option, "!")) { $option = ltrim($option, '!'); $option = $this->expressionLanguage->evaluate($option, $inputOptions); } } - if (strtolower($config['operation']) == 'subchain') { + if (strtolower((string) $config['operation']) == 'subchain') { $subChain = $config['options']['name']; if (!isset($this->subChainProcessors[$subChain])) { throw new UnknownOperationException("No subchain '$subChain' was found to create operation"); diff --git a/src/Oliverde8/Component/PhpEtl/ChainBuilderV2.php b/src/Oliverde8/Component/PhpEtl/ChainBuilderV2.php new file mode 100644 index 0000000..d1bdb07 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/ChainBuilderV2.php @@ -0,0 +1,39 @@ + $factories + */ + public function __construct( + private readonly ExecutionContextFactoryInterface $contextFactory, + private readonly iterable $factories = [], + ) {} + + public function createChain(ChainConfig $chainConfig): ChainProcessorInterface + { + $operations = []; + foreach ($chainConfig->getConfigs() as $linkConfig) { + $operations[] = $this->getOperationFromConfig($linkConfig); + } + return new ChainProcessor($operations, $this->contextFactory, $chainConfig->maxAsynchronousItems); + } + + private function getOperationFromConfig(OperationConfigInterface $linkConfig): ConfigurableChainOperationInterface + { + foreach ($this->factories as $factory) { + if ($factory->supports($linkConfig)) { + return $factory->build($linkConfig, $this); + } + } + + throw new \RuntimeException('No factory found to link config of type ' . $linkConfig::class); + } +} \ No newline at end of file diff --git a/src/Oliverde8/Component/PhpEtl/ChainConfig.php b/src/Oliverde8/Component/PhpEtl/ChainConfig.php new file mode 100644 index 0000000..68659ef --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/ChainConfig.php @@ -0,0 +1,38 @@ + */ + private array $configs = []; + + /** + * @param OperationConfigInterface[] $configs + */ + public function __construct(public readonly int $maxAsynchronousItems = 1) + {} + + + public function addLink(OperationConfigInterface $linkConfig): self { + $this->configs[] = $linkConfig; + return $this; + } + + /** + * @return array + */ + public function getConfigs(): array + { + return $this->configs; + } + + #[\Override] + public function getFlavor(): string + { + // Main chain does not have a flavor. + return ""; + } +} diff --git a/src/Oliverde8/Component/PhpEtl/ChainObserver/ChainObserver.php b/src/Oliverde8/Component/PhpEtl/ChainObserver/ChainObserver.php index db6128b..0985d56 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainObserver/ChainObserver.php +++ b/src/Oliverde8/Component/PhpEtl/ChainObserver/ChainObserver.php @@ -24,6 +24,7 @@ public function __construct(callable $closure) } + #[\Override] public function init(array $chainLinks, array $chainNames): void { foreach ($chainLinks as $id => $operation) { @@ -33,6 +34,7 @@ public function init(array $chainLinks, array $chainNames): void $this->callback(); } + #[\Override] public function onBeforeProcess($operationId, ChainOperationInterface $operation, ItemInterface $item): void { $this->ended = false; @@ -42,6 +44,7 @@ public function onBeforeProcess($operationId, ChainOperationInterface $operation $this->callback(); } + #[\Override] public function onAfterProcess($operationId, ChainOperationInterface $operation, ItemInterface $item): void { $this->returnItem($operation, $item); @@ -50,6 +53,7 @@ public function onAfterProcess($operationId, ChainOperationInterface $operation, $this->callback(); } + #[\Override] public function onFinish() { $this->ended = true; diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/AbstractChainOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/AbstractChainOperation.php index b87f847..d763dc7 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/AbstractChainOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/AbstractChainOperation.php @@ -23,12 +23,13 @@ abstract class AbstractChainOperation implements ChainOperationInterface /** * @inheritdoc */ + #[\Override] final public function process(ItemInterface $item, ExecutionContext $context): ItemInterface { - if (!isset($this->methodResolutionCache[get_class($item)])) { - $this->methodResolutionCache[get_class($item)] = $this->resolveMethodName($item); + if (!isset($this->methodResolutionCache[$item::class])) { + $this->methodResolutionCache[$item::class] = $this->resolveMethodName($item); } - $method = $this->methodResolutionCache[get_class($item)]; + $method = $this->methodResolutionCache[$item::class]; if (!is_null($method)) { return $this->$method($item, $context); @@ -53,7 +54,7 @@ private function resolveMethodName(ItemInterface $item): ?string continue; } - if ($this->checkIsA(get_class($item), $expecting)) { + if ($this->checkIsA($item::class, $expecting)) { return $method->getName(); } }; @@ -85,7 +86,7 @@ private function validateMethod(\ReflectionMethod $method) if (in_array($method->getName(), ['process', 'resolveMethodName'])) { return false; } - if (strpos($method->getName(), "process") !== 0) { + if (!str_starts_with($method->getName(), "process")) { return false; } diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainMergeOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainMergeOperation.php index f6c1d3a..9785ea7 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainMergeOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainMergeOperation.php @@ -4,58 +4,57 @@ namespace Oliverde8\Component\PhpEtl\ChainOperation; +use Oliverde8\Component\PhpEtl\ChainBuilderV2; use Oliverde8\Component\PhpEtl\ChainProcessor; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Item\MixItem; use Oliverde8\Component\PhpEtl\Item\StopItem; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainMergeConfig; /** - * Class ChainSplitOperation + * Class ChainMergeOperation * * @author de Cramer Oliver * @copyright 2018 Oliverde8 * @package Oliverde8\Component\PhpEtl\ChainOperation */ -class ChainMergeOperation extends AbstractChainOperation implements DataChainOperationInterface, DetailedObservableOperation +class ChainMergeOperation extends AbstractChainOperation implements DataChainOperationInterface, DetailedObservableOperation, ConfigurableChainOperationInterface { use SplittedChainOperationTrait; - /** @var ChainProcessor[] */ - protected array $chainProcessors; - /** - * ChainSplitOperation constructor. - * - * @param ChainProcessor[] $chainProcessors + * @var ChainProcessor[] */ - public function __construct(array $chainProcessors) + private array $chainProcessors = []; + + public function __construct(ChainBuilderV2 $chainBuilder, ChainMergeConfig $config) { - $this->chainProcessors = $chainProcessors; - $this->onSplittedChainOperationConstruct($chainProcessors); + foreach ($config->getChainConfigs() as $chainConfig) { + $this->chainProcessors[] = $chainBuilder->createChain($chainConfig); + } + $this->onSplittedChainOperationConstruct($this->chainProcessors); } + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface { $returnItems = []; foreach ($this->chainProcessors as $chainProcessor) { - $returnItems[] = $chainProcessor->processItemWithChain($item, 0, $context); + foreach ($chainProcessor->processGenerator($item, $context, withStop: false) as $newItem) { + $returnItems[] = $newItem; + } } - // Nothing to process. + // Return all items merged together. return new MixItem($returnItems); } public function processStop(StopItem $item, ExecutionContext $context): ItemInterface { foreach ($this->chainProcessors as $chainProcessor) { - $result = $chainProcessor->processItemWithChain($item, 0, $context); - - if ($result !== $item) { - // Return a new stop item in order to continue flushing out data with stop items. - $item = new StopItem(); - } + foreach ($chainProcessor->processGenerator($item, $context) as $newItem) {} } return $item; @@ -68,4 +67,9 @@ public function getChainProcessors(): array { return $this->chainProcessors; } + + public function getConfigurationClass(): string + { + return ChainMergeConfig::class; + } } diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainRepeatOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainRepeatOperation.php index 1327b78..0ccb2c7 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainRepeatOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainRepeatOperation.php @@ -3,25 +3,31 @@ namespace Oliverde8\Component\PhpEtl\ChainOperation; +use Oliverde8\Component\PhpEtl\ChainBuilderV2; use Oliverde8\Component\PhpEtl\ChainProcessor; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\GroupedItem; use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainRepeatConfig; use Symfony\Component\ExpressionLanguage\ExpressionLanguage; -class ChainRepeatOperation extends AbstractChainOperation implements DetailedObservableOperation +class ChainRepeatOperation extends AbstractChainOperation implements DetailedObservableOperation, ConfigurableChainOperationInterface { use SplittedChainOperationTrait; private ExpressionLanguage $expressionLanguage; + private ChainProcessor $chainProcessor; + private bool $allowAsynchronous; + private string $validationExpression; - public function __construct( - protected readonly ChainProcessor $chainProcessor, - protected readonly string $validationExpression, - protected readonly bool $allowAsynchronous = false, - ) { - $this->onSplittedChainOperationConstruct([$chainProcessor]); + public function __construct(ChainBuilderV2 $chainBuilder, ChainRepeatConfig $config) + { + $this->chainProcessor = $chainBuilder->createChain($config->getChainConfig()); + $this->validationExpression = $config->validationExpression; + $this->allowAsynchronous = $config->allowAsynchronous; + + $this->onSplittedChainOperationConstruct([$this->chainProcessor]); $this->expressionLanguage = new ExpressionLanguage(); } @@ -56,4 +62,9 @@ public function itemIsValid(ItemInterface $item, ExecutionContext $context): boo // If not a data, then it's valid. return true; } + + public function getConfigurationClass(): string + { + return ChainRepeatConfig::class; + } } diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainSplitOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainSplitOperation.php index 5e519ae..3184a98 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainSplitOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/ChainSplitOperation.php @@ -4,12 +4,14 @@ namespace Oliverde8\Component\PhpEtl\ChainOperation; +use Oliverde8\Component\PhpEtl\ChainBuilderV2; use Oliverde8\Component\PhpEtl\ChainProcessor; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Item\StopItem; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; use Oliverde8\Component\PhpEtl\Model\State\OperationState; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainSplitConfig; /** * Class ChainSplitOperation @@ -18,26 +20,25 @@ * @copyright 2018 Oliverde8 * @package Oliverde8\Component\PhpEtl\ChainOperation */ -class ChainSplitOperation extends AbstractChainOperation implements DataChainOperationInterface, DetailedObservableOperation +class ChainSplitOperation extends AbstractChainOperation implements DataChainOperationInterface, DetailedObservableOperation, ConfigurableChainOperationInterface { use SplittedChainOperationTrait; /** * @var ChainProcessor[] */ - private array $chainProcessors; + private array $chainProcessors = []; - /** - * ChainSplitOperation constructor. - * - * @param ChainProcessor[] $chainProcessors - */ - public function __construct(array $chainProcessors) + + public function __construct(ChainBuilderV2 $chainProcessors, ChainSplitConfig $config) { - $this->chainProcessors = $chainProcessors; - $this->onSplittedChainOperationConstruct($chainProcessors); + foreach ($config->getChainConfigs() as $chainConfig) { + $this->chainProcessors[] = $chainProcessors->createChain($chainConfig); + } + $this->onSplittedChainOperationConstruct($this->chainProcessors); } + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface { foreach ($this->chainProcessors as $chainProcessor) { diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/ConfigurableChainOperationInterface.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/ConfigurableChainOperationInterface.php new file mode 100644 index 0000000..d95bb0f --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/ConfigurableChainOperationInterface.php @@ -0,0 +1,6 @@ +delimiter = $delimiter; - $this->enclosure = $enclosure; - $this->escape = $escape; - $this->fileKey = $fileKey; - $this->scoped = $scoped; - } + public function __construct(protected readonly CsvExtractConfig $config) + {} + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface { $filename = $item->getData(); if (is_array($filename)) { - $filename = AssociativeArray::getFromKey($filename, $this->fileKey); + $filename = AssociativeArray::getFromKey($filename, $this->config->fileKey); } - $fileIterator = new Csv($context->getFileSystem()->readStream($filename), $this->delimiter, $this->enclosure, $this->escape); + $fileIterator = new Csv($context->getFileSystem()->readStream($filename), $this->config->delimiter, $this->config->enclosure, $this->config->escape); return new MixItem([new GroupedItem($fileIterator), new FileExtractedItem($filename)]); } -} \ No newline at end of file + + public function getConfigurationClass(): string + { + return CsvExtractConfig::class; + } +} diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/Extract/ExternalFileFinderOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/Extract/ExternalFileFinderOperation.php index dd9626b..f683d43 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/Extract/ExternalFileFinderOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/Extract/ExternalFileFinderOperation.php @@ -4,6 +4,7 @@ namespace Oliverde8\Component\PhpEtl\ChainOperation\Extract; use Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation; +use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface; use Oliverde8\Component\PhpEtl\ChainOperation\DataChainOperationInterface; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\ExternalFileItem; @@ -11,39 +12,35 @@ use Oliverde8\Component\PhpEtl\Item\MixItem; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; use Oliverde8\Component\PhpEtl\Model\File\FileSystemInterface; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\ExternalFileFinderConfig; use Symfony\Component\ExpressionLanguage\ExpressionLanguage; -class ExternalFileFinderOperation extends AbstractChainOperation implements DataChainOperationInterface +class ExternalFileFinderOperation extends AbstractChainOperation implements DataChainOperationInterface, ConfigurableChainOperationInterface { - protected FileSystemInterface $fileSystem; - - protected string $directory; - - private ExpressionLanguage $expressionLanguage; - - - public function __construct(FileSystemInterface $fileSystem, string $directory) - { - $this->fileSystem = $fileSystem; - $this->directory = $directory; + private readonly ExpressionLanguage $expressionLanguage; + public function __construct( + private readonly FileSystemInterface $fileSystem, + private readonly ExternalFileFinderConfig $config + ) { $this->expressionLanguage = new ExpressionLanguage(); } - function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface + #[\Override] + public function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface { $pattern = $item->getData(); $files = []; - $directory = $this->directory; - if (strpos($this->directory, "@") === 0) { - $directory = ltrim($this->directory, '@'); + $directory = $this->config->directory; + if (str_starts_with($this->config->directory, "@")) { + $directory = ltrim($this->config->directory, '@'); $directory = $this->expressionLanguage->evaluate($directory, ['context' => $context->getParameters()]); } foreach ($this->fileSystem->listContents($directory) as $file) { - if (preg_match($pattern, $file) !== 0) { + if (preg_match($pattern, (string) $file) !== 0) { $files[] = new ExternalFileItem($directory . "/" . $file, $this->fileSystem); } } diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/Extract/JsonExtractOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/Extract/JsonExtractOperation.php index 5bfb6bd..858e1a2 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/Extract/JsonExtractOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/Extract/JsonExtractOperation.php @@ -4,6 +4,7 @@ use oliverde8\AssociativeArraySimplified\AssociativeArray; use Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation; +use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface; use Oliverde8\Component\PhpEtl\ChainOperation\DataChainOperationInterface; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\FileExtractedItem; @@ -11,32 +12,23 @@ use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Item\MixItem; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\Extract\JsonExtractConfig; -class JsonExtractOperation extends AbstractChainOperation implements DataChainOperationInterface +class JsonExtractOperation extends AbstractChainOperation implements DataChainOperationInterface, ConfigurableChainOperationInterface { - protected string $fileKey; - - protected bool $scoped; - - public function __construct(string $fileKey, bool $scoped) - { - $this->fileKey = $fileKey; - $this->scoped = $scoped; - } + public function __construct(private readonly JsonExtractConfig $config) + {} + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface { $filename = $item->getData(); if (is_array($filename)) { - $filename = AssociativeArray::getFromKey($filename, $this->fileKey); + $filename = AssociativeArray::getFromKey($filename, $this->config->fileKey); } - if ($this->scoped) { - $data = json_decode($context->getFileSystem()->read($filename), true); - } else { - $data = json_decode(file_get_contents($filename), true); - } + $data = json_decode($context->getFileSystem()->read($filename), true); - return new MixItem([new GroupedItem(new \ArrayIterator($data)), new FileExtractedItem($item->getData())]); + return new MixItem([new GroupedItem(new \ArrayIterator($data)), new FileExtractedItem($filename)]); } } diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/FailSafeOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/FailSafeOperation.php index d622892..b290097 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/FailSafeOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/FailSafeOperation.php @@ -3,35 +3,44 @@ namespace Oliverde8\Component\PhpEtl\ChainOperation; +use Oliverde8\Component\PhpEtl\ChainBuilderV2; use Oliverde8\Component\PhpEtl\ChainProcessor; +use Oliverde8\Component\PhpEtl\Exception\ChainOperationException; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\GroupedItem; use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Item\StopItem; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; -class FailSafeOperation implements ChainOperationInterface, DetailedObservableOperation +class FailSafeOperation extends AbstractChainOperation implements DataChainOperationInterface, DetailedObservableOperation, ConfigurableChainOperationInterface { use SplittedChainOperationTrait; - protected int $count = 0; + private ChainProcessor $chainProcessor; + private array $exceptionsToCatch = []; + private int $nbAttempts = 1; - public function __construct( - protected readonly ChainProcessor $chainProcessor, - protected readonly array $exceptionsToCatch, - protected readonly int $nbAttempts, - ){} - - public function process(ItemInterface $item, ExecutionContext $context) + public function __construct(ChainBuilderV2 $chainBuilder, FailSafeConfig $config) { - if ($item instanceof StopItem) { - foreach ($this->repeatOnItem($item, $context) as $newItem) {} - return $item; - } + $this->chainProcessor = $chainBuilder->createChain($config->getChainConfig()); + $this->exceptionsToCatch = $config->exceptionsToCatch; + $this->nbAttempts = $config->nbAttempts; + $this->onSplittedChainOperationConstruct([$this->chainProcessor]); + } + #[\Override] + public function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface + { return new GroupedItem($this->repeatOnItem($item, $context)); } + public function processStop(StopItem $item, ExecutionContext $context): ItemInterface + { + foreach ($this->repeatOnItem($item, $context) as $ignored) {} + return $item; + } + public function repeatOnItem(ItemInterface $inputItem, ExecutionContext $context): \Generator { $nbAttempts = 0; @@ -40,23 +49,31 @@ public function repeatOnItem(ItemInterface $inputItem, ExecutionContext $context foreach ($this->chainProcessor->processGenerator($inputItem, $context, withStop: false) as $newItem) { yield $newItem; } - return; + return; // success - stop retrying } catch (\Exception $exception) { + $exceptionToHandle = $exception; + if ($exception instanceof ChainOperationException) { + $exceptionToHandle = $exception->getPrevious() ?? $exception; + } + $nbAttempts++; - $exceptionHandled = false; + $handled = false; foreach ($this->exceptionsToCatch as $exceptionType) { - if ($exception instanceof $exceptionType) { - $exceptionHandled = true; + if ($exceptionToHandle instanceof $exceptionType) { + $handled = true; break; } } - - if (!$exceptionHandled || $nbAttempts >= $this->nbAttempts) { - $context->getLogger()->error("Failed to handle exception in fail safe!", ['exception' => $exception]); + if (!$handled || $nbAttempts >= $this->nbAttempts) { + $context->getLogger()->error('FailSafeOperation giving up', ['attempts' => $nbAttempts, 'exception' => $exception]); throw $exception; - } else { - $context->getLogger()->warning("Handling exception with fail safe!", ['exception' => $exception, 'nbAttempts' => $nbAttempts]); } + $context->getLogger()->warning('FailSafeOperation retrying', ['attempts' => $nbAttempts, 'exception' => $exception]); } } while ($nbAttempts < $this->nbAttempts); } + + public function getConfigurationClass(): string + { + return FailSafeConfig::class; + } } \ No newline at end of file diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/Grouping/SimpleGroupingOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/Grouping/SimpleGroupingOperation.php index 2fab9e7..6fd7b5c 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/Grouping/SimpleGroupingOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/Grouping/SimpleGroupingOperation.php @@ -6,6 +6,7 @@ use oliverde8\AssociativeArraySimplified\AssociativeArray; use Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation; +use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface; use Oliverde8\Component\PhpEtl\ChainOperation\DataChainOperationInterface; use Oliverde8\Component\PhpEtl\Item\ChainBreakItem; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; @@ -13,51 +14,25 @@ use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Item\StopItem; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\Grouping\SimpleGroupingConfig; -/** - * Class SimpleGrouping - * - * @author de Cramer Oliver - * @copyright 2018 Oliverde8 - * @package Oliverde8\Component\PhpEtl\ChainOperation\Grouping - */ -class SimpleGroupingOperation extends AbstractChainOperation implements DataChainOperationInterface +class SimpleGroupingOperation extends AbstractChainOperation implements DataChainOperationInterface, ConfigurableChainOperationInterface { - /** @var string[] Key to use for grouping, if array it will be used to read recursively inside the array. */ - protected $groupKey = []; - - /** @var string[] Key to identify each individual data inside the group. */ - protected $groupIdentifierKey = []; - - /** - * Grouped data kept in memory. - * - * @var array - */ - protected $data = []; - - /** - * SimpleGroupingOperation constructor. - * - * @param string[] $groupKey Key to use for grouping, if array it will be used to read recursively inside the array. - * @param string[] $groupIdentifierKey key to identify each individual data inside the group. - */ - public function __construct(array $groupKey, array $groupIdentifierKey = []) - { - $this->groupKey = $groupKey; - $this->groupIdentifierKey = $groupIdentifierKey; - } + private array $data = []; + public function __construct(private readonly SimpleGroupingConfig $config) + {} /** * @inheritdoc */ + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): ChainBreakItem { - $groupingValue = AssociativeArray::getFromKey($item->getData(), $this->groupKey); + $groupingValue = AssociativeArray::getFromKey($item->getData(), $this->config->groupKey); - if (!empty($this->groupIdentifierKey)) { - $groupIdValue = AssociativeArray::getFromKey($item->getData(), $this->groupIdentifierKey); + if (!empty($this->config->groupIdentifierKey)) { + $groupIdValue = AssociativeArray::getFromKey($item->getData(), $this->config->groupIdentifierKey); $this->data[$groupingValue][$groupIdValue] = $item->getData(); } else { $this->data[$groupingValue][] = $item->getData(); diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/Loader/FileWriterOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/Loader/FileWriterOperation.php index e4a463a..be90abe 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/Loader/FileWriterOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/Loader/FileWriterOperation.php @@ -5,6 +5,7 @@ namespace Oliverde8\Component\PhpEtl\ChainOperation\Loader; use Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation; +use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface; use Oliverde8\Component\PhpEtl\ChainOperation\DataChainOperationInterface; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\FileLoadedItem; @@ -13,32 +14,25 @@ use Oliverde8\Component\PhpEtl\Item\StopItem; use Oliverde8\Component\PhpEtl\Load\File\FileWriterInterface; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\FileWriterConfigInterface; -/** - * Class FileWriter - * - * @author de Cramer Oliver - * @copyright 2022 Oliverde8 - * @package Oliverde8\Component\PhpEtl\ChainOperation\Loader - */ -class FileWriterOperation extends AbstractChainOperation implements DataChainOperationInterface +class FileWriterOperation extends AbstractChainOperation implements DataChainOperationInterface, ConfigurableChainOperationInterface { - /** @var FileWriterInterface */ - protected $writer; + private ?FileWriterInterface $writer = null; - protected string $fileName; - - public function __construct(FileWriterInterface $writer, string $fileName) - { - $this->writer = $writer; - $this->fileName = $fileName; - } + public function __construct(private readonly FileWriterConfigInterface $config) + {} /** * @inheritdoc */ + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): DataItemInterface { + if ($this->writer === null) { + $this->writer = $this->config->getFile(); + } + $this->writer->write($item->getData()); return $item; @@ -46,18 +40,22 @@ public function processData(DataItemInterface $item, ExecutionContext $context): public function processStop(StopItem $stopItem, ExecutionContext $context): ItemInterface { + if ($this->writer === null) { + $this->writer = $this->config->getFile(); + } + $resource = $this->writer->getResource(); if(is_resource($resource) && $stopItem->isFinal) { $meta_data = stream_get_meta_data($resource); $filename = $meta_data["uri"]; - $context->getFileSystem()->writeStream($this->fileName, fopen($filename, 'r')); + $context->getFileSystem()->writeStream($this->config->getFileName(), fopen($filename, 'r')); fclose($resource); unlink($filename); } - return new MixItem([new FileLoadedItem($this->fileName), $stopItem]); + return new MixItem([new FileLoadedItem($this->config->getFileName(),), $stopItem]); } } diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/SplittedChainOperationTrait.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/SplittedChainOperationTrait.php index a2e5c28..68757c0 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/SplittedChainOperationTrait.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/SplittedChainOperationTrait.php @@ -17,7 +17,7 @@ protected function onSplittedChainOperationConstruct(array $chainProcessors): vo $lastObservedState = &$this->lastObservedState; foreach ($chainProcessors as $subChainPart => $chainProcessor) { if ($chainProcessor instanceof ChainProcessor) { - $observer = $chainProcessor->initObserver(function (array $operationStates) use (&$lastObservedState, $subChainPart) { + $observer = $chainProcessor->initObserver(function (array $operationStates) use (&$lastObservedState, $subChainPart): void { $lastObservedState[$subChainPart] = $operationStates; }); diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/CallbackTransformerOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/CallbackTransformerOperation.php index b188ad1..2dfda23 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/CallbackTransformerOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/CallbackTransformerOperation.php @@ -5,38 +5,23 @@ namespace Oliverde8\Component\PhpEtl\ChainOperation\Transformer; use Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation; +use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface; use Oliverde8\Component\PhpEtl\ChainOperation\DataChainOperationInterface; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\ItemInterface; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; -/** - * Class CallbackTransformerOperation - * - * @author de Cramer Oliver - * @copyright 2018 Oliverde8 - * @package Oliverde8\Component\PhpEtl\ChainOperation\Transformer - */ -class CallbackTransformerOperation extends AbstractChainOperation implements DataChainOperationInterface +class CallbackTransformerOperation extends AbstractChainOperation implements DataChainOperationInterface, ConfigurableChainOperationInterface { - protected $callback; - - /** - * CallbackTransformerOperation constructor. - * - * @param $callback - */ - public function __construct($callback) + public function __construct(private readonly CallBackTransformerConfig $config) { - $this->callback = $callback; } - /** - * @inheritdoc - */ + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface { - $method = $this->callback; + $method = $this->config->getCallable(); return $method($item, $context); } -} \ No newline at end of file +} diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/ExternalFileProcessorOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/ExternalFileProcessorOperation.php index 41da6f0..f84eba0 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/ExternalFileProcessorOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/ExternalFileProcessorOperation.php @@ -4,14 +4,19 @@ namespace Oliverde8\Component\PhpEtl\ChainOperation\Transformer; use Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation; +use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface; use Oliverde8\Component\PhpEtl\Item\DataItem; use Oliverde8\Component\PhpEtl\Item\ExternalFileItem; use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Item\MixItem; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\ExternalFileProcessorConfig; -class ExternalFileProcessorOperation extends AbstractChainOperation +class ExternalFileProcessorOperation extends AbstractChainOperation implements ConfigurableChainOperationInterface { + public function __construct(private readonly ExternalFileProcessorConfig $config) + {} + public function processFile(ExternalFileItem $item, ExecutionContext $context): ItemInterface { $externalFilePath = $item->getFilePath(); diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/FilterDataOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/FilterDataOperation.php index 850494c..1d6abd1 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/FilterDataOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/FilterDataOperation.php @@ -3,37 +3,33 @@ namespace Oliverde8\Component\PhpEtl\ChainOperation\Transformer; use Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation; +use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface; use Oliverde8\Component\PhpEtl\ChainOperation\DataChainOperationInterface; use Oliverde8\Component\PhpEtl\Item\ChainBreakItem; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\FilterDataConfig; use Oliverde8\Component\RuleEngine\RuleApplier; -class FilterDataOperation extends AbstractChainOperation implements DataChainOperationInterface +class FilterDataOperation extends AbstractChainOperation implements DataChainOperationInterface, ConfigurableChainOperationInterface { - protected RuleApplier $ruleApplier; - protected array $rule; - - protected bool $negate; - - public function __construct(RuleApplier $ruleApplier, array $rule, bool $negate) - { - $this->ruleApplier = $ruleApplier; - $this->rule = $rule; - $this->negate = $negate; - } + public function __construct( + private readonly RuleApplier $ruleApplier, + private readonly FilterDataConfig $config + ) {} + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface { $data = $item->getData(); $resultData = []; - $result = $this->ruleApplier->apply($data, $resultData, $this->rule); + $result = $this->ruleApplier->apply($data, $resultData, $this->config->rules); - if (($this->negate && $result == false) || (!$this->negate && $result == true)) { + if (($this->config->negate && $result == false) || (!$this->config->negate && $result == true)) { return $item; } diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/LogOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/LogOperation.php index 06c10ac..12b7d44 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/LogOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/LogOperation.php @@ -5,64 +5,49 @@ use oliverde8\AssociativeArraySimplified\AssociativeArray; use Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation; +use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface; use Oliverde8\Component\PhpEtl\ChainOperation\DataChainOperationInterface; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\LogConfig; use Symfony\Component\ExpressionLanguage\ExpressionLanguage; -class LogOperation extends AbstractChainOperation implements DataChainOperationInterface +class LogOperation extends AbstractChainOperation implements DataChainOperationInterface, ConfigurableChainOperationInterface { protected readonly ExpressionLanguage $expressionLanguage; - public function __construct( - protected readonly string $message, - protected readonly string $level, - protected readonly array $context, - ){ + public function __construct(protected readonly LogConfig $config) + { $this->expressionLanguage = new ExpressionLanguage(); } + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface { $data = new AssociativeArray($item->getData()); - $message = $this->message; - if (strpos($message, "@") === 0) { + $message = $this->config->message; + if (str_starts_with($message, "@")) { $message = ltrim($message, "@"); $message = $this->expressionLanguage->evaluate($message, ['data' => $item->getData(), 'context' => $context->getParameters()]); } $logContext = []; - foreach ($this->context as $key => $valueKey) { + foreach ($this->config->context as $key => $valueKey) { $logContext[$key] = $data->get($valueKey); } - switch ($this->level) { - case 'debug': - $context->getLogger()->debug($message, $logContext); - break; - case 'info': - $context->getLogger()->info($message, $logContext); - break; - case 'notice': - $context->getLogger()->notice($message, $logContext); - break; - case 'warning': - $context->getLogger()->warning($message, $logContext); - break; - case 'error': - $context->getLogger()->error($message, $logContext); - break; - case 'critical': - $context->getLogger()->critical($message, $logContext); - break; - case 'alert': - $context->getLogger()->alert($message, $logContext); - break; - case 'emergency': - $context->getLogger()->emergency($message, $logContext); - break; - } + match ($this->config->level) { + 'debug' => $context->getLogger()->debug($message, $logContext), + 'info' => $context->getLogger()->info($message, $logContext), + 'notice' => $context->getLogger()->notice($message, $logContext), + 'warning' => $context->getLogger()->warning($message, $logContext), + 'error' => $context->getLogger()->error($message, $logContext), + 'critical' => $context->getLogger()->critical($message, $logContext), + 'alert' => $context->getLogger()->alert($message, $logContext), + 'emergency' => $context->getLogger()->emergency($message, $logContext), + default => $item, + }; return $item; } diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/RuleTransformOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/RuleTransformOperation.php index f74915e..f07c1f9 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/RuleTransformOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/RuleTransformOperation.php @@ -6,54 +6,38 @@ use oliverde8\AssociativeArraySimplified\AssociativeArray; use Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation; +use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface; use Oliverde8\Component\PhpEtl\ChainOperation\DataChainOperationInterface; use Oliverde8\Component\PhpEtl\Item\DataItem; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; use Oliverde8\Component\RuleEngine\RuleApplier; -/** - * Class RuleTransformOperation - * - * @author de Cramer Oliver - * @copyright 2018 Oliverde8 - * @package Oliverde8\Component\PhpEtl\ChainOperation\Transformer - */ -class RuleTransformOperation extends AbstractChainOperation implements DataChainOperationInterface + +class RuleTransformOperation extends AbstractChainOperation implements DataChainOperationInterface, ConfigurableChainOperationInterface { const VARIABLE_MATCH_REGEX = '/{(?[^{}]+)}/'; /** @var string[] */ protected array $parsedColumns = []; - /** @var RuleApplier */ - protected RuleApplier $ruleApplier; - - /** @var array */ - protected array $rules; - - /** @var boolean */ - protected bool $add; - - public function __construct(RuleApplier $ruleApplier, array $rules, bool $add) - { - $this->ruleApplier = $ruleApplier; - $this->rules = $rules; - $this->add = $add; - } + public function __construct(private readonly RuleApplier $ruleApplier, private readonly RuleTransformConfig $config) + {} + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): DataItemInterface { $data = $item->getData(); $newData = []; // We add data and don't send new data. - if ($this->add) { + if ($this->config->add) { $newData = $data; } - foreach ($this->rules as $column => $rule) { + foreach ($this->config->getRules() as $column => $rule) { // Add context to the data. $data['@context'] = array_merge($context->getParameters(), $rule['context'] ?? []); diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/SimpleHttpOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/SimpleHttpOperation.php index 7559522..38b86be 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/SimpleHttpOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/SimpleHttpOperation.php @@ -6,67 +6,44 @@ use oliverde8\AssociativeArraySimplified\AssociativeArray; use Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation; +use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface; use Oliverde8\Component\PhpEtl\ChainOperation\DataChainOperationInterface; use Oliverde8\Component\PhpEtl\Item\AsyncHttpClientResponseItem; -use Oliverde8\Component\PhpEtl\Item\DataItem; use Oliverde8\Component\PhpEtl\Item\DataItemInterface; use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; use Symfony\Component\ExpressionLanguage\ExpressionLanguage; use Symfony\Contracts\HttpClient\HttpClientInterface; -class SimpleHttpOperation extends AbstractChainOperation implements DataChainOperationInterface +class SimpleHttpOperation extends AbstractChainOperation implements DataChainOperationInterface, ConfigurableChainOperationInterface { - private HttpClientInterface $client; - - private string $method = "GET"; - private string $url; - - private bool $responseIsJson; - - private ?string $optionsKey; - - protected ?string $responseKey; - - private ExpressionLanguage $expressionLanguage; - - public function __construct( - HttpClientInterface $client, - string $method, - string $url, - bool $responseIsJson, - ?string $optionsKey, - ?string $responseKey - ) { - $this->client = $client; - $this->method = $method; - $this->url = $url; - $this->responseIsJson = $responseIsJson; - $this->optionsKey = $optionsKey; - $this->responseKey = $responseKey; + private readonly ExpressionLanguage $expressionLanguage; + public function __construct(private readonly HttpClientInterface $client, private readonly SimpleHttpConfig $config) + { $this->expressionLanguage = new ExpressionLanguage(); } - + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface { $data = $item->getData(); - if ($this->optionsKey) { - $options = AssociativeArray::getFromKey($data, $this->optionsKey, []); + if ($this->config->optionKey) { + $options = AssociativeArray::getFromKey($data, $this->config->optionKey, []); } else { $options = $data; } - $url = $this->url; - if (strpos($url, "@") === 0) { + $url = $this->config->url; + if (str_starts_with($url, "@")) { $url = ltrim($url, '@'); $url = $this->expressionLanguage->evaluate($url, ['data' => $data]); } - $response = $this->client->request($this->method, $url, $options); + $response = $this->client->request($this->config->method, $url, $options); $response->getInfo(); - return new AsyncHttpClientResponseItem($this->client, $response, $this->responseIsJson, $this->responseKey, $data); + return new AsyncHttpClientResponseItem($this->client, $response, $this->config->responseIsJson, $this->config->responseKey, $data); } } diff --git a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/SplitItemOperation.php b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/SplitItemOperation.php index 5194a15..a30f9f9 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/SplitItemOperation.php +++ b/src/Oliverde8/Component/PhpEtl/ChainOperation/Transformer/SplitItemOperation.php @@ -4,6 +4,7 @@ use oliverde8\AssociativeArraySimplified\AssociativeArray; use Oliverde8\Component\PhpEtl\ChainOperation\AbstractChainOperation; +use Oliverde8\Component\PhpEtl\ChainOperation\ConfigurableChainOperationInterface; use Oliverde8\Component\PhpEtl\ChainOperation\DataChainOperationInterface; use Oliverde8\Component\PhpEtl\Exception\ChainOperationException; use Oliverde8\Component\PhpEtl\Item\ChainBreakItem; @@ -12,35 +13,22 @@ use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Item\MixItem; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SplitItemConfig; -class SplitItemOperation extends AbstractChainOperation implements DataChainOperationInterface +class SplitItemOperation extends AbstractChainOperation implements DataChainOperationInterface, ConfigurableChainOperationInterface { - protected bool $singleElement; - - protected bool $keepKeys; - - protected array $keys; - - protected ?string $keyName; - - protected array $duplicateKeys; - - public function __construct(bool $singleElement, array $keys, bool $keepKeys, string $keyName = null, array $duplicatekeys = []) + public function __construct(protected readonly SplitItemConfig $config) { - $this->singleElement = $singleElement; - $this->keepKeys = $keepKeys; - $this->keys = $keys; - $this->keyName = $keyName; - $this->duplicateKeys = $duplicatekeys; } /** * @throws ChainOperationException */ + #[\Override] public function processData(DataItemInterface $item, ExecutionContext $context): ItemInterface { - if ($this->singleElement) { - $data = AssociativeArray::getFromKey($item->getData(), $this->keys[0], new ChainBreakItem()); + if ($this->config->singleElement) { + $data = AssociativeArray::getFromKey($item->getData(), $this->config->keys[0], new ChainBreakItem()); if ($data instanceof ItemInterface) { return $data; } @@ -49,14 +37,14 @@ public function processData(DataItemInterface $item, ExecutionContext $context): } $newItemData = []; - foreach ($this->keys as $key) { + foreach ($this->config->keys as $key) { $newItemData[] = AssociativeArray::getFromKey($item->getData(), $key, []); } return $this->createItem($newItemData); } - protected function createItem($itemData, $fullData): ItemInterface + protected function createItem($itemData, $fullData = []): ItemInterface { if (!is_array($itemData)) { throw new ChainOperationException(sprintf('Split operation expects an array to split; "%s', gettype($itemData))); @@ -68,17 +56,17 @@ protected function createItem($itemData, $fullData): ItemInterface $items[] = $datum; } else { $dataItem = []; - if ($this->keyName) { - AssociativeArray::setFromKey($dataItem, $this->keyName, $datum); + if ($this->config->keyName) { + AssociativeArray::setFromKey($dataItem, $this->config->keyName, $datum); } else { $dataItem = $datum; } - foreach ($this->duplicateKeys as $keyStore => $keyFetch) { + foreach ($this->config->duplicateKeys as $keyStore => $keyFetch) { AssociativeArray::setFromKey($dataItem, $keyStore, AssociativeArray::getFromKey($fullData, $keyFetch)); } - if ($this->keepKeys) { + if ($this->config->keepKeys) { $dataItem = ['key' => $datumKey, 'value' => $dataItem]; } $items[] = new DataItem($dataItem); diff --git a/src/Oliverde8/Component/PhpEtl/ChainProcessor.php b/src/Oliverde8/Component/PhpEtl/ChainProcessor.php index 17c305e..37701a2 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainProcessor.php +++ b/src/Oliverde8/Component/PhpEtl/ChainProcessor.php @@ -21,7 +21,7 @@ final class ChainProcessor extends LoggerContext implements ChainProcessorInterface { - const KEY_LOGGER_ETL_IDENTIFIER = 'etl.identifier'; + const string KEY_LOGGER_ETL_IDENTIFIER = 'etl.identifier'; /** @var ChainOperationInterface[] */ protected readonly array $chainLinks; @@ -53,6 +53,7 @@ public function __construct( } + #[\Override] public function process(\Iterator|ItemInterface $item, array $parameters, ?callable $observerCallback = null, $withStop = true): void { $context = $this->contextFactory->get($parameters); @@ -76,6 +77,7 @@ public function process(\Iterator|ItemInterface $item, array $parameters, ?calla } } + #[\Override] public function processGenerator( \Iterator|ItemInterface $item, ExecutionContext $context, @@ -181,7 +183,7 @@ protected function getItemsFromGroupItem(GroupedItem $item): \Generator } } - protected function handleAsyncItems(int $maxItems = null): \Generator + protected function handleAsyncItems(?int $maxItems = null): \Generator { if ($maxItems === null) { $maxItems = $this->maxAsynchronousItems; @@ -258,7 +260,7 @@ public function initObserver(?callable $observerCallback = null): ChainObserver } if (!$observerCallback) { - $observerCallback = function (){}; + $observerCallback = function (): void{}; } $this->chainObserver = new ChainObserver($observerCallback); $this->chainObserver->init($this->chainLinks, $this->chainLinkNames); @@ -266,11 +268,13 @@ public function initObserver(?callable $observerCallback = null): ChainObserver return $this->chainObserver; } + #[\Override] public function getChainLinkNames(): array { return $this->chainLinkNames; } + #[\Override] public function getChainLinks(): array { return $this->chainLinks; diff --git a/src/Oliverde8/Component/PhpEtl/ChainWorkDirManager.php b/src/Oliverde8/Component/PhpEtl/ChainWorkDirManager.php index cf3a627..4425ad6 100644 --- a/src/Oliverde8/Component/PhpEtl/ChainWorkDirManager.php +++ b/src/Oliverde8/Component/PhpEtl/ChainWorkDirManager.php @@ -8,13 +8,10 @@ class ChainWorkDirManager { - private string $baseDir; + private readonly LocalFileSystem $fileSystem; - private LocalFileSystem $fileSystem; - - public function __construct(string $baseDir) + public function __construct(private readonly string $baseDir) { - $this->baseDir = $baseDir; $this->fileSystem = new LocalFileSystem('/'); } diff --git a/src/Oliverde8/Component/PhpEtl/Exception/ChainBuilderException.php b/src/Oliverde8/Component/PhpEtl/Exception/ChainBuilderException.php new file mode 100644 index 0000000..70e69e3 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Exception/ChainBuilderException.php @@ -0,0 +1,8 @@ +chainOperationName = $chainOperationName; - parent::__construct($message, $code, $previous); } diff --git a/src/Oliverde8/Component/PhpEtl/ExecutionContextFactory.php b/src/Oliverde8/Component/PhpEtl/ExecutionContextFactory.php index 924731c..8a77ed5 100644 --- a/src/Oliverde8/Component/PhpEtl/ExecutionContextFactory.php +++ b/src/Oliverde8/Component/PhpEtl/ExecutionContextFactory.php @@ -7,6 +7,7 @@ class ExecutionContextFactory implements ExecutionContextFactoryInterface { + #[\Override] public function get(array $parameters): ExecutionContext { return new ExecutionContext($parameters, new LocalFileSystem()); diff --git a/src/Oliverde8/Component/PhpEtl/Extract/File/Csv.php b/src/Oliverde8/Component/PhpEtl/Extract/File/Csv.php index 4abc5a4..43feca6 100644 --- a/src/Oliverde8/Component/PhpEtl/Extract/File/Csv.php +++ b/src/Oliverde8/Component/PhpEtl/Extract/File/Csv.php @@ -83,6 +83,7 @@ public function setHeaders(array $headers): void * @return mixed Can return any type. * @since 5.0.0 */ + #[\Override] public function current(): mixed { $this->init(); @@ -97,6 +98,7 @@ public function current(): mixed * @return void Any returned value is ignored. * @since 5.0.0 */ + #[\Override] public function next(): void { $this->init(); @@ -119,6 +121,7 @@ public function next(): void * @return mixed scalar on success, or null on failure. * @since 5.0.0 */ + #[\Override] public function key(): mixed { $this->init(); @@ -134,6 +137,7 @@ public function key(): mixed * Returns true on success or false on failure. * @since 5.0.0 */ + #[\Override] public function valid(): bool { $this->init(); @@ -148,6 +152,7 @@ public function valid(): bool * @return void Any returned value is ignored. * @since 5.0.0 */ + #[\Override] public function rewind(): void { if (!is_null($this->file)) { diff --git a/src/Oliverde8/Component/PhpEtl/Factory/LocalFileSystemFactory.php b/src/Oliverde8/Component/PhpEtl/Factory/LocalFileSystemFactory.php index 9a2e57f..9acdf88 100644 --- a/src/Oliverde8/Component/PhpEtl/Factory/LocalFileSystemFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Factory/LocalFileSystemFactory.php @@ -10,16 +10,14 @@ class LocalFileSystemFactory implements FileSystemFactoryInterface { - private ChainWorkDirManager $chainWorkDirManager; - /** @var FileSystemInterface[] */ private array $fileSystems; - public function __construct(ChainWorkDirManager $chainWorkDirManager) + public function __construct(private readonly ChainWorkDirManager $chainWorkDirManager) { - $this->chainWorkDirManager = $chainWorkDirManager; } + #[\Override] public function get(ExecutionInterface $execution): FileSystemInterface { if (!isset($this->fileSystems[$execution->getId()])) { diff --git a/src/Oliverde8/Component/PhpEtl/Factory/NullLoggerFactory.php b/src/Oliverde8/Component/PhpEtl/Factory/NullLoggerFactory.php index 2a7efa6..d21defe 100644 --- a/src/Oliverde8/Component/PhpEtl/Factory/NullLoggerFactory.php +++ b/src/Oliverde8/Component/PhpEtl/Factory/NullLoggerFactory.php @@ -9,6 +9,7 @@ class NullLoggerFactory implements LoggerFactoryInterface { + #[\Override] public function get(ExecutionInterface $execution): LoggerInterface { return new NullLogger(); diff --git a/src/Oliverde8/Component/PhpEtl/GenericChainFactory.php b/src/Oliverde8/Component/PhpEtl/GenericChainFactory.php new file mode 100644 index 0000000..af891b8 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/GenericChainFactory.php @@ -0,0 +1,108 @@ +isOfType($this->operationClassName, ConfigurableChainOperationInterface::class)) { + throw new ChainBuilderException("Operation class '{$this->operationClassName}' must implement ConfigurableChainOperationInterface"); + } + } + + public function build(OperationConfigInterface $linkConfig, ChainBuilderV2 $chainBuilder): ConfigurableChainOperationInterface + { + $refClass = new \ReflectionClass($this->operationClassName); + $constructor = $refClass->getConstructor(); + + if ($constructor) { + $params = $constructor->getParameters(); + + // Build arguments in the correct order + $args = []; + foreach ($params as $param) { + $name = $param->getName(); + if ($param->getType() !== null && $this->reflectionIsOfType($param->getType(), OperationConfigInterface::class)) { + $args[] = $linkConfig; + } elseif ($param->getType() !== null && $param->getType()->getName() === 'string' && $name === 'flavor') { + $args[] = $this->flavor; + } elseif ($param->getType() !== null && $this->reflectionIsOfType($param->getType(), ChainBuilderV2::class)) { + $args[] = $chainBuilder; + } elseif (array_key_exists($name, $this->injections)) { + $args[] = $this->injections[$name]; + } elseif ($param->isDefaultValueAvailable()) { + $args[] = $param->getDefaultValue(); + } else { + throw new \InvalidArgumentException("Missing parameter '$name' while creating instance of '{$this->operationClassName}' with flavor '{$this->flavor}'"); + } + } + + return $refClass->newInstanceArgs($args); + } else { + return $refClass->newInstance(); + } + } + + public function supports(OperationConfigInterface $linkConfig): bool + { + if (!$this->isOfType($linkConfig::class, $this->configClassName)) { + return false; + } + if ($linkConfig->getFlavor() !== $this->flavor) { + return false; + } + return true; + } + + private function reflectionIsOfType(ReflectionType $type, string $expectedClassName): bool + { + foreach ($this->flattenTypes($type) as $reflectionType) { + if ($this->isOfType($reflectionType->getName(), $expectedClassName)) { + return true; + } + } + return false; + } + + private function isOfType(string $className, string $expectedClassName): bool + { + if (!class_exists($className) && !interface_exists($className)) { + return false; + } + + if (!class_exists($expectedClassName) && !interface_exists($expectedClassName)) { + return false; + } + + if (!is_a($className, $expectedClassName, true)) { + return false; + } + + return true; + } + + + private function flattenTypes(ReflectionType $type): array + { + if ($type instanceof ReflectionUnionType) { + return array_values(array_filter($type->getTypes(), fn($t) => $t instanceof ReflectionNamedType)); + } + if ($type instanceof ReflectionNamedType) { + return [$type]; + } + return []; + } +} \ No newline at end of file diff --git a/src/Oliverde8/Component/PhpEtl/Item/AsyncHttpClientResponseItem.php b/src/Oliverde8/Component/PhpEtl/Item/AsyncHttpClientResponseItem.php index f90de31..b1b4efa 100644 --- a/src/Oliverde8/Component/PhpEtl/Item/AsyncHttpClientResponseItem.php +++ b/src/Oliverde8/Component/PhpEtl/Item/AsyncHttpClientResponseItem.php @@ -10,16 +10,6 @@ class AsyncHttpClientResponseItem implements AsyncItemInterface { - private HttpClientInterface $client; - - private ResponseInterface $response; - - private bool $responseIsJson; - - private ?string $responseKey; - - private array $baseData; - private bool $isRunning = true; /** @@ -28,21 +18,12 @@ class AsyncHttpClientResponseItem implements AsyncItemInterface * @param string|null $responseKey * @param array $baseData */ - public function __construct( - HttpClientInterface $client, - ResponseInterface $response, - bool $responseIsJson, - ?string $responseKey, - array $baseData - ) { - $this->client = $client; - $this->response = $response; - $this->responseIsJson = $responseIsJson; - $this->responseKey = $responseKey; - $this->baseData = $baseData; + public function __construct(private readonly HttpClientInterface $client, private readonly ResponseInterface $response, private readonly bool $responseIsJson, private readonly ?string $responseKey, private readonly array $baseData) + { } + #[\Override] public function isRunning(): bool { try { @@ -51,13 +32,14 @@ public function isRunning(): bool return false; } } - } catch (TimeoutException $exception) { + } catch (TimeoutException) { // This is normal, we have used a very low stream timeout because we wish to continue processing // other items while this item is being downloaded. return true; } } + #[\Override] public function getItem(): ItemInterface { $responseData = [ diff --git a/src/Oliverde8/Component/PhpEtl/Item/DataItem.php b/src/Oliverde8/Component/PhpEtl/Item/DataItem.php index ba38bc7..07a8369 100644 --- a/src/Oliverde8/Component/PhpEtl/Item/DataItem.php +++ b/src/Oliverde8/Component/PhpEtl/Item/DataItem.php @@ -13,21 +13,14 @@ */ class DataItem implements DataItemInterface { - /** - * @var mixed - */ - protected $data; - /** * DataItem constructor. - * - * @param mixed $data */ - public function __construct($data) + public function __construct(protected mixed $data) { - $this->data = $data; } + #[\Override] public function getData() { return $this->data; diff --git a/src/Oliverde8/Component/PhpEtl/Item/ExternalFileItem.php b/src/Oliverde8/Component/PhpEtl/Item/ExternalFileItem.php index 95e7d0b..56006d1 100644 --- a/src/Oliverde8/Component/PhpEtl/Item/ExternalFileItem.php +++ b/src/Oliverde8/Component/PhpEtl/Item/ExternalFileItem.php @@ -12,16 +12,10 @@ class ExternalFileItem implements ItemInterface const STATE_PROCESSED = "processed"; - protected string $filePath; - - protected FileSystemInterface $fileSystem; - protected string $state; - public function __construct(string $filePath, FileSystemInterface $fileSystem) + public function __construct(protected string $filePath, protected FileSystemInterface $fileSystem) { - $this->fileSystem = $fileSystem; - $this->filePath = $filePath; $this->state = self::STATE_NEW; } diff --git a/src/Oliverde8/Component/PhpEtl/Item/FileExtractedItem.php b/src/Oliverde8/Component/PhpEtl/Item/FileExtractedItem.php index 59c5866..6cc9b19 100644 --- a/src/Oliverde8/Component/PhpEtl/Item/FileExtractedItem.php +++ b/src/Oliverde8/Component/PhpEtl/Item/FileExtractedItem.php @@ -4,11 +4,8 @@ class FileExtractedItem extends FileItem { - private string $filePath; - - public function __construct(string $filePath) + public function __construct(private readonly string $filePath) { - $this->filePath = $filePath; } public function getFilePath(): string diff --git a/src/Oliverde8/Component/PhpEtl/Item/FileLoadedItem.php b/src/Oliverde8/Component/PhpEtl/Item/FileLoadedItem.php index 5d6cd35..9716886 100644 --- a/src/Oliverde8/Component/PhpEtl/Item/FileLoadedItem.php +++ b/src/Oliverde8/Component/PhpEtl/Item/FileLoadedItem.php @@ -4,11 +4,8 @@ class FileLoadedItem extends FileItem { - private string $filePath; - - public function __construct(string $filePath) + public function __construct(private readonly string $filePath) { - $this->filePath = $filePath; } public function getFilePath(): string diff --git a/src/Oliverde8/Component/PhpEtl/Item/GroupedItem.php b/src/Oliverde8/Component/PhpEtl/Item/GroupedItem.php index 0d62981..e329e82 100644 --- a/src/Oliverde8/Component/PhpEtl/Item/GroupedItem.php +++ b/src/Oliverde8/Component/PhpEtl/Item/GroupedItem.php @@ -14,16 +14,14 @@ class GroupedItem implements GroupedItemInterface { - protected \Iterator $iterator; - - public function __construct(\Iterator $iterator) + public function __construct(protected \Iterator $iterator) { - $this->iterator = $iterator; } /** * @inheritdoc */ + #[\Override] public function getIterator(): \Iterator { return $this->iterator; diff --git a/src/Oliverde8/Component/PhpEtl/Item/MixItem.php b/src/Oliverde8/Component/PhpEtl/Item/MixItem.php index b46ca63..9767989 100644 --- a/src/Oliverde8/Component/PhpEtl/Item/MixItem.php +++ b/src/Oliverde8/Component/PhpEtl/Item/MixItem.php @@ -4,11 +4,8 @@ class MixItem implements ItemInterface { - private array $items; - - public function __construct(array $items) + public function __construct(private readonly array $items) { - $this->items = $items; } public function getItems(): array diff --git a/src/Oliverde8/Component/PhpEtl/Load/File/Csv.php b/src/Oliverde8/Component/PhpEtl/Load/File/Csv.php index 97ad3c6..3972ff4 100644 --- a/src/Oliverde8/Component/PhpEtl/Load/File/Csv.php +++ b/src/Oliverde8/Component/PhpEtl/Load/File/Csv.php @@ -13,21 +13,17 @@ */ class Csv extends AbstractCsvFile implements FileWriterInterface { - /** @var bool */ - protected $hasHeader; - /** * Writer constructor. * * @param $filePath - * @param bool $hasHeaders + * @param bool $hasHeader * @param string $delimiter * @param string $enclosure * @param string $escape */ - public function __construct($filePath, $hasHeaders = true, $delimiter = ';', $enclosure = '"', $escape = '\\') + public function __construct($filePath, protected $hasHeader = true, $delimiter = ';', $enclosure = '"', $escape = '\\') { - $this->hasHeader = $hasHeaders; parent::__construct($filePath, $delimiter, $enclosure, $escape); } @@ -46,6 +42,7 @@ protected function init($rowData) } } + #[\Override] public function getResource() { $this->init([]); @@ -58,6 +55,7 @@ public function getResource() * * @param array $rowData data to wrinte in a line of the csv. */ + #[\Override] public function write($rowData) { $this->init($rowData); diff --git a/src/Oliverde8/Component/PhpEtl/Load/File/Json.php b/src/Oliverde8/Component/PhpEtl/Load/File/Json.php index 1bf1c5b..e39b8a3 100644 --- a/src/Oliverde8/Component/PhpEtl/Load/File/Json.php +++ b/src/Oliverde8/Component/PhpEtl/Load/File/Json.php @@ -4,14 +4,11 @@ class Json implements FileWriterInterface { - protected string $filePath; - /** @var resource File pointer */ protected $file = null; - public function __construct(string $filePath) + public function __construct(protected string $filePath) { - $this->filePath = $filePath; } protected function init() @@ -21,12 +18,14 @@ protected function init() } } + #[\Override] public function write($rowData) { $this->init(); fputs($this->file, json_encode($rowData) . "\n"); } + #[\Override] public function getResource() { $this->init(); diff --git a/src/Oliverde8/Component/PhpEtl/Model/ExecutionContext.php b/src/Oliverde8/Component/PhpEtl/Model/ExecutionContext.php index 523d6da..26fda79 100644 --- a/src/Oliverde8/Component/PhpEtl/Model/ExecutionContext.php +++ b/src/Oliverde8/Component/PhpEtl/Model/ExecutionContext.php @@ -9,18 +9,12 @@ class ExecutionContext extends LoggerContext { - protected array $parameters; - - protected FileSystemInterface $fileSystem; - /** * @param array $parameters * @param FileSystemInterface $fileSystem */ - public function __construct(array $parameters, FileSystemInterface $fileSystem) + public function __construct(protected array $parameters, protected FileSystemInterface $fileSystem) { - $this->parameters = $parameters; - $this->fileSystem = $fileSystem; } diff --git a/src/Oliverde8/Component/PhpEtl/Model/File/AbstractCsvFile.php b/src/Oliverde8/Component/PhpEtl/Model/File/AbstractCsvFile.php index 6cecae8..77800f1 100644 --- a/src/Oliverde8/Component/PhpEtl/Model/File/AbstractCsvFile.php +++ b/src/Oliverde8/Component/PhpEtl/Model/File/AbstractCsvFile.php @@ -11,9 +11,6 @@ */ class AbstractCsvFile { - /** @var String */ - protected $filePath; - /** @var string */ protected $delimiter; @@ -34,9 +31,8 @@ class AbstractCsvFile * @param string $enclosure Enclosure to use for the csv file. * @param string $escape Escape to use for the csv file. */ - public function __construct($filePath, string $delimiter = ';', string $enclosure = '"', string $escape = '\\') + public function __construct(protected $filePath, string $delimiter = ';', string $enclosure = '"', string $escape = '\\') { - $this->filePath = $filePath; $this->delimiter = $delimiter; $this->enclosure = $enclosure; $this->escape = $escape; diff --git a/src/Oliverde8/Component/PhpEtl/Model/File/FlySystemFileSystem.php b/src/Oliverde8/Component/PhpEtl/Model/File/FlySystemFileSystem.php index 4686ddf..0728a7b 100644 --- a/src/Oliverde8/Component/PhpEtl/Model/File/FlySystemFileSystem.php +++ b/src/Oliverde8/Component/PhpEtl/Model/File/FlySystemFileSystem.php @@ -17,62 +17,74 @@ public function __construct(Filesystem $filesystem) $this->filesystem = $filesystem; } + #[\Override] public function getRootPath(): string { return "/"; } + #[\Override] public function fileExists(string $path): bool { return $this->filesystem->fileExists($path); } + #[\Override] public function write(string $path, string $contents, array $config = []): void { $this->filesystem->write($path, $contents, $config); } + #[\Override] public function writeStream(string $path, $contents, array $config = []): void { $this->filesystem->writeStream($path, $contents, $config); } + #[\Override] public function read(string $path): string { return $this->filesystem->read($path); } + #[\Override] public function readStream(string $path) { return $this->filesystem->readStream($path); } + #[\Override] public function delete(string $path): void { $this->filesystem->delete($path); } + #[\Override] public function deleteDirectory(string $path): void { $this->filesystem->deleteDirectory($path); } + #[\Override] public function createDirectory(string $path, array $config = []): void { $this->filesystem->createDirectory($path, $config); } + #[\Override] public function listContents(string $path): array { $listing = $this->filesystem->listContents($path); return $listing->toArray(); } + #[\Override] public function move(string $source, string $destination, array $config = []) { $this->filesystem->move($source, $destination, $config); } + #[\Override] public function copy(string $source, string $destination, array $config = []) { $this->filesystem->copy($source, $destination, $config); diff --git a/src/Oliverde8/Component/PhpEtl/Model/File/LocalFileSystem.php b/src/Oliverde8/Component/PhpEtl/Model/File/LocalFileSystem.php index ee408e0..a0e5d72 100644 --- a/src/Oliverde8/Component/PhpEtl/Model/File/LocalFileSystem.php +++ b/src/Oliverde8/Component/PhpEtl/Model/File/LocalFileSystem.php @@ -28,62 +28,74 @@ public function __construct($rootPath = null) /** * @return string */ + #[\Override] public function getRootPath(): string { return $this->rootPath; } + #[\Override] public function fileExists(string $path): bool { return $this->filesystem->exists($this->rootPath . "/" . $path); } + #[\Override] public function write(string $path, string $contents, array $config = []): void { $this->filesystem->dumpFile($this->rootPath . "/" . $path, $contents); } + #[\Override] public function writeStream(string $path, $contents, array $config = []): void { stream_copy_to_stream($contents, fopen($this->rootPath . "/" . $path, 'w+')); } + #[\Override] public function read(string $path): string { return file_get_contents($this->rootPath . "/" . $path); } + #[\Override] public function readStream(string $path) { return fopen($this->rootPath . "/" . $path, 'r'); } + #[\Override] public function delete(string $path): void { $this->filesystem->remove($this->rootPath . "/" . $path); } + #[\Override] public function deleteDirectory(string $path): void { $this->filesystem->remove($this->rootPath . "/" . $path); } + #[\Override] public function createDirectory(string $path, array $config = []): void { $this->filesystem->mkdir($this->rootPath . "/" . $path); } + #[\Override] public function listContents(string $path): array { $files = scandir($this->rootPath . "/" .$path); return $files ?: []; } + #[\Override] public function move(string $source, string $destination, array $config = []) { $this->filesystem->rename($this->rootPath . "/" . $source, $this->rootPath . "/" . $destination); } + #[\Override] public function copy(string $source, string $destination, array $config = []) { $this->filesystem->copy($this->rootPath . "/" . $source, $this->rootPath . "/" . $destination); diff --git a/src/Oliverde8/Component/PhpEtl/Model/File/ProxySubFileSystem.php b/src/Oliverde8/Component/PhpEtl/Model/File/ProxySubFileSystem.php index 31b97c3..63e9707 100644 --- a/src/Oliverde8/Component/PhpEtl/Model/File/ProxySubFileSystem.php +++ b/src/Oliverde8/Component/PhpEtl/Model/File/ProxySubFileSystem.php @@ -6,73 +6,82 @@ class ProxySubFileSystem implements FileSystemInterface { protected $subDir; - protected FileSystemInterface $fileSystem; - /** * @param $subDir * @param FileSystemInterface $fileSystem */ - public function __construct($subDir, FileSystemInterface $fileSystem) + public function __construct($subDir, protected FileSystemInterface $fileSystem) { - $this->subDir = "/" . trim($subDir, "/") . "/"; - $this->fileSystem = $fileSystem; + $this->subDir = "/" . trim((string) $subDir, "/") . "/"; } + #[\Override] public function getRootPath(): string { return $this->fileSystem->getRootPath() . $this->subDir; } + #[\Override] public function fileExists(string $path): bool { return $this->fileSystem->fileExists($this->subDir . $path); } + #[\Override] public function write(string $path, string $contents, array $config = []): void { $this->fileSystem->write($this->subDir . $path, $contents, $config); } + #[\Override] public function writeStream(string $path, $contents, array $config = []): void { $this->fileSystem->writeStream($this->subDir . $path, $contents, $config); } + #[\Override] public function read(string $path): string { return $this->fileSystem->read($this->subDir . $path); } + #[\Override] public function readStream(string $path) { return $this->fileSystem->readStream($this->subDir . $path); } + #[\Override] public function delete(string $path): void { $this->fileSystem->delete($this->subDir . $path); } + #[\Override] public function deleteDirectory(string $path): void { $this->fileSystem->deleteDirectory($this->subDir . $path); } + #[\Override] public function createDirectory(string $path, array $config = []): void { $this->fileSystem->createDirectory($this->subDir . $path, $config); } + #[\Override] public function listContents(string $path): array { return $this->fileSystem->listContents($this->subDir . $path); } + #[\Override] public function move(string $source, string $destination, array $config = []) { $this->fileSystem->move($this->subDir . $source, $this->subDir . $destination, $config); } + #[\Override] public function copy(string $source, string $destination, array $config = []) { $this->fileSystem->copy($this->subDir . $source, $this->subDir . $destination, $config); diff --git a/src/Oliverde8/Component/PhpEtl/Model/PerExecutionExecutionContext.php b/src/Oliverde8/Component/PhpEtl/Model/PerExecutionExecutionContext.php index d617f6f..76958b7 100644 --- a/src/Oliverde8/Component/PhpEtl/Model/PerExecutionExecutionContext.php +++ b/src/Oliverde8/Component/PhpEtl/Model/PerExecutionExecutionContext.php @@ -9,15 +9,13 @@ class PerExecutionExecutionContext extends ExecutionContext { - protected string $workDir; - - public function __construct(array $parameters, FileSystemInterface $fileSystem, LoggerInterface $logger, string $logPath) + public function __construct(array $parameters, FileSystemInterface $fileSystem, LoggerInterface $logger, protected string $workDir) { parent::__construct($parameters, $fileSystem); $this->logger = $logger; - $this->workDir = $logPath; } + #[\Override] protected function finalise(): void { if (class_exists("Monolog\Logger") && $this->logger instanceof \Monolog\Logger) { diff --git a/src/Oliverde8/Component/PhpEtl/Model/PockExecution.php b/src/Oliverde8/Component/PhpEtl/Model/PockExecution.php index 67a46ba..1cd9950 100644 --- a/src/Oliverde8/Component/PhpEtl/Model/PockExecution.php +++ b/src/Oliverde8/Component/PhpEtl/Model/PockExecution.php @@ -5,22 +5,21 @@ class PockExecution implements ExecutionInterface { - protected \DateTime $createdAt; - protected string $id; - public function __construct(\DateTime $createdAt) + public function __construct(protected \DateTime $createdAt) { - $this->createdAt = $createdAt; - $this->id = (string) rand(1, 100000); + $this->id = (string) random_int(1, 100000); } + #[\Override] public function getId(): string { return $this->id; } + #[\Override] public function GetCreateTime(): \DateTime { return $this->createdAt; diff --git a/src/Oliverde8/Component/PhpEtl/Model/State/OperationState.php b/src/Oliverde8/Component/PhpEtl/Model/State/OperationState.php index 16fe5a5..e13fa11 100644 --- a/src/Oliverde8/Component/PhpEtl/Model/State/OperationState.php +++ b/src/Oliverde8/Component/PhpEtl/Model/State/OperationState.php @@ -12,8 +12,6 @@ class OperationState implements \JsonSerializable { - private readonly string $operationName; - private OperationStateEnum $state = OperationStateEnum::Waiting; private int $itemsProcessed = 0; @@ -29,10 +27,8 @@ class OperationState implements \JsonSerializable private array $subStates = []; - public function __construct(string $operationName, ChainOperationInterface $operation) + public function __construct(private readonly string $operationName, ChainOperationInterface $operation) { - $this->operationName = $operationName; - if ($operation instanceof DetailedObservableOperation) { $this->subStates = $operation->getLastObservedState(); } @@ -126,6 +122,7 @@ protected function returnItem(ChainOperationInterface $operation, ItemInterface } } + #[\Override] public function jsonSerialize(): mixed { $vars = get_object_vars($this); diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/AbstractOperationConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/AbstractOperationConfig.php new file mode 100644 index 0000000..df934a5 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/AbstractOperationConfig.php @@ -0,0 +1,32 @@ +constructed = true; + $this->validate(true); + } + + /** + * @throws ChainBuilderValidationException + */ + abstract protected function validate(bool $constructOnly): void; + + #[\Override] + public function getFlavor(): string + { + if (!$this->constructed) { + throw new ChainBuilderException("Impossible to get flavor are you sure the config calls it's parent constructor?"); + } + return $this->flavor; + } +} \ No newline at end of file diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/ChainMergeConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/ChainMergeConfig.php new file mode 100644 index 0000000..6551c3f --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/ChainMergeConfig.php @@ -0,0 +1,46 @@ +chainConfigs; + } + + public function addMerge(ChainConfig $chainConfig): self + { + $this->chainConfigs[] = $chainConfig; + return $this; + } + + #[\Override] + protected function validate(bool $constructOnly): void + { + if ($constructOnly) { + return; + } + + if (empty($this->chainConfigs)) { + throw new ChainBuilderException("At least one chain config must be provided for ChainMergeConfig"); + } + foreach ($this->chainConfigs as $chainConfig) { + if (!$chainConfig instanceof ChainConfig) { + throw new ChainBuilderException("All chain configs must be instances of ChainConfig"); + } + } + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/ChainRepeatConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/ChainRepeatConfig.php new file mode 100644 index 0000000..eb42157 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/ChainRepeatConfig.php @@ -0,0 +1,34 @@ +chainConfig; + } + + #[\Override] + protected function validate(bool $constructOnly): void + { + + if (empty($this->validationExpression)) { + throw new \InvalidArgumentException("Validation expression cannot be empty"); + } + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/ChainSplitConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/ChainSplitConfig.php new file mode 100644 index 0000000..543fb0a --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/ChainSplitConfig.php @@ -0,0 +1,43 @@ +chainConfigs; + } + + public function addSplit(ChainConfig $chainConfig): self + { + $this->chainConfigs[] = $chainConfig; + return $this; + } + + #[\Override] + protected function validate(bool $constructOnly): void + { + if ($constructOnly) { + return; + } + + if (empty($this->chainConfigs)) { + throw new ChainBuilderException("At least one chain config must be provided for ChainSplitConfig"); + } + foreach ($this->chainConfigs as $chainConfig) { + if (!$chainConfig instanceof ChainConfig) { + throw new ChainBuilderException("All chain configs must be instances of ChainConfig"); + } + } + } +} diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/Extract/CsvExtractConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/Extract/CsvExtractConfig.php new file mode 100644 index 0000000..ccac403 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/Extract/CsvExtractConfig.php @@ -0,0 +1,26 @@ +enclosure, ["'", '"'], true)) { + throw new \InvalidArgumentException("Enclosure must be a single or double quote"); + } + } +} \ No newline at end of file diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/Extract/ExternalFileFinderConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/Extract/ExternalFileFinderConfig.php new file mode 100644 index 0000000..cc0f265 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/Extract/ExternalFileFinderConfig.php @@ -0,0 +1,27 @@ +directory)) { + throw new \InvalidArgumentException('Directory cannot be empty'); + } + if (!is_string($this->directory)) { + throw new \InvalidArgumentException('Directory must be a string'); + } + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/Extract/JsonExtractConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/Extract/JsonExtractConfig.php new file mode 100644 index 0000000..861519a --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/Extract/JsonExtractConfig.php @@ -0,0 +1,17 @@ +chainConfig; + } + + #[\Override] + protected function validate(bool $constructOnly): void + { + if ($this->nbAttempts < 1) { + throw new \InvalidArgumentException('nbAttempts must be >= 1'); + } + foreach ($this->exceptionsToCatch as $ex) { + if (!is_string($ex) || (!class_exists($ex) && !interface_exists($ex))) { + throw new \InvalidArgumentException('exceptionsToCatch must be class/interface names'); + } + } + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/Grouping/SimpleGroupingConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/Grouping/SimpleGroupingConfig.php new file mode 100644 index 0000000..524600e --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/Grouping/SimpleGroupingConfig.php @@ -0,0 +1,35 @@ +groupKey)) { + throw new \InvalidArgumentException('Group key cannot be empty'); + } + foreach ($this->groupKey as $groupKey) { + if (!is_string($groupKey)) { + throw new \InvalidArgumentException('Group key must be an array of strings'); + } + } + foreach ($this->groupIdentifierKey as $groupIdentifierKey) { + if (!is_string($groupIdentifierKey)) { + throw new \InvalidArgumentException('Group identifier key must be an array of strings'); + } + } + } +} \ No newline at end of file diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/Loader/CsvFileWriterConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/Loader/CsvFileWriterConfig.php new file mode 100644 index 0000000..119c739 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/Loader/CsvFileWriterConfig.php @@ -0,0 +1,38 @@ +fileName; + } + + #[\Override] + public function getFile(): Csv + { + $tmp = tempnam(sys_get_temp_dir(), 'etl'); + return new Csv($tmp, $this->hasHeader, $this->delimiter, $this->enclosure, $this->escape); + } + + #[\Override] + protected function validate(bool $constructOnly): void + {} +} \ No newline at end of file diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/Loader/FileWriterConfigInterface.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/Loader/FileWriterConfigInterface.php new file mode 100644 index 0000000..657548a --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/Loader/FileWriterConfigInterface.php @@ -0,0 +1,13 @@ +callable = $callable; + parent::__construct($flavor); + } + + public function getCallable(): callable + { + return $this->callable; + } + + #[\Override] + function validate(bool $constructOnly): void + { + // All callables are valid. Maybe add check on signature in the future. + } +} \ No newline at end of file diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/ExternalFileProcessorConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/ExternalFileProcessorConfig.php new file mode 100644 index 0000000..6d7e0c2 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/ExternalFileProcessorConfig.php @@ -0,0 +1,21 @@ +level, $validLevels, true)) { + throw new \InvalidArgumentException( + "Log level must be one of: " . implode(', ', $validLevels) . ". Got: {$this->level}" + ); + } + + if (empty($this->message)) { + throw new \InvalidArgumentException("Message cannot be empty"); + } + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/RuleTransformConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/RuleTransformConfig.php new file mode 100644 index 0000000..71a160b --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/RuleTransformConfig.php @@ -0,0 +1,31 @@ +rules[$columnName]['rules'] = $rules; + return $this; + } + + public function getRules(): array + { + return $this->rules; + } + + #[\Override] + protected function validate(bool $constructOnly): void + { + } +} diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/SimpleHttpConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/SimpleHttpConfig.php new file mode 100644 index 0000000..be19a3f --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/SimpleHttpConfig.php @@ -0,0 +1,37 @@ +method, $validMethods, true)) { + throw new \InvalidArgumentException( + "Method must be one of: " . implode(', ', $validMethods) . ". Got: {$this->method}" + ); + } + + if (empty($this->url)) { + throw new \InvalidArgumentException("URL cannot be empty"); + } + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/SplitItemConfig.php b/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/SplitItemConfig.php new file mode 100644 index 0000000..f349372 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/OperationConfig/Transformer/SplitItemConfig.php @@ -0,0 +1,30 @@ +keys)) { + throw new \InvalidArgumentException("Keys cannot be empty"); + } + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/PerExecutionContextFactory.php b/src/Oliverde8/Component/PhpEtl/PerExecutionContextFactory.php index 5cbd346..3f56f63 100644 --- a/src/Oliverde8/Component/PhpEtl/PerExecutionContextFactory.php +++ b/src/Oliverde8/Component/PhpEtl/PerExecutionContextFactory.php @@ -11,23 +11,12 @@ class PerExecutionContextFactory implements ExecutionContextFactoryInterface { - private ChainWorkDirManager $dirManager; - - private FileSystemFactoryInterface $fileSystemFactory; - - private LoggerFactoryInterface $loggerFactory; - - public function __construct( - ChainWorkDirManager $dirManager, - FileSystemFactoryInterface $fileSystemFactory, - LoggerFactoryInterface $loggerFactory - ) { - $this->dirManager = $dirManager; - $this->fileSystemFactory = $fileSystemFactory; - $this->loggerFactory = $loggerFactory; + public function __construct(private readonly ChainWorkDirManager $dirManager, private readonly FileSystemFactoryInterface $fileSystemFactory, private readonly LoggerFactoryInterface $loggerFactory) + { } + #[\Override] public function get(array $parameters): ExecutionContext { $execution = $parameters['etl']['execution'] ?? null; diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainBuilderTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainBuilderTest.php index 0337260..80df8d6 100644 --- a/src/Oliverde8/Component/PhpEtl/Tests/ChainBuilderTest.php +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainBuilderTest.php @@ -15,6 +15,7 @@ class ChainBuilderTest extends TestCase private MockObject $mockFactory2; private ChainBuilder $chainBuilder; + #[\Override] protected function setUp(): void { parent::setUp(); // TODO: Change the autogenerated stub @@ -22,18 +23,14 @@ protected function setUp(): void $this->mockFactory1 = $this->getMockBuilder(CsvFileWriterFactory::class)->disableOriginalConstructor()->getMock(); $this->mockFactory1 ->expects($this->any())->method("supports") - ->willReturnCallback(function (string $operation, array $options) { - return $operation == 'op1'; - }); + ->willReturnCallback(fn(string $operation, array $options) => $operation == 'op1'); $this->mockFactory2 = $this->getMockBuilder(CsvFileWriterFactory::class)->disableOriginalConstructor()->getMock(); $this->mockFactory2->expects($this->any())->method("supports") - ->willReturnCallback(function (string $operation, array $options) { - return $operation == 'op2'; - }); + ->willReturnCallback(fn(string $operation, array $options) => $operation == 'op2'); $this->chainBuilder = new ChainBuilder(new ExecutionContextFactory()); - $this->chainBuilder->registerFactory($this->mockFactory1, 'op1'); - $this->chainBuilder->registerFactory($this->mockFactory2, 'op2'); + $this->chainBuilder->registerFactory($this->mockFactory1); + $this->chainBuilder->registerFactory($this->mockFactory2); } diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainBuilderV2Test.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainBuilderV2Test.php new file mode 100644 index 0000000..ce640c2 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainBuilderV2Test.php @@ -0,0 +1,260 @@ +contextFactory = new ExecutionContextFactory(); + } + + public function testCreateChainWithNoFactories() + { + $chainBuilder = new ChainBuilderV2($this->contextFactory, []); + $chainConfig = new ChainConfig(); + + $chain = $chainBuilder->createChain($chainConfig); + + $this->assertInstanceOf(ChainProcessorInterface::class, $chain); + } + + public function testCreateChainWithSingleOperation() + { + $mockConfig = $this->createMock(OperationConfigInterface::class); + $mockConfig->method('getFlavor')->willReturn('default'); + + $mockOperation = $this->createMock(ConfigurableChainOperationInterface::class); + + $mockFactory = $this->createMock(GenericChainFactory::class); + $mockFactory->expects($this->once()) + ->method('supports') + ->with($mockConfig) + ->willReturn(true); + $mockFactory->expects($this->once()) + ->method('build') + ->with($mockConfig, $this->isInstanceOf(ChainBuilderV2::class)) + ->willReturn($mockOperation); + + $chainBuilder = new ChainBuilderV2($this->contextFactory, [$mockFactory]); + + $chainConfig = new ChainConfig(); + $chainConfig->addLink($mockConfig); + + $chain = $chainBuilder->createChain($chainConfig); + + $this->assertInstanceOf(ChainProcessorInterface::class, $chain); + } + + public function testCreateChainWithMultipleOperations() + { + $mockConfig1 = $this->createMock(OperationConfigInterface::class); + $mockConfig1->method('getFlavor')->willReturn('default'); + + $mockConfig2 = $this->createMock(OperationConfigInterface::class); + $mockConfig2->method('getFlavor')->willReturn('default'); + + $mockOperation1 = $this->createMock(ConfigurableChainOperationInterface::class); + $mockOperation2 = $this->createMock(ConfigurableChainOperationInterface::class); + + $mockFactory = $this->createMock(GenericChainFactory::class); + $mockFactory->expects($this->exactly(2)) + ->method('supports') + ->willReturn(true); + $mockFactory->expects($this->exactly(2)) + ->method('build') + ->willReturnOnConsecutiveCalls($mockOperation1, $mockOperation2); + + $chainBuilder = new ChainBuilderV2($this->contextFactory, [$mockFactory]); + + $chainConfig = new ChainConfig(); + $chainConfig->addLink($mockConfig1)->addLink($mockConfig2); + + $chain = $chainBuilder->createChain($chainConfig); + + $this->assertInstanceOf(ChainProcessorInterface::class, $chain); + } + + public function testCreateChainWithMultipleFactories() + { + $mockConfig1 = $this->createMock(OperationConfigInterface::class); + $mockConfig1->method('getFlavor')->willReturn('default'); + + $mockConfig2 = $this->createMock(OperationConfigInterface::class); + $mockConfig2->method('getFlavor')->willReturn('custom'); + + $mockOperation1 = $this->createMock(ConfigurableChainOperationInterface::class); + $mockOperation2 = $this->createMock(ConfigurableChainOperationInterface::class); + + $mockFactory1 = $this->createMock(GenericChainFactory::class); + $mockFactory1->method('supports') + ->willReturnCallback(fn($config) => $config === $mockConfig1); + $mockFactory1->expects($this->once()) + ->method('build') + ->with($mockConfig1) + ->willReturn($mockOperation1); + + $mockFactory2 = $this->createMock(GenericChainFactory::class); + $mockFactory2->method('supports') + ->willReturnCallback(fn($config) => $config === $mockConfig2); + $mockFactory2->expects($this->once()) + ->method('build') + ->with($mockConfig2) + ->willReturn($mockOperation2); + + $chainBuilder = new ChainBuilderV2($this->contextFactory, [$mockFactory1, $mockFactory2]); + + $chainConfig = new ChainConfig(); + $chainConfig->addLink($mockConfig1)->addLink($mockConfig2); + + $chain = $chainBuilder->createChain($chainConfig); + + $this->assertInstanceOf(ChainProcessorInterface::class, $chain); + } + + public function testCreateChainWithFactoryNotSupporting() + { + $mockConfig = $this->createMock(OperationConfigInterface::class); + $mockConfig->method('getFlavor')->willReturn('default'); + + $mockFactory = $this->createMock(GenericChainFactory::class); + $mockFactory->method('supports')->willReturn(false); + + $chainBuilder = new ChainBuilderV2($this->contextFactory, [$mockFactory]); + + $chainConfig = new ChainConfig(); + $chainConfig->addLink($mockConfig); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('No factory found'); + + $chainBuilder->createChain($chainConfig); + } + + public function testCreateChainWithNoMatchingFactory() + { + $mockConfig = $this->createMock(OperationConfigInterface::class); + $mockConfig->method('getFlavor')->willReturn('default'); + + $chainBuilder = new ChainBuilderV2($this->contextFactory, []); + + $chainConfig = new ChainConfig(); + $chainConfig->addLink($mockConfig); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessageMatches('/No factory found.*' . preg_quote($mockConfig::class) . '/'); + + $chainBuilder->createChain($chainConfig); + } + + public function testCreateChainWithMaxAsynchronousItems() + { + $chainBuilder = new ChainBuilderV2($this->contextFactory, []); + $chainConfig = new ChainConfig(maxAsynchronousItems: 5); + + $chain = $chainBuilder->createChain($chainConfig); + + $this->assertInstanceOf(ChainProcessorInterface::class, $chain); + } + + public function testFactoryReceivesChainBuilderInstance() + { + $mockConfig = $this->createMock(OperationConfigInterface::class); + $mockConfig->method('getFlavor')->willReturn('default'); + + $mockOperation = $this->createMock(ConfigurableChainOperationInterface::class); + + $chainBuilderReceived = null; + + $mockFactory = $this->createMock(GenericChainFactory::class); + $mockFactory->method('supports')->willReturn(true); + $mockFactory->expects($this->once()) + ->method('build') + ->willReturnCallback(function($config, $builder) use ($mockOperation, &$chainBuilderReceived) { + $chainBuilderReceived = $builder; + return $mockOperation; + }); + + $chainBuilder = new ChainBuilderV2($this->contextFactory, [$mockFactory]); + + $chainConfig = new ChainConfig(); + $chainConfig->addLink($mockConfig); + + $chainBuilder->createChain($chainConfig); + + $this->assertSame($chainBuilder, $chainBuilderReceived); + } + + public function testCreateChainWithIterableFactories() + { + $mockConfig = $this->createMock(OperationConfigInterface::class); + $mockConfig->method('getFlavor')->willReturn('default'); + + $mockOperation = $this->createMock(ConfigurableChainOperationInterface::class); + + $factoryGenerator = function() use ($mockConfig, $mockOperation) { + $mockFactory = $this->createMock(GenericChainFactory::class); + $mockFactory->method('supports')->willReturn(true); + $mockFactory->method('build')->willReturn($mockOperation); + yield $mockFactory; + }; + + $chainBuilder = new ChainBuilderV2($this->contextFactory, $factoryGenerator()); + + $chainConfig = new ChainConfig(); + $chainConfig->addLink($mockConfig); + + $chain = $chainBuilder->createChain($chainConfig); + + $this->assertInstanceOf(ChainProcessorInterface::class, $chain); + } + + public function testCreateChainPreservesOperationOrder() + { + $configs = []; + $operations = []; + + for ($i = 0; $i < 3; $i++) { + $configs[$i] = $this->createMock(OperationConfigInterface::class); + $configs[$i]->method('getFlavor')->willReturn('default'); + $operations[$i] = $this->createMock(ConfigurableChainOperationInterface::class); + } + + $callOrder = []; + + $mockFactory = $this->createMock(GenericChainFactory::class); + $mockFactory->method('supports')->willReturn(true); + $mockFactory->method('build') + ->willReturnCallback(function($config) use ($configs, $operations, &$callOrder) { + $index = array_search($config, $configs, true); + $callOrder[] = $index; + return $operations[$index]; + }); + + $chainBuilder = new ChainBuilderV2($this->contextFactory, [$mockFactory]); + + $chainConfig = new ChainConfig(); + foreach ($configs as $config) { + $chainConfig->addLink($config); + } + + $chainBuilder->createChain($chainConfig); + + $this->assertEquals([0, 1, 2], $callOrder); + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/ChainMergeOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/ChainMergeOperationTest.php new file mode 100644 index 0000000..c8c2ee2 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/ChainMergeOperationTest.php @@ -0,0 +1,328 @@ +context = new ExecutionContext([], new LocalFileSystem()); + + $executionFactory = new ExecutionContextFactory(); + $this->chainBuilder = new ChainBuilderV2( + $executionFactory, + [new GenericChainFactory(CallbackTransformerOperation::class, CallBackTransformerConfig::class)] + ); + } + + public function testMergeTwoChains() + { + $chain1Config = new ChainConfig(); + $chain1Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem(['chain' => 1, 'data' => $item->getData()]))); + + $chain2Config = new ChainConfig(); + $chain2Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem(['chain' => 2, 'data' => $item->getData()]))); + + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge($chain1Config)->addMerge($chain2Config); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $item = new DataItem('test'); + $result = $operation->process($item, $this->context); + + $this->assertInstanceOf(MixItem::class, $result); + $items = $result->getItems(); + $this->assertCount(2, $items); + $this->assertEquals(['chain' => 1, 'data' => 'test'], $items[0]->getData()); + $this->assertEquals(['chain' => 2, 'data' => 'test'], $items[1]->getData()); + } + + public function testMergeThreeChains() + { + $chain1Config = new ChainConfig(); + $chain1Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem('chain1'))); + + $chain2Config = new ChainConfig(); + $chain2Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem('chain2'))); + + $chain3Config = new ChainConfig(); + $chain3Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem('chain3'))); + + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge($chain1Config)->addMerge($chain2Config)->addMerge($chain3Config); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $item = new DataItem('input'); + $result = $operation->process($item, $this->context); + + $items = $result->getItems(); + $this->assertCount(3, $items); + $this->assertEquals('chain1', $items[0]->getData()); + $this->assertEquals('chain2', $items[1]->getData()); + $this->assertEquals('chain3', $items[2]->getData()); + } + + public function testMergeWithMultipleItemsPerChain() + { + $chain1Config = new ChainConfig(); + $chain1Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new MixItem([ + new DataItem('a1'), + new DataItem('a2') + ]))); + + $chain2Config = new ChainConfig(); + $chain2Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new MixItem([ + new DataItem('b1'), + new DataItem('b2') + ]))); + + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge($chain1Config)->addMerge($chain2Config); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $item = new DataItem('input'); + $result = $operation->process($item, $this->context); + + $items = $result->getItems(); + $this->assertCount(4, $items); + $this->assertEquals('a1', $items[0]->getData()); + $this->assertEquals('a2', $items[1]->getData()); + $this->assertEquals('b1', $items[2]->getData()); + $this->assertEquals('b2', $items[3]->getData()); + } + + public function testMergePreservesOrder() + { + $results = []; + + $chain1Config = new ChainConfig(); + $chain1Config->addLink(new CallBackTransformerConfig(function(ItemInterface $item) use (&$results) { + $results[] = 'chain1'; + return new DataItem('data1'); + })); + + $chain2Config = new ChainConfig(); + $chain2Config->addLink(new CallBackTransformerConfig(function(ItemInterface $item) use (&$results) { + $results[] = 'chain2'; + return new DataItem('data2'); + })); + + $chain3Config = new ChainConfig(); + $chain3Config->addLink(new CallBackTransformerConfig(function(ItemInterface $item) use (&$results) { + $results[] = 'chain3'; + return new DataItem('data3'); + })); + + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge($chain1Config)->addMerge($chain2Config)->addMerge($chain3Config); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $item = new DataItem('input'); + $operation->process($item, $this->context); + + $this->assertEquals(['chain1', 'chain2', 'chain3'], $results); + } + + public function testProcessStopCallsAllChains() + { + $chain1Config = new ChainConfig(); + $chain1Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem('chain1'))); + + $chain2Config = new ChainConfig(); + $chain2Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem('chain2'))); + + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge($chain1Config)->addMerge($chain2Config); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $stopItem = new StopItem(); + $result = $operation->processStop($stopItem, $this->context); + + $this->assertSame($stopItem, $result); + $this->assertCount(2, $operation->getChainProcessors()); + } + + public function testMergeWithEmptyChain() + { + $chain1Config = new ChainConfig(); + $chain1Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem('result'))); + + $chain2Config = new ChainConfig(); + + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge($chain1Config)->addMerge($chain2Config); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $item = new DataItem('input'); + $result = $operation->process($item, $this->context); + + $items = $result->getItems(); + $this->assertCount(2, $items); + $this->assertEquals('result', $items[0]->getData()); + $this->assertEquals('input', $items[1]->getData()); + } + + public function testGetChainProcessors() + { + $chain1Config = new ChainConfig(); + $chain2Config = new ChainConfig(); + + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge($chain1Config)->addMerge($chain2Config); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $processors = $operation->getChainProcessors(); + $this->assertCount(2, $processors); + $this->assertContainsOnlyInstancesOf(\Oliverde8\Component\PhpEtl\ChainProcessorInterface::class, $processors); + } + + public function testMergeWithDifferentDataTypes() + { + $chain1Config = new ChainConfig(); + $chain1Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem(['type' => 'array', 'value' => 1]))); + + $chain2Config = new ChainConfig(); + $chain2Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem('string_value'))); + + $chain3Config = new ChainConfig(); + $chain3Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem(123))); + + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge($chain1Config)->addMerge($chain2Config)->addMerge($chain3Config); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $item = new DataItem('input'); + $result = $operation->process($item, $this->context); + + $items = $result->getItems(); + $this->assertCount(3, $items); + $this->assertEquals(['type' => 'array', 'value' => 1], $items[0]->getData()); + $this->assertEquals('string_value', $items[1]->getData()); + $this->assertEquals(123, $items[2]->getData()); + } + + public function testMergeWithChainTransformations() + { + $chain1Config = new ChainConfig(); + $chain1Config->addLink(new CallBackTransformerConfig(function(ItemInterface $item) { + $data = $item->getData(); + return new DataItem($data * 2); + })); + + $chain2Config = new ChainConfig(); + $chain2Config->addLink(new CallBackTransformerConfig(function(ItemInterface $item) { + $data = $item->getData(); + return new DataItem($data * 3); + })); + + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge($chain1Config)->addMerge($chain2Config); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $item = new DataItem(5); + $result = $operation->process($item, $this->context); + + $items = $result->getItems(); + $this->assertCount(2, $items); + $this->assertEquals(10, $items[0]->getData()); + $this->assertEquals(15, $items[1]->getData()); + } + + public function testGetConfigurationClass() + { + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge(new ChainConfig()); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $this->assertEquals(ChainMergeConfig::class, $operation->getConfigurationClass()); + } + + public function testMergeDoesNotStopOnFirstChain() + { + $processedChains = []; + + $chain1Config = new ChainConfig(); + $chain1Config->addLink(new CallBackTransformerConfig(function(ItemInterface $item) use (&$processedChains) { + $processedChains[] = 1; + return new DataItem('chain1'); + })); + + $chain2Config = new ChainConfig(); + $chain2Config->addLink(new CallBackTransformerConfig(function(ItemInterface $item) use (&$processedChains) { + $processedChains[] = 2; + return new DataItem('chain2'); + })); + + $chain3Config = new ChainConfig(); + $chain3Config->addLink(new CallBackTransformerConfig(function(ItemInterface $item) use (&$processedChains) { + $processedChains[] = 3; + return new DataItem('chain3'); + })); + + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge($chain1Config)->addMerge($chain2Config)->addMerge($chain3Config); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $item = new DataItem('input'); + $operation->process($item, $this->context); + + $this->assertEquals([1, 2, 3], $processedChains); + } + + public function testMergeWithComplexChainOperations() + { + $chain1Config = new ChainConfig(); + $chain1Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem(strtoupper($item->getData())))); + $chain1Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem($item->getData() . '_PROCESSED'))); + + $chain2Config = new ChainConfig(); + $chain2Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem(strtolower($item->getData())))); + $chain2Config->addLink(new CallBackTransformerConfig(fn(ItemInterface $item) => new DataItem($item->getData() . '_processed'))); + + $mergeConfig = new ChainMergeConfig(); + $mergeConfig->addMerge($chain1Config)->addMerge($chain2Config); + + $operation = new ChainMergeOperation($this->chainBuilder, $mergeConfig); + + $item = new DataItem('Test'); + $result = $operation->process($item, $this->context); + + $items = $result->getItems(); + $this->assertCount(2, $items); + $this->assertEquals('TEST_PROCESSED', $items[0]->getData()); + $this->assertEquals('test_processed', $items[1]->getData()); + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/ChainRepeatOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/ChainRepeatOperationTest.php index 0016919..d05f18c 100644 --- a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/ChainRepeatOperationTest.php +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/ChainRepeatOperationTest.php @@ -3,14 +3,19 @@ namespace Oliverde8\Component\PhpEtl\Tests\ChainOperation; +use Oliverde8\Component\PhpEtl\ChainBuilderV2; +use Oliverde8\Component\PhpEtl\ChainConfig; use Oliverde8\Component\PhpEtl\ChainOperation\ChainRepeatOperation; use Oliverde8\Component\PhpEtl\ChainOperation\Transformer\CallbackTransformerOperation; use Oliverde8\Component\PhpEtl\ChainProcessor; use Oliverde8\Component\PhpEtl\ExecutionContextFactory; +use Oliverde8\Component\PhpEtl\GenericChainFactory; use Oliverde8\Component\PhpEtl\Item\DataItem; use Oliverde8\Component\PhpEtl\Item\GroupedItem; use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Item\MixItem; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainRepeatConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; use Oliverde8\Component\PhpEtl\Tests\Item\TestAsyncItem; use PHPUnit\Framework\TestCase; @@ -20,13 +25,13 @@ public function testRepeatDataItem() { $callNum = 0; $results = []; - $repeatedOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { + $repeatedOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$callNum) { return new DataItem(['val' => $callNum++]); - }); - $endOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { + })); + $endOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$results) { $results[] = $item->getData(); return $item; - }); + })); $chain = $this->createChain([$repeatedOperation], [$endOperation], 'data["val"] != 3'); $chain->process(new \ArrayIterator([['var' => 1]]), []); @@ -38,15 +43,15 @@ public function testRepeatGroupedItem() { $callNum = 0; $results = []; - $repeatedOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { + $repeatedOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$callNum) { return new GroupedItem(new \ArrayIterator([['val' => $callNum++], ['val' => $callNum++]])); - }); - $endOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { + })); + $endOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$results) { if ($item instanceof DataItem) { $results[] = $item->getData(); } return $item; - }); + })); $chain = $this->createChain([$repeatedOperation], [$endOperation], 'data["val"] != 3'); $chain->process(new \ArrayIterator([['var' => 1]]), []); @@ -57,16 +62,16 @@ public function testAsyncDisabled() { $results = []; $callNum = 0; - $repeatedOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { + $repeatedOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$callNum) { return new MixItem([ new TestAsyncItem(new DataItem(['val' => $callNum++, 'speed' => 'slow']), 2), new TestAsyncItem(new DataItem(['val' => $callNum++, 'speed' => 'fast']), 1), ]); - }); - $endOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { + })); + $endOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$results) { $results[] = $item->getData(); return $item; - }); + })); $chain = $this->createChain([$repeatedOperation], [$endOperation], 'data["val"] != 3'); $chain->process(new \ArrayIterator([['var' => 1]]), []); @@ -79,16 +84,16 @@ public function testAsyncEnabled() { $results = []; $callNum = 0; - $repeatedOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { + $repeatedOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$callNum) { return new MixItem([ new TestAsyncItem(new DataItem(['val' => $callNum++, 'speed' => 'slow']), 2), new TestAsyncItem(new DataItem(['val' => $callNum++, 'speed' => 'fast']), 1), ]); - }); - $endOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { + })); + $endOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$results) { $results[] = $item->getData(); return $item; - }); + })); $chain = $this->createChain([$repeatedOperation], [$endOperation], 'data["val"] != 5', true); $chain->process(new \ArrayIterator([['var' => 1]]), []); @@ -106,10 +111,32 @@ public function testAsyncEnabled() protected function createChain(array $repeatedOperations, array $afterOperations, string $expression, bool $allowAsync = false): ChainProcessor { $executionFactory = new ExecutionContextFactory(); + + // Create a chain config for the repeated operations + $repeatChainConfig = new ChainConfig(2); + + // Extract configs from the operations using reflection + $reflection = new \ReflectionClass(CallbackTransformerOperation::class); + $configProperty = $reflection->getProperty('config'); + $configProperty->setAccessible(true); + + foreach ($repeatedOperations as $operation) { + if ($operation instanceof CallbackTransformerOperation) { + $config = $configProperty->getValue($operation); + $repeatChainConfig->addLink($config); + } + } + + // Create a simple ChainBuilderV2 with CallbackTransformer factory + $chainBuilder = new ChainBuilderV2( + $executionFactory, + [new GenericChainFactory(CallbackTransformerOperation::class, CallBackTransformerConfig::class)] + ); + + // Create the repeat operation with the new config $repeatOperation = new ChainRepeatOperation( - new ChainProcessor($repeatedOperations, $executionFactory), - $expression, - $allowAsync, + $chainBuilder, + new ChainRepeatConfig($repeatChainConfig, $expression, $allowAsync) ); array_unshift($afterOperations, $repeatOperation); diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/ChainSplitOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/ChainSplitOperationTest.php index 025be19..7c128eb 100644 --- a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/ChainSplitOperationTest.php +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/ChainSplitOperationTest.php @@ -8,12 +8,14 @@ namespace Oliverde8\Component\PhpEtl\Tests\ChainOperation; +use Oliverde8\Component\PhpEtl\ChainBuilderV2; +use Oliverde8\Component\PhpEtl\ChainConfig; use Oliverde8\Component\PhpEtl\ChainOperation\ChainSplitOperation; -use Oliverde8\Component\PhpEtl\ChainProcessor; use Oliverde8\Component\PhpEtl\ChainProcessorInterface; use Oliverde8\Component\PhpEtl\Item\DataItem; use Oliverde8\Component\PhpEtl\Item\StopItem; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainSplitConfig; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; @@ -21,7 +23,6 @@ class ChainSplitOperationTest extends TestCase { public function testDataProcessing() { - /** @var \PHPUnit_Framework_MockObject_MockObject[] $processors */ $processors = [ $this->getMockBuilder(ChainProcessorInterface::class)->getMock(), $this->getMockBuilder(ChainProcessorInterface::class)->getMock(), @@ -39,8 +40,21 @@ public function testDataProcessing() ->withConsecutive([$datas[0], $context], [$datas[1], $context]); } + // Create ChainSplitConfig with empty ChainConfigs (since we're mocking the processors) + $splitConfig = new ChainSplitConfig(); + $splitConfig->addSplit(new ChainConfig()); + $splitConfig->addSplit(new ChainConfig()); - $splitOperation = new ChainSplitOperation($processors); + // Mock ChainBuilderV2 to return our mocked processors + $chainBuilder = $this->getMockBuilder(ChainBuilderV2::class) + ->disableOriginalConstructor() + ->getMock(); + + $chainBuilder->expects($this->exactly(2)) + ->method('createChain') + ->willReturnOnConsecutiveCalls($processors[0], $processors[1]); + + $splitOperation = new ChainSplitOperation($chainBuilder, $splitConfig); $splitOperation->process($datas[0], $context); $splitOperation->process($datas[1], $context); } @@ -67,7 +81,21 @@ public function testStopProcess() ->method('processGenerator') ->withConsecutive([$datas[0], $context], [$datas[1], $context], [$stopItem, $context, null, true]); - $splitOperation = new ChainSplitOperation($processors); + // Create ChainSplitConfig with empty ChainConfigs (since we're mocking the processors) + $splitConfig = new ChainSplitConfig(); + $splitConfig->addSplit(new ChainConfig()); + $splitConfig->addSplit(new ChainConfig()); + + // Mock ChainBuilderV2 to return our mocked processors + $chainBuilder = $this->getMockBuilder(ChainBuilderV2::class) + ->disableOriginalConstructor() + ->getMock(); + + $chainBuilder->expects($this->exactly(2)) + ->method('createChain') + ->willReturnOnConsecutiveCalls($processors[0], $processors[1]); + + $splitOperation = new ChainSplitOperation($chainBuilder, $splitConfig); $splitOperation->process($datas[0], $context); $splitOperation->process($datas[1], $context); diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Extract/CsvExtractOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Extract/CsvExtractOperationTest.php new file mode 100644 index 0000000..43891ed --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Extract/CsvExtractOperationTest.php @@ -0,0 +1,262 @@ +testCsvFile = tempnam(sys_get_temp_dir(), 'csv_test_'); + } + + #[\Override] + protected function tearDown(): void + { + if (file_exists($this->testCsvFile)) { + unlink($this->testCsvFile); + } + parent::tearDown(); + } + + public function testExtractCsvWithDefaultDelimiter() + { + file_put_contents($this->testCsvFile, "name;age;city\nJohn;30;NYC\nJane;25;LA"); + + $config = new CsvExtractConfig(); + $operation = new CsvExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('readStream')->willReturn(fopen($this->testCsvFile, 'r')); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testCsvFile); + + $result = $operation->process($item, $context); + + $this->assertInstanceOf(MixItem::class, $result); + + $items = $result->getItems(); + $this->assertCount(2, $items); + $this->assertInstanceOf(GroupedItem::class, $items[0]); + $this->assertInstanceOf(FileExtractedItem::class, $items[1]); + + $csvData = iterator_to_array($items[0]->getIterator(), false); + $this->assertIsArray($csvData); + $this->assertCount(2, $csvData, 'Expected 2 rows of CSV data'); + $this->assertArrayHasKey(0, $csvData); + $this->assertEquals(['name' => 'John', 'age' => '30', 'city' => 'NYC'], $csvData[0]); + $this->assertEquals(['name' => 'Jane', 'age' => '25', 'city' => 'LA'], $csvData[1]); + } + + public function testExtractCsvWithCustomDelimiter() + { + file_put_contents($this->testCsvFile, "name,age,city\nJohn,30,NYC\nJane,25,LA"); + + $config = new CsvExtractConfig(delimiter: ','); + $operation = new CsvExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('readStream')->willReturn(fopen($this->testCsvFile, 'r')); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testCsvFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $csvData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(2, $csvData); + $this->assertEquals('John', $csvData[0]['name']); + $this->assertEquals('30', $csvData[0]['age']); + } + + public function testExtractCsvWithCustomEnclosure() + { + file_put_contents($this->testCsvFile, "'name';'age';'city'\n'John';'30';'NYC'"); + + $config = new CsvExtractConfig(enclosure: "'"); + $operation = new CsvExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('readStream')->willReturn(fopen($this->testCsvFile, 'r')); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testCsvFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $csvData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertEquals('John', $csvData[0]['name']); + } + + public function testExtractCsvFromArrayWithFileKey() + { + file_put_contents($this->testCsvFile, "name;age\nJohn;30"); + + $config = new CsvExtractConfig(fileKey: 'filepath'); + $operation = new CsvExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('readStream')->willReturn(fopen($this->testCsvFile, 'r')); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem(['filepath' => $this->testCsvFile, 'other' => 'data']); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $csvData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(1, $csvData); + $this->assertEquals('John', $csvData[0]['name']); + } + + public function testExtractCsvWithNestedFileKey() + { + file_put_contents($this->testCsvFile, "name;age\nJohn;30"); + + $config = new CsvExtractConfig(fileKey: 'data/file'); + $operation = new CsvExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('readStream')->willReturn(fopen($this->testCsvFile, 'r')); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem(['data' => ['file' => $this->testCsvFile]]); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $csvData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(1, $csvData); + } + + public function testExtractEmptyCsv() + { + file_put_contents($this->testCsvFile, "name;age\n"); + + $config = new CsvExtractConfig(); + $operation = new CsvExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('readStream')->willReturn(fopen($this->testCsvFile, 'r')); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testCsvFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $csvData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(0, $csvData); + } + + public function testExtractCsvWithQuotedFields() + { + file_put_contents($this->testCsvFile, "name;description\n\"John\";\";special;chars;\"\n"); + + $config = new CsvExtractConfig(); + $operation = new CsvExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('readStream')->willReturn(fopen($this->testCsvFile, 'r')); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testCsvFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $csvData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertEquals('John', $csvData[0]['name']); + $this->assertEquals(';special;chars;', $csvData[0]['description']); + } + + public function testFileExtractedItemContainsCorrectFilename() + { + file_put_contents($this->testCsvFile, "name;age\nJohn;30"); + + $config = new CsvExtractConfig(); + $operation = new CsvExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('readStream')->willReturn(fopen($this->testCsvFile, 'r')); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testCsvFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + + $this->assertInstanceOf(FileExtractedItem::class, $items[1]); + $this->assertEquals($this->testCsvFile, $items[1]->getFilePath()); + } + + public function testExtractMultipleRows() + { + $csvContent = "id;name;email\n1;John;john@test.com\n2;Jane;jane@test.com\n3;Bob;bob@test.com"; + file_put_contents($this->testCsvFile, $csvContent); + + $config = new CsvExtractConfig(); + $operation = new CsvExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('readStream')->willReturn(fopen($this->testCsvFile, 'r')); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testCsvFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $csvData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(3, $csvData); + $this->assertEquals('1', $csvData[0]['id']); + $this->assertEquals('2', $csvData[1]['id']); + $this->assertEquals('3', $csvData[2]['id']); + } + + public function testExtractCsvWithEscapeCharacter() + { + file_put_contents($this->testCsvFile, "name;value\n\"test\";\"escaped\\\"quote\""); + + $config = new CsvExtractConfig(escape: '\\'); + $operation = new CsvExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('readStream')->willReturn(fopen($this->testCsvFile, 'r')); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testCsvFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $csvData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(1, $csvData); + $this->assertEquals('test', $csvData[0]['name']); + } + + public function testGetConfigurationClass() + { + $config = new CsvExtractConfig(); + $operation = new CsvExtractOperation($config); + + $this->assertEquals(CsvExtractConfig::class, $operation->getConfigurationClass()); + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Extract/ExternalFileFinderOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Extract/ExternalFileFinderOperationTest.php new file mode 100644 index 0000000..dad676e --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Extract/ExternalFileFinderOperationTest.php @@ -0,0 +1,303 @@ +createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->with('/data/files') + ->willReturn(['file1.csv', 'file2.csv', 'file3.txt', 'file4.csv']); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem('/\.csv$/'); + + $result = $operation->process($item, $context); + + $this->assertInstanceOf(MixItem::class, $result); + + $files = $result->getItems(); + $this->assertCount(3, $files); + + foreach ($files as $file) { + $this->assertInstanceOf(ExternalFileItem::class, $file); + } + + $this->assertEquals('/data/files/file1.csv', $files[0]->getFilePath()); + $this->assertEquals('/data/files/file2.csv', $files[1]->getFilePath()); + $this->assertEquals('/data/files/file4.csv', $files[2]->getFilePath()); + } + + public function testFindFilesWithNoMatches() + { + $config = new ExternalFileFinderConfig('/data/files'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->with('/data/files') + ->willReturn(['file1.txt', 'file2.doc', 'file3.pdf']); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem('/\.csv$/'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertCount(0, $files); + } + + public function testFindFilesWithComplexPattern() + { + $config = new ExternalFileFinderConfig('/data/files'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->willReturn([ + 'report_2023_01.csv', + 'report_2023_02.csv', + 'summary_2023_01.csv', + 'report_2024_01.csv', + 'data.csv' + ]); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem('/^report_2023_/'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertCount(2, $files); + $this->assertEquals('/data/files/report_2023_01.csv', $files[0]->getFilePath()); + $this->assertEquals('/data/files/report_2023_02.csv', $files[1]->getFilePath()); + } + + public function testFindFilesInEmptyDirectory() + { + $config = new ExternalFileFinderConfig('/empty/dir'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->with('/empty/dir') + ->willReturn([]); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem('/.*$/'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertCount(0, $files); + } + + public function testFindAllFilesWithMatchAllPattern() + { + $config = new ExternalFileFinderConfig('/data'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->willReturn(['file1.csv', 'file2.txt', 'file3.json']); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem('/.*$/'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertCount(3, $files); + } + + public function testFileSystemIsPassedToExternalFileItem() + { + $config = new ExternalFileFinderConfig('/data'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->willReturn(['test.csv']); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem('/\.csv$/'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertCount(1, $files); + $this->assertSame($fileSystem, $files[0]->getFileSystem()); + } + + public function testExternalFileItemInitialState() + { + $config = new ExternalFileFinderConfig('/data'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->willReturn(['test.csv']); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem('/\.csv$/'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertEquals(ExternalFileItem::STATE_NEW, $files[0]->getState()); + } + + public function testFindFilesWithCaseInsensitivePattern() + { + $config = new ExternalFileFinderConfig('/data'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->willReturn(['File1.CSV', 'file2.csv', 'FILE3.TXT']); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem('/\.csv$/i'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertCount(2, $files); + $this->assertEquals('/data/File1.CSV', $files[0]->getFilePath()); + $this->assertEquals('/data/file2.csv', $files[1]->getFilePath()); + } + + public function testFindFilesWithNumericPattern() + { + $config = new ExternalFileFinderConfig('/data'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->willReturn(['file001.csv', 'file002.csv', 'fileabc.csv', 'file999.csv']); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem('/file\d+\.csv$/'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertCount(3, $files); + $this->assertEquals('/data/file001.csv', $files[0]->getFilePath()); + $this->assertEquals('/data/file002.csv', $files[1]->getFilePath()); + $this->assertEquals('/data/file999.csv', $files[2]->getFilePath()); + } + + public function testDirectoryWithExpression() + { + $config = new ExternalFileFinderConfig('@context["directory"]'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->with('/dynamic/path') + ->willReturn(['file1.csv', 'file2.csv']); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext(['directory' => '/dynamic/path'], $fileSystem); + $item = new DataItem('/\.csv$/'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertCount(2, $files); + $this->assertEquals('/dynamic/path/file1.csv', $files[0]->getFilePath()); + } + + public function testDirectoryExpressionWithComplexPath() + { + $config = new ExternalFileFinderConfig('@context["base_path"] ~ "/" ~ context["sub_dir"]'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->with('/base/subdir') + ->willReturn(['test.csv']); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext(['base_path' => '/base', 'sub_dir' => 'subdir'], $fileSystem); + $item = new DataItem('/\.csv$/'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertCount(1, $files); + $this->assertEquals('/base/subdir/test.csv', $files[0]->getFilePath()); + } + + public function testFindFilesWithSpecialCharactersInName() + { + $config = new ExternalFileFinderConfig('/data'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->willReturn(['file-1.csv', 'file_2.csv', 'file.3.csv', 'file@4.csv']); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem('/file[-_.].*\.csv$/'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertCount(3, $files); + } + + public function testMultipleExtensionsPattern() + { + $config = new ExternalFileFinderConfig('/data'); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('listContents') + ->willReturn([ + 'data.csv', + 'data.json', + 'data.xml', + 'data.txt', + 'data.xlsx' + ]); + + $operation = new ExternalFileFinderOperation($fileSystem, $config); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem('/\.(csv|json|xml)$/'); + + $result = $operation->process($item, $context); + + $files = $result->getItems(); + $this->assertCount(3, $files); + $this->assertEquals('/data/data.csv', $files[0]->getFilePath()); + $this->assertEquals('/data/data.json', $files[1]->getFilePath()); + $this->assertEquals('/data/data.xml', $files[2]->getFilePath()); + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Extract/JsonExtractOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Extract/JsonExtractOperationTest.php new file mode 100644 index 0000000..c385f0b --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Extract/JsonExtractOperationTest.php @@ -0,0 +1,334 @@ +testJsonFile = tempnam(sys_get_temp_dir(), 'json_test_'); + } + + #[\Override] + protected function tearDown(): void + { + if (file_exists($this->testJsonFile)) { + unlink($this->testJsonFile); + } + parent::tearDown(); + } + + public function testExtractSimpleJsonArray() + { + $jsonData = [ + ['id' => 1, 'name' => 'John'], + ['id' => 2, 'name' => 'Jane'] + ]; + file_put_contents($this->testJsonFile, json_encode($jsonData)); + + $config = new JsonExtractConfig(); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testJsonFile); + + $result = $operation->process($item, $context); + + $this->assertInstanceOf(MixItem::class, $result); + + $items = $result->getItems(); + $this->assertCount(2, $items); + $this->assertInstanceOf(GroupedItem::class, $items[0]); + $this->assertInstanceOf(FileExtractedItem::class, $items[1]); + + $extractedData = iterator_to_array($items[0]->getIterator(), false); + $this->assertCount(2, $extractedData); + $this->assertEquals(['id' => 1, 'name' => 'John'], $extractedData[0]); + $this->assertEquals(['id' => 2, 'name' => 'Jane'], $extractedData[1]); + } + + public function testExtractJsonObject() + { + $jsonData = ['user' => 'John', 'age' => 30, 'active' => true]; + file_put_contents($this->testJsonFile, json_encode($jsonData)); + + $config = new JsonExtractConfig(); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testJsonFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $extractedData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertEquals('John', $extractedData[0]); + $this->assertEquals(30, $extractedData[1]); + $this->assertTrue($extractedData[2]); + } + + public function testExtractFromArrayWithFileKey() + { + $jsonData = [['name' => 'Test']]; + file_put_contents($this->testJsonFile, json_encode($jsonData)); + + $config = new JsonExtractConfig(fileKey: 'filepath'); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem(['filepath' => $this->testJsonFile, 'other' => 'data']); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $extractedData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(1, $extractedData); + $this->assertEquals('Test', $extractedData[0]['name']); + } + + public function testExtractWithNestedFileKey() + { + $jsonData = [['id' => 1]]; + file_put_contents($this->testJsonFile, json_encode($jsonData)); + + $config = new JsonExtractConfig(fileKey: 'data/file'); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem(['data' => ['file' => $this->testJsonFile]]); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $extractedData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(1, $extractedData); + $this->assertEquals(1, $extractedData[0]['id']); + } + + public function testExtractEmptyJsonArray() + { + file_put_contents($this->testJsonFile, json_encode([])); + + $config = new JsonExtractConfig(); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testJsonFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $extractedData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(0, $extractedData); + } + + public function testExtractNestedJsonStructure() + { + $jsonData = [ + [ + 'id' => 1, + 'details' => ['address' => 'NYC', 'zip' => '10001'], + 'tags' => ['premium', 'active'] + ], + [ + 'id' => 2, + 'details' => ['address' => 'LA', 'zip' => '90001'], + 'tags' => ['basic'] + ] + ]; + file_put_contents($this->testJsonFile, json_encode($jsonData)); + + $config = new JsonExtractConfig(); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testJsonFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $extractedData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(2, $extractedData); + $this->assertEquals('NYC', $extractedData[0]['details']['address']); + $this->assertEquals(['premium', 'active'], $extractedData[0]['tags']); + $this->assertEquals('LA', $extractedData[1]['details']['address']); + } + + public function testExtractJsonWithSpecialCharacters() + { + $jsonData = [ + ['name' => 'Test "quoted"', 'description' => "Line1\nLine2"], + ['name' => 'Test\'s', 'description' => 'Special: ā‚¬, Ā£, Ā„'] + ]; + file_put_contents($this->testJsonFile, json_encode($jsonData)); + + $config = new JsonExtractConfig(); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testJsonFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $extractedData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertEquals('Test "quoted"', $extractedData[0]['name']); + $this->assertEquals("Line1\nLine2", $extractedData[0]['description']); + $this->assertEquals('Special: ā‚¬, Ā£, Ā„', $extractedData[1]['description']); + } + + public function testFileExtractedItemContainsOriginalData() + { + $jsonData = [['test' => 'value']]; + file_put_contents($this->testJsonFile, json_encode($jsonData)); + + $config = new JsonExtractConfig(); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testJsonFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + + $this->assertInstanceOf(FileExtractedItem::class, $items[1]); + $this->assertEquals($this->testJsonFile, $items[1]->getFilePath()); + } + + public function testFileExtractedItemWithArrayInput() + { + $jsonData = [['test' => 'value']]; + file_put_contents($this->testJsonFile, json_encode($jsonData)); + + $config = new JsonExtractConfig(fileKey: 'file'); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $inputData = ['file' => $this->testJsonFile, 'metadata' => 'info']; + $item = new DataItem($inputData); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + + $this->assertInstanceOf(FileExtractedItem::class, $items[1]); + $this->assertEquals($this->testJsonFile, $items[1]->getFilePath()); + } + + public function testExtractJsonWithNumericKeys() + { + $jsonData = ['first' => 1, 'second' => 2, 'third' => 3]; + file_put_contents($this->testJsonFile, json_encode($jsonData)); + + $config = new JsonExtractConfig(); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testJsonFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $extractedData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(3, $extractedData); + $this->assertEquals(1, $extractedData[0]); + $this->assertEquals(2, $extractedData[1]); + $this->assertEquals(3, $extractedData[2]); + } + + public function testExtractJsonWithBooleanAndNull() + { + $jsonData = [ + ['active' => true, 'deleted' => false, 'optional' => null] + ]; + file_put_contents($this->testJsonFile, json_encode($jsonData)); + + $config = new JsonExtractConfig(); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testJsonFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $extractedData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertTrue($extractedData[0]['active']); + $this->assertFalse($extractedData[0]['deleted']); + $this->assertNull($extractedData[0]['optional']); + } + + public function testExtractLargeJsonArray() + { + $jsonData = []; + for ($i = 0; $i < 100; $i++) { + $jsonData[] = ['id' => $i, 'value' => 'item_' . $i]; + } + file_put_contents($this->testJsonFile, json_encode($jsonData)); + + $config = new JsonExtractConfig(); + $operation = new JsonExtractOperation($config); + + $fileSystem = $this->createMock(FileSystemInterface::class); + $fileSystem->method('read')->willReturn(file_get_contents($this->testJsonFile)); + + $context = new ExecutionContext([], $fileSystem); + $item = new DataItem($this->testJsonFile); + + $result = $operation->process($item, $context); + $items = $result->getItems(); + $extractedData = iterator_to_array($items[0]->getIterator(), false); + + $this->assertCount(100, $extractedData); + $this->assertEquals(0, $extractedData[0]['id']); + $this->assertEquals(99, $extractedData[99]['id']); + $this->assertEquals('item_50', $extractedData[50]['value']); + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/FailSafeOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/FailSafeOperationTest.php index 6d817df..85c30b5 100644 --- a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/FailSafeOperationTest.php +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/FailSafeOperationTest.php @@ -3,14 +3,18 @@ namespace Oliverde8\Component\PhpEtl\Tests\ChainOperation; -use Oliverde8\Component\PhpEtl\ChainOperation\ChainRepeatOperation; +use Oliverde8\Component\PhpEtl\ChainBuilderV2; +use Oliverde8\Component\PhpEtl\ChainConfig; use Oliverde8\Component\PhpEtl\ChainOperation\FailSafeOperation; use Oliverde8\Component\PhpEtl\ChainOperation\Transformer\CallbackTransformerOperation; use Oliverde8\Component\PhpEtl\ChainProcessor; use Oliverde8\Component\PhpEtl\Exception\ChainOperationException; use Oliverde8\Component\PhpEtl\ExecutionContextFactory; +use Oliverde8\Component\PhpEtl\GenericChainFactory; use Oliverde8\Component\PhpEtl\Item\DataItem; use Oliverde8\Component\PhpEtl\Item\ItemInterface; +use Oliverde8\Component\PhpEtl\OperationConfig\FailSafeConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; use PHPUnit\Framework\TestCase; class FailSafeOperationTest extends TestCase @@ -18,7 +22,7 @@ class FailSafeOperationTest extends TestCase public function testSingleFail() { $callNum = 0; - $failOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { + $failOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$callNum) { if ($callNum == 0 || $callNum == 2) { return new DataItem(['val' => $callNum++]); } @@ -26,11 +30,12 @@ public function testSingleFail() $callNum++; throw new \Exception("Exception at $callNum"); } - }); - $endOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { + return $item; + })); + $endOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$results) { $results[] = $item->getData(); return $item; - }); + })); $chain = $this->createChain([$failOperation], [$endOperation]); $chain->process(new \ArrayIterator([['var' => 1],['var' => 2]]), []); @@ -41,15 +46,15 @@ public function testSingleFail() public function testToManyFail() { $callNum = 0; - $failOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { + $failOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$callNum): void { $callNum++; throw new \Exception("Exception at $callNum"); - }); + })); $results = []; - $endOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { + $endOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$results) { $results[] = $item->getData(); return $item; - }); + })); $chain = $this->createChain([$failOperation], [$endOperation]); $e = null; @@ -66,10 +71,28 @@ public function testToManyFail() protected function createChain(array $failSafeOperation, array $afterOperations): ChainProcessor { $executionFactory = new ExecutionContextFactory(); + + $failSafeChainConfig = new ChainConfig(); + + $reflection = new \ReflectionClass(CallbackTransformerOperation::class); + $configProperty = $reflection->getProperty('config'); + $configProperty->setAccessible(true); + + foreach ($failSafeOperation as $operation) { + if ($operation instanceof CallbackTransformerOperation) { + $config = $configProperty->getValue($operation); + $failSafeChainConfig->addLink($config); + } + } + + $chainBuilder = new ChainBuilderV2( + $executionFactory, + [new GenericChainFactory(CallbackTransformerOperation::class, CallBackTransformerConfig::class)] + ); + $repeatOperation = new FailSafeOperation( - new ChainProcessor($failSafeOperation, $executionFactory), - [\Exception::class], - 2, + $chainBuilder, + new FailSafeConfig($failSafeChainConfig, [\Exception::class], 2) ); array_unshift($afterOperations, $repeatOperation); diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Grouping/SimpleGroupingOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Grouping/SimpleGroupingOperationTest.php new file mode 100644 index 0000000..a2f48f0 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Grouping/SimpleGroupingOperationTest.php @@ -0,0 +1,250 @@ +context = new ExecutionContext([], new LocalFileSystem()); + } + + public function testGroupBySingleKey() + { + $config = new SimpleGroupingConfig(['category']); + $operation = new SimpleGroupingOperation($config); + + $item1 = new DataItem(['category' => 'A', 'value' => 1]); + $item2 = new DataItem(['category' => 'B', 'value' => 2]); + $item3 = new DataItem(['category' => 'A', 'value' => 3]); + + $result1 = $operation->process($item1, $this->context); + $result2 = $operation->process($item2, $this->context); + $result3 = $operation->process($item3, $this->context); + + $this->assertInstanceOf(ChainBreakItem::class, $result1); + $this->assertInstanceOf(ChainBreakItem::class, $result2); + $this->assertInstanceOf(ChainBreakItem::class, $result3); + + $stopResult = $operation->processStop(new StopItem(), $this->context); + $this->assertInstanceOf(GroupedItem::class, $stopResult); + + $groupedData = iterator_to_array($stopResult->getIterator()); + + $this->assertCount(2, $groupedData); + $this->assertArrayHasKey('A', $groupedData); + $this->assertArrayHasKey('B', $groupedData); + $this->assertCount(2, $groupedData['A']); + $this->assertCount(1, $groupedData['B']); + } + + public function testGroupByNestedKey() + { + $config = new SimpleGroupingConfig(['user', 'type']); + $operation = new SimpleGroupingOperation($config); + + $item1 = new DataItem(['user' => ['type' => 'admin'], 'name' => 'John']); + $item2 = new DataItem(['user' => ['type' => 'user'], 'name' => 'Jane']); + $item3 = new DataItem(['user' => ['type' => 'admin'], 'name' => 'Bob']); + + $operation->process($item1, $this->context); + $operation->process($item2, $this->context); + $operation->process($item3, $this->context); + + $stopResult = $operation->processStop(new StopItem(), $this->context); + $groupedData = iterator_to_array($stopResult->getIterator()); + + $this->assertCount(2, $groupedData); + $this->assertArrayHasKey('admin', $groupedData); + $this->assertArrayHasKey('user', $groupedData); + $this->assertCount(2, $groupedData['admin']); + $this->assertCount(1, $groupedData['user']); + } + + public function testGroupWithIdentifierKey() + { + $config = new SimpleGroupingConfig(['category'], ['id']); + $operation = new SimpleGroupingOperation($config); + + $item1 = new DataItem(['category' => 'A', 'id' => 1, 'value' => 'first']); + $item2 = new DataItem(['category' => 'A', 'id' => 2, 'value' => 'second']); + $item3 = new DataItem(['category' => 'B', 'id' => 1, 'value' => 'third']); + + $operation->process($item1, $this->context); + $operation->process($item2, $this->context); + $operation->process($item3, $this->context); + + $stopResult = $operation->processStop(new StopItem(), $this->context); + $groupedData = iterator_to_array($stopResult->getIterator()); + + $this->assertArrayHasKey('A', $groupedData); + $this->assertArrayHasKey('B', $groupedData); + $this->assertArrayHasKey(1, $groupedData['A']); + $this->assertArrayHasKey(2, $groupedData['A']); + $this->assertEquals('first', $groupedData['A'][1]['value']); + $this->assertEquals('second', $groupedData['A'][2]['value']); + } + + public function testGroupWithNestedIdentifierKey() + { + $config = new SimpleGroupingConfig(['category'], ['meta', 'id']); + $operation = new SimpleGroupingOperation($config); + + $item1 = new DataItem(['category' => 'A', 'meta' => ['id' => 100], 'value' => 'test1']); + $item2 = new DataItem(['category' => 'A', 'meta' => ['id' => 200], 'value' => 'test2']); + + $operation->process($item1, $this->context); + $operation->process($item2, $this->context); + + $stopResult = $operation->processStop(new StopItem(), $this->context); + $groupedData = iterator_to_array($stopResult->getIterator()); + + $this->assertArrayHasKey(100, $groupedData['A']); + $this->assertArrayHasKey(200, $groupedData['A']); + $this->assertEquals('test1', $groupedData['A'][100]['value']); + } + + public function testMultipleGroupsWithoutIdentifier() + { + $config = new SimpleGroupingConfig(['status']); + $operation = new SimpleGroupingOperation($config); + + $item1 = new DataItem(['status' => 'active', 'name' => 'A']); + $item2 = new DataItem(['status' => 'active', 'name' => 'B']); + $item3 = new DataItem(['status' => 'inactive', 'name' => 'C']); + + $operation->process($item1, $this->context); + $operation->process($item2, $this->context); + $operation->process($item3, $this->context); + + $stopResult = $operation->processStop(new StopItem(), $this->context); + $groupedData = iterator_to_array($stopResult->getIterator()); + + $this->assertCount(2, $groupedData['active']); + $this->assertCount(1, $groupedData['inactive']); + $this->assertEquals('A', $groupedData['active'][0]['name']); + $this->assertEquals('B', $groupedData['active'][1]['name']); + } + + public function testEmptyDataReturnsStopItem() + { + $config = new SimpleGroupingConfig(['category']); + $operation = new SimpleGroupingOperation($config); + + $stopItem = new StopItem(); + $result = $operation->processStop($stopItem, $this->context); + + $this->assertSame($stopItem, $result); + } + + public function testDataClearedAfterStop() + { + $config = new SimpleGroupingConfig(['type']); + $operation = new SimpleGroupingOperation($config); + + $item1 = new DataItem(['type' => 'X', 'value' => 1]); + $operation->process($item1, $this->context); + + $stopResult1 = $operation->processStop(new StopItem(), $this->context); + $groupedData1 = iterator_to_array($stopResult1->getIterator()); + $this->assertCount(1, $groupedData1); + + $stopResult2 = $operation->processStop(new StopItem(), $this->context); + $this->assertInstanceOf(StopItem::class, $stopResult2); + } + + public function testMultipleStopCycles() + { + $config = new SimpleGroupingConfig(['batch']); + $operation = new SimpleGroupingOperation($config); + + $item1 = new DataItem(['batch' => 1, 'value' => 'a']); + $operation->process($item1, $this->context); + $stopResult1 = $operation->processStop(new StopItem(), $this->context); + $groupedData1 = iterator_to_array($stopResult1->getIterator()); + $this->assertArrayHasKey(1, $groupedData1); + + $item2 = new DataItem(['batch' => 2, 'value' => 'b']); + $operation->process($item2, $this->context); + $stopResult2 = $operation->processStop(new StopItem(), $this->context); + $groupedData2 = iterator_to_array($stopResult2->getIterator()); + $this->assertArrayHasKey(2, $groupedData2); + $this->assertArrayNotHasKey(1, $groupedData2); + } + + public function testGroupingPreservesAllDataFields() + { + $config = new SimpleGroupingConfig(['category']); + $operation = new SimpleGroupingOperation($config); + + $item = new DataItem([ + 'category' => 'test', + 'id' => 123, + 'name' => 'Test Item', + 'metadata' => ['key' => 'value'] + ]); + + $operation->process($item, $this->context); + $stopResult = $operation->processStop(new StopItem(), $this->context); + $groupedData = iterator_to_array($stopResult->getIterator()); + + $this->assertEquals(123, $groupedData['test'][0]['id']); + $this->assertEquals('Test Item', $groupedData['test'][0]['name']); + $this->assertEquals(['key' => 'value'], $groupedData['test'][0]['metadata']); + } + + public function testOverwriteWithSameIdentifier() + { + $config = new SimpleGroupingConfig(['category'], ['id']); + $operation = new SimpleGroupingOperation($config); + + $item1 = new DataItem(['category' => 'A', 'id' => 1, 'value' => 'first']); + $item2 = new DataItem(['category' => 'A', 'id' => 1, 'value' => 'second']); + + $operation->process($item1, $this->context); + $operation->process($item2, $this->context); + + $stopResult = $operation->processStop(new StopItem(), $this->context); + $groupedData = iterator_to_array($stopResult->getIterator()); + + $this->assertCount(1, $groupedData['A']); + $this->assertEquals('second', $groupedData['A'][1]['value']); + } + + public function testNumericGroupKeys() + { + $config = new SimpleGroupingConfig(['priority']); + $operation = new SimpleGroupingOperation($config); + + $item1 = new DataItem(['priority' => 1, 'task' => 'high']); + $item2 = new DataItem(['priority' => 2, 'task' => 'medium']); + $item3 = new DataItem(['priority' => 1, 'task' => 'critical']); + + $operation->process($item1, $this->context); + $operation->process($item2, $this->context); + $operation->process($item3, $this->context); + + $stopResult = $operation->processStop(new StopItem(), $this->context); + $groupedData = iterator_to_array($stopResult->getIterator()); + + $this->assertArrayHasKey(1, $groupedData); + $this->assertArrayHasKey(2, $groupedData); + $this->assertCount(2, $groupedData[1]); + $this->assertCount(1, $groupedData[2]); + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Loader/FileWriterOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Loader/FileWriterOperationTest.php index 66d2457..a5f5848 100644 --- a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Loader/FileWriterOperationTest.php +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Loader/FileWriterOperationTest.php @@ -14,6 +14,7 @@ use Oliverde8\Component\PhpEtl\Load\File\Csv; use Oliverde8\Component\PhpEtl\Load\File\FileWriterInterface; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; +use Oliverde8\Component\PhpEtl\OperationConfig\Loader\FileWriterConfigInterface; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; @@ -25,6 +26,7 @@ class FileWriterOperationTest extends TestCase /** @var FileWriterOperation */ protected $writerOperation; + #[\Override] protected function setUp(): void { parent::setUp(); @@ -32,7 +34,13 @@ protected function setUp(): void $this->writerMock = $this->getMockBuilder(FileWriterInterface::class) ->getMock(); - $this->writerOperation = new FileWriterOperation($this->writerMock, 'fileName'); + $configMock = $this->getMockBuilder(FileWriterConfigInterface::class) + ->getMock(); + + $configMock->method('getFile')->willReturn($this->writerMock); + $configMock->method('getFileName')->willReturn('fileName'); + + $this->writerOperation = new FileWriterOperation($configMock); } public function testWrite() @@ -50,7 +58,13 @@ public function testTmpFileDeleted() { $tmpFile = tempnam(sys_get_temp_dir(), 'etl'); $csv = new Csv($tmpFile); - $operation = new FileWriterOperation($csv, basename($tmpFile)); + + $configMock = $this->getMockBuilder(FileWriterConfigInterface::class) + ->getMock(); + $configMock->method('getFile')->willReturn($csv); + $configMock->method('getFileName')->willReturn(basename($tmpFile)); + + $operation = new FileWriterOperation($configMock); $context = $this->getMockBuilder(ExecutionContext::class)->disableOriginalConstructor()->getMock(); $operation->process(new DataItem(['test' => 'val1']), $context); diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/SimpleHttpOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/SimpleHttpOperationTest.php index 714507a..27f9a58 100644 --- a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/SimpleHttpOperationTest.php +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/SimpleHttpOperationTest.php @@ -8,6 +8,8 @@ use Oliverde8\Component\PhpEtl\ChainProcessor; use Oliverde8\Component\PhpEtl\ExecutionContextFactory; use Oliverde8\Component\PhpEtl\Item\ItemInterface; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\SimpleHttpConfig; use PHPUnit\Framework\TestCase; use Symfony\Component\HttpClient\Exception\TransportException; use Symfony\Component\HttpClient\HttpClient; @@ -16,10 +18,10 @@ class SimpleHttpOperationTest extends TestCase { public function testTimeout() { - $endOperation = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { + $endOperation = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$results) { $results[] = $item->getData(); return $item; - }); + })); $chain = $this->createChain( 'http://www.google.com:81', @@ -38,11 +40,13 @@ protected function createChain(string $url, string $method, array $afterOperatio $executionFactory = new ExecutionContextFactory(); $repeatOperation = new SimpleHttpOperation( $httpClient, - $method, - $url, - $responseIsJson, - 'options', - 'result', + new SimpleHttpConfig( + method: $method, + url: $url, + responseIsJson: $responseIsJson, + optionKey: 'options', + responseKey: 'result' + ) ); array_unshift($afterOperations, $repeatOperation); diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Transformer/ExternalFileProcessorOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Transformer/ExternalFileProcessorOperationTest.php new file mode 100644 index 0000000..2ed618e --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Transformer/ExternalFileProcessorOperationTest.php @@ -0,0 +1,353 @@ +createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->expects($this->once()) + ->method('createDirectory') + ->with('/data/files/processing'); + + $externalFileSystem->expects($this->once()) + ->method('move') + ->with('/data/files/test.csv', '/data/files/processing/test.csv'); + + $externalFileSystem->expects($this->once()) + ->method('readStream') + ->with('/data/files/processing/test.csv') + ->willReturn(fopen('php://memory', 'r')); + + $localFileSystem->expects($this->once()) + ->method('writeStream') + ->with('test.csv', $this->anything()); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/data/files/test.csv', $externalFileSystem); + + $this->assertEquals(ExternalFileItem::STATE_NEW, $item->getState()); + + $result = $operation->processFile($item, $context); + + $this->assertInstanceOf(MixItem::class, $result); + $items = $result->getItems(); + $this->assertCount(2, $items); + $this->assertInstanceOf(DataItem::class, $items[0]); + $this->assertInstanceOf(ExternalFileItem::class, $items[1]); + $this->assertEquals('test.csv', $items[0]->getData()); + $this->assertEquals(ExternalFileItem::STATE_PROCESSING, $item->getState()); + } + + public function testProcessProcessingFileMovesToProcessed() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->expects($this->once()) + ->method('createDirectory') + ->with('/data/files/processed'); + + $externalFileSystem->expects($this->once()) + ->method('move') + ->with('/data/files/processing/test.csv', '/data/files/processed/test.csv'); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/data/files/test.csv', $externalFileSystem); + $item->setState(ExternalFileItem::STATE_PROCESSING); + + $result = $operation->processFile($item, $context); + + $this->assertInstanceOf(ExternalFileItem::class, $result); + $this->assertEquals(ExternalFileItem::STATE_PROCESSED, $item->getState()); + } + + public function testProcessNewFileCreatesLocalCopy() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $fileContent = 'test,content'; + $stream = fopen('php://memory', 'r+'); + fwrite($stream, $fileContent); + rewind($stream); + + $externalFileSystem->method('readStream')->willReturn($stream); + $externalFileSystem->method('createDirectory'); + $externalFileSystem->method('move'); + + $capturedStream = null; + $localFileSystem->expects($this->once()) + ->method('writeStream') + ->with('test.csv', $this->anything()) + ->willReturnCallback(function($filename, $stream) use (&$capturedStream): void { + $capturedStream = $stream; + }); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/data/files/test.csv', $externalFileSystem); + + $operation->processFile($item, $context); + + $this->assertNotNull($capturedStream); + } + + public function testProcessFileWithNestedDirectory() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->expects($this->once()) + ->method('createDirectory') + ->with('/data/nested/dir/processing'); + + $externalFileSystem->expects($this->once()) + ->method('move') + ->with('/data/nested/dir/file.csv', '/data/nested/dir/processing/file.csv'); + + $externalFileSystem->method('readStream')->willReturn(fopen('php://memory', 'r')); + $localFileSystem->method('writeStream'); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/data/nested/dir/file.csv', $externalFileSystem); + + $result = $operation->processFile($item, $context); + + $items = $result->getItems(); + $this->assertEquals('file.csv', $items[0]->getData()); + } + + public function testProcessFilePreservesFilename() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->method('createDirectory'); + $externalFileSystem->method('move'); + $externalFileSystem->method('readStream')->willReturn(fopen('php://memory', 'r')); + $localFileSystem->method('writeStream'); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/data/special-file_name.123.csv', $externalFileSystem); + + $result = $operation->processFile($item, $context); + + $items = $result->getItems(); + $this->assertEquals('special-file_name.123.csv', $items[0]->getData()); + } + + public function testProcessMultipleFilesInSequence() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->method('createDirectory'); + $externalFileSystem->method('move'); + $externalFileSystem->method('readStream')->willReturn(fopen('php://memory', 'r')); + $localFileSystem->method('writeStream'); + + $context = new ExecutionContext([], $localFileSystem); + + $item1 = new ExternalFileItem('/data/file1.csv', $externalFileSystem); + $result1 = $operation->processFile($item1, $context); + $this->assertEquals(ExternalFileItem::STATE_PROCESSING, $item1->getState()); + + $result2 = $operation->processFile($item1, $context); + $this->assertEquals(ExternalFileItem::STATE_PROCESSED, $item1->getState()); + $this->assertInstanceOf(ExternalFileItem::class, $result2); + } + + public function testNewFileResultContainsDataItemAndExternalFileItem() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->method('createDirectory'); + $externalFileSystem->method('move'); + $externalFileSystem->method('readStream')->willReturn(fopen('php://memory', 'r')); + $localFileSystem->method('writeStream'); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/data/test.csv', $externalFileSystem); + + $result = $operation->processFile($item, $context); + + $this->assertInstanceOf(MixItem::class, $result); + $items = $result->getItems(); + $this->assertCount(2, $items); + $this->assertSame($item, $items[1]); + } + + public function testProcessingFileOnlyReturnsExternalFileItem() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->method('createDirectory'); + $externalFileSystem->method('move'); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/data/test.csv', $externalFileSystem); + $item->setState(ExternalFileItem::STATE_PROCESSING); + + $result = $operation->processFile($item, $context); + + $this->assertInstanceOf(ExternalFileItem::class, $result); + $this->assertNotInstanceOf(MixItem::class, $result); + $this->assertSame($item, $result); + } + + public function testProcessFileCreatesProcessingDirectory() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->expects($this->once()) + ->method('createDirectory') + ->with('/path/to/files/processing'); + + $externalFileSystem->method('move'); + $externalFileSystem->method('readStream')->willReturn(fopen('php://memory', 'r')); + $localFileSystem->method('writeStream'); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/path/to/files/myfile.csv', $externalFileSystem); + + $operation->processFile($item, $context); + } + + public function testProcessFileCreatesProcessedDirectory() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->expects($this->once()) + ->method('createDirectory') + ->with('/path/to/files/processed'); + + $externalFileSystem->method('move'); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/path/to/files/myfile.csv', $externalFileSystem); + $item->setState(ExternalFileItem::STATE_PROCESSING); + + $operation->processFile($item, $context); + } + + public function testStateTransitionFromNewToProcessing() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->method('createDirectory'); + $externalFileSystem->method('move'); + $externalFileSystem->method('readStream')->willReturn(fopen('php://memory', 'r')); + $localFileSystem->method('writeStream'); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/data/test.csv', $externalFileSystem); + + $initialState = $item->getState(); + $this->assertEquals(ExternalFileItem::STATE_NEW, $initialState); + + $operation->processFile($item, $context); + + $this->assertEquals(ExternalFileItem::STATE_PROCESSING, $item->getState()); + } + + public function testStateTransitionFromProcessingToProcessed() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->method('createDirectory'); + $externalFileSystem->method('move'); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/data/test.csv', $externalFileSystem); + $item->setState(ExternalFileItem::STATE_PROCESSING); + + $initialState = $item->getState(); + $this->assertEquals(ExternalFileItem::STATE_PROCESSING, $initialState); + + $operation->processFile($item, $context); + + $this->assertEquals(ExternalFileItem::STATE_PROCESSED, $item->getState()); + } + + public function testFilenameExtraction() + { + $config = new ExternalFileProcessorConfig(); + $operation = new ExternalFileProcessorOperation($config); + + $externalFileSystem = $this->createMock(FileSystemInterface::class); + $localFileSystem = $this->createMock(FileSystemInterface::class); + + $externalFileSystem->method('createDirectory'); + $externalFileSystem->method('move'); + $externalFileSystem->method('readStream')->willReturn(fopen('php://memory', 'r')); + + $capturedFilename = null; + $localFileSystem->expects($this->once()) + ->method('writeStream') + ->willReturnCallback(function($filename) use (&$capturedFilename): void { + $capturedFilename = $filename; + }); + + $context = new ExecutionContext([], $localFileSystem); + $item = new ExternalFileItem('/very/deep/path/to/important-file.csv', $externalFileSystem); + + $operation->processFile($item, $context); + + $this->assertEquals('important-file.csv', $capturedFilename); + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Transformer/RuleTransformOperationTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Transformer/RuleTransformOperationTest.php index 86b29d0..604c879 100644 --- a/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Transformer/RuleTransformOperationTest.php +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainOperation/Transformer/RuleTransformOperationTest.php @@ -12,6 +12,7 @@ use Oliverde8\Component\PhpEtl\Item\DataItem; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; use Oliverde8\Component\PhpEtl\Model\File\LocalFileSystem; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\RuleTransformConfig; use Oliverde8\Component\RuleEngine\RuleApplier; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; @@ -24,6 +25,7 @@ class RuleTransformOperationTest extends TestCase /** @var MockObject */ protected $context; + #[\Override] protected function setUp(): void { parent::setUp(); @@ -38,11 +40,12 @@ public function testMultiColumnRulesAddingData() { $this->ruleApplierMock->expects($this->exactly(3))->method('apply')->willReturn(1); - $transform = new RuleTransformOperation( - $this->ruleApplierMock, - ['test1' => ['rules' => ''], 'test2' => ['rules' => ''], 'test3' => ['rules' => '']], - true - ); + $config = new RuleTransformConfig(add: true); + $config->addColumn('test1', ['rules' => '']) + ->addColumn('test2', ['rules' => '']) + ->addColumn('test3', ['rules' => '']); + + $transform = new RuleTransformOperation($this->ruleApplierMock, $config); $data = new DataItem(['test' => 'test']); $data = $transform->process($data, $this->context); @@ -54,11 +57,12 @@ public function testMultiColumnRulesReplacingData() { $this->ruleApplierMock->expects($this->exactly(3))->method('apply')->willReturn(1); - $transform = new RuleTransformOperation( - $this->ruleApplierMock, - ['test1' => ['rules' => ''], 'test2' => ['rules' => ''], 'test3' => ['rules' => '']], - false - ); + $config = new RuleTransformConfig(add: false); + $config->addColumn('test1', ['rules' => '']) + ->addColumn('test2', ['rules' => '']) + ->addColumn('test3', ['rules' => '']); + + $transform = new RuleTransformOperation($this->ruleApplierMock, $config); $data = new DataItem(['test' => 'test']); $data = $transform->process($data, $this->context); @@ -72,11 +76,11 @@ public function testDynamicColumn() $context = clone $this->context; $context->setParameter('locales', ['fr_FR', 'en_GB']); - $transform = new RuleTransformOperation( - $this->ruleApplierMock, - ['test1-{@context/locales}' => ['rules' => '']], - false - ); + + $config = new RuleTransformConfig(add: false); + $config->addColumn('test1-{@context/locales}', ['rules' => '']); + + $transform = new RuleTransformOperation($this->ruleApplierMock, $config); $data = new DataItem(['test' => 'test']); $data = $transform->process($data,$context); @@ -92,11 +96,10 @@ public function testMultipleDynamic() $context->setParameter('locales', ['fr_FR', 'en_GB']); $context->setParameter('scopes', ['master', 'mobile']); - $transform = new RuleTransformOperation( - $this->ruleApplierMock, - ['test1-{@context/scopes}-{@context/locales}' => ['rules' => '']], - false - ); + $config = new RuleTransformConfig(add: false); + $config->addColumn('test1-{@context/scopes}-{@context/locales}', ['rules' => '']); + + $transform = new RuleTransformOperation($this->ruleApplierMock, $config); $data = new DataItem(['test' => 'test']); $data = $transform->process($data,$context); @@ -111,14 +114,13 @@ public function testSimpleDynamic() { $this->ruleApplierMock->expects($this->exactly(1))->method('apply')->willReturn(1); - $context = clone $this->context; $context->setParameter('locales', ['fr_FR']); - $transform = new RuleTransformOperation( - $this->ruleApplierMock, - ['test1-{@context/locales}' => ['rules' => '']], - false - ); + + $config = new RuleTransformConfig(add: false); + $config->addColumn('test1-{@context/locales}', ['rules' => '']); + + $transform = new RuleTransformOperation($this->ruleApplierMock, $config); $data = new DataItem(['test' => 'test']); $data = $transform->process($data,$context); diff --git a/src/Oliverde8/Component/PhpEtl/Tests/ChainProcessorTest.php b/src/Oliverde8/Component/PhpEtl/Tests/ChainProcessorTest.php index eda7968..502f243 100644 --- a/src/Oliverde8/Component/PhpEtl/Tests/ChainProcessorTest.php +++ b/src/Oliverde8/Component/PhpEtl/Tests/ChainProcessorTest.php @@ -8,6 +8,9 @@ namespace Oliverde8\Component\PhpEtl\Tests; +use Oliverde8\Component\PhpEtl\ChainBuilderV2; +use Oliverde8\Component\PhpEtl\ChainConfig; +use Oliverde8\Component\PhpEtl\GenericChainFactory; use Oliverde8\Component\PhpEtl\Model\ExecutionContext; use Oliverde8\Component\PhpEtl\Model\File\LocalFileSystem; use Oliverde8\Component\PhpEtl\ChainOperation\ChainSplitOperation; @@ -23,6 +26,9 @@ use Oliverde8\Component\PhpEtl\Item\GroupedItem; use Oliverde8\Component\PhpEtl\Item\ItemInterface; use Oliverde8\Component\PhpEtl\Item\MixItem; +use Oliverde8\Component\PhpEtl\OperationConfig\ChainSplitConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Grouping\SimpleGroupingConfig; +use Oliverde8\Component\PhpEtl\OperationConfig\Transformer\CallBackTransformerConfig; use PHPUnit\Framework\TestCase; class ChainProcessorTest extends TestCase @@ -33,16 +39,16 @@ public function testSimpleChain() $count1 = 0; $count2 = 0; - $mock1 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$count1) { + $mock1 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$count1) { $count1++; return $item; - }); - $mock2 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$count2) { + })); + $mock2 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$count2) { $count2++; return $item; - }); + })); $chainProcessor = new ChainProcessor([$mock1, $mock2], new ExecutionContextFactory()); $chainProcessor->process(new \ArrayIterator([1,2]), ['toto']); @@ -56,16 +62,16 @@ public function testChainBreak() $count1 = 0; $count2 = 0; - $mock1 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$count1) { + $mock1 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$count1) { $count1++; return new ChainBreakItem(); - }); - $mock2 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$count2) { + })); + $mock2 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$count2) { $count2++; return $item; - }); + })); $chainProcessor = new ChainProcessor([$mock1, $mock2], new ExecutionContextFactory()); $chainProcessor->process(new \ArrayIterator([1,2]), ['toto']); @@ -79,12 +85,12 @@ public function testSingleGrouping() $count1 = 0; $count2 = 0; - $mock1 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$count1) { + $mock1 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$count1) { $count1++; return $item; - }); - $mock2 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$count2) { + })); + $mock2 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$count2) { $count2++; $data = $item->getData(); @@ -100,7 +106,7 @@ public function testSingleGrouping() } return $item; - }); + })); $data = [ ['group' => 100, 'id' => 200, 'value' => 'test01'], @@ -109,7 +115,7 @@ public function testSingleGrouping() ['group' => 101, 'id' => 201, 'value' => 'test12'], ]; - $chainProcessor = new ChainProcessor([$mock1, new SimpleGroupingOperation(['group'], ['id']), $mock2], new ExecutionContextFactory()); + $chainProcessor = new ChainProcessor([$mock1, new SimpleGroupingOperation(new SimpleGroupingConfig(['group'], ['id'])), $mock2], new ExecutionContextFactory()); $chainProcessor->process(new \ArrayIterator($data), ['toto']); $this->assertEquals(4, $count1); @@ -122,23 +128,23 @@ public function testDoubleGrouping() $count2 = 0; $count3 = 0; - $mock1 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$count1) { + $mock1 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$count1) { $count1++; return $item; - }); - $mock2 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$count2) { + })); + $mock2 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$count2) { $count2++; $data = ['value' => 0]; return new DataItem($data); - }); - $mock3 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$count3) { + })); + $mock3 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$count3) { $count3++; return $item; - }); + })); $data = [ ['group' => 100, 'id' => 200, 'value' => 'test01'], @@ -150,9 +156,9 @@ public function testDoubleGrouping() $chainProcessor = new ChainProcessor( [ $mock1, - new SimpleGroupingOperation(['group'], ['id']), + new SimpleGroupingOperation(new SimpleGroupingConfig(['group'], ['id'])), $mock2, - new SimpleGroupingOperation(['value']), + new SimpleGroupingOperation(new SimpleGroupingConfig(['value'])), $mock3, ], new ExecutionContextFactory() @@ -169,14 +175,14 @@ public function testMixItems() $values1 = []; $values2 = []; - $mock1 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$values1) { + $mock1 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$values1) { $values1[] = $item->getData(); return new MixItem([$item, new GroupedItem(new \ArrayIterator([3, 4])), new DataItem(5)]); - }); - $mock2 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$values2) { + })); + $mock2 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$values2) { $values2[] = $item->getData(); return $item; - }); + })); $chainProcessor = new ChainProcessor([$mock1, $mock2], new ExecutionContextFactory()); $chainProcessor->process(new \ArrayIterator([1,2]), ['toto']); @@ -190,18 +196,18 @@ public function testAsyncItems() $results = []; $callNum = 0; - $mock = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { + $mock = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$callNum) { $callNum++; if ($callNum == 1) { return new TestAsyncItem(new DataItem('I am slow'), 2); } else { return new TestAsyncItem(new DataItem('I am fast'), 1); } - }); - $mockEnd = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { + })); + $mockEnd = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$results) { $results[] = $item->getData(); return $item; - }); + })); $chainProcessor = new ChainProcessor(['mock' => $mock, 'mocked_end' => $mockEnd], new ExecutionContextFactory()); $chainProcessor->process(new \ArrayIterator([1, 2]), ['toto']); @@ -214,17 +220,15 @@ public function testReturnOfAsyncItems() { $callNum = 0; - $mock = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { + $mock = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$callNum) { $callNum++; if ($callNum == 1) { return new TestAsyncItem(new DataItem('I am slow'), 2); } else { return new TestAsyncItem(new DataItem('I am fast'), 1); } - }); - $mockEnd = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { - return $item; - }); + })); + $mockEnd = new CallbackTransformerOperation(new CallBackTransformerConfig(fn(ItemInterface $item) => $item)); $chainProcessor = new ChainProcessor(['mock' => $mock, 'mocked_end' => $mockEnd], new ExecutionContextFactory()); foreach ($chainProcessor->processGenerator(new \ArrayIterator([1, 2]), new ExecutionContext([], new LocalFileSystem())) as $item) { @@ -242,7 +246,7 @@ public function testMaxAsyncItems() $results = []; $callNum = 0; - $mock = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { + $mock = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$callNum) { $callNum++; if ($callNum == 1) { return new TestAsyncItem(new DataItem('I am slow1'), 2); @@ -251,11 +255,11 @@ public function testMaxAsyncItems() } else { return new TestAsyncItem(new DataItem('I am speed'), 1); } - }); - $mockEnd = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { + })); + $mockEnd = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$results) { $results[] = $item->getData(); return $item; - }); + })); $chainProcessor = new ChainProcessor( ['mock' => $mock, 'mocked_end' => $mockEnd], @@ -272,16 +276,14 @@ public function testAsyncItemsWithMixItem() { $results = []; - $mock = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { - return new MixItem([ - new TestAsyncItem(new DataItem('I am slow'), 2), - new TestAsyncItem(new DataItem('I am fast'), 1), - ]); - }); - $mockEnd = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { + $mock = new CallbackTransformerOperation(new CallBackTransformerConfig(fn(ItemInterface $item) => new MixItem([ + new TestAsyncItem(new DataItem('I am slow'), 2), + new TestAsyncItem(new DataItem('I am fast'), 1), + ]))); + $mockEnd = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$results) { $results[] = $item->getData(); return $item; - }); + })); $chainProcessor = new ChainProcessor( ['mock' => $mock, 'mocked_end' => $mockEnd], @@ -296,25 +298,41 @@ public function testAsyncItemsWithMixItem() public function testAsyncDisabledWithSplit() { $results = []; - $mockBranch1 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { - return new TestAsyncItem(new DataItem('I am slow'), 2); - }); - $mockBranch2 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$callNum) { - return new TestAsyncItem(new DataItem('I am fast'), 1); - }); - $mockEnd = new CallbackTransformerOperation(function (ItemInterface $item) use (&$results) { + $mockBranch1 = new CallbackTransformerOperation(new CallBackTransformerConfig(fn(ItemInterface $item) => new TestAsyncItem(new DataItem('I am slow'), 2))); + $mockBranch2 = new CallbackTransformerOperation(new CallBackTransformerConfig(fn(ItemInterface $item) => new TestAsyncItem(new DataItem('I am fast'), 1))); + $mockEnd = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$results) { $results[] = $item->getData(); return $item; - }); - + })); $executionContext = new ExecutionContextFactory(); + + // Extract configs from operations using reflection + $reflection = new \ReflectionClass(CallbackTransformerOperation::class); + $configProperty = $reflection->getProperty('config'); + $configProperty->setAccessible(true); + + // Create chain configs for each split branch + $branch1Config = new ChainConfig(); + $branch1Config->addLink($configProperty->getValue($mockBranch1)) + ->addLink($configProperty->getValue($mockEnd)); + + $branch2Config = new ChainConfig(); + $branch2Config->addLink($configProperty->getValue($mockBranch2)) + ->addLink($configProperty->getValue($mockEnd)); + + $splitConfig = new ChainSplitConfig(); + $splitConfig->addSplit($branch1Config)->addSplit($branch2Config); + + // Create ChainBuilderV2 with CallbackTransformer factory + $chainBuilder = new ChainBuilderV2( + $executionContext, + [new GenericChainFactory(CallbackTransformerOperation::class, CallBackTransformerConfig::class)] + ); + $chainProcessor = new ChainProcessor( [ - 'split' => new ChainSplitOperation([ - new ChainProcessor(["mock1" => $mockBranch1, 'mockEnd' => $mockEnd], $executionContext, 0), - new ChainProcessor(["mock2" => $mockBranch2, 'mockEnd' => $mockEnd], $executionContext, 0), - ]) + 'split' => new ChainSplitOperation($chainBuilder, $splitConfig) ], $executionContext ); @@ -326,9 +344,9 @@ public function testAsyncDisabledWithSplit() public function testException() { - $mock1 = new CallbackTransformerOperation(function (ItemInterface $item) use (&$count1) { + $mock1 = new CallbackTransformerOperation(new CallBackTransformerConfig(function (ItemInterface $item) use (&$count1): void { throw new \Exception('Test exception'); - }); + })); try { $chainProcessor = new ChainProcessor(["op1" => $mock1], new ExecutionContextFactory()); diff --git a/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestNonConfigurableOperation.php b/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestNonConfigurableOperation.php new file mode 100644 index 0000000..7dad74b --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestNonConfigurableOperation.php @@ -0,0 +1,10 @@ +config; + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithConfigAndChainBuilder.php b/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithConfigAndChainBuilder.php new file mode 100644 index 0000000..168aa1e --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithConfigAndChainBuilder.php @@ -0,0 +1,28 @@ +chainBuilder; + } + + public function getConfig(): OperationConfigInterface + { + return $this->config; + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithConfigAndInjection.php b/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithConfigAndInjection.php new file mode 100644 index 0000000..c4a08c0 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithConfigAndInjection.php @@ -0,0 +1,27 @@ +service; + } + + public function getConfig(): OperationConfigInterface + { + return $this->config; + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithDefaults.php b/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithDefaults.php new file mode 100644 index 0000000..1e91bb8 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithDefaults.php @@ -0,0 +1,22 @@ +optionalParam; + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithFlavor.php b/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithFlavor.php new file mode 100644 index 0000000..0284010 --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/Fixtures/TestOperationWithFlavor.php @@ -0,0 +1,22 @@ +flavor; + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/GenericChainFactoryTest.php b/src/Oliverde8/Component/PhpEtl/Tests/GenericChainFactoryTest.php new file mode 100644 index 0000000..070406c --- /dev/null +++ b/src/Oliverde8/Component/PhpEtl/Tests/GenericChainFactoryTest.php @@ -0,0 +1,247 @@ +chainBuilder = $this->createMock(ChainBuilderV2::class); + } + + public function testConstructorThrowsExceptionForNonConfigurableOperation() + { + $this->expectException(ChainBuilderException::class); + $this->expectExceptionMessage('must implement ConfigurableChainOperationInterface'); + + new GenericChainFactory( + TestNonConfigurableOperation::class, + TestOperationConfig::class + ); + } + + public function testSupportsReturnsTrueForMatchingConfigAndFlavor() + { + $factory = new GenericChainFactory( + TestOperationWithConfig::class, + TestOperationConfig::class, + 'default' + ); + + $config = new TestOperationConfig('default'); + + $this->assertTrue($factory->supports($config)); + } + + public function testSupportsReturnsFalseForDifferentConfigClass() + { + $factory = new GenericChainFactory( + TestOperationWithConfig::class, + TestOperationConfig::class, + 'default' + ); + + $config = $this->createMock(OperationConfigInterface::class); + $config->method('getFlavor')->willReturn('default'); + + $this->assertFalse($factory->supports($config)); + } + + public function testSupportsReturnsFalseForDifferentFlavor() + { + $factory = new GenericChainFactory( + TestOperationWithConfig::class, + TestOperationConfig::class, + 'custom' + ); + + $config = new TestOperationConfig('default'); + + $this->assertFalse($factory->supports($config)); + } + + public function testSupportsReturnsTrueForMatchingSubclass() + { + $factory = new GenericChainFactory( + TestOperationWithConfig::class, + OperationConfigInterface::class, + 'default' + ); + + $config = new TestOperationConfig('default'); + + $this->assertTrue($factory->supports($config)); + } + + public function testBuildCreatesOperationWithConfig() + { + $factory = new GenericChainFactory( + TestOperationWithConfig::class, + TestOperationConfig::class + ); + + $config = new TestOperationConfig('default'); + + $operation = $factory->build($config, $this->chainBuilder); + + $this->assertInstanceOf(TestOperationWithConfig::class, $operation); + $this->assertSame($config, $operation->getConfig()); + } + + public function testBuildInjectsChainBuilder() + { + $factory = new GenericChainFactory( + TestOperationWithConfigAndChainBuilder::class, + TestOperationConfig::class + ); + + $config = new TestOperationConfig('default'); + + $operation = $factory->build($config, $this->chainBuilder); + + $this->assertInstanceOf(TestOperationWithConfigAndChainBuilder::class, $operation); + $this->assertSame($this->chainBuilder, $operation->getChainBuilder()); + $this->assertSame($config, $operation->getConfig()); + } + + public function testBuildInjectsCustomDependencies() + { + $injectedService = new \stdClass(); + $injectedService->value = 'test'; + + $factory = new GenericChainFactory( + TestOperationWithConfigAndInjection::class, + TestOperationConfig::class, + injections: ['service' => $injectedService] + ); + + $config = new TestOperationConfig('default'); + + $operation = $factory->build($config, $this->chainBuilder); + + $this->assertInstanceOf(TestOperationWithConfigAndInjection::class, $operation); + $this->assertSame($injectedService, $operation->getService()); + } + + public function testBuildInjectsFlavor() + { + $factory = new GenericChainFactory( + TestOperationWithFlavor::class, + TestOperationConfig::class, + flavor: 'custom' + ); + + $config = new TestOperationConfig('custom'); + + $operation = $factory->build($config, $this->chainBuilder); + + $this->assertInstanceOf(TestOperationWithFlavor::class, $operation); + $this->assertEquals('custom', $operation->getFlavor()); + } + + public function testBuildUsesDefaultParameterValues() + { + $factory = new GenericChainFactory( + TestOperationWithDefaults::class, + TestOperationConfig::class + ); + + $config = new TestOperationConfig('default'); + + $operation = $factory->build($config, $this->chainBuilder); + + $this->assertInstanceOf(TestOperationWithDefaults::class, $operation); + $this->assertEquals('default_value', $operation->getOptionalParam()); + } + + public function testBuildThrowsExceptionForMissingParameter() + { + $factory = new GenericChainFactory( + TestOperationWithConfigAndInjection::class, + TestOperationConfig::class + ); + + $config = new TestOperationConfig('default'); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage("Missing parameter 'service'"); + + $factory->build($config, $this->chainBuilder); + } + + public function testBuildInjectsMultipleParameters() + { + $service1 = new \stdClass(); + $service2 = new \stdClass(); + + $factory = new GenericChainFactory( + TestOperationWithConfigAndInjection::class, + TestOperationConfig::class, + injections: ['service' => $service1, 'unused' => $service2] + ); + + $config = new TestOperationConfig('default'); + + $operation = $factory->build($config, $this->chainBuilder); + + $this->assertSame($service1, $operation->getService()); + } + + public function testBuildWithNoConstructor() + { + $factory = new GenericChainFactory( + TestOperationWithConfig::class, + TestOperationConfig::class + ); + + $config = new TestOperationConfig('default'); + + $operation = $factory->build($config, $this->chainBuilder); + + $this->assertInstanceOf(ConfigurableChainOperationInterface::class, $operation); + } + + public function testSupportsChecksFlavorMatch() + { + $factory = new GenericChainFactory( + TestOperationWithConfig::class, + TestOperationConfig::class, + 'production' + ); + + $defaultConfig = new TestOperationConfig('default'); + $productionConfig = new TestOperationConfig('production'); + + $this->assertFalse($factory->supports($defaultConfig)); + $this->assertTrue($factory->supports($productionConfig)); + } + + public function testConstructorValidatesOperationClass() + { + $this->expectException(ChainBuilderException::class); + + new GenericChainFactory( + 'NonExistentClass', + TestOperationConfig::class + ); + } +} + diff --git a/src/Oliverde8/Component/PhpEtl/Tests/Item/TestAsyncItem.php b/src/Oliverde8/Component/PhpEtl/Tests/Item/TestAsyncItem.php index 66b602c..3cb7c25 100644 --- a/src/Oliverde8/Component/PhpEtl/Tests/Item/TestAsyncItem.php +++ b/src/Oliverde8/Component/PhpEtl/Tests/Item/TestAsyncItem.php @@ -7,26 +7,21 @@ class TestAsyncItem implements AsyncItemInterface { - private ItemInterface $item; + private readonly int $startTime; - private int $runTime; - - private int $startTime; - - public function __construct(ItemInterface $item, int $runTime) + public function __construct(private readonly ItemInterface $item, private readonly int $runTime) { - $this->item = $item; - $this->runTime = $runTime; - $this->startTime = time(); } + #[\Override] public function isRunning(): bool { return $this->startTime + $this->runTime > time(); } + #[\Override] public function getItem(): ItemInterface { return $this->item; diff --git a/src/Oliverde8/Component/RuleEngine/RuleApplier.php b/src/Oliverde8/Component/RuleEngine/RuleApplier.php index 0e7769a..253ecf4 100644 --- a/src/Oliverde8/Component/RuleEngine/RuleApplier.php +++ b/src/Oliverde8/Component/RuleEngine/RuleApplier.php @@ -18,13 +18,9 @@ */ class RuleApplier { - protected LoggerInterface $logger; - /** @var RuleInterface[] */ protected array $rules; - protected bool $validate; - protected array $currentIdentity; /** @@ -35,11 +31,8 @@ class RuleApplier * @param boolean $validate When validation is active rules are validated and clear messages are displayed * when a rule is not well coded. But performance is worsened. */ - public function __construct(LoggerInterface $logger, array $rules, bool $validate = false) + public function __construct(protected LoggerInterface $logger, array $rules, protected bool $validate = false) { - $this->logger = $logger; - $this->validate = $validate; - foreach ($rules as $rule) { $this->registerRule($rule); } diff --git a/src/Oliverde8/Component/RuleEngine/Rules/AbstractRule.php b/src/Oliverde8/Component/RuleEngine/Rules/AbstractRule.php index ba4630b..b25c6d1 100644 --- a/src/Oliverde8/Component/RuleEngine/Rules/AbstractRule.php +++ b/src/Oliverde8/Component/RuleEngine/Rules/AbstractRule.php @@ -19,16 +19,13 @@ abstract class AbstractRule implements RuleInterface { protected RuleApplier $ruleApplier; - protected LoggerInterface $logger; - /** * AbstractRule constructor. * * @param LoggerInterface $logger Standard logger to log information. */ - public function __construct(LoggerInterface $logger) + public function __construct(protected LoggerInterface $logger) { - $this->logger = $logger; } /** @@ -37,6 +34,7 @@ public function __construct(LoggerInterface $logger) * @param RuleApplier $ruleApplier The rule applier that is using this rule. * @return $this */ + #[\Override] public function setApplier(RuleApplier $ruleApplier): self { $this->ruleApplier = $ruleApplier; diff --git a/src/Oliverde8/Component/RuleEngine/Rules/Constant.php b/src/Oliverde8/Component/RuleEngine/Rules/Constant.php index d759d69..e370306 100644 --- a/src/Oliverde8/Component/RuleEngine/Rules/Constant.php +++ b/src/Oliverde8/Component/RuleEngine/Rules/Constant.php @@ -25,6 +25,7 @@ class Constant extends AbstractRule * * @return string|null */ + #[\Override] public function apply(array $rowData, array &$transformedData, array $options = []) { return $options['value']; @@ -33,6 +34,7 @@ public function apply(array $rowData, array &$transformedData, array $options = /** * @inheritdoc */ + #[\Override] public function validate($options): void { $this->requireOption('value', $options); @@ -43,6 +45,7 @@ public function validate($options): void * * @return string */ + #[\Override] public function getRuleCode(): string { return 'constant'; diff --git a/src/Oliverde8/Component/RuleEngine/Rules/ExpressionLanguage.php b/src/Oliverde8/Component/RuleEngine/Rules/ExpressionLanguage.php index 1f2de5e..3e2eb72 100644 --- a/src/Oliverde8/Component/RuleEngine/Rules/ExpressionLanguage.php +++ b/src/Oliverde8/Component/RuleEngine/Rules/ExpressionLanguage.php @@ -16,6 +16,7 @@ class ExpressionLanguage extends AbstractRule /** * @inheritdoc */ + #[\Override] public function apply(array $rowData, array &$transformedData, array $options = []) { $values = [ @@ -40,6 +41,7 @@ public function apply(array $rowData, array &$transformedData, array $options = * * @return string */ + #[\Override] public function getRuleCode(): string { return 'expression_language'; @@ -49,6 +51,7 @@ public function getRuleCode(): string /** * @inheritdoc */ + #[\Override] public function validate(array $options): void { $this->requireOption('expression', $options); diff --git a/src/Oliverde8/Component/RuleEngine/Rules/Get.php b/src/Oliverde8/Component/RuleEngine/Rules/Get.php index dc490c0..9d3af1b 100644 --- a/src/Oliverde8/Component/RuleEngine/Rules/Get.php +++ b/src/Oliverde8/Component/RuleEngine/Rules/Get.php @@ -18,6 +18,7 @@ class Get extends AbstractRule /** * @inheritdoc */ + #[\Override] public function apply(array $rowData, array &$transformedData, array $options = []) { if (!is_array($options['field'])) { @@ -35,6 +36,7 @@ public function apply(array $rowData, array &$transformedData, array $options = /** * @inheritdoc */ + #[\Override] public function validate(array $options): void { $this->requireOption('field', $options); @@ -43,6 +45,7 @@ public function validate(array $options): void /** * @inheritdoc */ + #[\Override] public function getRuleCode(): string { return 'get'; diff --git a/src/Oliverde8/Component/RuleEngine/Rules/Implode.php b/src/Oliverde8/Component/RuleEngine/Rules/Implode.php index 42c6bf1..30e6a53 100644 --- a/src/Oliverde8/Component/RuleEngine/Rules/Implode.php +++ b/src/Oliverde8/Component/RuleEngine/Rules/Implode.php @@ -16,6 +16,7 @@ class Implode extends AbstractRule /** * @inheritdoc */ + #[\Override] public function apply(array $rowData, array &$transformedData, array $options = []) { $subOptions = $options; @@ -63,6 +64,7 @@ protected function flatten(array $array): \Generator /** * @inheritdoc */ + #[\Override] public function validate(array $options): void { $this->requireOption('values', $options); @@ -72,6 +74,7 @@ public function validate(array $options): void /** * @inheritdoc */ + #[\Override] public function getRuleCode(): string { return 'implode'; diff --git a/src/Oliverde8/Component/RuleEngine/Rules/StrToLower.php b/src/Oliverde8/Component/RuleEngine/Rules/StrToLower.php index 2d79ca0..fc453d0 100644 --- a/src/Oliverde8/Component/RuleEngine/Rules/StrToLower.php +++ b/src/Oliverde8/Component/RuleEngine/Rules/StrToLower.php @@ -16,17 +16,19 @@ class StrToLower extends AbstractRule /** * @inheritdoc */ + #[\Override] public function apply(array $rowData, array &$transformedData, array $options = []): string { $value = $options['value']; unset($options['value']); - return strtolower($this->ruleApplier->apply($rowData, $transformedData, $value, $options)); + return strtolower((string) $this->ruleApplier->apply($rowData, $transformedData, $value, $options)); } /** * @inheritdoc */ + #[\Override] public function validate(array $options): void { $this->requireOption('value', $options); @@ -35,6 +37,7 @@ public function validate(array $options): void /** * @inheritdoc */ + #[\Override] public function getRuleCode(): string { return 'str_lower'; diff --git a/src/Oliverde8/Component/RuleEngine/Rules/StrToUpper.php b/src/Oliverde8/Component/RuleEngine/Rules/StrToUpper.php index dd8f0a8..1d78365 100644 --- a/src/Oliverde8/Component/RuleEngine/Rules/StrToUpper.php +++ b/src/Oliverde8/Component/RuleEngine/Rules/StrToUpper.php @@ -18,17 +18,19 @@ class StrToUpper extends AbstractRule /** * @inheritdoc */ + #[\Override] public function apply(array $rowData, array &$transformedData, array $options = []): string { $value = $options['value']; unset($options['value']); - return strtoupper($this->ruleApplier->apply($rowData, $transformedData, $value, $options)); + return strtoupper((string) $this->ruleApplier->apply($rowData, $transformedData, $value, $options)); } /** * @inheritdoc */ + #[\Override] public function validate(array $options): void { $this->requireOption('value', $options); @@ -37,6 +39,7 @@ public function validate(array $options): void /** * @inheritdoc */ + #[\Override] public function getRuleCode(): string { return 'str_upper'; diff --git a/src/Oliverde8/Component/RuleEngine/Tests/RuleApplierTest.php b/src/Oliverde8/Component/RuleEngine/Tests/RuleApplierTest.php index 02e3697..351f543 100644 --- a/src/Oliverde8/Component/RuleEngine/Tests/RuleApplierTest.php +++ b/src/Oliverde8/Component/RuleEngine/Tests/RuleApplierTest.php @@ -33,6 +33,7 @@ class RuleApplierTest extends TestCase /** * @inheritdoc */ + #[\Override] protected function setUp(): void { parent::setUp(); diff --git a/src/Oliverde8/Component/RuleEngine/Tests/Rules/AbstractRule.php b/src/Oliverde8/Component/RuleEngine/Tests/Rules/AbstractRule.php index 66d9b1e..1e98c78 100644 --- a/src/Oliverde8/Component/RuleEngine/Tests/Rules/AbstractRule.php +++ b/src/Oliverde8/Component/RuleEngine/Tests/Rules/AbstractRule.php @@ -19,6 +19,7 @@ abstract class AbstractRule extends TestCase /** @var RuleInterface */ protected $rule; + #[\Override] protected function setUp(): void { parent::setUp(); diff --git a/src/Oliverde8/Component/RuleEngine/Tests/Rules/ConstantTest.php b/src/Oliverde8/Component/RuleEngine/Tests/Rules/ConstantTest.php index 4aaf8fa..4774240 100644 --- a/src/Oliverde8/Component/RuleEngine/Tests/Rules/ConstantTest.php +++ b/src/Oliverde8/Component/RuleEngine/Tests/Rules/ConstantTest.php @@ -20,6 +20,7 @@ public function testConstants() $this->assertRuleResults(['test' => 'toto 1'], [], ['value' => ['1' => ['toto']]], ['1' => ['toto']]); } + #[\Override] protected function getRule() { return new Constant(new NullLogger()); diff --git a/src/Oliverde8/Component/RuleEngine/Tests/Rules/GetTest.php b/src/Oliverde8/Component/RuleEngine/Tests/Rules/GetTest.php index d9beae7..f4ce75b 100644 --- a/src/Oliverde8/Component/RuleEngine/Tests/Rules/GetTest.php +++ b/src/Oliverde8/Component/RuleEngine/Tests/Rules/GetTest.php @@ -33,6 +33,7 @@ public function testUnfound() /** * @inheritdoc */ + #[\Override] protected function getRule() { return new Get(new NullLogger()); diff --git a/src/Oliverde8/Component/RuleEngine/Tests/Rules/ImplodeTest.php b/src/Oliverde8/Component/RuleEngine/Tests/Rules/ImplodeTest.php index be2f644..1d149de 100644 --- a/src/Oliverde8/Component/RuleEngine/Tests/Rules/ImplodeTest.php +++ b/src/Oliverde8/Component/RuleEngine/Tests/Rules/ImplodeTest.php @@ -52,6 +52,7 @@ public function testFourLevelImplode() /** * @inheritdoc */ + #[\Override] function getRule() { return new Implode(new NullLogger()); diff --git a/src/Oliverde8/Component/RuleEngine/Tests/Rules/StrToLowerTest.php b/src/Oliverde8/Component/RuleEngine/Tests/Rules/StrToLowerTest.php index eb3695e..aa20cb1 100644 --- a/src/Oliverde8/Component/RuleEngine/Tests/Rules/StrToLowerTest.php +++ b/src/Oliverde8/Component/RuleEngine/Tests/Rules/StrToLowerTest.php @@ -25,6 +25,7 @@ public function testStrToLower() /** * @inheritdoc */ + #[\Override] protected function getRule() { return new StrToLower(new NullLogger()); diff --git a/src/Oliverde8/Component/RuleEngine/Tests/Rules/StrToUpperTest.php b/src/Oliverde8/Component/RuleEngine/Tests/Rules/StrToUpperTest.php index 85c98fc..8a1ee8b 100644 --- a/src/Oliverde8/Component/RuleEngine/Tests/Rules/StrToUpperTest.php +++ b/src/Oliverde8/Component/RuleEngine/Tests/Rules/StrToUpperTest.php @@ -25,6 +25,7 @@ public function testStrToUpper() /** * @inheritdoc */ + #[\Override] protected function getRule() { return new StrToUpper(new NullLogger()); diff --git a/src/Oliverde8/Component/RuleEngine/Tests/Rules/TestRuleApplier.php b/src/Oliverde8/Component/RuleEngine/Tests/Rules/TestRuleApplier.php index b28dd59..1805f1d 100644 --- a/src/Oliverde8/Component/RuleEngine/Tests/Rules/TestRuleApplier.php +++ b/src/Oliverde8/Component/RuleEngine/Tests/Rules/TestRuleApplier.php @@ -14,6 +14,7 @@ */ class TestRuleApplier extends RuleApplier { + #[\Override] public function apply($rowData, $transformedData, $rules, $options = [], $identifier = []) { return $rules;