diff --git a/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java b/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java index 23d19ebc0af..5600a65210d 100644 --- a/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java +++ b/engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java @@ -1417,6 +1417,7 @@ public void handlePutRowTo(IRowMeta rowMeta, Object[] row, IRowSet rowSet) try { Thread.sleep(1); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new HopTransformException(e); } } @@ -1523,7 +1524,7 @@ public void handlePutError( } } incrementLinesRejected(); - if (!Utils.isEmpty(errorDescriptions)) { + if (isLoggingErrorDescriptions() && !Utils.isEmpty(errorDescriptions)) { logError(errorDescriptions); } } else if (transformErrorMeta.isEnabled()) { @@ -2723,6 +2724,19 @@ public void logRowlevel(String message, Object... arguments) { log.logRowlevel(message, arguments); } + /** + * Whether rejected-row error descriptions should be written to the log. + * + *

Transforms that can produce a very large number of validation/rejection errors may override + * this to reduce log volume. Error rows are still sent to the error handling hop when configured. + * + * @return true when error descriptions should be logged (default) + * @since 2.19.0 + */ + protected boolean isLoggingErrorDescriptions() { + return true; + } + /** * Log error. * diff --git a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/Validator.java b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/Validator.java index f4257dde018..191fdcc4c8b 100644 --- a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/Validator.java +++ b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/Validator.java @@ -47,6 +47,9 @@ public class Validator extends BaseTransform implements ITransform { private static final Class PKG = ValidatorMeta.class; + /** Placeholder used in log/error messages when failed data must not be exposed. */ + static final String OMITTED_VALUE = "?"; + public Validator( TransformMeta transformMeta, ValidatorMeta meta, @@ -279,7 +282,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.NullNotAllowed", field.getFieldName(), - inputRowMeta.getString(r)), + getRowForMessage(inputRowMeta, r)), field.getFieldName()); exceptions.add(exception); if (!meta.isValidatingAll()) { @@ -297,7 +300,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.OnlyNullAllowed", field.getFieldName(), - inputRowMeta.getString(r)), + getRowForMessage(inputRowMeta, r)), field.getFieldName()); exceptions.add(exception); if (!meta.isValidatingAll()) { @@ -367,7 +370,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.ShorterThanMininumLength", field.getFieldName(), - valueMeta.getString(valueData), + getValueForMessage(valueMeta, valueData), Integer.toString(stringValue.length()), field.getMinimumLength()), field.getFieldName()); @@ -390,7 +393,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.LongerThanMaximumLength", field.getFieldName(), - valueMeta.getString(valueData), + getValueForMessage(valueMeta, valueData), Integer.toString(stringValue.length()), field.getMaximumLength()), field.getFieldName()); @@ -413,7 +416,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.LowerThanMinimumValue", field.getFieldName(), - valueMeta.getString(valueData), + getValueForMessage(valueMeta, valueData), data.constantsMeta[i].getString(data.minimumValue[i])), field.getFieldName()); exceptions.add(exception); @@ -435,7 +438,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.HigherThanMaximumValue", field.getFieldName(), - valueMeta.getString(valueData), + getValueForMessage(valueMeta, valueData), data.constantsMeta[i].getString(data.maximumValue[i])), field.getFieldName()); exceptions.add(exception); @@ -465,7 +468,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.NotInList", field.getFieldName(), - valueMeta.getString(valueData)), + getValueForMessage(valueMeta, valueData)), field.getFieldName()); exceptions.add(exception); if (!meta.isValidatingAll()) { @@ -498,7 +501,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.DoesNotStartWithString", field.getFieldName(), - valueMeta.getString(valueData), + getValueForMessage(valueMeta, valueData), field.getStartString()), field.getFieldName()); exceptions.add(exception); @@ -519,7 +522,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.DoesNotEndWithString", field.getFieldName(), - valueMeta.getString(valueData), + getValueForMessage(valueMeta, valueData), field.getEndString()), field.getFieldName()); exceptions.add(exception); @@ -541,7 +544,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.StartsWithString", field.getFieldName(), - valueMeta.getString(valueData), + getValueForMessage(valueMeta, valueData), field.getStartStringNotAllowed()), field.getFieldName()); exceptions.add(exception); @@ -563,7 +566,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.EndsWithString", field.getFieldName(), - valueMeta.getString(valueData), + getValueForMessage(valueMeta, valueData), field.getEndStringNotAllowed()), field.getFieldName()); exceptions.add(exception); @@ -586,7 +589,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.MatchingRegExpExpected", field.getFieldName(), - valueMeta.getString(valueData), + getValueForMessage(valueMeta, valueData), data.regularExpression[i]), field.getFieldName()); exceptions.add(exception); @@ -610,7 +613,7 @@ private List validateFields(IRowMeta inputRowMeta, Object PKG, "Validator.Exception.MatchingRegExpNotAllowed", field.getFieldName(), - valueMeta.getString(valueData), + getValueForMessage(valueMeta, valueData), data.regularExpressionNotAllowed[i]), field.getFieldName()); exceptions.add(exception); @@ -625,6 +628,28 @@ private List validateFields(IRowMeta inputRowMeta, Object return exceptions; } + /** + * Returns the field value for inclusion in validation messages, or a placeholder when logging of + * failed data is suppressed. + */ + String getValueForMessage(IValueMeta valueMeta, Object valueData) throws HopValueException { + if (meta.isSuppressingLogFailedData()) { + return OMITTED_VALUE; + } + return valueMeta.getString(valueData); + } + + /** + * Returns the row for inclusion in validation messages, or a placeholder when logging of failed + * data is suppressed. + */ + String getRowForMessage(IRowMeta inputRowMeta, Object[] r) throws HopValueException { + if (meta.isSuppressingLogFailedData()) { + return OMITTED_VALUE; + } + return inputRowMeta.getString(r); + } + // package-local visibility for testing purposes HopValidatorException assertNumeric(IValueMeta valueMeta, Object valueData, Validation field) throws HopValueException { @@ -640,7 +665,7 @@ HopValidatorException assertNumeric(IValueMeta valueMeta, Object valueData, Vali "Validator.Exception.NonNumericDataNotAllowed", field.getFieldName(), valueMeta.toStringMeta(), - valueMeta.getString(valueData)), + getValueForMessage(valueMeta, valueData)), field.getFieldName()); } diff --git a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialog.java b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialog.java index c6cbf240341..861318cb156 100644 --- a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialog.java +++ b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialog.java @@ -113,6 +113,7 @@ public class ValidatorDialog extends BaseTransformDialog { private TextVar wErrorDescription; private Button wConcatErrors; private TextVar wConcatSeparator; + private Button wSuppressLogFailedData; public ValidatorDialog( Shell parent, IVariables variables, ValidatorMeta transformMeta, PipelineMeta pipelineMeta) { @@ -207,6 +208,20 @@ public String open() { fdConcatSeparator.top = new FormAttachment(wValidateAll, margin); wConcatSeparator.setLayoutData(fdConcatSeparator); + // Optionally suppress logging of failed validation data (reduces log volume) + // + wSuppressLogFailedData = new Button(shell, SWT.CHECK); + wSuppressLogFailedData.setText( + BaseMessages.getString(PKG, "ValidatorDialog.SuppressLogFailedData.Label")); + wSuppressLogFailedData.setToolTipText( + BaseMessages.getString(PKG, "ValidatorDialog.SuppressLogFailedData.Tooltip")); + PropsUi.setLook(wSuppressLogFailedData); + FormData fdSuppressLogFailedData = new FormData(); + fdSuppressLogFailedData.left = new FormAttachment(middle, 0); + fdSuppressLogFailedData.right = new FormAttachment(100, 0); + fdSuppressLogFailedData.top = new FormAttachment(wConcatErrors, margin); + wSuppressLogFailedData.setLayoutData(fdSuppressLogFailedData); + // Create a scrolled composite on the right side... // ScrolledComposite wSComp = new ScrolledComposite(shell, SWT.H_SCROLL | SWT.V_SCROLL); @@ -214,7 +229,7 @@ public String open() { wSComp.setLayout(new FillLayout()); FormData fdComp = new FormData(); fdComp.left = new FormAttachment(middle / 2, margin); - fdComp.top = new FormAttachment(wConcatSeparator, margin); + fdComp.top = new FormAttachment(wSuppressLogFailedData, margin); fdComp.right = new FormAttachment(100, -margin); fdComp.bottom = new FormAttachment(wOk, -margin); // Limit viewport size so the dialog opens at a reasonable size; content scrolls inside. @@ -344,8 +359,8 @@ public String open() { PropsUi.setLook(wgType); wgType.setText(BaseMessages.getString(PKG, "ValidatorDialog.TypeGroup.Label")); FormLayout typeGroupLayout = new FormLayout(); - typeGroupLayout.marginHeight = Const.FORM_MARGIN; - typeGroupLayout.marginWidth = Const.FORM_MARGIN; + typeGroupLayout.marginHeight = PropsUi.getFormMargin(); + typeGroupLayout.marginWidth = PropsUi.getFormMargin(); wgType.setLayout(typeGroupLayout); FormData fdType = new FormData(); fdType.left = new FormAttachment(0, 0); @@ -893,10 +908,6 @@ private void saveChanges() { selectedField.setSourcingField(wSourceField.getText()); selectedField.setSourcingTransformName(wSourceTransform.getText()); selectedField.setSourcingTransform(pipelineMeta.findTransform(wSourceTransform.getText())); - - // Save the old info in the map - // - // selectionList.add(selectedField); } } @@ -1044,11 +1055,12 @@ public void getData() { wValidateAll.setSelection(input.isValidatingAll()); wConcatErrors.setSelection(input.isConcatenatingErrors()); wConcatSeparator.setText(Const.NVL(input.getConcatenationSeparator(), "")); + wSuppressLogFailedData.setSelection(input.isSuppressingLogFailedData()); // Select the first available field... // if (!input.getValidations().isEmpty()) { - Validation validatorField = input.getValidations().get(0); + Validation validatorField = input.getValidations().getFirst(); String description = validatorField.getName(); int index = wValidationsList.indexOf(description); if (index >= 0) { @@ -1085,6 +1097,7 @@ private void ok() { input.setValidatingAll(wValidateAll.getSelection()); input.setConcatenatingErrors(wConcatErrors.getSelection()); input.setConcatenationSeparator(wConcatSeparator.getText()); + input.setSuppressingLogFailedData(wSuppressLogFailedData.getSelection()); input.setValidations(selectionList); @@ -1169,17 +1182,37 @@ private void newValidation() { /** Clear the validation rules for a certain field. */ private void clearValidation() { - int index = wValidationsList.getSelectionIndex(); - if (index >= 0) { - selectionList.remove(index); - selectedField = null; - wValidationsList.remove(index); + if (index < 0) { + return; + } + + selectionList.remove(index); + selectedField = null; + wValidationsList.remove(index); + + if (selectionList.isEmpty()) { enableFields(); + return; + } - if (!selectionList.isEmpty()) { - wValidationsList.select(selectionList.size() - 1); - } + // Keep showing the nearest remaining rule (same index, or previous when deleting the last) + int newIndex = indexAfterRemoval(index, selectionList.size()); + wValidationsList.select(newIndex); + showSelectedValidatorField(wValidationsList.getItem(newIndex)); + } + + /** + * Calculate which list index to select after removing an item. + * + * @param removedIndex the index that was removed + * @param remainingCount the number of items left after removal + * @return the index to select, or -1 when nothing remains + */ + static int indexAfterRemoval(int removedIndex, int remainingCount) { + if (remainingCount <= 0) { + return -1; } + return Math.min(removedIndex, remainingCount - 1); } } diff --git a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorMeta.java b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorMeta.java index c6cf5813ac0..c8ccb6407a3 100644 --- a/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorMeta.java +++ b/plugins/transforms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorMeta.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.List; +import lombok.Getter; +import lombok.Setter; import org.apache.hop.core.CheckResult; import org.apache.hop.core.Const; import org.apache.hop.core.ICheckResult; @@ -38,6 +40,8 @@ import org.apache.hop.pipeline.transform.stream.Stream; import org.apache.hop.pipeline.transform.stream.StreamIcon; +@Getter +@Setter @Transform( id = "Validator", name = "i18n::ValidatorDialog.Transform.Name", @@ -76,6 +80,27 @@ public class ValidatorMeta extends BaseTransformMeta { injectionKeyDescription = "Validator.Injection.CONCATENATION_SEPARATOR") private String concatenationSeparator; + /** + * When true, failed field/row values are omitted from validation error messages. Errors are still + * logged and error rows are still produced for error handling. Defaults to false so existing + * pipelines keep including failed values in messages. + */ + @HopMetadataProperty( + key = "suppress_log_failed_data", + injectionKey = "SUPPRESS_LOG_FAILED_DATA", + injectionKeyDescription = "Validator.Injection.SUPPRESS_LOG_FAILED_DATA") + private boolean suppressingLogFailedData; + + /** The standard new validation stream */ + @Getter @Setter + private static IStream newValidation = + new Stream( + IStream.StreamType.INFO, + null, + BaseMessages.getString(PKG, "ValidatorMeta.NewValidation.Description"), + StreamIcon.INFO, + null); + public ValidatorMeta() { this.validations = new ArrayList<>(); } @@ -86,6 +111,7 @@ public ValidatorMeta(ValidatorMeta m) { this.validatingAll = m.validatingAll; this.concatenatingErrors = m.concatenatingErrors; this.concatenationSeparator = m.concatenationSeparator; + this.suppressingLogFailedData = m.suppressingLogFailedData; } @Override @@ -170,7 +196,7 @@ public ITransformIOMeta getTransformIOMeta() { } @Override - public void searchInfoAndTargetTransforms(List transforms) { + public void searchInfoAndTargetTransforms(List transforms) { for (Validation validation : validations) { TransformMeta transformMeta = TransformMeta.findTransform(transforms, validation.getSourcingTransformName()); @@ -179,15 +205,6 @@ public void searchInfoAndTargetTransforms(List transforms) { resetTransformIoMeta(); } - /** The standard new validation stream */ - private static IStream newValidation = - new Stream( - IStream.StreamType.INFO, - null, - BaseMessages.getString(PKG, "ValidatorMeta.NewValidation.Description"), - StreamIcon.INFO, - null); - @Override public void handleStreamSelection(IStream stream) { // A hack to prevent us from losing information in the UI because @@ -212,94 +229,4 @@ public void handleStreamSelection(IStream stream) { // Force the IO to be recreated when it is next needed. resetTransformIoMeta(); } - - /** - * Gets validations - * - * @return value of validations - */ - public List getValidations() { - return validations; - } - - /** - * Sets validations - * - * @param validations value of validations - */ - public void setValidations(List validations) { - this.validations = validations; - } - - /** - * Gets validatingAll - * - * @return value of validatingAll - */ - public boolean isValidatingAll() { - return validatingAll; - } - - /** - * Sets validatingAll - * - * @param validatingAll value of validatingAll - */ - public void setValidatingAll(boolean validatingAll) { - this.validatingAll = validatingAll; - } - - /** - * Gets concatenatingErrors - * - * @return value of concatenatingErrors - */ - public boolean isConcatenatingErrors() { - return concatenatingErrors; - } - - /** - * Sets concatenatingErrors - * - * @param concatenatingErrors value of concatenatingErrors - */ - public void setConcatenatingErrors(boolean concatenatingErrors) { - this.concatenatingErrors = concatenatingErrors; - } - - /** - * Gets concatenationSeparator - * - * @return value of concatenationSeparator - */ - public String getConcatenationSeparator() { - return concatenationSeparator; - } - - /** - * Sets concatenationSeparator - * - * @param concatenationSeparator value of concatenationSeparator - */ - public void setConcatenationSeparator(String concatenationSeparator) { - this.concatenationSeparator = concatenationSeparator; - } - - /** - * Gets newValidation - * - * @return value of newValidation - */ - public static IStream getNewValidation() { - return newValidation; - } - - /** - * Sets newValidation - * - * @param newValidation value of newValidation - */ - public static void setNewValidation(IStream newValidation) { - ValidatorMeta.newValidation = newValidation; - } } diff --git a/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_en_US.properties b/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_en_US.properties index b79d97aa41a..f81ecbec9ab 100644 --- a/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_en_US.properties +++ b/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_en_US.properties @@ -38,6 +38,7 @@ Validator.Exception.StartsWithString=During validation of field ''{0}'' we found Validator.Exception.UnexpectedDataType=During validation of field ''{0}'' we found that its data type [{1}] is different from the expected [{2}]. Validator.Injection.CONCATENATE_ERRORS=This option causes all errors to be combined into a single output row. Validator.Injection.CONCATENATION_SEPARATOR=Specify the error separator when the errors are in one output row. +Validator.Injection.SUPPRESS_LOG_FAILED_DATA=When enabled, failed field/row values are omitted from validation error messages written to the log. Error rows are still sent to the error handling hop. Validator.Injection.CONVERSION_MASK=Specify the mask to use to convert the data specified in this validation rule. Validator.Injection.DATA_TYPE=Specify the data type to verify. Validator.Injection.DATA_TYPE_VERIFIED=This option causes the specified data type to be verified. @@ -70,6 +71,8 @@ ValidatorDialog.ButtonAddAllowed.Label=\ Add ValidatorDialog.ButtonRemoveAllowed.Label=\ Remove ValidatorDialog.ClearButton.Label=Remove validation ValidatorDialog.ConcatErrors.Label=Output one row, concatenate errors with separator +ValidatorDialog.SuppressLogFailedData.Label=Do not log failed validation data +ValidatorDialog.SuppressLogFailedData.Tooltip=When enabled, failed field/row values are omitted from validation error messages (shown as "?"). Error messages are still logged. Error rows are still sent to the error handling hop when configured. ValidatorDialog.ConversionMask.Label=Conversion mask ValidatorDialog.DataGroup.Label=Data ValidatorDialog.DataType.Label=Data type diff --git a/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_zh_CN.properties b/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_zh_CN.properties index 9f7a1300795..f590eb1006d 100644 --- a/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_zh_CN.properties +++ b/plugins/transforms/validator/src/main/resources/org/apache/hop/pipeline/transforms/validator/messages/messages_zh_CN.properties @@ -23,3 +23,6 @@ Validator.Exception.DataConversionErrorEncountered=\u5728\u8FDB\u884C\u9A8C\u8BC Validator.Exception.DoesNotEndWithString=\u5728\u9A8C\u8BC1\u5B57\u6BB5 ''{0}'' \u65F6\uFF0C\u6211\u4EEC\u53D1\u73B0\u5B57\u7B26\u4E32\u503C [{1}] \u5E76\u672A\u4EE5\u5B57\u7B26\u4E32 [{2}] \u7ED3\u5C3E Validator.Exception.DoesNotStartWithString=\u5728\u9A8C\u8BC1\u5B57\u6BB5 ''{0}'' \u65F6\uFF0C\u5B57\u7B26\u4E32\u503C [{1}] \u5E76\u672A\u4EE5\u5B57\u7B26\u4E32 [{2}] \u5F00\u5934 Validator.Exception.EndsWithString=\u5728\u9A8C\u8BC1\u5B57\u6BB5\u201C{0}\u201D\u65F6\uFF0C\u6211\u4EEC\u53D1\u73B0\u5B57\u7B26\u4E32\u503C [{1}] \u4EE5\u5B57\u7B26\u4E32 [{2}] \u7ED3\u5C3E +Validator.Injection.SUPPRESS_LOG_FAILED_DATA=\u542F\u7528\u540E\uFF0C\u5199\u5165\u65E5\u5FD7\u7684\u6821\u9A8C\u9519\u8BEF\u6D88\u606F\u4E2D\u4E0D\u5305\u542B\u5931\u8D25\u5B57\u6BB5/\u884C\u6570\u636E\u3002\u9519\u8BEF\u884C\u4ECD\u4F1A\u53D1\u9001\u5230\u9519\u8BEF\u5904\u7406\u8DEF\u5F84\u3002 +ValidatorDialog.SuppressLogFailedData.Label=\u4E0D\u8BB0\u5F55\u6821\u9A8C\u5931\u8D25\u7684\u6570\u636E +ValidatorDialog.SuppressLogFailedData.Tooltip=\u542F\u7528\u540E\uFF0C\u6821\u9A8C\u9519\u8BEF\u6D88\u606F\u4E2D\u7684\u5931\u8D25\u5B57\u6BB5/\u884C\u6570\u636E\u4F1A\u88AB\u7701\u7565\uFF08\u663E\u793A\u4E3A\u201C?\u201D\uFF09\uFF0C\u4F46\u4ECD\u4F1A\u6253\u5370\u9519\u8BEF\u65E5\u5FD7\u3002\u82E5\u914D\u7F6E\u4E86\u9519\u8BEF\u5904\u7406\uFF0C\u9519\u8BEF\u884C\u4ECD\u4F1A\u53D1\u9001\u5230\u9519\u8BEF\u8DEF\u5F84\u3002 diff --git a/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/HopValidatorExceptionTest.java b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/HopValidatorExceptionTest.java new file mode 100644 index 00000000000..fa8cd4d77af --- /dev/null +++ b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/HopValidatorExceptionTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.validator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Unit tests for {@link HopValidatorException}. */ +class HopValidatorExceptionTest { + + private Validator validator; + + @BeforeEach + void setUp() { + validator = mock(Validator.class); + when(validator.resolve(anyString())).thenAnswer(invocation -> invocation.getArgument(0)); + } + + @Test + void exposesCodeFieldNameAndValidation() { + Validation field = new Validation(); + field.setName("rule"); + HopValidatorException exception = + new HopValidatorException( + validator, + field, + HopValidatorException.ERROR_NULL_VALUE_NOT_ALLOWED, + "null not allowed", + "value"); + + assertEquals(HopValidatorException.ERROR_NULL_VALUE_NOT_ALLOWED, exception.getCode()); + assertEquals("value", exception.getFieldName()); + assertSame(field, exception.getValidatorField()); + assertTrue(exception.getMessage().contains("null not allowed")); + } + + @Test + void getCodeDescUsesBuiltInCodeWhenNoCustomErrorCode() { + Validation field = new Validation(); + HopValidatorException exception = + new HopValidatorException( + validator, + field, + HopValidatorException.ERROR_LONGER_THAN_MAXIMUM_LENGTH, + "too long", + "value"); + + assertEquals("KVD002", exception.getCodeDesc()); + } + + @Test + void getCodeDescUsesResolvedCustomErrorCode() { + Validation field = new Validation(); + field.setErrorCode("${ERR_CODE}"); + when(validator.resolve("${ERR_CODE}")).thenReturn("CUSTOM_CODE"); + + HopValidatorException exception = + new HopValidatorException( + validator, + field, + HopValidatorException.ERROR_VALUE_NOT_IN_LIST, + "not in list", + "value"); + + assertEquals("CUSTOM_CODE", exception.getCodeDesc()); + } + + @Test + void getMessageUsesResolvedCustomErrorDescription() { + Validation field = new Validation(); + field.setErrorDescription("${ERR_DESC}"); + when(validator.resolve("${ERR_DESC}")).thenReturn("custom description"); + + HopValidatorException exception = + new HopValidatorException( + validator, + field, + HopValidatorException.ERROR_SHORTER_THAN_MINIMUM_LENGTH, + "built-in message", + "value"); + + assertTrue(exception.getMessage().contains("custom description")); + } + + @Test + void getMessageFallsBackToBuiltInMessage() { + Validation field = new Validation(); + HopValidatorException exception = + new HopValidatorException( + validator, + field, + HopValidatorException.ERROR_ONLY_NULL_VALUE_ALLOWED, + "built-in message", + "value"); + + assertTrue(exception.getMessage().contains("built-in message")); + } + + @Test + void allBuiltInErrorCodesAreMapped() { + Validation field = new Validation(); + String[] expected = { + "KVD000", "KVD001", "KVD002", "KVD003", "KVD004", "KVD005", "KVD006", "KVD007", "KVD008", + "KVD009", "KVD010", "KVD011", "KVD012", "KVD013", "KVD014", "KVD015", + }; + + for (int code = 0; code < expected.length; code++) { + HopValidatorException exception = + new HopValidatorException(validator, field, code, "msg", "field"); + assertEquals(expected[code], exception.getCodeDesc(), "code " + code); + } + } +} diff --git a/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidationTest.java b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidationTest.java new file mode 100644 index 00000000000..02e2a782dae --- /dev/null +++ b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidationTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.validator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.junit.jupiter.api.Test; + +/** Unit tests for {@link Validation}. */ +class ValidationTest { + + @Test + void defaultConstructorSetsSafeDefaults() { + Validation validation = new Validation(); + + assertTrue(validation.isNullAllowed()); + assertFalse(validation.isOnlyNullAllowed()); + assertFalse(validation.isOnlyNumericAllowed()); + assertEquals("", validation.getMaximumLength()); + assertEquals("", validation.getMinimumLength()); + assertNotNull(validation.getAllowedValues()); + assertTrue(validation.getAllowedValues().isEmpty()); + } + + @Test + void nameConstructorSetsFieldName() { + Validation validation = new Validation("email"); + assertEquals("email", validation.getFieldName()); + assertTrue(validation.isNullAllowed()); + } + + @Test + void copyConstructorAndCloneAreDeepForAllowedValues() { + Validation original = new Validation(); + original.setName("rule"); + original.setFieldName("field"); + original.setMaximumLength("10"); + original.setMinimumLength("1"); + original.setNullAllowed(false); + original.setOnlyNullAllowed(true); + original.setOnlyNumericAllowed(true); + original.setDataType("String"); + original.setDataTypeVerified(true); + original.setConversionMask("#"); + original.setDecimalSymbol("."); + original.setGroupingSymbol(","); + original.setSourcingValues(true); + original.setSourcingTransformName("source"); + original.setSourcingField("allowed"); + original.setMinimumValue("1"); + original.setMaximumValue("9"); + original.setStartString("A"); + original.setEndString("Z"); + original.setStartStringNotAllowed("X"); + original.setEndStringNotAllowed("Y"); + original.setRegularExpression(".*"); + original.setRegularExpressionNotAllowed("bad"); + original.setErrorCode("E1"); + original.setErrorDescription("desc"); + original.setAllowedValues(new ArrayList<>(List.of("A", "B"))); + TransformMeta sourcing = new TransformMeta(); + sourcing.setName("source"); + original.setSourcingTransform(sourcing); + + Validation copy = new Validation(original); + Validation clone = original.clone(); + + assertNotSame(original, copy); + assertNotSame(original.getAllowedValues(), copy.getAllowedValues()); + assertEquals(original.getName(), copy.getName()); + assertEquals(original.getFieldName(), copy.getFieldName()); + assertEquals(original.getMaximumLength(), copy.getMaximumLength()); + assertEquals(original.getMinimumLength(), copy.getMinimumLength()); + assertEquals(original.isNullAllowed(), copy.isNullAllowed()); + assertEquals(original.isOnlyNullAllowed(), copy.isOnlyNullAllowed()); + assertEquals(original.isOnlyNumericAllowed(), copy.isOnlyNumericAllowed()); + assertEquals(original.getDataType(), copy.getDataType()); + assertEquals(original.isDataTypeVerified(), copy.isDataTypeVerified()); + assertEquals(original.getConversionMask(), copy.getConversionMask()); + assertEquals(original.getDecimalSymbol(), copy.getDecimalSymbol()); + assertEquals(original.getGroupingSymbol(), copy.getGroupingSymbol()); + assertEquals(original.isSourcingValues(), copy.isSourcingValues()); + assertEquals(original.getSourcingTransformName(), copy.getSourcingTransformName()); + assertEquals(original.getSourcingField(), copy.getSourcingField()); + assertEquals(original.getMinimumValue(), copy.getMinimumValue()); + assertEquals(original.getMaximumValue(), copy.getMaximumValue()); + assertEquals(original.getStartString(), copy.getStartString()); + assertEquals(original.getEndString(), copy.getEndString()); + assertEquals(original.getStartStringNotAllowed(), copy.getStartStringNotAllowed()); + assertEquals(original.getEndStringNotAllowed(), copy.getEndStringNotAllowed()); + assertEquals(original.getRegularExpression(), copy.getRegularExpression()); + assertEquals(original.getRegularExpressionNotAllowed(), copy.getRegularExpressionNotAllowed()); + assertEquals(original.getErrorCode(), copy.getErrorCode()); + assertEquals(original.getErrorDescription(), copy.getErrorDescription()); + assertEquals(original.getAllowedValues(), copy.getAllowedValues()); + assertSameSourcingTransform(original, copy); + + assertEquals(original.getName(), clone.getName()); + assertNotSame(original.getAllowedValues(), clone.getAllowedValues()); + + copy.getAllowedValues().add("C"); + assertEquals(2, original.getAllowedValues().size()); + assertEquals(3, copy.getAllowedValues().size()); + } + + @Test + void equalsIgnoresCaseOnName() { + Validation left = new Validation(); + left.setName("RuleOne"); + Validation right = new Validation(); + right.setName("ruleone"); + + assertTrue(left.equals(right)); + assertTrue(right.equals(left)); + } + + @Test + void findValidationMatchesCaseInsensitively() { + Validation first = new Validation(); + first.setName("Alpha"); + Validation second = new Validation(); + second.setName("Beta"); + List list = List.of(first, second); + + assertEquals(first, Validation.findValidation(list, "alpha")); + assertEquals(second, Validation.findValidation(list, "BETA")); + assertNull(Validation.findValidation(list, "gamma")); + } + + private static void assertSameSourcingTransform(Validation original, Validation copy) { + assertSame(original.getSourcingTransform(), copy.getSourcingTransform()); + } +} diff --git a/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorDataTest.java b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorDataTest.java new file mode 100644 index 00000000000..479853ccb81 --- /dev/null +++ b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorDataTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.validator; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.apache.hop.pipeline.transform.ITransformData; +import org.junit.jupiter.api.Test; + +/** Unit tests for {@link ValidatorData}. */ +class ValidatorDataTest { + + @Test + void constructorCreatesUsableTransformData() { + ValidatorData data = new ValidatorData(); + + assertNotNull(data); + assertInstanceOf(ITransformData.class, data); + assertNull(data.fieldIndexes); + assertNull(data.constantsMeta); + assertNull(data.inputRowMeta); + assertNull(data.patternExpected); + assertNull(data.patternDisallowed); + } +} diff --git a/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialogSelectionTest.java b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialogSelectionTest.java new file mode 100644 index 00000000000..00fcd704f82 --- /dev/null +++ b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialogSelectionTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.validator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for non-UI logic in {@link ValidatorDialog}. Full SWT dialog coverage is out of scope + * for headless unit tests. + */ +class ValidatorDialogSelectionTest { + + private ValidatorDialog dialog; + + @BeforeEach + void setUp() { + dialog = mock(ValidatorDialog.class); + when(dialog.spacesValidation(anyString())).thenCallRealMethod(); + when(dialog.spacesValidation(null)).thenCallRealMethod(); + when(dialog.trailingSpacesValidation(anyString())).thenCallRealMethod(); + when(dialog.trailingSpacesValidation(null)).thenCallRealMethod(); + } + + @Test + void indexAfterRemovalSelectsSameIndexWhenItemsRemainAfter() { + // Remove index 0 from [A,B,C] -> remaining [B,C], select B (index 0) + assertEquals(0, ValidatorDialog.indexAfterRemoval(0, 2)); + } + + @Test + void indexAfterRemovalSelectsPreviousWhenLastItemRemoved() { + // Remove index 2 from [A,B,C] -> remaining [A,B], select B (index 1) + assertEquals(1, ValidatorDialog.indexAfterRemoval(2, 2)); + } + + @Test + void indexAfterRemovalSelectsMiddleNeighbor() { + // Remove index 1 from [A,B,C] -> remaining [A,C], select C (index 1) + assertEquals(1, ValidatorDialog.indexAfterRemoval(1, 2)); + } + + @Test + void indexAfterRemovalReturnsMinusOneWhenEmpty() { + assertEquals(-1, ValidatorDialog.indexAfterRemoval(0, 0)); + } + + @Test + void indexAfterRemovalKeepsSingleRemainingItem() { + assertEquals(0, ValidatorDialog.indexAfterRemoval(0, 1)); + assertEquals(0, ValidatorDialog.indexAfterRemoval(5, 1)); + } + + @Test + void spacesValidationDetectsOnlySpaces() { + assertTrue(dialog.spacesValidation(" ")); + assertFalse(dialog.spacesValidation("a")); + assertFalse(dialog.spacesValidation(" a ")); + assertFalse(dialog.spacesValidation("")); + assertFalse(dialog.spacesValidation(null)); + } + + @Test + void trailingSpacesValidationDetectsTrailingSpace() { + assertTrue(dialog.trailingSpacesValidation("value ")); + assertTrue(dialog.trailingSpacesValidation(" ")); + assertFalse(dialog.trailingSpacesValidation("value")); + assertFalse(dialog.trailingSpacesValidation("")); + assertFalse(dialog.trailingSpacesValidation(null)); + } +} diff --git a/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorMetaTest.java b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorMetaTest.java new file mode 100644 index 00000000000..c65f32fbca2 --- /dev/null +++ b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorMetaTest.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.validator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.ITransformIOMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transform.TransformSerializationTestUtil; +import org.apache.hop.pipeline.transform.stream.IStream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** Unit test for {@link ValidatorMeta} */ +class ValidatorMetaTest { + @RegisterExtension + static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension(); + + @BeforeEach + void setUp() throws Exception { + HopEnvironment.init(); + PluginRegistry.init(); + } + + @Test + void testSerialization() throws Exception { + ValidatorMeta meta = + TransformSerializationTestUtil.testSerialization( + "/validator-transform.xml", ValidatorMeta.class); + + assertTrue(meta.isValidatingAll()); + assertTrue(meta.isConcatenatingErrors()); + assertEquals(";", meta.getConcatenationSeparator()); + assertTrue(meta.isSuppressingLogFailedData()); + assertEquals(1, meta.getValidations().size()); + + Validation validation = meta.getValidations().getFirst(); + assertEquals("value rules", validation.getName()); + assertEquals("value", validation.getFieldName()); + assertFalse(validation.isNullAllowed()); + assertEquals("10", validation.getMaximumLength()); + assertEquals("1", validation.getMinimumLength()); + assertEquals("CODE_NULL", validation.getErrorCode()); + assertEquals(2, validation.getAllowedValues().size()); + assertEquals("A", validation.getAllowedValues().get(0)); + assertEquals("B", validation.getAllowedValues().get(1)); + } + + @Test + void testCloneCopiesOptionsAndValidations() { + ValidatorMeta meta = new ValidatorMeta(); + meta.setValidatingAll(true); + meta.setConcatenatingErrors(true); + meta.setConcatenationSeparator("|"); + meta.setSuppressingLogFailedData(true); + + Validation validation = new Validation(); + validation.setName("rule1"); + validation.setFieldName("field1"); + validation.setNullAllowed(false); + meta.getValidations().add(validation); + + ValidatorMeta clone = meta.clone(); + assertNotSame(meta, clone); + assertNotSame(meta.getValidations(), clone.getValidations()); + assertNotSame(meta.getValidations().getFirst(), clone.getValidations().getFirst()); + assertTrue(clone.isValidatingAll()); + assertTrue(clone.isConcatenatingErrors()); + assertEquals("|", clone.getConcatenationSeparator()); + assertTrue(clone.isSuppressingLogFailedData()); + assertEquals(1, clone.getValidations().size()); + assertEquals("rule1", clone.getValidations().getFirst().getName()); + assertEquals("field1", clone.getValidations().getFirst().getFieldName()); + } + + @Test + void testSupportsErrorHandling() { + assertTrue(new ValidatorMeta().supportsErrorHandling()); + } + + @Test + void testSuppressLogFailedDataDefaultsToFalse() { + assertFalse(new ValidatorMeta().isSuppressingLogFailedData()); + } + + @Test + void checkNoPrevFieldsAddsWarning() { + ValidatorMeta meta = new ValidatorMeta(); + List remarks = new ArrayList<>(); + meta.check( + remarks, + mock(PipelineMeta.class), + mock(TransformMeta.class), + null, + new String[] {"in"}, + new String[0], + mock(IRowMeta.class), + null, + mock(IHopMetadataProvider.class)); + + assertTrue(remarks.stream().anyMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_WARNING)); + } + + @Test + void checkEmptyPrevFieldsAddsWarning() { + ValidatorMeta meta = new ValidatorMeta(); + List remarks = new ArrayList<>(); + meta.check( + remarks, + mock(PipelineMeta.class), + mock(TransformMeta.class), + new RowMeta(), + new String[] {"in"}, + new String[0], + mock(IRowMeta.class), + null, + mock(IHopMetadataProvider.class)); + + assertTrue(remarks.stream().anyMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_WARNING)); + } + + @Test + void checkWithPrevAndInputAddsOkRemarks() { + ValidatorMeta meta = new ValidatorMeta(); + RowMeta prev = new RowMeta(); + prev.addValueMeta(new ValueMetaString("field1")); + + List remarks = new ArrayList<>(); + meta.check( + remarks, + mock(PipelineMeta.class), + mock(TransformMeta.class), + prev, + new String[] {"in"}, + new String[0], + mock(IRowMeta.class), + null, + mock(IHopMetadataProvider.class)); + + assertEquals(2, remarks.size()); + assertEquals(ICheckResult.TYPE_RESULT_OK, remarks.get(0).getType()); + assertEquals(ICheckResult.TYPE_RESULT_OK, remarks.get(1).getType()); + } + + @Test + void checkNoInputAddsError() { + ValidatorMeta meta = new ValidatorMeta(); + RowMeta prev = new RowMeta(); + prev.addValueMeta(new ValueMetaString("field1")); + + List remarks = new ArrayList<>(); + meta.check( + remarks, + mock(PipelineMeta.class), + mock(TransformMeta.class), + prev, + new String[0], + new String[0], + mock(IRowMeta.class), + null, + mock(IHopMetadataProvider.class)); + + assertTrue(remarks.stream().anyMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_ERROR)); + } + + @Test + void getTransformIOMetaCreatesInfoStreamPerValidation() { + ValidatorMeta meta = new ValidatorMeta(); + Validation validation = new Validation(); + validation.setName("rule1"); + validation.setSourcingTransformName("source"); + meta.getValidations().add(validation); + + ITransformIOMeta ioMeta = meta.getTransformIOMeta(); + assertNotNull(ioMeta); + assertEquals(1, ioMeta.getInfoStreams().size()); + assertEquals(1, meta.getTransformIOMeta().getInfoStreams().size()); + } + + @Test + void searchInfoAndTargetTransformsResolvesSourcingTransform() { + ValidatorMeta meta = new ValidatorMeta(); + Validation validation = new Validation(); + validation.setName("rule1"); + validation.setSourcingTransformName("source"); + meta.getValidations().add(validation); + + TransformMeta source = new TransformMeta(); + source.setName("source"); + meta.searchInfoAndTargetTransforms(List.of(source)); + + assertSame(source, validation.getSourcingTransform()); + } + + @Test + void searchInfoAndTargetTransformsClearsMissingSource() { + ValidatorMeta meta = new ValidatorMeta(); + Validation validation = new Validation(); + validation.setName("rule1"); + validation.setSourcingTransformName("missing"); + meta.getValidations().add(validation); + + meta.searchInfoAndTargetTransforms(List.of()); + assertNull(validation.getSourcingTransform()); + } + + @Test + void handleStreamSelectionAddsValidationForNewValidationStream() { + ValidatorMeta meta = new ValidatorMeta(); + TransformMeta source = new TransformMeta(); + source.setName("allowed_values"); + + IStream newValidation = ValidatorMeta.getNewValidation(); + newValidation.setTransformMeta(source); + + meta.handleStreamSelection(newValidation); + + assertEquals(1, meta.getValidations().size()); + Validation created = meta.getValidations().getFirst(); + assertEquals("allowed_values", created.getName()); + assertTrue(created.isSourcingValues()); + assertSame(source, created.getSourcingTransform()); + } + + @Test + void handleStreamSelectionIgnoresUnrelatedStream() { + ValidatorMeta meta = new ValidatorMeta(); + IStream other = mock(IStream.class); + + meta.handleStreamSelection(other); + assertTrue(meta.getValidations().isEmpty()); + } + + @Test + void newValidationStreamAccessorRoundTrip() { + IStream original = ValidatorMeta.getNewValidation(); + assertNotNull(original); + + IStream replacement = mock(IStream.class); + try { + ValidatorMeta.setNewValidation(replacement); + assertSame(replacement, ValidatorMeta.getNewValidation()); + } finally { + ValidatorMeta.setNewValidation(original); + } + assertSame(original, ValidatorMeta.getNewValidation()); + } +} diff --git a/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorTest.java b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorTest.java new file mode 100644 index 00000000000..8d7b8b203cf --- /dev/null +++ b/plugins/transforms/validator/src/test/java/org/apache/hop/pipeline/transforms/validator/ValidatorTest.java @@ -0,0 +1,865 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.validator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.ILoggingObject; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.apache.hop.pipeline.transforms.mock.TransformMockHelper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** Unit test for {@link Validator} */ +class ValidatorTest { + + @RegisterExtension + static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension(); + + private TransformMockHelper helper; + + @BeforeAll + static void setUpBeforeClass() throws HopException { + HopEnvironment.init(); + } + + @BeforeEach + void setUp() throws Exception { + helper = new TransformMockHelper<>("Validator", ValidatorMeta.class, ValidatorData.class); + when(helper.logChannelFactory.create(any(), any(ILoggingObject.class))) + .thenReturn(helper.iLogChannel); + when(helper.pipeline.isRunning()).thenReturn(true); + when(helper.transformMeta.getName()).thenReturn("Validator"); + } + + @AfterEach + void tearDown() { + helper.cleanUp(); + } + + @Test + void testValidRowIsPassedThrough() throws Exception { + ValidatorMeta meta = createMeta(nullNotAllowed()); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {"ok"}).doReturn(null).when(validator).getRow(); + doNothing().when(validator).putRow(any(IRowMeta.class), any(Object[].class)); + + assertTrue(validator.processRow()); + verify(validator).putRow(any(IRowMeta.class), eq(new Object[] {"ok"})); + verify(validator, never()) + .putError( + any(IRowMeta.class), + any(Object[].class), + anyLong(), + anyString(), + anyString(), + anyString()); + assertFalse(validator.processRow()); + } + + @Test + void testNullNotAllowedPutsError() throws Exception { + ValidatorMeta meta = createMeta(nullNotAllowed()); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {null}).doReturn(null).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("value"), + eq("KVD001")); + } + + @Test + void testNullNotAllowedWithoutErrorHandlingThrows() throws Exception { + ValidatorMeta meta = createMeta(nullNotAllowed()); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, false); + + doReturn(new Object[] {null}).when(validator).getRow(); + + HopException exception = assertThrows(HopException.class, validator::processRow); + assertTrue(exception.getMessage().contains("null")); + } + + @Test + void testMaximumLengthViolation() throws Exception { + Validation rule = new Validation(); + rule.setName("max length"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setMaximumLength("3"); + + ValidatorMeta meta = createMeta(rule); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {"toolong"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + contains("toolong"), + eq("value"), + eq("KVD002")); + } + + @Test + void testMinimumLengthViolation() throws Exception { + Validation rule = new Validation(); + rule.setName("min length"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setMinimumLength("5"); + + ValidatorMeta meta = createMeta(rule); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {"abc"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + contains("abc"), + eq("value"), + eq("KVD003")); + } + + @Test + void testValueNotInAllowedList() throws Exception { + Validation rule = new Validation(); + rule.setName("list"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setDataType("String"); + rule.setAllowedValues(List.of("A", "B")); + + ValidatorMeta meta = createMeta(rule); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {"C"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + contains("C"), + eq("value"), + eq("KVD007")); + } + + @Test + void testOnlyNumericAllowed() throws Exception { + Validation rule = new Validation(); + rule.setName("numeric"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setOnlyNumericAllowed(true); + + ValidatorMeta meta = createMeta(rule); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {"12a"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("value"), + eq("KVD008")); + } + + @Test + void testRegularExpressionExpected() throws Exception { + Validation rule = new Validation(); + rule.setName("regex"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setRegularExpression("^[0-9]+$"); + + ValidatorMeta meta = createMeta(rule); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {"abc"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("value"), + eq("KVD013")); + } + + @Test + void testMinimumValueViolation() throws Exception { + Validation rule = new Validation(); + rule.setName("min value"); + rule.setFieldName("amount"); + rule.setNullAllowed(true); + rule.setDataType("Integer"); + rule.setMinimumValue("10"); + + ValidatorMeta meta = createMeta(rule); + RowMeta rowMeta = new RowMeta(); + rowMeta.addValueMeta(new ValueMetaInteger("amount")); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {5L}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("amount"), + eq("KVD005")); + } + + @Test + void testValidateAllConcatenatesErrorsIntoOneRow() throws Exception { + Validation lengthRule = new Validation(); + lengthRule.setName("length"); + lengthRule.setFieldName("value"); + lengthRule.setNullAllowed(true); + lengthRule.setMaximumLength("1"); + + Validation onlyNull = new Validation(); + onlyNull.setName("only null"); + onlyNull.setFieldName("value"); + onlyNull.setNullAllowed(true); + onlyNull.setOnlyNullAllowed(true); + + ValidatorMeta meta = createMeta(lengthRule, onlyNull); + meta.setValidatingAll(true); + meta.setConcatenatingErrors(true); + meta.setConcatenationSeparator(";"); + + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {"ab"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(2L), + contains(";"), + contains(";"), + contains(";")); + } + + @Test + void testCustomErrorCodeAndDescription() throws Exception { + Validation rule = nullNotAllowed(); + rule.setErrorCode("MY_CODE"); + rule.setErrorDescription("custom description"); + + ValidatorMeta meta = createMeta(rule); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {null}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + eq("custom description"), + eq("value"), + eq("MY_CODE")); + } + + @Test + void testSuppressLogFailedDataOmitsValuesFromErrorMessages() throws Exception { + Validation rule = new Validation(); + rule.setName("max length"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setMaximumLength("3"); + + ValidatorMeta meta = createMeta(rule); + meta.setSuppressingLogFailedData(true); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {"toolong"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + contains(Validator.OMITTED_VALUE), + eq("value"), + eq("KVD002")); + verify(validator, never()) + .putError( + any(IRowMeta.class), + any(Object[].class), + anyLong(), + contains("toolong"), + anyString(), + anyString()); + } + + @Test + void testSuppressLogFailedDataStillLogsCustomErrorDescription() throws Exception { + Validation rule = nullNotAllowed(); + rule.setErrorCode("name_valid_code"); + rule.setErrorDescription("name_valid_desc"); + + ValidatorMeta meta = createMeta(rule); + meta.setSuppressingLogFailedData(true); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {null}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + eq("name_valid_desc"), + eq("value"), + eq("name_valid_code")); + } + + @Test + void testValueAndRowForMessageRespectSuppressFlag() throws Exception { + ValidatorMeta meta = new ValidatorMeta(); + ValidatorData data = new ValidatorData(); + Validator validator = + new Validator(helper.transformMeta, meta, data, 0, helper.pipelineMeta, helper.pipeline); + + ValueMetaString valueMeta = new ValueMetaString("value"); + RowMeta rowMeta = new RowMeta(); + rowMeta.addValueMeta(valueMeta); + Object[] row = new Object[] {"secret"}; + + meta.setSuppressingLogFailedData(false); + assertEquals("secret", validator.getValueForMessage(valueMeta, "secret")); + assertEquals(rowMeta.getString(row), validator.getRowForMessage(rowMeta, row)); + + meta.setSuppressingLogFailedData(true); + assertEquals(Validator.OMITTED_VALUE, validator.getValueForMessage(valueMeta, "secret")); + assertEquals(Validator.OMITTED_VALUE, validator.getRowForMessage(rowMeta, row)); + } + + @Test + void testAssertNumericAllowsDigitsAndNumericTypes() throws Exception { + ValidatorMeta meta = new ValidatorMeta(); + ValidatorData data = new ValidatorData(); + Validator validator = + new Validator(helper.transformMeta, meta, data, 0, helper.pipelineMeta, helper.pipeline); + + Validation field = new Validation(); + field.setFieldName("value"); + ValueMetaString stringMeta = new ValueMetaString("value"); + ValueMetaInteger integerMeta = new ValueMetaInteger("value"); + + assertNull(validator.assertNumeric(stringMeta, "12345", field)); + assertNull(validator.assertNumeric(integerMeta, 42L, field)); + HopValidatorException exception = validator.assertNumeric(stringMeta, "12a", field); + assertEquals(HopValidatorException.ERROR_NON_NUMERIC_DATA, exception.getCode()); + } + + @Test + void testStartAndEndStringRules() throws Exception { + Validation rule = new Validation(); + rule.setName("prefix"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setStartString("AB"); + rule.setEndString("Z"); + + ValidatorMeta meta = createMeta(rule); + RowMeta rowMeta = stringRowMeta(); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {"ABX"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("value"), + eq("KVD010")); + } + + @Test + void testDoesNotStartWithExpectedString() throws Exception { + Validation rule = new Validation(); + rule.setName("start"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setStartString("AB"); + + ValidatorMeta meta = createMeta(rule); + Validator validator = createInitializedValidator(meta, stringRowMeta(), true); + + doReturn(new Object[] {"XX"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("value"), + eq("KVD009")); + } + + @Test + void testMaximumValueViolation() throws Exception { + Validation rule = new Validation(); + rule.setName("max value"); + rule.setFieldName("amount"); + rule.setNullAllowed(true); + rule.setDataType("Integer"); + rule.setMaximumValue("10"); + + ValidatorMeta meta = createMeta(rule); + RowMeta rowMeta = new RowMeta(); + rowMeta.addValueMeta(new ValueMetaInteger("amount")); + Validator validator = createInitializedValidator(meta, rowMeta, true); + + doReturn(new Object[] {11L}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("amount"), + eq("KVD006")); + } + + @Test + void testOnlyNullAllowedViolation() throws Exception { + Validation rule = new Validation(); + rule.setName("only null"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setOnlyNullAllowed(true); + + ValidatorMeta meta = createMeta(rule); + Validator validator = createInitializedValidator(meta, stringRowMeta(), true); + + doReturn(new Object[] {"x"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("value"), + eq("KVD015")); + } + + @Test + void testUnexpectedDataTypeViolation() throws Exception { + Validation rule = new Validation(); + rule.setName("type"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setDataType("Integer"); + rule.setDataTypeVerified(true); + + ValidatorMeta meta = createMeta(rule); + Validator validator = createInitializedValidator(meta, stringRowMeta(), true); + + doReturn(new Object[] {"abc"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("value"), + eq("KVD004")); + } + + @Test + void testAllowedValueInListPasses() throws Exception { + Validation rule = new Validation(); + rule.setName("list"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setDataType("String"); + rule.setAllowedValues(List.of("A", "B")); + + ValidatorMeta meta = createMeta(rule); + Validator validator = createInitializedValidator(meta, stringRowMeta(), true); + + doReturn(new Object[] {"A"}).doReturn(null).when(validator).getRow(); + doNothing().when(validator).putRow(any(IRowMeta.class), any(Object[].class)); + + assertTrue(validator.processRow()); + verify(validator).putRow(any(IRowMeta.class), eq(new Object[] {"A"})); + verify(validator, never()) + .putError( + any(IRowMeta.class), + any(Object[].class), + anyLong(), + anyString(), + anyString(), + anyString()); + } + + @Test + void testDisallowedRegularExpression() throws Exception { + Validation rule = new Validation(); + rule.setName("bad regex"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setRegularExpressionNotAllowed("^[0-9]+$"); + + ValidatorMeta meta = createMeta(rule); + Validator validator = createInitializedValidator(meta, stringRowMeta(), true); + + doReturn(new Object[] {"123"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("value"), + eq("KVD014")); + } + + @Test + void testStartsWithNotAllowedString() throws Exception { + Validation rule = new Validation(); + rule.setName("bad start"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setStartStringNotAllowed("XX"); + + ValidatorMeta meta = createMeta(rule); + Validator validator = createInitializedValidator(meta, stringRowMeta(), true); + + doReturn(new Object[] {"XXY"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("value"), + eq("KVD011")); + } + + @Test + void testEndsWithNotAllowedString() throws Exception { + Validation rule = new Validation(); + rule.setName("bad end"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setEndStringNotAllowed("YY"); + + ValidatorMeta meta = createMeta(rule); + Validator validator = createInitializedValidator(meta, stringRowMeta(), true); + + doReturn(new Object[] {"aYY"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("value"), + eq("KVD012")); + } + + @Test + void testValidateAllWithoutConcatenateEmitsMultipleErrors() throws Exception { + Validation lengthRule = new Validation(); + lengthRule.setName("length"); + lengthRule.setFieldName("value"); + lengthRule.setNullAllowed(true); + lengthRule.setMaximumLength("1"); + + Validation onlyNull = new Validation(); + onlyNull.setName("only null"); + onlyNull.setFieldName("value"); + onlyNull.setNullAllowed(true); + onlyNull.setOnlyNullAllowed(true); + + ValidatorMeta meta = createMeta(lengthRule, onlyNull); + meta.setValidatingAll(true); + meta.setConcatenatingErrors(false); + + Validator validator = createInitializedValidator(meta, stringRowMeta(), true); + doReturn(new Object[] {"ab"}).when(validator).getRow(); + stubPutError(validator); + + assertTrue(validator.processRow()); + verify(validator, times(2)) + .putError( + any(IRowMeta.class), + any(Object[].class), + eq(1L), + anyString(), + eq("value"), + anyString()); + } + + @Test + void testNullInputEndsProcessing() throws Exception { + ValidatorMeta meta = createMeta(nullNotAllowed()); + Validator validator = createInitializedValidator(meta, stringRowMeta(), true); + + doReturn(null).when(validator).getRow(); + + assertFalse(validator.processRow()); + } + + @Test + void testMissingFieldNameFailsInitCalculation() throws Exception { + Validation rule = new Validation(); + rule.setName("missing field"); + rule.setFieldName("does_not_exist"); + rule.setNullAllowed(true); + + ValidatorMeta meta = createMeta(rule); + Validator validator = createInitializedValidator(meta, stringRowMeta(), true); + + doReturn(new Object[] {"x"}).when(validator).getRow(); + + HopException exception = assertThrows(HopException.class, validator::processRow); + assertTrue(exception.getMessage().contains("does_not_exist")); + } + + @Test + void testEmptyFieldNameFailsOnProcess() throws Exception { + Validation rule = new Validation(); + rule.setName("empty field name"); + rule.setFieldName(""); + rule.setNullAllowed(true); + + ValidatorMeta meta = createMeta(rule); + when(helper.transformMeta.isDoingErrorHandling()).thenReturn(true); + when(helper.pipelineMeta.getTransformFields(any(IVariables.class), anyString())) + .thenReturn(stringRowMeta()); + + ValidatorData data = new ValidatorData(); + Validator validator = + spy( + new Validator( + helper.transformMeta, meta, data, 0, helper.pipelineMeta, helper.pipeline)); + validator.setInputRowMeta(stringRowMeta()); + assertTrue(validator.init()); + + doReturn(new Object[] {"x"}).when(validator).getRow(); + + HopException exception = assertThrows(HopException.class, validator::processRow); + assertTrue(exception.getMessage().toLowerCase().contains("no name")); + } + + @Test + void testInitFailsForInvalidMinimumLength() throws Exception { + Validation rule = new Validation(); + rule.setName("bad length"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setMinimumLength("abc"); + + ValidatorMeta meta = createMeta(rule); + when(helper.pipelineMeta.getTransformFields(any(IVariables.class), anyString())) + .thenReturn(stringRowMeta()); + + Validator validator = + new Validator( + helper.transformMeta, + meta, + new ValidatorData(), + 0, + helper.pipelineMeta, + helper.pipeline); + + assertFalse(validator.init()); + } + + @Test + void testValidStartAndEndStringPasses() throws Exception { + Validation rule = new Validation(); + rule.setName("prefix-suffix"); + rule.setFieldName("value"); + rule.setNullAllowed(true); + rule.setStartString("AB"); + rule.setEndString("Z"); + + ValidatorMeta meta = createMeta(rule); + Validator validator = createInitializedValidator(meta, stringRowMeta(), true); + + doReturn(new Object[] {"ABZ"}).doReturn(null).when(validator).getRow(); + doNothing().when(validator).putRow(any(IRowMeta.class), any(Object[].class)); + + assertTrue(validator.processRow()); + verify(validator).putRow(any(IRowMeta.class), eq(new Object[] {"ABZ"})); + } + + private Validator createInitializedValidator( + ValidatorMeta meta, RowMeta rowMeta, boolean errorHandling) throws Exception { + when(helper.transformMeta.isDoingErrorHandling()).thenReturn(errorHandling); + when(helper.pipelineMeta.getTransformFields(any(IVariables.class), anyString())) + .thenReturn(rowMeta); + + ValidatorData data = new ValidatorData(); + Validator validator = + spy( + new Validator( + helper.transformMeta, meta, data, 0, helper.pipelineMeta, helper.pipeline)); + validator.setInputRowMeta(rowMeta); + + assertTrue(validator.init()); + return validator; + } + + private static void stubPutError(Validator validator) throws HopException { + doNothing() + .when(validator) + .putError( + any(IRowMeta.class), + any(Object[].class), + anyLong(), + anyString(), + nullable(String.class), + anyString()); + } + + private static ValidatorMeta createMeta(Validation... validations) { + ValidatorMeta meta = new ValidatorMeta(); + meta.setValidations(new ArrayList<>(List.of(validations))); + return meta; + } + + private static Validation nullNotAllowed() { + Validation rule = new Validation(); + rule.setName("value" + " null check"); + rule.setFieldName("value"); + rule.setNullAllowed(false); + return rule; + } + + private static RowMeta stringRowMeta() { + RowMeta rowMeta = new RowMeta(); + rowMeta.addValueMeta(new ValueMetaString("value")); + return rowMeta; + } +} diff --git a/plugins/transforms/validator/src/test/resources/validator-transform.xml b/plugins/transforms/validator/src/test/resources/validator-transform.xml new file mode 100644 index 00000000000..770f34184fc --- /dev/null +++ b/plugins/transforms/validator/src/test/resources/validator-transform.xml @@ -0,0 +1,60 @@ + + + + Data validator + Validator + + Y + + 1 + + none + + + Y + ; + Y + Y + + + A + B + + + String + N + + CODE_NULL + Field value cannot be null + + N + 10 + 1 + value + N + N + N + value rules + + + + 100 + 100 + +