Skip to content

Fix#7453 Add option to omit failed values from Data Validator log messages #7570

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hansva merged 1 commit into from
Jul 20, 2026
Merged

Fix#7453 Add option to omit failed values from Data Validator log messages #7570

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion engine/src/main/java/org/apache/hop/pipeline/transform/BaseTransform.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1417,6 +1417,7 @@ public void handlePutRowTo(IRowMeta rowMeta, Object[] row, IRowSet rowSet)
try {
Thread.sleep(1);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new HopTransformException(e);
}
}
Expand Down Expand Up @@ -1523,7 +1524,7 @@ public void handlePutError(
}
}
incrementLinesRejected();
if (!Utils.isEmpty(errorDescriptions)) {
if (isLoggingErrorDescriptions() && !Utils.isEmpty(errorDescriptions)) {
logError(errorDescriptions);
}
} else if (transformErrorMeta.isEnabled()) {
Expand Down Expand Up @@ -2723,6 +2724,19 @@ public void logRowlevel(String message, Object... arguments) {
log.logRowlevel(message, arguments);
}

/**
* Whether rejected-row error descriptions should be written to the log.
*
* <p>Transforms that can produce a very large number of validation/rejection errors may override
* this to reduce log volume. Error rows are still sent to the error handling hop when configured.
*
* @return true when error descriptions should be logged (default)
* @since 2.19.0
*/
protected boolean isLoggingErrorDescriptions() {
return true;
}

/**
* Log error.
*
Expand Down
53 changes: 39 additions & 14 deletions ...forms/validator/src/main/java/org/apache/hop/pipeline/transforms/validator/Validator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@
public class Validator extends BaseTransform<ValidatorMeta, ValidatorData> implements ITransform {
private static final Class<?> PKG = ValidatorMeta.class;

/** Placeholder used in log/error messages when failed data must not be exposed. */
static final String OMITTED_VALUE = "?";

public Validator(
TransformMeta transformMeta,
ValidatorMeta meta,
Expand Down Expand Up @@ -279,7 +282,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.NullNotAllowed",
field.getFieldName(),
inputRowMeta.getString(r)),
getRowForMessage(inputRowMeta, r)),
field.getFieldName());
exceptions.add(exception);
if (!meta.isValidatingAll()) {
Expand All @@ -297,7 +300,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.OnlyNullAllowed",
field.getFieldName(),
inputRowMeta.getString(r)),
getRowForMessage(inputRowMeta, r)),
field.getFieldName());
exceptions.add(exception);
if (!meta.isValidatingAll()) {
Expand Down Expand Up @@ -367,7 +370,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.ShorterThanMininumLength",
field.getFieldName(),
valueMeta.getString(valueData),
getValueForMessage(valueMeta, valueData),
Integer.toString(stringValue.length()),
field.getMinimumLength()),
field.getFieldName());
Expand All @@ -390,7 +393,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.LongerThanMaximumLength",
field.getFieldName(),
valueMeta.getString(valueData),
getValueForMessage(valueMeta, valueData),
Integer.toString(stringValue.length()),
field.getMaximumLength()),
field.getFieldName());
Expand All @@ -413,7 +416,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.LowerThanMinimumValue",
field.getFieldName(),
valueMeta.getString(valueData),
getValueForMessage(valueMeta, valueData),
data.constantsMeta[i].getString(data.minimumValue[i])),
field.getFieldName());
exceptions.add(exception);
Expand All @@ -435,7 +438,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.HigherThanMaximumValue",
field.getFieldName(),
valueMeta.getString(valueData),
getValueForMessage(valueMeta, valueData),
data.constantsMeta[i].getString(data.maximumValue[i])),
field.getFieldName());
exceptions.add(exception);
Expand Down Expand Up @@ -465,7 +468,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.NotInList",
field.getFieldName(),
valueMeta.getString(valueData)),
getValueForMessage(valueMeta, valueData)),
field.getFieldName());
exceptions.add(exception);
if (!meta.isValidatingAll()) {
Expand Down Expand Up @@ -498,7 +501,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.DoesNotStartWithString",
field.getFieldName(),
valueMeta.getString(valueData),
getValueForMessage(valueMeta, valueData),
field.getStartString()),
field.getFieldName());
exceptions.add(exception);
Expand All @@ -519,7 +522,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.DoesNotEndWithString",
field.getFieldName(),
valueMeta.getString(valueData),
getValueForMessage(valueMeta, valueData),
field.getEndString()),
field.getFieldName());
exceptions.add(exception);
Expand All @@ -541,7 +544,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.StartsWithString",
field.getFieldName(),
valueMeta.getString(valueData),
getValueForMessage(valueMeta, valueData),
field.getStartStringNotAllowed()),
field.getFieldName());
exceptions.add(exception);
Expand All @@ -563,7 +566,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.EndsWithString",
field.getFieldName(),
valueMeta.getString(valueData),
getValueForMessage(valueMeta, valueData),
field.getEndStringNotAllowed()),
field.getFieldName());
exceptions.add(exception);
Expand All @@ -586,7 +589,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.MatchingRegExpExpected",
field.getFieldName(),
valueMeta.getString(valueData),
getValueForMessage(valueMeta, valueData),
data.regularExpression[i]),
field.getFieldName());
exceptions.add(exception);
Expand All @@ -610,7 +613,7 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
PKG,
"Validator.Exception.MatchingRegExpNotAllowed",
field.getFieldName(),
valueMeta.getString(valueData),
getValueForMessage(valueMeta, valueData),
data.regularExpressionNotAllowed[i]),
field.getFieldName());
exceptions.add(exception);
Expand All @@ -625,6 +628,28 @@ private List<HopValidatorException> validateFields(IRowMeta inputRowMeta, Object
return exceptions;
}

/**
* Returns the field value for inclusion in validation messages, or a placeholder when logging of
* failed data is suppressed.
*/
String getValueForMessage(IValueMeta valueMeta, Object valueData) throws HopValueException {
if (meta.isSuppressingLogFailedData()) {
return OMITTED_VALUE;
}
return valueMeta.getString(valueData);
}

/**
* Returns the row for inclusion in validation messages, or a placeholder when logging of failed
* data is suppressed.
*/
String getRowForMessage(IRowMeta inputRowMeta, Object[] r) throws HopValueException {
if (meta.isSuppressingLogFailedData()) {
return OMITTED_VALUE;
}
return inputRowMeta.getString(r);
}

// package-local visibility for testing purposes
HopValidatorException assertNumeric(IValueMeta valueMeta, Object valueData, Validation field)
throws HopValueException {
Expand All @@ -640,7 +665,7 @@ HopValidatorException assertNumeric(IValueMeta valueMeta, Object valueData, Vali
"Validator.Exception.NonNumericDataNotAllowed",
field.getFieldName(),
valueMeta.toStringMeta(),
valueMeta.getString(valueData)),
getValueForMessage(valueMeta, valueData)),
field.getFieldName());
}

Expand Down
65 changes: 49 additions & 16 deletions ...validator/src/main/java/org/apache/hop/pipeline/transforms/validator/ValidatorDialog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ public class ValidatorDialog extends BaseTransformDialog {
private TextVar wErrorDescription;
private Button wConcatErrors;
private TextVar wConcatSeparator;
private Button wSuppressLogFailedData;

public ValidatorDialog(
Shell parent, IVariables variables, ValidatorMeta transformMeta, PipelineMeta pipelineMeta) {
Expand Down Expand Up @@ -207,14 +208,28 @@ public String open() {
fdConcatSeparator.top = new FormAttachment(wValidateAll, margin);
wConcatSeparator.setLayoutData(fdConcatSeparator);

// Optionally suppress logging of failed validation data (reduces log volume)
//
wSuppressLogFailedData = new Button(shell, SWT.CHECK);
wSuppressLogFailedData.setText(
BaseMessages.getString(PKG, "ValidatorDialog.SuppressLogFailedData.Label"));
wSuppressLogFailedData.setToolTipText(
BaseMessages.getString(PKG, "ValidatorDialog.SuppressLogFailedData.Tooltip"));
PropsUi.setLook(wSuppressLogFailedData);
FormData fdSuppressLogFailedData = new FormData();
fdSuppressLogFailedData.left = new FormAttachment(middle, 0);
fdSuppressLogFailedData.right = new FormAttachment(100, 0);
fdSuppressLogFailedData.top = new FormAttachment(wConcatErrors, margin);
wSuppressLogFailedData.setLayoutData(fdSuppressLogFailedData);

// Create a scrolled composite on the right side...
//
ScrolledComposite wSComp = new ScrolledComposite(shell, SWT.H_SCROLL | SWT.V_SCROLL);
PropsUi.setLook(wSComp);
wSComp.setLayout(new FillLayout());
FormData fdComp = new FormData();
fdComp.left = new FormAttachment(middle / 2, margin);
fdComp.top = new FormAttachment(wConcatSeparator, margin);
fdComp.top = new FormAttachment(wSuppressLogFailedData, margin);
fdComp.right = new FormAttachment(100, -margin);
fdComp.bottom = new FormAttachment(wOk, -margin);
// Limit viewport size so the dialog opens at a reasonable size; content scrolls inside.
Expand Down Expand Up @@ -344,8 +359,8 @@ public String open() {
PropsUi.setLook(wgType);
wgType.setText(BaseMessages.getString(PKG, "ValidatorDialog.TypeGroup.Label"));
FormLayout typeGroupLayout = new FormLayout();
typeGroupLayout.marginHeight = Const.FORM_MARGIN;
typeGroupLayout.marginWidth = Const.FORM_MARGIN;
typeGroupLayout.marginHeight = PropsUi.getFormMargin();
typeGroupLayout.marginWidth = PropsUi.getFormMargin();
wgType.setLayout(typeGroupLayout);
FormData fdType = new FormData();
fdType.left = new FormAttachment(0, 0);
Expand Down Expand Up @@ -893,10 +908,6 @@ private void saveChanges() {
selectedField.setSourcingField(wSourceField.getText());
selectedField.setSourcingTransformName(wSourceTransform.getText());
selectedField.setSourcingTransform(pipelineMeta.findTransform(wSourceTransform.getText()));

// Save the old info in the map
//
// selectionList.add(selectedField);
}
}

Expand Down Expand Up @@ -1044,11 +1055,12 @@ public void getData() {
wValidateAll.setSelection(input.isValidatingAll());
wConcatErrors.setSelection(input.isConcatenatingErrors());
wConcatSeparator.setText(Const.NVL(input.getConcatenationSeparator(), ""));
wSuppressLogFailedData.setSelection(input.isSuppressingLogFailedData());

// Select the first available field...
//
if (!input.getValidations().isEmpty()) {
Validation validatorField = input.getValidations().get(0);
Validation validatorField = input.getValidations().getFirst();
String description = validatorField.getName();
int index = wValidationsList.indexOf(description);
if (index >= 0) {
Expand Down Expand Up @@ -1085,6 +1097,7 @@ private void ok() {
input.setValidatingAll(wValidateAll.getSelection());
input.setConcatenatingErrors(wConcatErrors.getSelection());
input.setConcatenationSeparator(wConcatSeparator.getText());
input.setSuppressingLogFailedData(wSuppressLogFailedData.getSelection());

input.setValidations(selectionList);

Expand Down Expand Up @@ -1169,17 +1182,37 @@ private void newValidation() {

/** Clear the validation rules for a certain field. */
private void clearValidation() {

int index = wValidationsList.getSelectionIndex();
if (index >= 0) {
selectionList.remove(index);
selectedField = null;
wValidationsList.remove(index);
if (index < 0) {
return;
}

selectionList.remove(index);
selectedField = null;
wValidationsList.remove(index);

if (selectionList.isEmpty()) {
enableFields();
return;
}

if (!selectionList.isEmpty()) {
wValidationsList.select(selectionList.size() - 1);
}
// Keep showing the nearest remaining rule (same index, or previous when deleting the last)
int newIndex = indexAfterRemoval(index, selectionList.size());
wValidationsList.select(newIndex);
showSelectedValidatorField(wValidationsList.getItem(newIndex));
}

/**
* Calculate which list index to select after removing an item.
*
* @param removedIndex the index that was removed
* @param remainingCount the number of items left after removal
* @return the index to select, or -1 when nothing remains
*/
static int indexAfterRemoval(int removedIndex, int remainingCount) {
if (remainingCount <= 0) {
return -1;
}
return Math.min(removedIndex, remainingCount - 1);
}
}
Loading
Loading