Skip to content

NIFI-15588 - Improve date, time, and timestamp type compatibility with ISO 8601 default formatters #10891

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pvillard31 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

NIFI-15588 - Improve date, time, and timestamp type compatibility with ISO 8601 default formatters #10891

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b1fa423
NIFI-15588 - Improve date, time, and timestamp type compatibility wit…
pvillard31 Feb 12, 2026
88a1960
Merge branch 'main' into NIFI-15588
pvillard31 Feb 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 21 additions & 6 deletions ...c/main/java/org/apache/nifi/serialization/record/field/ObjectLocalDateFieldConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,19 @@
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.format.ResolverStyle;
import java.util.Optional;

/**
* Convert Object to java.time.LocalDate using instanceof evaluation and optional format pattern for DateTimeFormatter
*/
class ObjectLocalDateFieldConverter implements FieldConverter<Object, LocalDate> {
private static final DateTimeFormatter[] DEFAULT_DATE_FORMATTERS = new DateTimeFormatter[]{
DateTimeFormatter.ISO_LOCAL_DATE.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_OFFSET_DATE.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_DATE.withResolverStyle(ResolverStyle.STRICT)
};

/**
* Convert Object field to java.sql.Timestamp using optional format supported in DateTimeFormatter
*
Expand Down Expand Up @@ -72,15 +79,23 @@ public LocalDate convertField(final Object field, final Optional<String> pattern
} catch (final DateTimeParseException e) {
throw new FieldConversionException(LocalDate.class, field, name, e);
}
} else {
}

for (final DateTimeFormatter formatter : DEFAULT_DATE_FORMATTERS) {
try {
final long number = Long.parseLong(string);
final Instant instant = Instant.ofEpochMilli(number);
return ofInstant(instant);
} catch (final NumberFormatException e) {
throw new FieldConversionException(LocalDate.class, field, name, e);
return LocalDate.parse(string, formatter);
} catch (final DateTimeParseException e) {
continue;
}
}

try {
final long number = Long.parseLong(string);
final Instant instant = Instant.ofEpochMilli(number);
return ofInstant(instant);
} catch (final NumberFormatException e) {
throw new FieldConversionException(LocalDate.class, field, name, e);
}
}
default -> {
}
Expand Down
18 changes: 16 additions & 2 deletions ...in/java/org/apache/nifi/serialization/record/field/ObjectLocalDateTimeFieldConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.format.ResolverStyle;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQueries;
Expand All @@ -44,6 +45,11 @@ class ObjectLocalDateTimeFieldConverter implements FieldConverter<Object, LocalD
private static final long SECONDS_TO_MICROSECONDS = 1_000_000;

private static final TemporalQuery<LocalDateTime> LOCAL_DATE_TIME_TEMPORAL_QUERY = new LocalDateTimeQuery();
private static final DateTimeFormatter[] DEFAULT_DATE_TIME_FORMATTERS = new DateTimeFormatter[]{
DateTimeFormatter.ISO_OFFSET_DATE_TIME.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_LOCAL_DATE_TIME.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_INSTANT.withResolverStyle(ResolverStyle.STRICT)
};

/**
* Convert Object field to java.sql.Timestamp using optional format supported in DateTimeFormatter
Expand Down Expand Up @@ -93,9 +99,17 @@ public LocalDateTime convertField(final Object field, final Optional<String> pat
} catch (final DateTimeParseException e) {
return tryParseAsNumber(string, name);
}
} else {
return tryParseAsNumber(string, name);
}

for (final DateTimeFormatter formatter : DEFAULT_DATE_TIME_FORMATTERS) {
try {
return formatter.parse(string, LOCAL_DATE_TIME_TEMPORAL_QUERY);
} catch (final DateTimeParseException e) {
continue;
}
}

return tryParseAsNumber(string, name);
}
default -> {
}
Expand Down
26 changes: 20 additions & 6 deletions ...c/main/java/org/apache/nifi/serialization/record/field/ObjectLocalTimeFieldConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.format.ResolverStyle;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQueries;
import java.time.temporal.TemporalQuery;
Expand All @@ -38,6 +39,11 @@
class ObjectLocalTimeFieldConverter implements FieldConverter<Object, LocalTime> {

private static final TemporalQuery<LocalTime> LOCAL_TIME_TEMPORAL_QUERY = new LocalTimeQuery();
private static final DateTimeFormatter[] DEFAULT_TIME_FORMATTERS = new DateTimeFormatter[]{
DateTimeFormatter.ISO_LOCAL_TIME.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_OFFSET_TIME.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_TIME.withResolverStyle(ResolverStyle.STRICT)
};

/**
* Convert Object field to java.time.LocalTime using optional format supported in DateTimeFormatter
Expand Down Expand Up @@ -83,15 +89,23 @@ public LocalTime convertField(final Object field, final Optional<String> pattern
} catch (final DateTimeParseException e) {
throw new FieldConversionException(LocalTime.class, field, name, e);
}
} else {
}

for (final DateTimeFormatter formatter : DEFAULT_TIME_FORMATTERS) {
try {
final long number = Long.parseLong(string);
final Instant instant = Instant.ofEpochMilli(number);
return ofInstant(instant);
} catch (final NumberFormatException e) {
throw new FieldConversionException(LocalTime.class, field, name, e);
return formatter.parse(string, LOCAL_TIME_TEMPORAL_QUERY);
} catch (final DateTimeParseException e) {
continue;
}
}

try {
final long number = Long.parseLong(string);
final Instant instant = Instant.ofEpochMilli(number);
return ofInstant(instant);
} catch (final NumberFormatException e) {
throw new FieldConversionException(LocalTime.class, field, name, e);
}
}
default -> {
}
Expand Down
123 changes: 110 additions & 13 deletions ...ns/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.format.ResolverStyle;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
Expand Down Expand Up @@ -1143,7 +1144,29 @@ public static Date convertDateToUTC(Date dateLocalTZ) {
return new Date(zdtUTC.toInstant().toEpochMilli());
}

private static final DateTimeFormatter[] DEFAULT_DATE_FORMATTERS = new DateTimeFormatter[]{
DateTimeFormatter.ISO_LOCAL_DATE.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_OFFSET_DATE.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_DATE.withResolverStyle(ResolverStyle.STRICT)
};

private static final DateTimeFormatter[] DEFAULT_TIME_FORMATTERS = new DateTimeFormatter[]{
DateTimeFormatter.ISO_LOCAL_TIME.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_OFFSET_TIME.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_TIME.withResolverStyle(ResolverStyle.STRICT)
};

private static final DateTimeFormatter[] DEFAULT_DATETIME_FORMATTERS = new DateTimeFormatter[]{
DateTimeFormatter.ISO_OFFSET_DATE_TIME.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_LOCAL_DATE_TIME.withResolverStyle(ResolverStyle.STRICT),
DateTimeFormatter.ISO_INSTANT.withResolverStyle(ResolverStyle.STRICT)
};

public static boolean isDateTypeCompatible(final Object value, final String format) {
return isDateTypeCompatible(value, format, true);
}

public static boolean isDateTypeCompatible(final Object value, final String format, final boolean allowDefaultFormats) {
if (value == null) {
return false;
}
Expand All @@ -1152,17 +1175,87 @@ public static boolean isDateTypeCompatible(final Object value, final String form
return true;
}

if (value instanceof String) {
if (format == null) {
return isInteger((String) value);
if (value instanceof String stringValue) {
final String trimmed = stringValue.trim();
if (trimmed.isEmpty()) {
return false;
}

try {
DateTimeFormatter.ofPattern(format).parse(value.toString());
return true;
} catch (final DateTimeParseException e) {
if (format != null && !format.isBlank()) {
return isParsable(trimmed, DateTimeFormatter.ofPattern(format).withResolverStyle(ResolverStyle.STRICT));
}

if (!allowDefaultFormats) {
return isInteger(trimmed);
}

return isParsable(trimmed, DEFAULT_DATE_FORMATTERS);
}

return false;
}

public static boolean isTimeTypeCompatible(final Object value, final String format) {
return isTimeTypeCompatible(value, format, true);
}

public static boolean isTimeTypeCompatible(final Object value, final String format, final boolean allowDefaultFormats) {
if (value == null) {
return false;
}

if (value instanceof java.util.Date || value instanceof Number) {
return true;
}

if (value instanceof String stringValue) {
final String trimmed = stringValue.trim();
if (trimmed.isEmpty()) {
return false;
}

if (format != null && !format.isBlank()) {
return isParsable(trimmed, DateTimeFormatter.ofPattern(format).withResolverStyle(ResolverStyle.STRICT));
}

if (!allowDefaultFormats) {
return isInteger(trimmed);
}

return isParsable(trimmed, DEFAULT_TIME_FORMATTERS);
}

return false;
}

public static boolean isTimestampTypeCompatible(final Object value, final String format) {
return isTimestampTypeCompatible(value, format, true);
}

public static boolean isTimestampTypeCompatible(final Object value, final String format, final boolean allowDefaultFormats) {
if (value == null) {
return false;
}

if (value instanceof java.util.Date || value instanceof Number) {
return true;
}

if (value instanceof String stringValue) {
final String trimmed = stringValue.trim();
if (trimmed.isEmpty()) {
return false;
}

if (format != null && !format.isBlank()) {
return isParsable(trimmed, DateTimeFormatter.ofPattern(format).withResolverStyle(ResolverStyle.STRICT));
}

if (!allowDefaultFormats) {
return isInteger(trimmed);
}

return isParsable(trimmed, DEFAULT_DATETIME_FORMATTERS);
}

return false;
Expand All @@ -1182,12 +1275,16 @@ private static boolean isInteger(final String value) {
return true;
}

public static boolean isTimeTypeCompatible(final Object value, final String format) {
return isDateTypeCompatible(value, format);
}

public static boolean isTimestampTypeCompatible(final Object value, final String format) {
return isDateTypeCompatible(value, format);
private static boolean isParsable(final String value, final DateTimeFormatter... formatters) {
for (final DateTimeFormatter formatter : formatters) {
try {
formatter.parse(value);
return true;
} catch (final DateTimeParseException e) {
continue;
}
}
return false;
}

public static boolean isUUIDTypeCompatible(final Object value) {
Expand Down
66 changes: 66 additions & 0 deletions ...ons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1224,6 +1224,72 @@ void testNumberParsingWhereStringBlank() {
assertNull(DataTypeUtils.toShort("", fieldName));
}

@Test
void testIsoDateCompatibilityWithoutExplicitFormat() {
assertTrue(DataTypeUtils.isDateTypeCompatible("2024-01-15", null));
assertTrue(DataTypeUtils.isDateTypeCompatible(" 2024-01-15 ", null));
assertTrue(DataTypeUtils.isDateTypeCompatible("2024-01-15Z", null));

assertFalse(DataTypeUtils.isDateTypeCompatible("15/01/2024", null));
assertFalse(DataTypeUtils.isDateTypeCompatible("", null));
}

@Test
void testIsoTimeCompatibilityWithoutExplicitFormat() {
assertTrue(DataTypeUtils.isTimeTypeCompatible("13:45:00", null));
assertTrue(DataTypeUtils.isTimeTypeCompatible("13:45:00Z", null));
assertTrue(DataTypeUtils.isTimeTypeCompatible(" 13:45:00.123 ", null));

assertFalse(DataTypeUtils.isTimeTypeCompatible("25:61", null));
assertFalse(DataTypeUtils.isTimeTypeCompatible("", null));
}

@Test
void testIsoTimestampCompatibilityWithoutExplicitFormat() {
assertTrue(DataTypeUtils.isTimestampTypeCompatible("2024-01-15T13:45:00Z", null));
assertTrue(DataTypeUtils.isTimestampTypeCompatible("2024-01-15T13:45:00", null));
assertTrue(DataTypeUtils.isTimestampTypeCompatible(" 2024-01-15T13:45:00+01:00 ", null));

assertFalse(DataTypeUtils.isTimestampTypeCompatible("not-a-timestamp", null));
assertFalse(DataTypeUtils.isTimestampTypeCompatible("", null));
}

@Test
void testLegacyEpochCompatibilityWhenDefaultFormatsDisabled() {
// The three-argument overload keeps AbstractCSVRecordReader's non-coerce path aligned with legacy behaviour (see
// nifi-extension-bundles/.../csv/AbstractCSVRecordReader.java:122) by rejecting ISO strings when no explicit format
// is provided while still allowing epoch-based values.
assertFalse(DataTypeUtils.isDateTypeCompatible("2024-01-15", null, false));
assertFalse(DataTypeUtils.isTimeTypeCompatible("01:02:03", null, false));
assertFalse(DataTypeUtils.isTimestampTypeCompatible("2024-01-15T01:02:03Z", null, false));

assertTrue(DataTypeUtils.isDateTypeCompatible(String.valueOf(1700000000000L), null, false));
assertTrue(DataTypeUtils.isTimeTypeCompatible(String.valueOf(1700000000000L), null, false));
assertTrue(DataTypeUtils.isTimestampTypeCompatible(String.valueOf(1700000000000L), null, false));
}

@Test
void testDefaultIsoFormattersRejectInvalidDates() {
assertFalse(DataTypeUtils.isDateTypeCompatible("2024-02-30", null));
assertFalse(DataTypeUtils.isDateTypeCompatible("2024-13-01", null));
assertFalse(DataTypeUtils.isTimestampTypeCompatible("2024-02-30T10:00:00", null));

assertTrue(DataTypeUtils.isDateTypeCompatible("2024-02-29", null));
assertTrue(DataTypeUtils.isTimestampTypeCompatible("2024-02-29T10:00:00", null));
}

@Test
void testExplicitFormatWithYearOfEraPatternDoesNotRejectInvalidDates() {
// The yyyy pattern represents "year-of-era" which cannot fully resolve without an era under STRICT mode.
// As a result, formatter.parse() succeeds without validating day-of-month, unlike the default ISO formatters
// (tested in testDefaultIsoFormattersRejectInvalidDates) which use proleptic year (uuuu) and do reject invalid dates.
assertTrue(DataTypeUtils.isDateTypeCompatible("2024-02-30", "yyyy-MM-dd"));
assertTrue(DataTypeUtils.isTimestampTypeCompatible("2024-02-30 10:00:00", "yyyy-MM-dd HH:mm:ss"));

assertTrue(DataTypeUtils.isDateTypeCompatible("2024-02-29", "yyyy-MM-dd"));
assertTrue(DataTypeUtils.isTimestampTypeCompatible("2024-02-29 10:00:00", "yyyy-MM-dd HH:mm:ss"));
}

@Test
void testUuidCompatibilityAcrossSupportedRepresentations() {
final UUID uuid = UUID.randomUUID();
Expand Down
Loading
Loading