From 85b40407e49282dc866a50932c40fc86bbc1468e Mon Sep 17 00:00:00 2001 From: lance Date: Sun, 19 Jul 2026 21:45:09 +0800 Subject: [PATCH] Fix Database Lookup return type inference when Type is empty Signed-off-by: lance --- .../databaselookup/DatabaseLookup.java | 20 +- .../databaselookup/DatabaseLookupData.java | 4 + .../databaselookup/DatabaseLookupMeta.java | 58 ++-- .../messages/messages_en_US.properties | 2 + .../DatabaseLookupDataTest.java | 80 +++++ .../DatabaseLookupMetaTest.java | 268 +++++++++++++++- .../databaselookup/DatabaseLookupUTest.java | 297 +++++++++++++++++- .../databaselookup/DefaultCacheTest.java | 256 +++++++++++++++ .../databaselookup/KeyFieldTest.java | 80 +++++ .../transforms/databaselookup/LookupTest.java | 95 ++++++ .../databaselookup/ReturnValueTest.java | 98 ++++++ .../readallcache/IndexTestBase.java | 101 ++++-- .../readallcache/ReadAllCacheTest.java | 9 + .../readallcache/SearchingContextTest.java | 28 ++ 14 files changed, 1334 insertions(+), 62 deletions(-) create mode 100644 plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupDataTest.java create mode 100644 plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DefaultCacheTest.java create mode 100644 plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/KeyFieldTest.java create mode 100644 plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/LookupTest.java create mode 100644 plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/ReturnValueTest.java diff --git a/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookup.java b/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookup.java index 11152f77884..7ff16825004 100644 --- a/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookup.java +++ b/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookup.java @@ -223,6 +223,7 @@ void determineFieldsTypesQueryingDb() throws HopException { this, meta.getSchemaName(), meta.getTableName()); IRowMeta fields = data.db.getTableFields(schemaTable); + data.dbRowMeta = fields; if (fields != null) { // Fill in the types... for (int i = 0; i < keyFields.size(); i++) { @@ -300,9 +301,9 @@ private void initReturnMeta() { data.returnValueTypes = new int[returnValues.size()]; for (int i = 0; i < returnValues.size(); i++) { - data.returnValueTypes[i] = - ValueMetaFactory.getIdForValueMeta(returnValues.get(i).getDefaultType()); IValueMeta v = data.outputRowMeta.getValueMeta(getInputRowMeta().size() + i).clone(); + // Use resolved output type (may be inferred from the table when default type is empty) + data.returnValueTypes[i] = v.getType(); data.returnMeta.addValueMeta(v); } } @@ -318,10 +319,6 @@ public boolean processRow() throws HopException { if (first) { first = false; - // create the output metadata - data.outputRowMeta = getInputRowMeta().clone(); - meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); - Lookup lookup = meta.getLookup(); List keyFields = lookup.getKeyFields(); List returnValues = lookup.getReturnValues(); @@ -406,8 +403,19 @@ public boolean processRow() throws HopException { data.cache = DefaultCache.newCache(data, meta.getCacheSize()); } + // Query table metadata once: key types + return type inference for empty default types determineFieldsTypesQueryingDb(); + // create the output metadata (pass table fields so empty return types can be inferred) + data.outputRowMeta = getInputRowMeta().clone(); + meta.getFields( + data.outputRowMeta, + getTransformName(), + new IRowMeta[] {data.dbRowMeta}, + null, + this, + metadataProvider); + initNullIf(); initLookupMeta(); diff --git a/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupData.java b/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupData.java index cfccb998c1c..25e1c56a173 100644 --- a/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupData.java +++ b/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupData.java @@ -37,6 +37,10 @@ public class DatabaseLookupData extends BaseTransformData implements ITransformD public IRowMeta outputRowMeta; public IRowMeta lookupMeta; public IRowMeta returnMeta; + + /** Cached lookup-table field metadata (from getTableFields), reused for return type inference. */ + public IRowMeta dbRowMeta; + public boolean isCanceled; public boolean allEquals; public int[] conditions; diff --git a/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupMeta.java b/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupMeta.java index 83dca66a1e2..d0ba4ee9676 100644 --- a/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupMeta.java +++ b/plugins/transforms/databaselookup/src/main/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupMeta.java @@ -130,34 +130,48 @@ public void getFields( IHopMetadataProvider metadataProvider) throws HopTransformException { try { - // info row metadata: null or length 0 : no lookup row metadata from database + // Prefer an explicit return type. When none is configured, infer from table field + // metadata (info row from design-time / runtime, otherwise query the table). // - if (Utils.isEmpty(infoRowMeta) || infoRowMeta[0] == null) { - for (ReturnValue returnValue : lookup.getReturnValues()) { - IValueMeta v = - ValueMetaFactory.createValueMeta( - !Utils.isEmpty(returnValue.getNewName()) - ? returnValue.getNewName() - : returnValue.getTableField(), - ValueMetaFactory.getIdForValueMeta(returnValue.getDefaultType())); - v.setOrigin(name); - row.addValueMeta(v); - } - return; + IRowMeta tableFields = null; + if (!Utils.isEmpty(infoRowMeta) && infoRowMeta[0] != null) { + tableFields = infoRowMeta[0]; } for (ReturnValue returnValue : lookup.getReturnValues()) { - IValueMeta v = infoRowMeta[0].searchValueMeta(returnValue.getTableField()); - if (v != null) { - IValueMeta copy = v.clone(); // avoid renaming other value meta - copy.setName( - !Utils.isEmpty(returnValue.getNewName()) - ? returnValue.getNewName() - : returnValue.getTableField()); - copy.setOrigin(name); - row.addValueMeta(copy); + String fieldName = + !Utils.isEmpty(returnValue.getNewName()) + ? returnValue.getNewName() + : returnValue.getTableField(); + int typeId = ValueMetaFactory.getIdForValueMeta(returnValue.getDefaultType()); + IValueMeta v; + if (typeId != IValueMeta.TYPE_NONE) { + v = ValueMetaFactory.createValueMeta(fieldName, typeId); + } else { + if (tableFields == null) { + tableFields = getTableFields(variables); + } + if (tableFields == null) { + throw new HopTransformException( + BaseMessages.getString( + PKG, "DatabaseLookupMeta.Exception.UnableToRetrieveDataTypeOfReturnField")); + } + IValueMeta source = tableFields.searchValueMeta(returnValue.getTableField()); + if (source == null) { + throw new HopTransformException( + BaseMessages.getString( + PKG, + "DatabaseLookupMeta.Exception.UnableToFindReturnField", + returnValue.getTableField())); + } + v = source.clone(); // avoid renaming other value meta + v.setName(fieldName); } + v.setOrigin(name); + row.addValueMeta(v); } + } catch (HopTransformException e) { + throw e; } catch (HopException e) { throw new HopTransformException("Error getting fields metadata", e); } diff --git a/plugins/transforms/databaselookup/src/main/resources/org/apache/hop/pipeline/transforms/databaselookup/messages/messages_en_US.properties b/plugins/transforms/databaselookup/src/main/resources/org/apache/hop/pipeline/transforms/databaselookup/messages/messages_en_US.properties index 996582ad254..935f6446dec 100644 --- a/plugins/transforms/databaselookup/src/main/resources/org/apache/hop/pipeline/transforms/databaselookup/messages/messages_en_US.properties +++ b/plugins/transforms/databaselookup/src/main/resources/org/apache/hop/pipeline/transforms/databaselookup/messages/messages_en_US.properties @@ -97,6 +97,8 @@ DatabaseLookupMeta.Check.MissingReturnFieldsInLookupTable=Missing return fields DatabaseLookupMeta.Check.NoInputReceivedFromOtherTransforms=No input received from other transforms\! DatabaseLookupMeta.Check.TransformIsReceivingInfoFromOtherTransforms=Transform is receiving info from other transforms. DatabaseLookupMeta.ERROR0004.ErrorGettingTableFields=An error occurred\: +DatabaseLookupMeta.Exception.UnableToFindReturnField=Unable to find return field [{0}] in the lookup table. +DatabaseLookupMeta.Exception.UnableToRetrieveDataTypeOfReturnField=Unable to retrieve data type of return fields because table metadata is not available DatabaseLookupMeta.Impact.Key=Key DatabaseLookupMeta.Impact.ReturnValue=Return value DatabaseLookupMeta.Injection.Cache=Cache lookup data? diff --git a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupDataTest.java b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupDataTest.java new file mode 100644 index 00000000000..6cd9cbf1de2 --- /dev/null +++ b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupDataTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.databaselookup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hop.core.row.RowMeta; +import org.junit.jupiter.api.Test; + +class DatabaseLookupDataTest { + + @Test + void defaultConstructor_InitializesNullRuntimeState() { + DatabaseLookupData data = new DatabaseLookupData(); + + assertNull(data.db); + assertNull(data.cache); + assertNull(data.nullif); + assertNull(data.keynrs); + assertNull(data.keynrs2); + assertNull(data.keytypes); + assertNull(data.outputRowMeta); + assertNull(data.lookupMeta); + assertNull(data.returnMeta); + assertNull(data.dbRowMeta); + assertNull(data.conditions); + assertNull(data.returnValueTypes); + assertNull(data.returnTrimTypes); + assertNull(data.trimIndexes); + assertFalse(data.isCanceled); + assertFalse(data.allEquals); + assertFalse(data.hasDBCondition); + } + + @Test + void fields_CanBeAssignedAndRead() { + DatabaseLookupData data = new DatabaseLookupData(); + data.dbRowMeta = new RowMeta(); + data.lookupMeta = new RowMeta(); + data.returnMeta = new RowMeta(); + data.outputRowMeta = new RowMeta(); + data.keynrs = new int[] {0}; + data.keynrs2 = new int[] {-1}; + data.keytypes = new int[] {1}; + data.conditions = new int[] {DatabaseLookupMeta.CONDITION_EQ}; + data.returnValueTypes = new int[] {2}; + data.returnTrimTypes = new String[] {"none"}; + data.nullif = new Object[] {"N/A"}; + data.cache = DefaultCache.newCache(data, 8); + data.allEquals = true; + data.hasDBCondition = true; + data.isCanceled = true; + + assertTrue(data.allEquals); + assertTrue(data.hasDBCondition); + assertTrue(data.isCanceled); + assertEquals(0, data.keynrs[0]); + assertEquals(DatabaseLookupMeta.CONDITION_EQ, data.conditions[0]); + assertInstanceOf(DefaultCache.class, data.cache); + } +} diff --git a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupMetaTest.java b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupMetaTest.java index c3cd31453eb..58812681600 100644 --- a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupMetaTest.java +++ b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupMetaTest.java @@ -20,7 +20,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -30,6 +33,7 @@ import java.util.Arrays; import java.util.List; import org.apache.hop.core.HopClientEnvironment; +import org.apache.hop.core.ICheckResult; import org.apache.hop.core.RowMetaAndData; import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.exception.HopTransformException; @@ -39,11 +43,15 @@ import org.apache.hop.core.row.IValueMeta; import org.apache.hop.core.row.RowMeta; import org.apache.hop.core.row.RowMetaBuilder; +import org.apache.hop.core.row.value.ValueMetaInteger; import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.Variables; import org.apache.hop.core.xml.XmlHandler; import org.apache.hop.metadata.api.IHopMetadataProvider; import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider; import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil; +import org.apache.hop.pipeline.DatabaseImpact; +import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.transform.TransformMeta; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -112,14 +120,122 @@ void getFieldWithValueUsedTwice() throws HopTransformException { List expectedRow = Arrays.asList( new IValueMeta[] { - new ValueMetaString("value"), new ValueMetaString("v1"), new ValueMetaString("v2"), + new ValueMetaString("value"), + new ValueMetaString("v1"), + new ValueMetaString("v2"), + new ValueMetaString("v3"), }); - assertEquals(3, row.getValueMetaList().size()); - for (int i = 0; i < 3; i++) { + assertEquals(4, row.getValueMetaList().size()); + for (int i = 0; i < 4; i++) { assertEquals(expectedRow.get(i).getName(), row.getValueMetaList().get(i).getName()); } } + @Test + void getFieldsInfersTypeFromInfoWhenDefaultTypeEmpty() throws Exception { + Lookup lookup = databaseLookupMeta.getLookup(); + lookup + .getReturnValues() + .add( + new ReturnValue( + "stock_name", + "", + "", + "", + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE))); + + IRowMeta[] info = new IRowMeta[1]; + info[0] = new RowMeta(); + info[0].addValueMeta(new ValueMetaString("stock_name")); + + IRowMeta row = new RowMeta(); + row.addValueMeta(new ValueMetaString("id")); + + databaseLookupMeta.getFields(row, "Database lookup", info, null, null, null); + + assertEquals(2, row.size()); + IValueMeta stockName = row.searchValueMeta("stock_name"); + assertNotNull(stockName); + assertEquals(IValueMeta.TYPE_STRING, stockName.getType()); + } + + @Test + void getFieldsInfersTypeFromInfoWhenDefaultTypeNone() throws Exception { + Lookup lookup = databaseLookupMeta.getLookup(); + lookup + .getReturnValues() + .add( + new ReturnValue( + "amount", + "amt", + "", + "None", + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE))); + + IRowMeta[] info = new IRowMeta[1]; + info[0] = new RowMeta(); + info[0].addValueMeta(new ValueMetaInteger("amount")); + + IRowMeta row = new RowMeta(); + databaseLookupMeta.getFields(row, "Database lookup", info, null, null, null); + + assertEquals(1, row.size()); + IValueMeta amt = row.searchValueMeta("amt"); + assertNotNull(amt); + assertEquals(IValueMeta.TYPE_INTEGER, amt.getType()); + } + + @Test + void getFieldsExplicitTypeOverridesInfoType() throws Exception { + Lookup lookup = databaseLookupMeta.getLookup(); + lookup + .getReturnValues() + .add( + new ReturnValue( + "amount", + "", + "", + "String", + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE))); + + IRowMeta[] info = new IRowMeta[1]; + info[0] = new RowMeta(); + info[0].addValueMeta(new ValueMetaInteger("amount")); + + IRowMeta row = new RowMeta(); + databaseLookupMeta.getFields(row, "Database lookup", info, null, null, null); + + assertEquals(1, row.size()); + IValueMeta amount = row.searchValueMeta("amount"); + assertNotNull(amount); + assertEquals(IValueMeta.TYPE_STRING, amount.getType()); + } + + @Test + void getFieldsThrowsWhenTypeMissingAndFieldNotInInfo() { + Lookup lookup = databaseLookupMeta.getLookup(); + lookup + .getReturnValues() + .add( + new ReturnValue( + "missing_col", + "", + "", + "", + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE))); + + IRowMeta[] info = new IRowMeta[1]; + info[0] = new RowMeta(); + info[0].addValueMeta(new ValueMetaString("other")); + + IRowMeta row = new RowMeta(); + HopTransformException thrown = + assertThrows( + HopTransformException.class, + () -> databaseLookupMeta.getFields(row, "Database lookup", info, null, null, null)); + assertTrue(thrown.getMessage().contains("missing_col")); + } + @Test void cloneTest() throws Exception { DatabaseLookupMeta meta = new DatabaseLookupMeta(); @@ -358,4 +474,150 @@ void testInjection() throws Exception { assertEquals("?", meta.getLookup().getReturnValues().get(0).getDefaultValue()); assertEquals("String", meta.getLookup().getReturnValues().get(0).getDefaultType()); } + + @Test + void supportsErrorHandling_ReturnsTrue() { + assertTrue(databaseLookupMeta.supportsErrorHandling()); + } + + @Test + void getConditionStrings_ContainsAllOperators() { + List conditions = DatabaseLookupMeta.getConditionStrings(); + assertEquals(10, conditions.size()); + assertTrue(conditions.contains("=")); + assertTrue(conditions.contains("<>")); + assertTrue(conditions.contains("LIKE")); + assertTrue(conditions.contains("BETWEEN")); + assertTrue(conditions.contains("IS NULL")); + assertTrue(conditions.contains("IS NOT NULL")); + } + + @Test + void check_ReportsErrorWhenConnectionMissing() { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setConnection(""); + List remarks = new ArrayList<>(); + TransformMeta transformMeta = new TransformMeta("lookup", meta); + Variables variables = new Variables(); + + meta.check( + remarks, + mock(PipelineMeta.class), + transformMeta, + new RowMeta(), + new String[] {"prev"}, + new String[] {}, + null, + variables, + metadataProvider); + + assertTrue( + remarks.stream().anyMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_ERROR), + "Expected a missing-connection error remark"); + } + + @Test + void check_ReportsErrorWhenNoInputReceived() { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setConnection("postgres"); + meta.getLookup().setTableName(""); + List remarks = new ArrayList<>(); + TransformMeta transformMeta = new TransformMeta("lookup", meta); + Variables variables = new Variables(); + + // Connection exists in metadata but connecting to NONE DB may fail — still should report + // no-input when input array is empty. + meta.check( + remarks, + mock(PipelineMeta.class), + transformMeta, + new RowMeta(), + new String[] {}, + new String[] {}, + null, + variables, + metadataProvider); + + assertTrue( + remarks.stream() + .anyMatch( + r -> + r.getType() == ICheckResult.TYPE_RESULT_ERROR + && r.getText() != null + && r.getText().toLowerCase().contains("no input")), + "Expected a no-input error remark, got: " + remarks); + } + + @Test + void check_ReportsOkWhenReceivingInput() { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setConnection("postgres"); + meta.getLookup().setTableName(""); + List remarks = new ArrayList<>(); + TransformMeta transformMeta = new TransformMeta("lookup", meta); + Variables variables = new Variables(); + + meta.check( + remarks, + mock(PipelineMeta.class), + transformMeta, + new RowMeta(), + new String[] {"prev"}, + new String[] {}, + null, + variables, + metadataProvider); + + assertTrue( + remarks.stream() + .anyMatch( + r -> + r.getType() == ICheckResult.TYPE_RESULT_OK + && r.getText() != null + && r.getText().toLowerCase().contains("receiving")), + "Expected an OK remark about receiving input, got: " + remarks); + } + + @Test + void analyseImpact_AddsReadImpactForKeysAndReturns() throws Exception { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setConnection("postgres"); + Lookup lookup = meta.getLookup(); + lookup.setTableName("users"); + lookup.getKeyFields().add(new KeyField("in_id", "", "=", "id")); + lookup + .getReturnValues() + .add( + new ReturnValue( + "name", + "user_name", + "", + "String", + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE))); + + IRowMeta prev = new RowMeta(); + prev.addValueMeta(new ValueMetaString("in_id")); + + List impact = new ArrayList<>(); + TransformMeta transformMeta = new TransformMeta("lookup", meta); + PipelineMeta pipelineMeta = mock(PipelineMeta.class); + when(pipelineMeta.getName()).thenReturn("pipe"); + + meta.analyseImpact( + new Variables(), + impact, + pipelineMeta, + transformMeta, + prev, + new String[] {}, + new String[] {}, + null, + metadataProvider); + + assertEquals(2, impact.size()); + assertEquals(DatabaseImpact.TYPE_IMPACT_READ, impact.get(0).getType()); + assertEquals("id", impact.get(0).getField()); + assertEquals(DatabaseImpact.TYPE_IMPACT_READ, impact.get(1).getType()); + assertEquals("name", impact.get(1).getField()); + } } diff --git a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupUTest.java b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupUTest.java index be879a20739..13bf628aa98 100644 --- a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupUTest.java +++ b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DatabaseLookupUTest.java @@ -21,7 +21,11 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -35,12 +39,15 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockingDetails; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.concurrent.TimeUnit; @@ -358,9 +365,7 @@ private DatabaseLookup createSpiedTransform( DatabaseLookupData data) throws HopException { - NoneDatabaseMeta genericMeta = new NoneDatabaseMeta(); - DatabaseMeta dbMeta = new DatabaseMeta(); - dbMeta.setIDatabase(genericMeta); + DatabaseMeta dbMeta = createNoneDbMeta(); DatabaseLookup transform = spyLookup(mockHelper, meta, data, db, dbMeta); when(transform.getPipelineMeta().findDatabase(any(String.class), any(IVariables.class))) @@ -406,6 +411,292 @@ void createsReadDefaultCache_AndUsesOnlyNeededFieldsFromMeta() throws Exception assertNotNull(data.cache.getRowFromCache(data.lookupMeta, new Object[] {2L})); } + @Test + void infersReturnTypeFromTableWhenDefaultTypeEmpty() throws Exception { + Database db = mock(Database.class); + + RowMeta tableFields = new RowMeta(); + tableFields.addValueMeta(new ValueMetaInteger("ID")); + tableFields.addValueMeta(new ValueMetaString("stock_name")); + when(db.getTableFields(nullable(String.class))).thenReturn(tableFields); + + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setConnection("connection1"); + Lookup lookup = meta.getLookup(); + lookup.setTableName("VirtualTable"); + lookup.getKeyFields().add(new KeyField("id", "", "=", "ID")); + lookup + .getReturnValues() + .add( + new ReturnValue( + "stock_name", + "", + "", + "", + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE))); + + DatabaseLookupData data = new DatabaseLookupData(); + DatabaseMeta dbMeta = createNoneDbMeta(); + DatabaseLookup transform = spyLookup(mockHelper, meta, data, db, dbMeta); + when(transform.getPipelineMeta().findDatabase(any(String.class), any(IVariables.class))) + .thenReturn(dbMeta); + doReturn(null).when(transform).lookupValues(any(IRowMeta.class), any(Object[].class)); + + RowMeta input = new RowMeta(); + input.addValueMeta(new ValueMetaInteger("id")); + transform.setInputRowMeta(input); + + transform.init(); + data.db = db; + + transform.processRow(); + + IValueMeta stockName = data.outputRowMeta.searchValueMeta("stock_name"); + assertNotNull(stockName); + assertThat(stockName.getType(), is(IValueMeta.TYPE_STRING)); + assertThat(data.returnValueTypes[0], is(IValueMeta.TYPE_STRING)); + } + + @Test + void init_SetsHasDbConditionForLikeOperator() { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setConnection("connection1"); + meta.getLookup().getKeyFields().add(new KeyField("id", "", "LIKE", "ID")); + + DatabaseLookupData data = new DatabaseLookupData(); + Database db = mock(Database.class); + DatabaseLookup transform = spyLookup(mockHelper, meta, data, db, createNoneDbMeta()); + when(transform.getPipelineMeta().findDatabase(any(String.class), any(IVariables.class))) + .thenReturn(createNoneDbMeta()); + + assertTrue(transform.init()); + assertTrue(data.hasDBCondition); + assertFalse(data.allEquals); + assertEquals(DatabaseLookupMeta.CONDITION_LIKE, data.conditions[0]); + } + + @Test + void init_ReturnsFalseWhenConnectionMissing() { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setConnection(""); + DatabaseLookupData data = new DatabaseLookupData(); + DatabaseLookup transform = + new DatabaseLookup( + mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta, mockHelper.pipeline); + + assertFalse(transform.init()); + } + + @Test + void lookupValues_ReturnsDefaultNullIfWhenNoMatch() throws Exception { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setCached(false); + meta.getLookup().getKeyFields().add(new KeyField("id", "", "=", "ID")); + meta.getLookup() + .getReturnValues() + .add( + new ReturnValue( + "name", + "", + "N/A", + "String", + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE))); + + DatabaseLookupData data = new DatabaseLookupData(); + Database db = mock(Database.class); + when(db.getLookup(anyBoolean())).thenReturn(null); + + DatabaseLookup transform = spyLookup(mockHelper, meta, data, db, createNoneDbMeta()); + when(transform.getPipelineMeta().findDatabase(any(String.class), any(IVariables.class))) + .thenReturn(createNoneDbMeta()); + doReturn(false).when(transform).isRowLevel(); + + RowMeta input = new RowMeta(); + input.addValueMeta(new ValueMetaInteger("id")); + transform.setInputRowMeta(input); + + data.db = db; + data.keynrs = new int[] {0}; + data.keynrs2 = new int[] {-1}; + data.lookupMeta = new RowMeta(); + data.lookupMeta.addValueMeta(new ValueMetaInteger("ID")); + data.returnMeta = new RowMeta(); + data.returnMeta.addValueMeta(new ValueMetaString("name")); + data.outputRowMeta = input.clone(); + data.outputRowMeta.addValueMeta(new ValueMetaString("name")); + data.returnValueTypes = new int[] {IValueMeta.TYPE_STRING}; + data.nullif = new Object[] {"N/A"}; + data.trimIndexes = new ArrayList<>(); + + Object[] result = transform.lookupValues(input, new Object[] {1L}); + assertNotNull(result); + assertEquals("N/A", result[1]); + } + + @Test + void lookupValues_EatsRowWhenEatingRowOnLookupFailure() throws Exception { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setCached(false); + meta.getLookup().setEatingRowOnLookupFailure(true); + meta.getLookup().getKeyFields().add(new KeyField("id", "", "=", "ID")); + meta.getLookup() + .getReturnValues() + .add( + new ReturnValue( + "name", + "", + "", + "String", + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE))); + + DatabaseLookupData data = new DatabaseLookupData(); + Database db = mock(Database.class); + when(db.getLookup(anyBoolean())).thenReturn(null); + + DatabaseLookup transform = spyLookup(mockHelper, meta, data, db, createNoneDbMeta()); + doReturn(false).when(transform).isRowLevel(); + + RowMeta input = new RowMeta(); + input.addValueMeta(new ValueMetaInteger("id")); + transform.setInputRowMeta(input); + + data.db = db; + data.keynrs = new int[] {0}; + data.keynrs2 = new int[] {-1}; + data.lookupMeta = new RowMeta(); + data.lookupMeta.addValueMeta(new ValueMetaInteger("ID")); + data.returnMeta = new RowMeta(); + data.returnMeta.addValueMeta(new ValueMetaString("name")); + data.outputRowMeta = input.clone(); + data.outputRowMeta.addValueMeta(new ValueMetaString("name")); + data.returnValueTypes = new int[] {IValueMeta.TYPE_STRING}; + data.nullif = new Object[] {null}; + data.trimIndexes = new ArrayList<>(); + + assertNull(transform.lookupValues(input, new Object[] {1L})); + } + + @Test + void lookupValues_StoresInCacheOnDbHitWhenCached() throws Exception { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setCached(true); + meta.setLoadingAllDataInCache(false); + meta.getLookup().getKeyFields().add(new KeyField("id", "", "=", "ID")); + meta.getLookup() + .getReturnValues() + .add( + new ReturnValue( + "name", + "", + "", + "String", + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE))); + + DatabaseLookupData data = new DatabaseLookupData(); + data.allEquals = true; + data.cache = mock(DatabaseLookupData.ICache.class); + when(data.cache.getRowFromCache(any(IRowMeta.class), any(Object[].class))).thenReturn(null); + + Database db = mock(Database.class); + when(db.getLookup(anyBoolean())).thenReturn(new Object[] {"Alice"}); + RowMeta returnRowMeta = new RowMeta(); + returnRowMeta.addValueMeta(new ValueMetaString("name")); + when(db.getReturnRowMeta()).thenReturn(returnRowMeta); + + DatabaseLookup transform = spyLookup(mockHelper, meta, data, db, createNoneDbMeta()); + doReturn(false).when(transform).isRowLevel(); + + RowMeta input = new RowMeta(); + input.addValueMeta(new ValueMetaInteger("id")); + transform.setInputRowMeta(input); + + data.db = db; + data.keynrs = new int[] {0}; + data.keynrs2 = new int[] {-1}; + data.lookupMeta = new RowMeta(); + data.lookupMeta.addValueMeta(new ValueMetaInteger("ID")); + data.returnMeta = new RowMeta(); + data.returnMeta.addValueMeta(new ValueMetaString("name")); + data.outputRowMeta = input.clone(); + data.outputRowMeta.addValueMeta(new ValueMetaString("name")); + data.returnValueTypes = new int[] {IValueMeta.TYPE_STRING}; + data.nullif = new Object[] {null}; + data.trimIndexes = new ArrayList<>(); + + Object[] result = transform.lookupValues(input, new Object[] {1L}); + assertEquals("Alice", result[1]); + verify(data.cache, times(1)) + .storeRowInCache(any(DatabaseLookupMeta.class), any(IRowMeta.class), any(), any()); + } + + @Test + void determineFieldsTypesQueryingDb_ThrowsWhenKeyColumnMissingInTable() throws Exception { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setConnection("connection1"); + meta.getLookup().setTableName("t"); + meta.getLookup().getKeyFields().add(new KeyField("id", "", "=", "MISSING")); + + DatabaseLookupData data = new DatabaseLookupData(); + Database db = mock(Database.class); + RowMeta tableFields = new RowMeta(); + tableFields.addValueMeta(new ValueMetaInteger("ID")); + when(db.getTableFields(nullable(String.class))).thenReturn(tableFields); + + DatabaseLookup transform = spyLookup(mockHelper, meta, data, db, createNoneDbMeta()); + when(transform.getPipelineMeta().findDatabase(any(String.class), any(IVariables.class))) + .thenReturn(createNoneDbMeta()); + data.db = db; + + assertThrows(HopException.class, transform::determineFieldsTypesQueryingDb); + } + + @Test + void stopRunning_CancelsQueryOnce() throws Exception { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setConnection("connection1"); + DatabaseLookupData data = new DatabaseLookupData(); + Database db = mock(Database.class); + data.db = db; + data.isCanceled = false; + + DatabaseLookup transform = + new DatabaseLookup( + mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta, mockHelper.pipeline); + + transform.stopRunning(); + transform.stopRunning(); + + verify(db, times(1)).cancelQuery(); + assertTrue(data.isCanceled); + } + + @Test + void dispose_ClearsCacheAndDisconnects() { + DatabaseLookupMeta meta = new DatabaseLookupMeta(); + meta.setConnection("connection1"); + DatabaseLookupData data = new DatabaseLookupData(); + Database db = mock(Database.class); + data.db = db; + data.cache = DefaultCache.newCache(data, 16); + + DatabaseLookup transform = + new DatabaseLookup( + mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta, mockHelper.pipeline); + + transform.dispose(); + + verify(db, times(1)).disconnect(); + assertNull(data.db); + assertNull(data.cache); + } + + private DatabaseMeta createNoneDbMeta() { + NoneDatabaseMeta genericMeta = new NoneDatabaseMeta(); + DatabaseMeta dbMeta = new DatabaseMeta(); + dbMeta.setIDatabase(genericMeta); + return dbMeta; + } + public class MockDatabaseLookup extends DatabaseLookup { public MockDatabaseLookup( TransformMeta transformMeta, diff --git a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DefaultCacheTest.java b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DefaultCacheTest.java new file mode 100644 index 00000000000..84a63c210a4 --- /dev/null +++ b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/DefaultCacheTest.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.databaselookup; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class DefaultCacheTest { + + private DatabaseLookupData data; + private IRowMeta lookupMeta; + private DatabaseLookupMeta meta; + + @BeforeEach + void setUp() { + data = new DatabaseLookupData(); + data.lookupMeta = new RowMeta(); + data.lookupMeta.addValueMeta(new ValueMetaInteger("id")); + lookupMeta = data.lookupMeta.clone(); + + meta = new DatabaseLookupMeta(); + meta.setCached(true); + meta.setCacheSize(0); + meta.setLoadingAllDataInCache(false); + } + + @Test + void newCache_UsesDefaultCapacityWhenSizeNonPositive() { + DefaultCache cache = DefaultCache.newCache(data, 0); + assertNotNull(cache); + DefaultCache negative = DefaultCache.newCache(data, -5); + assertNotNull(negative); + } + + @Test + void newCache_AcceptsPositiveCacheSize() { + assertNotNull(DefaultCache.newCache(data, 10)); + } + + @Test + void getRowFromCache_AllEquals_ReturnsExactMapHit() throws Exception { + data.allEquals = true; + data.conditions = new int[] {DatabaseLookupMeta.CONDITION_EQ}; + DefaultCache cache = DefaultCache.newCache(data, 16); + + Object[] key = new Object[] {1L}; + Object[] value = new Object[] {"one"}; + cache.storeRowInCache(meta, lookupMeta, key, value); + + assertArrayEquals(value, cache.getRowFromCache(lookupMeta, new Object[] {1L})); + assertNull(cache.getRowFromCache(lookupMeta, new Object[] {2L})); + } + + @Test + void getRowFromCache_ConditionEq_ScansWhenNotAllEquals() throws Exception { + assertConditionMatch(DatabaseLookupMeta.CONDITION_EQ, 5L, 5L, true); + assertConditionMatch(DatabaseLookupMeta.CONDITION_EQ, 5L, 6L, false); + } + + @Test + void getRowFromCache_ConditionNe() throws Exception { + assertConditionMatch(DatabaseLookupMeta.CONDITION_NE, 5L, 6L, true); + assertConditionMatch(DatabaseLookupMeta.CONDITION_NE, 5L, 5L, false); + } + + @Test + void getRowFromCache_ConditionLt() throws Exception { + // cached key < lookup → match when lookup > key + assertConditionMatch(DatabaseLookupMeta.CONDITION_LT, 5L, 6L, true); + assertConditionMatch(DatabaseLookupMeta.CONDITION_LT, 5L, 4L, false); + } + + @Test + void getRowFromCache_ConditionLe() throws Exception { + assertConditionMatch(DatabaseLookupMeta.CONDITION_LE, 5L, 5L, true); + assertConditionMatch(DatabaseLookupMeta.CONDITION_LE, 5L, 6L, true); + assertConditionMatch(DatabaseLookupMeta.CONDITION_LE, 5L, 4L, false); + } + + @Test + void getRowFromCache_ConditionGt() throws Exception { + assertConditionMatch(DatabaseLookupMeta.CONDITION_GT, 5L, 4L, true); + assertConditionMatch(DatabaseLookupMeta.CONDITION_GT, 5L, 5L, false); + } + + @Test + void getRowFromCache_ConditionGe() throws Exception { + assertConditionMatch(DatabaseLookupMeta.CONDITION_GE, 5L, 5L, true); + assertConditionMatch(DatabaseLookupMeta.CONDITION_GE, 5L, 4L, true); + assertConditionMatch(DatabaseLookupMeta.CONDITION_GE, 5L, 6L, false); + } + + @Test + void getRowFromCache_ConditionIsNull() throws Exception { + data.allEquals = false; + data.hasDBCondition = false; + data.conditions = new int[] {DatabaseLookupMeta.CONDITION_IS_NULL}; + DefaultCache cache = DefaultCache.newCache(data, 16); + + Object[] nullKey = new Object[] {null}; + Object[] value = new Object[] {"null-row"}; + cache.storeRowInCache(meta, lookupMeta, nullKey, value); + cache.storeRowInCache(meta, lookupMeta, new Object[] {1L}, new Object[] {"one"}); + + assertArrayEquals(value, cache.getRowFromCache(lookupMeta, new Object[] {999L})); + } + + @Test + void getRowFromCache_ConditionIsNotNull() throws Exception { + data.allEquals = false; + data.hasDBCondition = false; + data.conditions = new int[] {DatabaseLookupMeta.CONDITION_IS_NOT_NULL}; + DefaultCache cache = DefaultCache.newCache(data, 16); + + cache.storeRowInCache(meta, lookupMeta, new Object[] {null}, new Object[] {"null-row"}); + Object[] value = new Object[] {"one"}; + cache.storeRowInCache(meta, lookupMeta, new Object[] {1L}, value); + + assertArrayEquals(value, cache.getRowFromCache(lookupMeta, new Object[] {999L})); + } + + @Test + void getRowFromCache_ConditionBetween() throws Exception { + data.allEquals = false; + data.hasDBCondition = false; + data.conditions = new int[] {DatabaseLookupMeta.CONDITION_BETWEEN}; + data.lookupMeta = new RowMeta(); + data.lookupMeta.addValueMeta(new ValueMetaInteger("from")); + data.lookupMeta.addValueMeta(new ValueMetaInteger("to")); + lookupMeta = data.lookupMeta.clone(); + + // BETWEEN stores a single key column in the map key + IRowMeta storeMeta = new RowMeta(); + storeMeta.addValueMeta(new ValueMetaInteger("id")); + + DefaultCache cache = DefaultCache.newCache(data, 16); + Object[] value = new Object[] {"mid"}; + cache.storeRowInCache(meta, storeMeta, new Object[] {5L}, value); + + IRowMeta lookupWithRange = new RowMeta(); + lookupWithRange.addValueMeta(new ValueMetaInteger("from")); + lookupWithRange.addValueMeta(new ValueMetaInteger("to")); + + assertArrayEquals(value, cache.getRowFromCache(lookupWithRange, new Object[] {1L, 10L})); + assertNull(cache.getRowFromCache(lookupWithRange, new Object[] {6L, 10L})); + } + + @Test + void getRowFromCache_UnknownCondition_SetsHasDbConditionAndStopsMatching() throws Exception { + data.allEquals = false; + data.hasDBCondition = false; + data.conditions = new int[] {DatabaseLookupMeta.CONDITION_LIKE}; + DefaultCache cache = DefaultCache.newCache(data, 16); + cache.storeRowInCache(meta, lookupMeta, new Object[] {1L}, new Object[] {"one"}); + + assertNull(cache.getRowFromCache(lookupMeta, new Object[] {1L})); + assertTrue(data.hasDBCondition); + } + + @Test + void getRowFromCache_HasDbCondition_SkipsScanAndReturnsNull() throws Exception { + data.allEquals = false; + data.hasDBCondition = true; + data.conditions = new int[] {DatabaseLookupMeta.CONDITION_LT}; + DefaultCache cache = DefaultCache.newCache(data, 16); + cache.storeRowInCache(meta, lookupMeta, new Object[] {1L}, new Object[] {"one"}); + + assertNull(cache.getRowFromCache(lookupMeta, new Object[] {10L})); + } + + @Test + void storeRowInCache_DoesNotEvictWhenLoadingAllDataInCache() throws Exception { + meta.setLoadingAllDataInCache(true); + meta.setCacheSize(2); + data.allEquals = true; + data.conditions = new int[] {DatabaseLookupMeta.CONDITION_EQ}; + DefaultCache cache = DefaultCache.newCache(data, 16); + + for (long i = 0; i < 5; i++) { + cache.storeRowInCache(meta, lookupMeta, new Object[] {i}, new Object[] {"v" + i}); + } + + // All entries remain when load-all is enabled + for (long i = 0; i < 5; i++) { + assertNotNull(cache.getRowFromCache(lookupMeta, new Object[] {i})); + } + } + + @Test + void storeRowInCache_EvictsWhenOverCacheSize() throws Exception { + meta.setLoadingAllDataInCache(false); + meta.setCacheSize(5); + data.allEquals = true; + data.conditions = new int[] {DatabaseLookupMeta.CONDITION_EQ}; + DefaultCache cache = DefaultCache.newCache(data, 16); + + for (long i = 0; i < 20; i++) { + cache.storeRowInCache(meta, lookupMeta, new Object[] {i}, new Object[] {"v" + i}); + // small delay so TimedRow timestamps differ enough for eviction sampling + Thread.sleep(2); + } + + int hits = 0; + for (long i = 0; i < 20; i++) { + if (cache.getRowFromCache(lookupMeta, new Object[] {i}) != null) { + hits++; + } + } + // Eviction is approximate (samples ~10%); still fewer than all 20 should remain + assertTrue(hits < 20, "Expected eviction to remove some entries, hits=" + hits); + assertTrue(hits > 0, "Expected some entries to remain"); + } + + private void assertConditionMatch(int condition, long cachedKey, long lookupValue, boolean expect) + throws Exception { + data.allEquals = false; + data.hasDBCondition = false; + data.conditions = new int[] {condition}; + DefaultCache cache = DefaultCache.newCache(data, 16); + + Object[] value = new Object[] {"hit"}; + cache.storeRowInCache(meta, lookupMeta, new Object[] {cachedKey}, value); + + Object[] found = cache.getRowFromCache(lookupMeta, new Object[] {lookupValue}); + if (expect) { + assertArrayEquals(value, found); + } else { + assertNull(found); + } + assertFalse(data.hasDBCondition); + } +} diff --git a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/KeyFieldTest.java b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/KeyFieldTest.java new file mode 100644 index 00000000000..58f5c4b5a9d --- /dev/null +++ b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/KeyFieldTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.databaselookup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; + +class KeyFieldTest { + + @Test + void defaultConstructor_LeavesFieldsNull() { + KeyField key = new KeyField(); + assertNull(key.getStreamField1()); + assertNull(key.getStreamField2()); + assertNull(key.getCondition()); + assertNull(key.getTableField()); + } + + @Test + void allArgsConstructor_SetsFields() { + KeyField key = new KeyField("stream1", "stream2", "=", "table_col"); + assertEquals("stream1", key.getStreamField1()); + assertEquals("stream2", key.getStreamField2()); + assertEquals("=", key.getCondition()); + assertEquals("table_col", key.getTableField()); + } + + @Test + void copyConstructor_CopiesAllFieldsIndependently() { + KeyField original = new KeyField("stream1", "stream2", "=", "table_col"); + KeyField copy = new KeyField(original); + + assertNotSame(original, copy); + assertEquals(original.getStreamField1(), copy.getStreamField1()); + assertEquals(original.getStreamField2(), copy.getStreamField2()); + assertEquals(original.getCondition(), copy.getCondition()); + assertEquals(original.getTableField(), copy.getTableField()); + + copy.setStreamField1("changed"); + copy.setStreamField2("changed2"); + copy.setCondition("<>"); + copy.setTableField("other"); + assertEquals("stream1", original.getStreamField1()); + assertEquals("stream2", original.getStreamField2()); + assertEquals("=", original.getCondition()); + assertEquals("table_col", original.getTableField()); + } + + @Test + void settersAndGetters() { + KeyField key = new KeyField(); + key.setStreamField1("a"); + key.setStreamField2("b"); + key.setCondition("LIKE"); + key.setTableField("c"); + + assertEquals("a", key.getStreamField1()); + assertEquals("b", key.getStreamField2()); + assertEquals("LIKE", key.getCondition()); + assertEquals("c", key.getTableField()); + } +} diff --git a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/LookupTest.java b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/LookupTest.java new file mode 100644 index 00000000000..cea13baf16c --- /dev/null +++ b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/LookupTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.databaselookup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.junit.jupiter.api.Test; + +class LookupTest { + + @Test + void defaultConstructor_StartsEmpty() { + Lookup lookup = new Lookup(); + assertTrue(lookup.getKeyFields().isEmpty()); + assertTrue(lookup.getReturnValues().isEmpty()); + assertFalse(lookup.isFailingOnMultipleResults()); + assertFalse(lookup.isEatingRowOnLookupFailure()); + } + + @Test + void settersAndGetters() { + Lookup lookup = new Lookup(); + lookup.setSchemaName("public"); + lookup.setTableName("users"); + lookup.setOrderByClause("id DESC"); + lookup.setFailingOnMultipleResults(true); + lookup.setEatingRowOnLookupFailure(true); + + assertEquals("public", lookup.getSchemaName()); + assertEquals("users", lookup.getTableName()); + assertEquals("id DESC", lookup.getOrderByClause()); + assertTrue(lookup.isFailingOnMultipleResults()); + assertTrue(lookup.isEatingRowOnLookupFailure()); + } + + @Test + void clone_DeepCopiesKeysAndReturns() { + Lookup original = new Lookup(); + original.setSchemaName("public"); + original.setTableName("users"); + original.setOrderByClause("id DESC"); + original.setFailingOnMultipleResults(true); + original.setEatingRowOnLookupFailure(true); + original.getKeyFields().add(new KeyField("in_id", "", "=", "id")); + original + .getReturnValues() + .add( + new ReturnValue( + "name", + "user_name", + "", + "String", + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE))); + + Lookup clone = original.clone(); + + assertNotSame(original, clone); + assertNotSame(original.getKeyFields(), clone.getKeyFields()); + assertNotSame(original.getReturnValues(), clone.getReturnValues()); + assertEquals("public", clone.getSchemaName()); + assertEquals("users", clone.getTableName()); + assertEquals("id DESC", clone.getOrderByClause()); + assertTrue(clone.isFailingOnMultipleResults()); + assertTrue(clone.isEatingRowOnLookupFailure()); + assertEquals(1, clone.getKeyFields().size()); + assertEquals("in_id", clone.getKeyFields().get(0).getStreamField1()); + assertEquals(1, clone.getReturnValues().size()); + assertEquals("user_name", clone.getReturnValues().get(0).getNewName()); + + clone.getKeyFields().get(0).setStreamField1("changed"); + clone.getReturnValues().get(0).setNewName("changed"); + assertEquals("in_id", original.getKeyFields().get(0).getStreamField1()); + assertEquals("user_name", original.getReturnValues().get(0).getNewName()); + } +} diff --git a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/ReturnValueTest.java b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/ReturnValueTest.java new file mode 100644 index 00000000000..de7c724c74f --- /dev/null +++ b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/ReturnValueTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.databaselookup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.junit.jupiter.api.Test; + +class ReturnValueTest { + + @Test + void defaultConstructor_LeavesFieldsNull() { + ReturnValue value = new ReturnValue(); + assertNull(value.getTableField()); + assertNull(value.getNewName()); + assertNull(value.getDefaultValue()); + assertNull(value.getDefaultType()); + assertNull(value.getTrimType()); + } + + @Test + void allArgsConstructor_SetsFields() { + String trim = ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_BOTH); + ReturnValue value = new ReturnValue("col", "rename", "?", "String", trim); + + assertEquals("col", value.getTableField()); + assertEquals("rename", value.getNewName()); + assertEquals("?", value.getDefaultValue()); + assertEquals("String", value.getDefaultType()); + assertEquals(trim, value.getTrimType()); + } + + @Test + void copyConstructor_CopiesAllFieldsIndependently() { + ReturnValue original = + new ReturnValue( + "col", + "rename", + "?", + "String", + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_BOTH)); + ReturnValue copy = new ReturnValue(original); + + assertNotSame(original, copy); + assertEquals(original.getTableField(), copy.getTableField()); + assertEquals(original.getNewName(), copy.getNewName()); + assertEquals(original.getDefaultValue(), copy.getDefaultValue()); + assertEquals(original.getDefaultType(), copy.getDefaultType()); + assertEquals(original.getTrimType(), copy.getTrimType()); + + copy.setTableField("t2"); + copy.setNewName("other"); + copy.setDefaultValue("x"); + copy.setDefaultType("Integer"); + copy.setTrimType(ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_LEFT)); + assertEquals("col", original.getTableField()); + assertEquals("rename", original.getNewName()); + assertEquals("?", original.getDefaultValue()); + assertEquals("String", original.getDefaultType()); + assertEquals( + ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_BOTH), original.getTrimType()); + } + + @Test + void settersAndGetters() { + ReturnValue value = new ReturnValue(); + value.setTableField("t"); + value.setNewName("n"); + value.setDefaultValue("d"); + value.setDefaultType("Integer"); + value.setTrimType(ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_LEFT)); + + assertEquals("t", value.getTableField()); + assertEquals("n", value.getNewName()); + assertEquals("d", value.getDefaultValue()); + assertEquals("Integer", value.getDefaultType()); + assertEquals(ValueMetaString.getTrimTypeCode(IValueMeta.TRIM_TYPE_LEFT), value.getTrimType()); + } +} diff --git a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/IndexTestBase.java b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/IndexTestBase.java index ab77dcc8951..52ab5e16e1a 100644 --- a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/IndexTestBase.java +++ b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/IndexTestBase.java @@ -24,25 +24,16 @@ import java.util.Arrays; import java.util.BitSet; import java.util.List; +import java.util.stream.Stream; import org.apache.hop.core.row.IValueMeta; import org.apache.hop.core.row.value.ValueMetaInteger; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; /** - * This is a base class for several similar cases. All of them are checking how indexes work with - * the same tuple of data: [0, 1, 2, 2, 3]. Since the data set is known, each subclass show - * implement tests for the following values: - * - *
    - *
  • -1 - *
  • 0 - *
  • 1 - *
  • 2 - *
  • 3 - *
  • 100 - *
+ * Base class for index cases against the shared data set family [0, 1, 2, 2, 3] (sorted, reverse, + * shuffled). Subclasses implement assertions for lookups of -1, 0, 1, 2, 3, and 100. */ public abstract class IndexTestBase { @@ -54,32 +45,43 @@ private static Long[][] toMatrix(long... values) { return result; } + static Stream sampleData() { + return Stream.of(toMatrix(0, 1, 2, 2, 3), toMatrix(3, 2, 2, 1, 0), toMatrix(1, 3, 2, 0, 2)); + } + + /** Retained for subclasses that still reference the old helper. */ static List createSampleData() { - // sorted, reversely sorted, and shuffled data return Arrays.asList( new Object[] {toMatrix(0, 1, 2, 2, 3)}, new Object[] {toMatrix(3, 2, 2, 1, 0)}, new Object[] {toMatrix(1, 3, 2, 0, 2)}); } - final Long[][] rows; private final Class clazz; + Long[][] rows; T index; SearchingContext context; - public IndexTestBase(Class clazz, Long[][] rows) { - this.rows = rows; + public IndexTestBase(Class clazz) { this.clazz = clazz; } - @BeforeEach - void setUp() throws Exception { - index = createIndexInstance(0, new ValueMetaInteger(), 5); + /** + * @deprecated use {@link #IndexTestBase(Class)} + */ + @Deprecated + public IndexTestBase(Class clazz, Long[][] ignoredRows) { + this(clazz); + } + + private void prepare(Long[][] sampleRows) throws Exception { + this.rows = sampleRows; + index = createIndexInstance(0, new ValueMetaInteger(), sampleRows.length); index.performIndexingOf(rows); context = new SearchingContext(); - context.init(5); + context.init(sampleRows.length); } T createIndexInstance(int column, IValueMeta meta, int rowsAmount) throws Exception { @@ -92,6 +94,7 @@ T createIndexInstance(int column, IValueMeta meta, int rowsAmount) throws Except void tearDown() { index = null; context = null; + rows = null; } void testFindsNothing(long value) { @@ -124,21 +127,63 @@ void testFindsCorrectly(long lookupValue, int expectedAmount) { abstract void doAssertMatches(BitSet candidates, long lookupValue, long actualValue); - @Test abstract void lookupFor_MinusOne(); - @Test abstract void lookupFor_Zero(); - @Test abstract void lookupFor_One(); - @Test abstract void lookupFor_Two(); - @Test abstract void lookupFor_Three(); - @Test abstract void lookupFor_Hundred(); + + @ParameterizedTest + @MethodSource( + "org.apache.hop.pipeline.transforms.databaselookup.readallcache.IndexTestBase#sampleData") + void runsLookupFor_MinusOne(Long[][] sampleRows) throws Exception { + prepare(sampleRows); + lookupFor_MinusOne(); + } + + @ParameterizedTest + @MethodSource( + "org.apache.hop.pipeline.transforms.databaselookup.readallcache.IndexTestBase#sampleData") + void runsLookupFor_Zero(Long[][] sampleRows) throws Exception { + prepare(sampleRows); + lookupFor_Zero(); + } + + @ParameterizedTest + @MethodSource( + "org.apache.hop.pipeline.transforms.databaselookup.readallcache.IndexTestBase#sampleData") + void runsLookupFor_One(Long[][] sampleRows) throws Exception { + prepare(sampleRows); + lookupFor_One(); + } + + @ParameterizedTest + @MethodSource( + "org.apache.hop.pipeline.transforms.databaselookup.readallcache.IndexTestBase#sampleData") + void runsLookupFor_Two(Long[][] sampleRows) throws Exception { + prepare(sampleRows); + lookupFor_Two(); + } + + @ParameterizedTest + @MethodSource( + "org.apache.hop.pipeline.transforms.databaselookup.readallcache.IndexTestBase#sampleData") + void runsLookupFor_Three(Long[][] sampleRows) throws Exception { + prepare(sampleRows); + lookupFor_Three(); + } + + @ParameterizedTest + @MethodSource( + "org.apache.hop.pipeline.transforms.databaselookup.readallcache.IndexTestBase#sampleData") + void runsLookupFor_Hundred(Long[][] sampleRows) throws Exception { + prepare(sampleRows); + lookupFor_Hundred(); + } } diff --git a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/ReadAllCacheTest.java b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/ReadAllCacheTest.java index bfb7917a16d..5844cb16510 100644 --- a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/ReadAllCacheTest.java +++ b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/ReadAllCacheTest.java @@ -199,4 +199,13 @@ void lookup_HandlesAbsenceOfLookupValue() throws Exception { Object[] found = cache.getRowFromCache(new RowMeta(), new Object[0]); assertArrayEquals(new Object[] {"one"}, found, "(keys[1] == 1L) --> row 2"); } + + @Test + void builder_BuildsEmptyCacheWhenNoRowsAdded() throws Exception { + transformData.conditions = new int[] {DatabaseLookupMeta.CONDITION_EQ}; + ReadAllCache.Builder builder = new ReadAllCache.Builder(transformData, 0); + builder.setKeysMeta(keysMeta); + ReadAllCache cache = builder.build(); + assertNull(cache.getRowFromCache(keysMeta.clone(), keys[0])); + } } diff --git a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/SearchingContextTest.java b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/SearchingContextTest.java index fc8468765cf..1634afe92fb 100644 --- a/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/SearchingContextTest.java +++ b/plugins/transforms/databaselookup/src/test/java/org/apache/hop/pipeline/transforms/databaselookup/readallcache/SearchingContextTest.java @@ -50,4 +50,32 @@ void intersectionDetectsBecomingEmpty() { ctx.intersect(set, false); assertTrue(ctx.isEmpty(), "Expected to be empty"); } + + @Test + void intersect_InverseModeRemovesCandidates() { + SearchingContext ctx = new SearchingContext(); + ctx.init(4); + + BitSet remove = ctx.getWorkingSet(); + remove.set(0); + remove.set(2); + ctx.intersect(remove, true); + + assertFalse(ctx.isEmpty()); + BitSet candidates = ctx.getCandidates(); + assertFalse(candidates.get(0)); + assertTrue(candidates.get(1)); + assertFalse(candidates.get(2)); + assertTrue(candidates.get(3)); + } + + @Test + void setEmpty_MarksContextEmpty() { + SearchingContext ctx = new SearchingContext(); + ctx.init(3); + assertFalse(ctx.isEmpty()); + + ctx.setEmpty(); + assertTrue(ctx.isEmpty()); + } }