From fbf2cc77834211728118f1d7401cb73afad48ee8 Mon Sep 17 00:00:00 2001 From: sashagavrilov Date: Tue, 28 Oct 2025 18:58:44 -0700 Subject: [PATCH] Fix fastserde to support classes generated by our own avro-builder --- .../avro/fastserde/FastSerdeCacheTest.java | 7 + .../test/ownAvro/TestOwnCodegenRecord.avsc | 398 ++++++++++++++++++ .../avro-fastserde-tests110/build.gradle | 33 +- .../avro-fastserde-tests111/build.gradle | 30 +- fastserde/avro-fastserde-tests14/build.gradle | 33 +- fastserde/avro-fastserde-tests15/build.gradle | 30 +- fastserde/avro-fastserde-tests16/build.gradle | 30 +- fastserde/avro-fastserde-tests17/build.gradle | 30 +- fastserde/avro-fastserde-tests18/build.gradle | 30 +- fastserde/avro-fastserde-tests19/build.gradle | 30 +- .../avro/fastserde/FastSerdeBase.java | 10 +- 11 files changed, 651 insertions(+), 10 deletions(-) create mode 100644 fastserde/avro-fastserde-tests-common/src/test/ownAvro/TestOwnCodegenRecord.avsc diff --git a/fastserde/avro-fastserde-tests-common/src/test/java/com/linkedin/avro/fastserde/FastSerdeCacheTest.java b/fastserde/avro-fastserde-tests-common/src/test/java/com/linkedin/avro/fastserde/FastSerdeCacheTest.java index 6a72890c6..bd4da01f8 100644 --- a/fastserde/avro-fastserde-tests-common/src/test/java/com/linkedin/avro/fastserde/FastSerdeCacheTest.java +++ b/fastserde/avro-fastserde-tests-common/src/test/java/com/linkedin/avro/fastserde/FastSerdeCacheTest.java @@ -2,6 +2,7 @@ import com.linkedin.avro.fastserde.generated.avro.SimpleRecord; import com.linkedin.avro.fastserde.generated.avro.TestRecord; +import com.linkedin.avro.fastserde.generated.ownavro.TestOwnCodegenRecord; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import java.io.ByteArrayOutputStream; @@ -81,6 +82,12 @@ public void testBuildFastSpecificDeserializerWithCorrectClasspath() { cache.buildFastSpecificDeserializer(TestRecord.SCHEMA$, TestRecord.SCHEMA$); } + @Test(groups = "deserializationTest") + public void testBuildFastSpecificDeserializerSupportsOurOwnGenerator() { + FastSerdeCache cache = FastSerdeCache.getDefaultInstance(); + cache.buildFastSpecificDeserializer(TestOwnCodegenRecord.SCHEMA$, TestOwnCodegenRecord.SCHEMA$); + } + @Test(groups = "serializationTest", timeOut = 5_000L, expectedExceptions = UnsupportedOperationException.class, expectedExceptionsMessageRegExp = "Fast specific serializer could not be generated.") diff --git a/fastserde/avro-fastserde-tests-common/src/test/ownAvro/TestOwnCodegenRecord.avsc b/fastserde/avro-fastserde-tests-common/src/test/ownAvro/TestOwnCodegenRecord.avsc new file mode 100644 index 000000000..2b21cb23f --- /dev/null +++ b/fastserde/avro-fastserde-tests-common/src/test/ownAvro/TestOwnCodegenRecord.avsc @@ -0,0 +1,398 @@ +{ + "type": "record", + "name": "TestOwnCodegenRecord", + "namespace": "com.linkedin.avro.fastserde.generated.ownavro", + "doc": "record used to test own codegen", + "fields": [ + { + "name": "testInt", + "type": "int", + "default": 42 + }, + { + "name": "testIntUnion", + "type": [ + "null", + "int" + ], + "default": null + }, + { + "name": "testLong", + "type": "long", + "default": 9223372036854775807 + }, + { + "name": "testLongUnion", + "type": [ + "null", + "long" + ], + "default": null + }, + { + "name": "testDouble", + "type": "double", + "default": 3.14 + }, + { + "name": "testDoubleUnion", + "type": [ + "null", + "double" + ], + "default": null + }, + { + "name": "testFloat", + "type": "float", + "default": 3.14 + }, + { + "name": "testFloatUnion", + "type": [ + "null", + "float" + ], + "default": null + }, + { + "name": "testBoolean", + "type": "boolean", + "default": true + }, + { + "name": "testBooleanUnion", + "type": [ + "null", + "boolean" + ], + "default": null + }, + { + "name": "testBytes", + "type": "bytes", + "default": "1234" + }, + { + "name": "testBytesUnion", + "type": [ + "null", + "bytes" + ], + "default": null + }, + { + "name": "testString", + "type": "string", + "default": "testStringValue" + }, + { + "name": "testStringable", + "type": { "type": "string", "java-class": "java.net.URL" }, + "default": "http://www.asdaldjaldladjal.sadjad" + }, + { + "name": "testStringUnion", + "type": [ + "null", + "string" + ], + "default": null, + "aliases": [ + "testStringAlias" + ] + }, + { + "name": "testFixed", + "type": { + "type": "fixed", + "name": "DefaultsFixed", + "size": 1 + }, + "default": "5" + }, + { + "name": "testFixedUnion", + "type": [ + "null", + "DefaultsFixed" + ], + "default": null + }, + { + "name": "testFixedArray", + "type": { + "type": "array", + "items": "DefaultsFixed" + }, + "default": ["6"] + }, + { + "name": "testFixedUnionArray", + "type": { + "type": "array", + "items": [ + "null", + "DefaultsFixed" + ] + }, + "default": [null] + }, + { + "name": "testEnum", + "type": { + "type": "enum", + "name": "DefaultsEnum", + "symbols": [ + "A", + "B", + "C", + "D", + "E" + ] + }, + "default": "C" + }, + { + "name": "testEnumUnion", + "type": [ + "null", + "DefaultsEnum" + ], + "default": null + }, + { + "name": "testEnumArray", + "type": { + "type": "array", + "items": "DefaultsEnum" + }, + "default": ["E", "B"] + }, + { + "name": "testEnumUnionArray", + "type": { + "type": "array", + "items": [ + "null", + "DefaultsEnum" + ] + }, + "default": [null] + }, + { + "name": "testNewEnumIntUnionArray", + "type": { + "type": "array", + "items": { + "type": "array", + "items": [ + { + "type": "enum", + "name": "DefaultsNewEnum", + "symbols": [ + "A", + "B" + ] + }, + "int" + ] + } + }, + "default": [["B"]] + }, + { + "name": "oldSubRecord", + "type": { + "type": "record", + "name": "OldSubRecord", + "fields": [ + { + "name": "oldSubField", + "type": "string" + } + ] + } + }, + { + "name": "newFieldWithOldSubRecord", + "type": "OldSubRecord", + "default": {"oldSubField": "defaultOldSubField" } + }, + { + "name": "subRecordUnion", + "type": [ + "null", + { + "type": "record", + "name": "DefaultsSubRecord", + "fields": [ + { + "name": "subField", + "type": "string" + }, + { + "name": "anotherField", + "type": [ + "null", + "string" + ], + "default": null + }, + { + "name": "arrayField", + "type": { + "type": "array", + "items": "DefaultsEnum" + }, + "default": [] + } + ] + } + ], + "default": null + }, + { + "name": "subRecord", + "type": "DefaultsSubRecord", + "default": { + "subField": "valueOfSubField", + "anotherField": null, + "arrayField": ["A"] + } + }, + { + "name": "recordArray", + "type": { + "type": "array", + "items": "DefaultsSubRecord" + }, + "default": [{ + "subField": "recordArrayValue", + "anotherField": null, + "arrayField": ["A"] + }] + }, + { + "name": "recordUnionArray", + "type": { + "type": "array", + "items": [ + "null", + "DefaultsSubRecord" + ] + }, + "default": [null] + }, + { + "name": "stringableMap", + "type": { + "type": "map", + "java-key-class": "java.net.URL", + "values": { "type": "string", "java-class": "java.math.BigInteger" } + }, + "default": {"http://www.asdaldjaldladjal.sadjad2": "123"} + }, + { + "name": "recordMap", + "type": { + "type": "map", + "values": "DefaultsSubRecord" + }, + "default": {"test": { + "subField": "recordMapValue", + "anotherField": null, + "arrayField": ["A"] + }} + }, + { + "name": "recordUnionMap", + "type": + { + "type": "map", + "values": [ + "null", + "DefaultsSubRecord" + ] + }, + "default": {"test": null} + }, + { + "name": "recordUnionMapArray", + "type": { + "type": "array", + "items": { + "type": "map", + "values": [ + "null", + "DefaultsSubRecord" + ] + } + }, + "default": [{"test": null}] + }, + { + "name": "recordUnionArrayMap", + "type": { + "type": "map", + "values": { + "type": "array", + "items": [ + "null", + "DefaultsSubRecord" + ] + } + }, + "default": {"test": [null]} + }, + { + "name": "booleanArray", + "type": { + "type": "array", + "items": "boolean" + }, + "default": [] + }, + { + "name": "doubleArray", + "type": { + "type": "array", + "items": "double" + }, + "default": [] + }, + { + "name": "floatArray", + "type": { + "type": "array", + "items": "float" + }, + "default": [] + }, + { + "name": "intArray", + "type": { + "type": "array", + "items": "int" + }, + "default": [] + }, + { + "name": "longArray", + "type": { + "type": "array", + "items": "long" + }, + "default": [] + }, + { + "name": "stringArray", + "type": { + "type": "array", + "items": "string" + }, + "default": [] + } + ] +} diff --git a/fastserde/avro-fastserde-tests110/build.gradle b/fastserde/avro-fastserde-tests110/build.gradle index a07ab584f..d08a6d7fd 100644 --- a/fastserde/avro-fastserde-tests110/build.gradle +++ b/fastserde/avro-fastserde-tests110/build.gradle @@ -11,6 +11,7 @@ plugins { configurations { codegen + ownCodegen } sourceSets { @@ -18,6 +19,7 @@ sourceSets { java { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/java" srcDir "$buildDir/generated/avro" + srcDir "$buildDir/generated/ownAvro" } resources { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/resources" @@ -27,6 +29,7 @@ sourceSets { java { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/java" srcDir "$buildDir/generated/avro" + srcDir "$buildDir/generated/ownAvro" } resources { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/resources" @@ -67,6 +70,8 @@ dependencies { codegen project(":helper:helper") codegen "org.apache.avro:avro-tools:1.10.2" codegen "org.apache.avro:avro-compiler:1.10.2" + + ownCodegen project(":avro-builder:builder") } test { @@ -100,5 +105,29 @@ task runVanillaAvroCodegen { } } -compileTestJava.dependsOn runVanillaAvroCodegen -compileTestFixturesJava.dependsOn runVanillaAvroCodegen +task runOwnAvroCodegen { + description = 'generate specific classes using own codegen utility' + + dependsOn configurations.ownCodegen + // define input and output files so we can have incremental build when nothing changes + inputs.dir("$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro") + outputs.dir("$buildDir/generated/ownAvro") + + fileTree(dir: "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro", include:'**/*.avsc').each { file -> + doLast { + javaexec { + classpath = configurations.ownCodegen + main = 'com.linkedin.avroutil1.builder.SchemaBuilder' + args = [ + "--input", file.getAbsolutePath(), + "--output", "$buildDir/generated/ownAvro", + "--generator", "AVRO_UTIL", + "--minAvroVer", "1.10" + ] + } + } + } +} + +compileTestJava.dependsOn runVanillaAvroCodegen, runOwnAvroCodegen +compileTestFixturesJava.dependsOn runVanillaAvroCodegen, runOwnAvroCodegen diff --git a/fastserde/avro-fastserde-tests111/build.gradle b/fastserde/avro-fastserde-tests111/build.gradle index c30c4078f..8138c2b01 100644 --- a/fastserde/avro-fastserde-tests111/build.gradle +++ b/fastserde/avro-fastserde-tests111/build.gradle @@ -10,6 +10,7 @@ plugins { configurations { codegen + ownCodegen } sourceSets { @@ -17,6 +18,7 @@ sourceSets { java { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/java" srcDir "$buildDir/generated/avro" + srcDir "$buildDir/generated/ownAvro" } resources { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/resources" @@ -57,6 +59,8 @@ dependencies { } codegen "org.apache.avro:trevni-core:1.11.4" codegen "org.apache.avro:avro-compiler:1.11.4" + + ownCodegen project(":avro-builder:builder") } test { @@ -90,4 +94,28 @@ task runVanillaAvroCodegen { } } -compileTestJava.dependsOn runVanillaAvroCodegen +task runOwnAvroCodegen { + description = 'generate specific classes using own codegen utility' + + dependsOn configurations.ownCodegen + // define input and output files so we can have incremental build when nothing changes + inputs.dir("$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro") + outputs.dir("$buildDir/generated/ownAvro") + + fileTree(dir: "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro", include:'**/*.avsc').each { file -> + doLast { + javaexec { + classpath = configurations.ownCodegen + main = 'com.linkedin.avroutil1.builder.SchemaBuilder' + args = [ + "--input", file.getAbsolutePath(), + "--output", "$buildDir/generated/ownAvro", + "--generator", "AVRO_UTIL", + "--minAvroVer", "1.11" + ] + } + } + } +} + +compileTestJava.dependsOn runVanillaAvroCodegen, runOwnAvroCodegen diff --git a/fastserde/avro-fastserde-tests14/build.gradle b/fastserde/avro-fastserde-tests14/build.gradle index 4e6de180b..8d77fb4ae 100644 --- a/fastserde/avro-fastserde-tests14/build.gradle +++ b/fastserde/avro-fastserde-tests14/build.gradle @@ -8,11 +8,16 @@ plugins { id "java-library" } +configurations { + ownCodegen +} + sourceSets { test { java { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/java" srcDir "$buildDir/generated/avro" + srcDir "$buildDir/generated/ownAvro" exclude "com/linkedin/avro/fastserde/logical/types/**" // LogicalType available since version 1.8 } resources { @@ -52,6 +57,8 @@ dependencies { testImplementation 'org.testng:testng:6.14.3' testImplementation 'org.slf4j:slf4j-simple:1.7.14' + + ownCodegen project(":avro-builder:builder") } test { @@ -91,4 +98,28 @@ task runVanillaAvroCodegen { } } -compileTestJava.dependsOn runVanillaAvroCodegen +task runOwnAvroCodegen { + description = 'generate specific classes using own codegen utility' + + dependsOn configurations.ownCodegen + // define input and output files so we can have incremental build when nothing changes + inputs.dir("$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro") + outputs.dir("$buildDir/generated/ownAvro") + + fileTree(dir: "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro", include:'**/*.avsc').each { file -> + doLast { + javaexec { + classpath = configurations.ownCodegen + main = 'com.linkedin.avroutil1.builder.SchemaBuilder' + args = [ + "--input", file.getAbsolutePath(), + "--output", "$buildDir/generated/ownAvro", + "--generator", "AVRO_UTIL", + "--minAvroVer", "1.4" + ] + } + } + } +} + +compileTestJava.dependsOn runVanillaAvroCodegen, runOwnAvroCodegen diff --git a/fastserde/avro-fastserde-tests15/build.gradle b/fastserde/avro-fastserde-tests15/build.gradle index 82295e243..4cb301826 100644 --- a/fastserde/avro-fastserde-tests15/build.gradle +++ b/fastserde/avro-fastserde-tests15/build.gradle @@ -10,6 +10,7 @@ plugins { configurations { codegen + ownCodegen } sourceSets { @@ -17,6 +18,7 @@ sourceSets { java { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/java" srcDir "$buildDir/generated/avro" + srcDir "$buildDir/generated/ownAvro" exclude "com/linkedin/avro/fastserde/logical/types/**" // LogicalType available since version 1.8 } resources { @@ -47,6 +49,8 @@ dependencies { codegen project(":helper:helper") codegen "org.apache.avro:avro-tools:1.5.4" codegen "org.apache.avro:avro-compiler:1.5.4" + + ownCodegen project(":avro-builder:builder") } test { @@ -80,4 +84,28 @@ task runVanillaAvroCodegen { } } -compileTestJava.dependsOn runVanillaAvroCodegen +task runOwnAvroCodegen { + description = 'generate specific classes using own codegen utility' + + dependsOn configurations.ownCodegen + // define input and output files so we can have incremental build when nothing changes + inputs.dir("$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro") + outputs.dir("$buildDir/generated/ownAvro") + + fileTree(dir: "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro", include:'**/*.avsc').each { file -> + doLast { + javaexec { + classpath = configurations.ownCodegen + main = 'com.linkedin.avroutil1.builder.SchemaBuilder' + args = [ + "--input", file.getAbsolutePath(), + "--output", "$buildDir/generated/ownAvro", + "--generator", "AVRO_UTIL", + "--minAvroVer", "1.5" + ] + } + } + } +} + +compileTestJava.dependsOn runVanillaAvroCodegen, runOwnAvroCodegen diff --git a/fastserde/avro-fastserde-tests16/build.gradle b/fastserde/avro-fastserde-tests16/build.gradle index df332f839..403a8a36c 100644 --- a/fastserde/avro-fastserde-tests16/build.gradle +++ b/fastserde/avro-fastserde-tests16/build.gradle @@ -10,6 +10,7 @@ plugins { configurations { codegen + ownCodegen } sourceSets { @@ -17,6 +18,7 @@ sourceSets { java { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/java" srcDir "$buildDir/generated/avro" + srcDir "$buildDir/generated/ownAvro" exclude "com/linkedin/avro/fastserde/logical/types/**" // LogicalType available since version 1.8 } resources { @@ -47,6 +49,8 @@ dependencies { codegen project(":helper:helper") codegen "org.apache.avro:avro-tools:1.6.3" codegen "org.apache.avro:avro-compiler:1.6.3" + + ownCodegen project(":avro-builder:builder") } test { @@ -80,4 +84,28 @@ task runVanillaAvroCodegen { } } -compileTestJava.dependsOn runVanillaAvroCodegen +task runOwnAvroCodegen { + description = 'generate specific classes using own codegen utility' + + dependsOn configurations.ownCodegen + // define input and output files so we can have incremental build when nothing changes + inputs.dir("$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro") + outputs.dir("$buildDir/generated/ownAvro") + + fileTree(dir: "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro", include:'**/*.avsc').each { file -> + doLast { + javaexec { + classpath = configurations.ownCodegen + main = 'com.linkedin.avroutil1.builder.SchemaBuilder' + args = [ + "--input", file.getAbsolutePath(), + "--output", "$buildDir/generated/ownAvro", + "--generator", "AVRO_UTIL", + "--minAvroVer", "1.6" + ] + } + } + } +} + +compileTestJava.dependsOn runVanillaAvroCodegen, runOwnAvroCodegen diff --git a/fastserde/avro-fastserde-tests17/build.gradle b/fastserde/avro-fastserde-tests17/build.gradle index 5e825c5ee..c4a0b5cd1 100644 --- a/fastserde/avro-fastserde-tests17/build.gradle +++ b/fastserde/avro-fastserde-tests17/build.gradle @@ -10,6 +10,7 @@ plugins { configurations { codegen + ownCodegen } sourceSets { @@ -17,6 +18,7 @@ sourceSets { java { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/java" srcDir "$buildDir/generated/avro" + srcDir "$buildDir/generated/ownAvro" exclude "com/linkedin/avro/fastserde/logical/types/**" // LogicalType available since version 1.8 } resources { @@ -47,6 +49,8 @@ dependencies { codegen project(":helper:helper") codegen "org.apache.avro:avro-tools:1.7.7" codegen "org.apache.avro:avro-compiler:1.7.7" + + ownCodegen project(":avro-builder:builder") } test { @@ -80,4 +84,28 @@ task runVanillaAvroCodegen { } } -compileTestJava.dependsOn runVanillaAvroCodegen +task runOwnAvroCodegen { + description = 'generate specific classes using own codegen utility' + + dependsOn configurations.ownCodegen + // define input and output files so we can have incremental build when nothing changes + inputs.dir("$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro") + outputs.dir("$buildDir/generated/ownAvro") + + fileTree(dir: "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro", include:'**/*.avsc').each { file -> + doLast { + javaexec { + classpath = configurations.ownCodegen + main = 'com.linkedin.avroutil1.builder.SchemaBuilder' + args = [ + "--input", file.getAbsolutePath(), + "--output", "$buildDir/generated/ownAvro", + "--generator", "AVRO_UTIL", + "--minAvroVer", "1.7" + ] + } + } + } +} + +compileTestJava.dependsOn runVanillaAvroCodegen, runOwnAvroCodegen diff --git a/fastserde/avro-fastserde-tests18/build.gradle b/fastserde/avro-fastserde-tests18/build.gradle index 36c06e1de..4a2446e80 100644 --- a/fastserde/avro-fastserde-tests18/build.gradle +++ b/fastserde/avro-fastserde-tests18/build.gradle @@ -10,6 +10,7 @@ plugins { configurations { codegen + ownCodegen } sourceSets { @@ -17,6 +18,7 @@ sourceSets { java { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/java" srcDir "$buildDir/generated/avro" + srcDir "$buildDir/generated/ownAvro" exclude "com/linkedin/avro/fastserde/logical/types/**" // due to joda.time instead of java.time } resources { @@ -48,6 +50,8 @@ dependencies { codegen project(":helper:helper") codegen "org.apache.avro:avro-tools:1.8.2" codegen "org.apache.avro:avro-compiler:1.8.2" + + ownCodegen project(":avro-builder:builder") } test { @@ -81,4 +85,28 @@ task runVanillaAvroCodegen { } } -compileTestJava.dependsOn runVanillaAvroCodegen +task runOwnAvroCodegen { + description = 'generate specific classes using own codegen utility' + + dependsOn configurations.ownCodegen + // define input and output files so we can have incremental build when nothing changes + inputs.dir("$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro") + outputs.dir("$buildDir/generated/ownAvro") + + fileTree(dir: "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro", include:'**/*.avsc').each { file -> + doLast { + javaexec { + classpath = configurations.ownCodegen + main = 'com.linkedin.avroutil1.builder.SchemaBuilder' + args = [ + "--input", file.getAbsolutePath(), + "--output", "$buildDir/generated/ownAvro", + "--generator", "AVRO_UTIL", + "--minAvroVer", "1.8" + ] + } + } + } +} + +compileTestJava.dependsOn runVanillaAvroCodegen, runOwnAvroCodegen diff --git a/fastserde/avro-fastserde-tests19/build.gradle b/fastserde/avro-fastserde-tests19/build.gradle index c4414f122..6991f2b3f 100644 --- a/fastserde/avro-fastserde-tests19/build.gradle +++ b/fastserde/avro-fastserde-tests19/build.gradle @@ -10,6 +10,7 @@ plugins { configurations { codegen + ownCodegen } sourceSets { @@ -17,6 +18,7 @@ sourceSets { java { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/java" srcDir "$buildDir/generated/avro" + srcDir "$buildDir/generated/ownAvro" } resources { srcDir "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/resources" @@ -46,6 +48,8 @@ dependencies { codegen project(":helper:helper") codegen "org.apache.avro:avro-tools:1.9.2" codegen "org.apache.avro:avro-compiler:1.9.2" + + ownCodegen project(":avro-builder:builder") } test { @@ -79,4 +83,28 @@ task runVanillaAvroCodegen { } } -compileTestJava.dependsOn runVanillaAvroCodegen +task runOwnAvroCodegen { + description = 'generate specific classes using own codegen utility' + + dependsOn configurations.ownCodegen + // define input and output files so we can have incremental build when nothing changes + inputs.dir("$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro") + outputs.dir("$buildDir/generated/ownAvro") + + fileTree(dir: "$rootProject.projectDir/fastserde/avro-fastserde-tests-common/src/test/ownAvro", include:'**/*.avsc').each { file -> + doLast { + javaexec { + classpath = configurations.ownCodegen + main = 'com.linkedin.avroutil1.builder.SchemaBuilder' + args = [ + "--input", file.getAbsolutePath(), + "--output", "$buildDir/generated/ownAvro", + "--generator", "AVRO_UTIL", + "--minAvroVer", "1.4" + ] + } + } + } +} + +compileTestJava.dependsOn runVanillaAvroCodegen, runOwnAvroCodegen diff --git a/fastserde/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeBase.java b/fastserde/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeBase.java index bf1dde9c3..34b1dea89 100644 --- a/fastserde/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeBase.java +++ b/fastserde/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastSerdeBase.java @@ -1,6 +1,8 @@ package com.linkedin.avro.fastserde; import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelperCommon; +import com.linkedin.avroutil1.compatibility.CustomDecoder; +import com.linkedin.avroutil1.compatibility.backports.SpecificRecordBaseExt; import com.sun.codemodel.JBlock; import com.sun.codemodel.JCodeModel; import com.sun.codemodel.JConditional; @@ -174,7 +176,13 @@ protected Class compileClass(final String className, Set knownUsedFullyQ throw new FastSerdeGeneratorException("Couldn't locate java compiler at runtime, please double check your env " + "setting for 'JAVA_HOME', and here is the value for 'System.getProperty(\"java.home\")': " + System.getProperty("java.home")); } - String compileClassPathForCurrentFile = Utils.inferCompileDependencies(compileClassPath, filePath, knownUsedFullyQualifiedClassNameSet); + Set knownClassNameSet = new HashSet<>(knownUsedFullyQualifiedClassNameSet); + + // Add classes that are used in generated code but may not be directly referenced in the schema + knownClassNameSet.add(SpecificRecordBaseExt.class.getName()); + knownClassNameSet.add(CustomDecoder.class.getName()); + + String compileClassPathForCurrentFile = Utils.inferCompileDependencies(compileClassPath, filePath, knownClassNameSet); int compileResult; try { /*